import requests
from bs4 import BeautifulSoup
import csv
import time
import random

BASE_URL = "https://www.cardumps.net"
START_URL = BASE_URL + "/ecu-files"
OUTPUT_FILE = "cardumps_full.csv"

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                  "AppleWebKit/537.36 (KHTML, like Gecko) "
                  "Chrome/128.0.0.0 Safari/537.36"
}

# exceptions pour Volvo et Vauxhall
SPECIAL_CASES = {
    "vauxhall": "/ecu-files/Vauxhall",
    "volvo": "/ecu-files/volvo"
}

def get_soup(url):
    """Télécharge une page et retourne un BeautifulSoup."""
    time.sleep(random.uniform(1.5, 3.0))  # pause anti-ban
    r = requests.get(url, headers=HEADERS)
    if r.status_code != 200:
        print(f"⚠️ Erreur {r.status_code} sur {url}")
        return None
    return BeautifulSoup(r.text, "html.parser")

def scrape_brand(brand_url, brand_name, writer):
    """Scrape toutes les pages d'une marque donnée."""
    page = 1
    while True:
        # ⚠️ La première page est sans paramètre
        if page == 1:
            url = f"{BASE_URL}{brand_url}"
        else:
            url = f"{BASE_URL}{brand_url}?page={page}"

        soup = get_soup(url)
        if not soup:
            break

        table = soup.select_one("#cardumps-table tbody")
        if not table:
            break

        rows = table.find_all("tr")
        if not rows:
            break

        for row in rows:
            cols = [c.get_text(" ", strip=True) for c in row.find_all("td")]
            if len(cols) >= 8:  # Car, Engine, ECU, HW, SW, Read, Other, Download
                writer.writerow([brand_name] + cols)

        # pagination : vérifie si bouton "Next" est actif
        next_button = soup.select_one("ul.pagination li.next:not(.disabled)")
        if next_button:
            page += 1
        else:
            break


def scrape_all():
    with open(OUTPUT_FILE, "w", newline="", encoding="utf-8-sig") as f:
        # séparateur tabulation = identique à un copier-coller HTML
        writer = csv.writer(f, delimiter="\t", quoting=csv.QUOTE_ALL)
        writer.writerow([
            "Brand", "Car", "Engine", "ECU", "HW", "SW",
            "Read", "Other", "Download"
        ])

        soup = get_soup(START_URL)
        if not soup:
            return

        brand_links = soup.select(".plist a")
        print(f"✅ {len(brand_links)} marques trouvées")

        for a in brand_links:
            brand_url = a["href"]
            brand_name = brand_url.strip("/").split("/")[-1]

            # appliquer correctif si nécessaire
            if brand_name.lower() in SPECIAL_CASES:
                brand_url = SPECIAL_CASES[brand_name.lower()]

            print(f"\n▶ Marque: {brand_name}")
            scrape_brand(brand_url, brand_name, writer)

    print(f"\n✅ Scraping terminé, résultats dans {OUTPUT_FILE}")


if __name__ == "__main__":
    scrape_all()