import requests
from bs4 import BeautifulSoup
import csv
import time
import random

BASE_URL = "https://www.mod-files.com/original-file/cars"
OUTPUT_FILE = "modfiles_all.csv"

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                  "AppleWebKit/537.36 (KHTML, like Gecko) "
                  "Chrome/128.0.0.0 Safari/537.36"
}

# Marques actives (pas disabled dans <select>)
BRANDS = [
    "abarth","acura","alfa-romeo","alpina","alpine","aston-martin","audi",
    "baic","bentley","bmw","borgward","buick","cadillac","catheram","changan",
    "chery","chevrolet","chrysler","citroen","cupra","dacia","dodge","dongfeng",
    "ds","ferrari","fiat","ford","genesis","gmc","gwm","holden","honda","hummer",
    "hyundai","infiniti","isuzu","iveco","jac","jaguar","kia","ktm","lada",
    "lamborghini","lancia","land-rover","lexus","lincoln","lotus","lynk-co",
    "mahindra","man","maserati","maxus","mazda","mercedes-benz","mercury","mg",
    "mini","mitsubishi","nissan","opel","piaggio","porsche","renault","roewe",
    "rover","saab","samsung","seat","smart","ssangyong","subaru","suzuki",
    "toyota","vauxhall","volkswagen","volvo"
]

def get_soup(url):
    """Télécharge une page et retourne BeautifulSoup"""
    time.sleep(random.uniform(1, 2))  # pause anti-ban
    r = requests.get(url, headers=HEADERS)
    if r.status_code != 200:
        print(f"⚠️ Erreur {r.status_code} → {url}")
        return None
    return BeautifulSoup(r.text, "html.parser")

def get_last_page(soup):
    """Retourne le numéro de la dernière page depuis la pagination"""
    pagis = soup.select("ul.pagination li[p]")
    if not pagis:
        return 1
    try:
        return int(pagis[-1]["p"])
    except:
        return 1

def scrape_brand(brand, writer):
    """Scrape toutes les pages pour une marque donnée"""
    url = f"{BASE_URL}/{brand}/_/_/_/_/"
    soup = get_soup(url)
    if not soup:
        return
    last_page = get_last_page(soup)

    for page in range(1, last_page + 1):
        page_url = f"{url}?p={page}" if page > 1 else url
        print(f"▶ {brand} - page {page}/{last_page}")
        soup = get_soup(page_url)
        if not soup:
            break

        table = soup.select_one("table")
        if not table:
            break

        rows = table.select("tr")[1:]  # skip header
        for row in rows:
            cols = [c.get_text(" ", strip=True) for c in row.find_all("td")]
            if len(cols) >= 6:
                writer.writerow([brand] + cols[:6])

def scrape_all():
    with open(OUTPUT_FILE, "w", newline="", encoding="utf-8-sig") as f:
        writer = csv.writer(f, delimiter=";")
        writer.writerow(["Brand", "Model", "Engine", "ECU", "HW", "SW"])

        print(f"✅ {len(BRANDS)} marques à scraper")
        for brand in BRANDS:
            scrape_brand(brand, writer)

    print(f"\n✅ Scraping terminé → {OUTPUT_FILE}")

if __name__ == "__main__":
    scrape_all()