import requests
import csv
import time
import random

BASE_URL = "https://www.dyno-chiptuningfiles.com/chiptuning-file/"
API_URL = "https://www.dyno-chiptuningfiles.com/api/type-loader"
OUTPUT_FILE = "dyno_full_scraping.csv"

session = requests.Session()
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                  "AppleWebKit/537.36 (KHTML, like Gecko) "
                  "Chrome/128.0.0.0 Safari/537.36",
    "X-Requested-With": "XMLHttpRequest",
    "Referer": BASE_URL,
}


def safe_post(payload: dict, context: str, retries=5):
    """POST vers l'API et retourne JSON ou {} avec retry si 429"""
    for attempt in range(retries):
        r = session.post(API_URL, headers=headers, data=payload)
        if r.status_code == 429:
            wait = 5 + attempt * 5
            print(f"⏳ 429 reçu pour {context}, attente {wait}s avant retry...")
            time.sleep(wait)
            continue
        try:
            return r.json()
        except Exception:
            print(f"⚠️ Réponse non-JSON pour {context}, status={r.status_code}")
            print("Texte brut:", r.text[:200].replace("\n", " "))
            return {}
    return {}


rows = []

# Étape 1 : récupérer les marques
data = safe_post({}, "marques")
makes = data.get("choices", {}).get("makes", [])
print(f"Nombre de marques trouvées : {len(makes)}")

for make in makes:
    make_id = make["id"]
    make_name = make["name"]
    print(f"▶ Make: {make_name}")

    # Étape 2 : modèles
    data_models = safe_post({"_method": "PUT", "make_id": make_id}, f"modèles de {make_name}")
    models = data_models.get("choices", {}).get("models", [])

    for model in models:
        model_id = model["id"]
        model_name = model["name"]
        print(f"   ├─ Model: {model_name}")

        # Étape 3 : générations
        data_gens = safe_post({"_method": "PUT", "model_id": model_id}, f"générations de {make_name} {model_name}")
        gens = data_gens.get("choices", {}).get("generations", [])

        for gen in gens:
            gen_id = gen["id"]
            gen_name = gen["name"]
            print(f"   │   ├─ Gen: {gen_name}")

            # Étape 4 : moteurs
            data_engs = safe_post({"_method": "PUT", "generation_id": gen_id},
                                  f"moteurs de {make_name} {model_name} {gen_name}")
            engines = data_engs.get("choices", {}).get("engines", [])

            for eng in engines:
                eng_name = eng["name"]
                print(f"   │   │   └─ Engine: {eng_name}")
                rows.append([make_name, model_name, gen_name, eng_name])

            # Pause aléatoire pour éviter détection bot
            time.sleep(random.uniform(1, 3))

# Sauvegarde CSV
with open(OUTPUT_FILE, "w", newline="", encoding="utf-8-sig") as f:
    writer = csv.writer(f, delimiter=";")
    writer.writerow(["Make", "Model", "Generation", "Engine"])
    writer.writerows(rows)

print(f"\n✅ Scraping terminé : {len(rows)} combinaisons enregistrées dans {OUTPUT_FILE}")