import requests
from bs4 import BeautifulSoup
import csv
import time

# URL de base
base_url = "https://www.dyno-chiptuningfiles.com/original-ecu-files-database/"
output_file = "dyno_chiptuning_data.csv"

# Initialisation du compteur ID
id_counter = 1

# Ouvrir le fichier CSV pour écrire les données
with open(output_file, mode="w", newline='', encoding='utf-8-sig') as f:
    writer = csv.writer(f, delimiter=';')
    writer.writerow(["ID", "Make and Model", "Engine", "Hardware Number", "Software Number", "ECU", "ECU Type", "Size", "Detail Link"])

    current_page = 1
    total_pages = 1042  # Vous pouvez ajuster ce nombre si nécessaire
    while current_page <= total_pages:
        print(f"Fetching page {current_page}...")

        # Construire l'URL de la page
        page_url = f"{base_url}?page={current_page}"
        response = requests.get(page_url)
        soup = BeautifulSoup(response.text, 'html.parser')

        # Rechercher le tableau des résultats
        table = soup.find("div", class_="datagrid-table-scroll")
        if not table:
            print(f"No table found on page {current_page}")
            break

        # Rechercher toutes les lignes de données
        rows = table.find_all("tr")
        if not rows:
            print(f"No rows found on page {current_page}")
            break

        for row in rows:
            try:
                cols = row.find_all("td")
                if not cols or len(cols) < 7:  # Sauter les lignes invalides
                    continue

                make_and_model = cols[0].get_text(strip=True)
                engine = cols[1].get_text(strip=True)
                hardware_number = cols[2].get_text(strip=True)
                software_number = cols[3].get_text(strip=True)
                ecu = cols[4].get_text(strip=True)
                ecu_type = cols[5].get_text(strip=True)
                size = cols[6].get_text(strip=True)
                detail_link = cols[0].find("a")["href"] if cols[0].find("a") else ""

                # Écrire les données dans le fichier CSV
                writer.writerow([
                    id_counter, make_and_model, engine, hardware_number, software_number, ecu, ecu_type, size, detail_link
                ])
                id_counter += 1
            except Exception as e:
                print(f"Error processing row: {e}")

        # Pause pour éviter de surcharger le serveur
        time.sleep(1)
        current_page += 1

print(f"Scraping terminé. Les données sont enregistrées dans {output_file}.")