import csv, time, os from selenium import webdriver from selenium.webdriver.chrome.service import Service from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from webdriver_manager.chrome import ChromeDriverManager from bs4 import BeautifulSoup BASE_URL = "https://www.dyno-chiptuningfiles.com/chiptuning-file/" OUTPUT_FILE = "dyno_full_data.csv" FIELD_MAP = { "Type of fuel": "Fuel", "Fuel type": "Fuel", "Method": "Method", "Tuning type": "Tuning", "Cylinder content": "Cylinder", "Engine number": "Engine Number", "Engine ECU": "ECU", "Engine Control Unit": "ECU", "Compression ratio": "Compression", "Bore X stroke": "BoreXStroke", } def get_driver(): options = webdriver.ChromeOptions() options.add_argument("--headless=new") options.add_argument("--disable-gpu") options.add_argument("--no-sandbox") return webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options) def safe_find_links(driver, selectors): if isinstance(selectors, str): selectors = [selectors] for sel in selectors: try: elems = driver.find_elements(By.CSS_SELECTOR, sel) if elems: return [(e.get_attribute("href"), e.text.strip()) for e in elems if e.get_attribute("href")] except: pass return [] def extract_specs(soup): details = {} for row in soup.select(".chiptuning-files-results-specs table tr"): cols = row.find_all("td") if len(cols) == 2: key = cols[0].get_text(strip=True) val = cols[1].get_text(strip=True) details[FIELD_MAP.get(key, key)] = val def extract_value(block, label): row = block.find("div", string=label) if row and row.find_next_sibling("div"): return row.find_next_sibling("div").get_text(strip=True).split()[0] return "" hp_block = soup.find("h3", string="Power hp") tq_block = soup.find("h3", string="Torque Nm") hp_stock = extract_value(hp_block.find_parent("div"), "Original") if hp_block else "" hp_tuned = extract_value(hp_block.find_parent("div"), "Dyno Chiptuning") if hp_block else "" tq_stock = extract_value(tq_block.find_parent("div"), "Original") if tq_block else "" tq_tuned = extract_value(tq_block.find_parent("div"), "Dyno Chiptuning") if tq_block else "" return details, hp_stock, hp_tuned, tq_stock, tq_tuned def write_row(row, unavailable=False): tag = "CSV (unavailable)" if unavailable else "CSV" print(f" → {tag}: {row}") with open(OUTPUT_FILE, "a", newline="", encoding="utf-8-sig") as f: csv.writer(f, delimiter=";").writerow(row) def scrape_level(driver, brand, model, generation, url): driver.get(url) time.sleep(1.2) soup = BeautifulSoup(driver.page_source, "html.parser") specs_block = soup.select_one(".chiptuning-files-results-specs") hp_block = soup.find("h3", string="Power hp") tq_block = soup.find("h3", string="Torque Nm") # --- Helper pour détecter une vraie génération --- def is_generation(text: str) -> bool: if not text: return False txt = text.replace(" ", "") return ("-" in txt or ">" in txt) and any(c.isdigit() for c in txt) # ✅ Cas moteur avec specs if specs_block and (hp_block or tq_block): page_title = soup.find("h1") full_title = page_title.get_text(strip=True) if page_title else url.split("/")[-2] # --- Déterminer la génération --- breadcrumb = soup.select("div.breadcrumbs a") gen_text = generation if breadcrumb and len(breadcrumb) >= 3: candidate = breadcrumb[-2].get_text(strip=True) if is_generation(candidate): gen_text = candidate if not gen_text: gen_text = "All" # --- Nettoyer le moteur --- engine = full_title for token in (brand, model, gen_text if is_generation(gen_text) else None): if token and token in engine: engine = engine.replace(token, "").strip() details, hp_stock, hp_tuned, tq_stock, tq_tuned = extract_specs(soup) row = [ brand, model, gen_text, engine, details.get("Fuel", ""), details.get("ECU", ""), details.get("Engine Number", ""), details.get("Cylinder", ""), details.get("Compression", ""), details.get("BoreXStroke", ""), hp_stock, hp_tuned, tq_stock, tq_tuned ] write_row(row) return # ✅ Cas moteur indispo unavailable_msg = soup.find( string=lambda t: t and "At this moment the tuning file is not available yet" in t ) if unavailable_msg: page_title = soup.find("h1") full_title = page_title.get_text(strip=True) if page_title else url.split("/")[-2] breadcrumb = soup.select("div.breadcrumbs a") gen_text = generation if breadcrumb and len(breadcrumb) >= 3: candidate = breadcrumb[-2].get_text(strip=True) if is_generation(candidate): gen_text = candidate if not gen_text: gen_text = "All" row = [brand, model, gen_text, full_title, "", "", "", "", "", "", "", "", "", ""] write_row(row, unavailable=True) return # ✅ Page intermédiaire (liste de générations OU moteurs) links = safe_find_links(driver, [ ".chiptuning-files-overview a", ".chiptuning-files-models-overview a" ]) if links: for sub_url, sub_text in links: if not sub_text: continue # ⚠️ Correction : si on est déjà dans une génération, ignorer les doublons de générations if is_generation(sub_text) and is_generation(generation): continue print(f" │ ├─ {sub_text}") # Si c'est une génération → on met à jour if is_generation(sub_text): new_gen = sub_text.strip() else: # sinon c’est un moteur, on garde la génération héritée new_gen = generation scrape_level(driver, brand, model, new_gen, sub_url) return # 🚫 Cas terminal sans specs if "all" in url: return def scrape(): driver = get_driver() # Init CSV if not os.path.exists(OUTPUT_FILE): with open(OUTPUT_FILE, "w", newline="", encoding="utf-8-sig") as f: writer = csv.writer(f, delimiter=";") writer.writerow([ "Make", "Model", "Generation", "Engine", "Fuel", "ECU", "Engine Number", "Cylinder", "Compression", "BoreXStroke", "HP Stock", "HP Tuned", "Torque Stock", "Torque Tuned" ]) driver.get(BASE_URL) WebDriverWait(driver, 20).until( EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".chiptuning-files-brands-overview-item a")) ) brand_links = [(a.get_attribute("href"), a.text.strip()) for a in driver.find_elements(By.CSS_SELECTOR, ".chiptuning-files-brands-overview-item a")] for brand_url, brand in brand_links: print(f"▶ {brand}") driver.get(brand_url) time.sleep(1.5) model_links = safe_find_links(driver, ".chiptuning-files-models-overview a") for model_url, model in model_links: print(f" ├─ {model}") scrape_level(driver, brand, model, "", model_url) driver.quit() if __name__ == "__main__": scrape()