Untitled

# core/management/commands/scrape_all.py
from __future__ import annotations

import importlib
import inspect
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import date
from pathlib import Path

from django.core.management.base import BaseCommand
from django.db import close_old_connections, transaction

from core.models import Article, Source
from core.utils.text import strip_nul


class Command(BaseCommand):
    help = "Scrape un scraper (--only) ou tous les scrapers de core/scrapers/."

    def _resolve_fn(self, module, module_name):
        module_key = module_name.lower()
        callables = {}
        for name in dir(module):
            if name.startswith("_"):
                continue
            fn = getattr(module, name, None)
            if callable(fn):
                callables[name.lower()] = fn

        preferred = [
            "scrape",
            f"scrape_{module_key}",
            module_key,
        ]
        for key in preferred:
            if key in callables:
                return callables[key]

        for key, fn in callables.items():
            if key.startswith("scrape"):
                return fn
        return None

    def add_arguments(self, parser):
        parser.add_argument("--all", action="store_true", help="Rouler tous les scrapers (*.py) du dossier core/scrapers.")
        parser.add_argument("--only", type=str, help="Nom du scraper (ex: CBC, lapresse, Arsenal).")
        parser.add_argument("--days", type=int, default=14, help="Passé seulement si le scraper accepte days.")
        parser.add_argument("--threads", type=int, default=5, help="Nombre de scrapers en parallèle.")

    def _run_one_scraper(self, module_name: str, days: int, existing_urls_snapshot: set[str]):
        """
        Exécute 1 scraper (import + run + upsert DB).
        Note: existing_urls_snapshot est un snapshot (read-only) utile aux scrapers pour filtrer.
        """
        # Important en multi-threads Django:
        close_old_connections()

        started_all = time.perf_counter()

        module = importlib.import_module(f"core.scrapers.{module_name}")
        fn = self._resolve_fn(module, module_name)
        if not callable(fn):
            return (module_name, 0, 0, 0, f"skip (fonction introuvable)")

        sig = inspect.signature(fn)
        params = sig.parameters
        kwargs = {}
        if "existing_urls" in params:
            kwargs["existing_urls"] = existing_urls_snapshot
        if "days" in params:
            kwargs["days"] = days
        if "cursor" in params:
            kwargs["cursor"] = None

        started_run = time.perf_counter()
        items = fn(**kwargs) or []
        run_elapsed = time.perf_counter() - started_run

        created = updated = skipped = 0

        with transaction.atomic():
            for it in items:
                source_name = strip_nul(it.get("source") or it.get("Media") or "")
                if not source_name:
                    source_name = "Source inconnue"
                source, _ = Source.objects.get_or_create(nom=source_name)

                d = None
                raw_date = strip_nul(it.get("date_publication") or it.get("Date") or "")
                if raw_date:
                    try:
                        d = date.fromisoformat(raw_date)
                    except ValueError:
                        pass

                url = strip_nul(it.get("url") or it.get("Lien") or "")
                if not url:
                    skipped += 1
                    continue

                _, was_created = Article.objects.update_or_create(
                    url=url,
                    defaults={
                        "source": source,
                        "titre": strip_nul(it.get("titre") or it.get("Titre") or ""),
                        "texte": strip_nul(it.get("texte") or it.get("Article") or ""),
                        "date_publication": d,
                    },
                )
                if was_created:
                    created += 1
                else:
                    updated += 1

        total_elapsed = time.perf_counter() - started_all
        msg = f"items={len(items)} run={run_elapsed:.2f}s total={total_elapsed:.2f}s"
        return (module_name, created, updated, skipped, msg)

    def handle(self, *args, **options):
        run_all = options["all"]
        only = options["only"]
        days = options["days"]
        threads = int(options["threads"] or 5)

        if not run_all and not only:
            run_all = True
            self.stdout.write("Aucun argument fourni: mode --all activé par défaut.")

        # Snapshot initial (anti-doublon)
        existing_urls = set(Article.objects.values_list("url", flat=True))
        self.stdout.write(f"Existing URLs in DB: {len(existing_urls)}")

        scrapers_dir = Path("core/scrapers")
        files = sorted(scrapers_dir.glob("*.py"))
        files = [p for p in files if p.name != "__init__.py" and not p.name.startswith("_")]

        if only:
            wanted = only.lower().replace(".py", "")
            files = [p for p in files if p.stem.lower() == wanted]
            if not files:
                self.stdout.write(self.style.ERROR(f"Scraper introuvable: {only}"))
                return

        if not files:
            self.stdout.write(self.style.WARNING("Aucun scraper trouvé dans core/scrapers."))
            return

        module_names = [p.stem for p in files]

        self.stdout.write(self.style.NOTICE(f"Running {len(module_names)} scrapers with threads={threads}"))

        total_created = total_updated = total_skipped = 0
        errors = 0

        # Important: on passe un snapshot read-only aux scrapers
        existing_urls_snapshot = frozenset(existing_urls)

        with ThreadPoolExecutor(max_workers=threads) as ex:
            futures = {
                ex.submit(self._run_one_scraper, mn, days, existing_urls_snapshot): mn
                for mn in module_names
            }

            for fut in as_completed(futures):
                mn = futures[fut]
                try:
                    module_name, created, updated, skipped, msg = fut.result()
                    total_created += created
                    total_updated += updated
                    total_skipped += skipped
                    self.stdout.write(
                        self.style.SUCCESS(
                            f"{module_name} done — created={created}, updated={updated}, skipped={skipped} ({msg})"
                        )
                    )
                except Exception as exc:
                    errors += 1
                    self.stdout.write(self.style.ERROR(f"{mn} ERROR: {exc}"))

        # Recompte final “vrai” des URLs en DB (plutôt que de tenter de sync un set cross-threads)
        final_known_urls = Article.objects.values("url").count()

        self.stdout.write(
            self.style.SUCCESS(
                f"\nALL done — created={total_created}, updated={total_updated}, skipped={total_skipped}, "
                f"errors={errors}, urls_in_db={final_known_urls}"
            )
        )