#!/usr/bin/env python3
"""
Extraction Sirene/RNE (recherche-entreprises.api.gouv.fr) par departement,
avec decoupage automatique commune -> section NAF pour contourner le plafond
de 10 000 resultats par requete de l'API.

Alimente, dans la base ENTREPRISES (instance mysql2) :
  - Entreprises<DEP>Phase1  (cree si absente, jamais ecrasee si presente)
  - Entreprises<DEP>Phase2  (cree si absente)
  - ENT_Communes            (table partagee, alimentee une fois par departement)

Ne touche jamais aux colonnes de qualification/prospection (ENT_StatutQualification,
ENT_CanalProspection, ENT_Phase2, ENT_ScoreCiblage, ENT_Notes) : seules les colonnes
issues de l'API sont mises a jour sur un SIREN deja present.

Usage :
  python3 sirene_extract.py --departements 56,17,64 --db-pass '...'

  (ou export ENTREPRISES_DB_PASS=... pour ne pas passer le mot de passe en argument)
"""
import argparse
import http.client
import json
import os
import sys
import time
import urllib.error
import urllib.parse
import urllib.request

import pymysql

API_SEARCH = "https://recherche-entreprises.api.gouv.fr/search"
API_GEO = "https://geo.api.gouv.fr/departements/{dep}/communes"
NAF_SECTIONS = list("ABCDEFGHIJKLMNOPQRSTU")

# Liste complete des 732 codes NAF (rev. 2) reconnus par l'API, recuperee depuis son
# propre message d'erreur de validation. Utilisee comme 3e niveau de repli (apres
# commune puis section NAF) quand une section reste plafonnee a 10000 malgre le 2e
# niveau -- rare (1 cas sur ~950 communes traitees au 12 septembre 2026 : Rennes,
# section L, tranche NN), mais le decoupage par section seule ne suffit alors plus.
NAF_CODES = [
    '01.11Z', '01.12Z', '01.13Z', '01.14Z', '01.15Z', '01.16Z', '01.19Z', '01.21Z', '01.22Z', '01.23Z',
    '01.24Z', '01.25Z', '01.26Z', '01.27Z', '01.28Z', '01.29Z', '01.30Z', '01.41Z', '01.42Z', '01.43Z',
    '01.44Z', '01.45Z', '01.46Z', '01.47Z', '01.49Z', '01.50Z', '01.61Z', '01.62Z', '01.63Z', '01.64Z',
    '01.70Z', '02.10Z', '02.20Z', '02.30Z', '02.40Z', '03.11Z', '03.12Z', '03.21Z', '03.22Z', '05.10Z',
    '05.20Z', '06.10Z', '06.20Z', '07.10Z', '07.21Z', '07.29Z', '08.11Z', '08.12Z', '08.91Z', '08.92Z',
    '08.93Z', '08.99Z', '09.10Z', '09.90Z', '10.11Z', '10.12Z', '10.13A', '10.13B', '10.20Z', '10.31Z',
    '10.32Z', '10.39A', '10.39B', '10.41A', '10.41B', '10.42Z', '10.51A', '10.51B', '10.51C', '10.51D',
    '10.52Z', '10.61A', '10.61B', '10.62Z', '10.71A', '10.71B', '10.71C', '10.71D', '10.72Z', '10.73Z',
    '10.81Z', '10.82Z', '10.83Z', '10.84Z', '10.85Z', '10.86Z', '10.89Z', '10.91Z', '10.92Z', '11.01Z',
    '11.02A', '11.02B', '11.03Z', '11.04Z', '11.05Z', '11.06Z', '11.07A', '11.07B', '12.00Z', '13.10Z',
    '13.20Z', '13.30Z', '13.91Z', '13.92Z', '13.93Z', '13.94Z', '13.95Z', '13.96Z', '13.99Z', '14.11Z',
    '14.12Z', '14.13Z', '14.14Z', '14.19Z', '14.20Z', '14.31Z', '14.39Z', '15.11Z', '15.12Z', '15.20Z',
    '16.10A', '16.10B', '16.21Z', '16.22Z', '16.23Z', '16.24Z', '16.29Z', '17.11Z', '17.12Z', '17.21A',
    '17.21B', '17.21C', '17.22Z', '17.23Z', '17.24Z', '17.29Z', '18.11Z', '18.12Z', '18.13Z', '18.14Z',
    '18.20Z', '19.10Z', '19.20Z', '20.11Z', '20.12Z', '20.13A', '20.13B', '20.14Z', '20.15Z', '20.16Z',
    '20.17Z', '20.20Z', '20.30Z', '20.41Z', '20.42Z', '20.51Z', '20.52Z', '20.53Z', '20.59Z', '20.60Z',
    '21.10Z', '21.20Z', '22.11Z', '22.19Z', '22.21Z', '22.22Z', '22.23Z', '22.29A', '22.29B', '23.11Z',
    '23.12Z', '23.13Z', '23.14Z', '23.19Z', '23.20Z', '23.31Z', '23.32Z', '23.41Z', '23.42Z', '23.43Z',
    '23.44Z', '23.49Z', '23.51Z', '23.52Z', '23.61Z', '23.62Z', '23.63Z', '23.64Z', '23.65Z', '23.69Z',
    '23.70Z', '23.91Z', '23.99Z', '24.10Z', '24.20Z', '24.31Z', '24.32Z', '24.33Z', '24.34Z', '24.41Z',
    '24.42Z', '24.43Z', '24.44Z', '24.45Z', '24.46Z', '24.51Z', '24.52Z', '24.53Z', '24.54Z', '25.11Z',
    '25.12Z', '25.21Z', '25.29Z', '25.30Z', '25.40Z', '25.50A', '25.50B', '25.61Z', '25.62A', '25.62B',
    '25.71Z', '25.72Z', '25.73A', '25.73B', '25.91Z', '25.92Z', '25.93Z', '25.94Z', '25.99A', '25.99B',
    '26.11Z', '26.12Z', '26.20Z', '26.30Z', '26.40Z', '26.51A', '26.51B', '26.52Z', '26.60Z', '26.70Z',
    '26.80Z', '27.11Z', '27.12Z', '27.20Z', '27.31Z', '27.32Z', '27.33Z', '27.40Z', '27.51Z', '27.52Z',
    '27.90Z', '28.11Z', '28.12Z', '28.13Z', '28.14Z', '28.15Z', '28.21Z', '28.22Z', '28.23Z', '28.24Z',
    '28.25Z', '28.29A', '28.29B', '28.30Z', '28.41Z', '28.49Z', '28.91Z', '28.92Z', '28.93Z', '28.94Z',
    '28.95Z', '28.96Z', '28.99A', '28.99B', '29.10Z', '29.20Z', '29.31Z', '29.32Z', '30.11Z', '30.12Z',
    '30.20Z', '30.30Z', '30.40Z', '30.91Z', '30.92Z', '30.99Z', '31.01Z', '31.02Z', '31.03Z', '31.09A',
    '31.09B', '32.11Z', '32.12Z', '32.13Z', '32.20Z', '32.30Z', '32.40Z', '32.50A', '32.50B', '32.91Z',
    '32.99Z', '33.11Z', '33.12Z', '33.13Z', '33.14Z', '33.15Z', '33.16Z', '33.17Z', '33.19Z', '33.20A',
    '33.20B', '33.20C', '33.20D', '35.11Z', '35.12Z', '35.13Z', '35.14Z', '35.21Z', '35.22Z', '35.23Z',
    '35.30Z', '36.00Z', '37.00Z', '38.11Z', '38.12Z', '38.21Z', '38.22Z', '38.31Z', '38.32Z', '39.00Z',
    '41.10A', '41.10B', '41.10C', '41.10D', '41.20A', '41.20B', '42.11Z', '42.12Z', '42.13A', '42.13B',
    '42.21Z', '42.22Z', '42.91Z', '42.99Z', '43.11Z', '43.12A', '43.12B', '43.13Z', '43.21A', '43.21B',
    '43.22A', '43.22B', '43.29A', '43.29B', '43.31Z', '43.32A', '43.32B', '43.32C', '43.33Z', '43.34Z',
    '43.39Z', '43.91A', '43.91B', '43.99A', '43.99B', '43.99C', '43.99D', '43.99E', '45.11Z', '45.19Z',
    '45.20A', '45.20B', '45.31Z', '45.32Z', '45.40Z', '46.11Z', '46.12A', '46.12B', '46.13Z', '46.14Z',
    '46.15Z', '46.16Z', '46.17A', '46.17B', '46.18Z', '46.19A', '46.19B', '46.21Z', '46.22Z', '46.23Z',
    '46.24Z', '46.31Z', '46.32A', '46.32B', '46.32C', '46.33Z', '46.34Z', '46.35Z', '46.36Z', '46.37Z',
    '46.38A', '46.38B', '46.39A', '46.39B', '46.41Z', '46.42Z', '46.43Z', '46.44Z', '46.45Z', '46.46Z',
    '46.47Z', '46.48Z', '46.49Z', '46.51Z', '46.52Z', '46.61Z', '46.62Z', '46.63Z', '46.64Z', '46.65Z',
    '46.66Z', '46.69A', '46.69B', '46.69C', '46.71Z', '46.72Z', '46.73A', '46.73B', '46.74A', '46.74B',
    '46.75Z', '46.76Z', '46.77Z', '46.90Z', '47.11A', '47.11B', '47.11C', '47.11D', '47.11E', '47.11F',
    '47.19A', '47.19B', '47.21Z', '47.22Z', '47.23Z', '47.24Z', '47.25Z', '47.26Z', '47.29Z', '47.30Z',
    '47.41Z', '47.42Z', '47.43Z', '47.51Z', '47.52A', '47.52B', '47.53Z', '47.54Z', '47.59A', '47.59B',
    '47.61Z', '47.62Z', '47.63Z', '47.64Z', '47.65Z', '47.71Z', '47.72A', '47.72B', '47.73Z', '47.74Z',
    '47.75Z', '47.76Z', '47.77Z', '47.78A', '47.78B', '47.78C', '47.79Z', '47.81Z', '47.82Z', '47.89Z',
    '47.91A', '47.91B', '47.99A', '47.99B', '49.10Z', '49.20Z', '49.31Z', '49.32Z', '49.39A', '49.39B',
    '49.39C', '49.41A', '49.41B', '49.41C', '49.42Z', '49.50Z', '50.10Z', '50.20Z', '50.30Z', '50.40Z',
    '51.10Z', '51.21Z', '51.22Z', '52.10A', '52.10B', '52.21Z', '52.22Z', '52.23Z', '52.24A', '52.24B',
    '52.29A', '52.29B', '53.10Z', '53.20Z', '55.10Z', '55.20Z', '55.30Z', '55.90Z', '56.10A', '56.10B',
    '56.10C', '56.21Z', '56.29A', '56.29B', '56.30Z', '58.11Z', '58.12Z', '58.13Z', '58.14Z', '58.19Z',
    '58.21Z', '58.29A', '58.29B', '58.29C', '59.11A', '59.11B', '59.11C', '59.12Z', '59.13A', '59.13B',
    '59.14Z', '59.20Z', '60.10Z', '60.20A', '60.20B', '61.10Z', '61.20Z', '61.30Z', '61.90Z', '62.01Z',
    '62.02A', '62.02B', '62.03Z', '62.09Z', '63.11Z', '63.12Z', '63.91Z', '63.99Z', '64.11Z', '64.19Z',
    '64.20Z', '64.30Z', '64.91Z', '64.92Z', '64.99Z', '65.11Z', '65.12Z', '65.20Z', '65.30Z', '66.11Z',
    '66.12Z', '66.19A', '66.19B', '66.21Z', '66.22Z', '66.29Z', '66.30Z', '68.10Z', '68.20A', '68.20B',
    '68.31Z', '68.32A', '68.32B', '69.10Z', '69.20Z', '70.10Z', '70.21Z', '70.22Z', '71.11Z', '71.12A',
    '71.12B', '71.20A', '71.20B', '72.11Z', '72.19Z', '72.20Z', '73.11Z', '73.12Z', '73.20Z', '74.10Z',
    '74.20Z', '74.30Z', '74.90A', '74.90B', '75.00Z', '77.11A', '77.11B', '77.12Z', '77.21Z', '77.22Z',
    '77.29Z', '77.31Z', '77.32Z', '77.33Z', '77.34Z', '77.35Z', '77.39Z', '77.40Z', '78.10Z', '78.20Z',
    '78.30Z', '79.11Z', '79.12Z', '79.90Z', '80.10Z', '80.20Z', '80.30Z', '81.10Z', '81.21Z', '81.22Z',
    '81.29A', '81.29B', '81.30Z', '82.11Z', '82.19Z', '82.20Z', '82.30Z', '82.91Z', '82.92Z', '82.99Z',
    '84.11Z', '84.12Z', '84.13Z', '84.21Z', '84.22Z', '84.23Z', '84.24Z', '84.25Z', '84.30A', '84.30B',
    '84.30C', '85.10Z', '85.20Z', '85.31Z', '85.32Z', '85.41Z', '85.42Z', '85.51Z', '85.52Z', '85.53Z',
    '85.59A', '85.59B', '85.60Z', '86.10Z', '86.21Z', '86.22A', '86.22B', '86.22C', '86.23Z', '86.90A',
    '86.90B', '86.90C', '86.90D', '86.90E', '86.90F', '87.10A', '87.10B', '87.10C', '87.20A', '87.20B',
    '87.30A', '87.30B', '87.90A', '87.90B', '88.10A', '88.10B', '88.10C', '88.91A', '88.91B', '88.99A',
    '88.99B', '90.01Z', '90.02Z', '90.03A', '90.03B', '90.04Z', '91.01Z', '91.02Z', '91.03Z', '91.04Z',
    '92.00Z', '93.11Z', '93.12Z', '93.13Z', '93.19Z', '93.21Z', '93.29Z', '94.11Z', '94.12Z', '94.20Z',
    '94.91Z', '94.92Z', '94.99Z', '95.11Z', '95.12Z', '95.21Z', '95.22Z', '95.23Z', '95.24Z', '95.25Z',
    '95.29Z', '96.01A', '96.01B', '96.02A', '96.02B', '96.03Z', '96.04Z', '96.09Z', '97.00Z', '98.10Z',
    '98.20Z', '99.00Z',
]

# Plage de divisions (2 premiers chiffres du code NAF) par section -- permet de
# limiter le repli par code NAF precis (3e niveau) aux seuls codes de la section
# en cours, au lieu de retester les 732 codes en entier a chaque section plafonnee
# (bug corrige le 13/09/2026 : provoquait un blocage de plusieurs heures sur les
# grandes villes, ex. Toulouse, ou plusieurs sections restent plafonnees).
SECTION_DIVISIONS = {
    "A": range(1, 4), "B": range(5, 10), "C": range(10, 34), "D": range(35, 36),
    "E": range(36, 40), "F": range(41, 44), "G": range(45, 48), "H": range(49, 54),
    "I": range(55, 57), "J": range(58, 64), "K": range(64, 67), "L": range(68, 69),
    "M": range(69, 76), "N": range(77, 83), "O": range(84, 85), "P": range(85, 86),
    "Q": range(86, 89), "R": range(90, 94), "S": range(94, 97), "T": range(97, 99),
    "U": range(99, 100),
}


def codes_for_section(section):
    divisions = SECTION_DIVISIONS[section]
    return [c for c in NAF_CODES if int(c[:2]) in divisions]


CAP = 10000
PAGE_SIZE = 25
RATE_DELAY = 0.15  # ~6-7 req/s, sous le quota documente de l'API

DDL_PHASE1 = """
CREATE TABLE IF NOT EXISTS `Entreprises{dep}Phase1` (
  `ENT_Siren` char(9) NOT NULL,
  `ENT_NomComplet` varchar(255) DEFAULT NULL,
  `ENT_NomRaisonSociale` varchar(255) DEFAULT NULL,
  `ENT_Sigle` varchar(50) DEFAULT NULL,
  `ENT_AdresseSiege` varchar(255) DEFAULT NULL,
  `ENT_CodePostal` varchar(20) DEFAULT NULL,
  `ENT_Commune` varchar(100) DEFAULT NULL,
  `ENT_CodeCommune` char(5) DEFAULT NULL,
  `ENT_Departement` char(3) DEFAULT NULL,
  `ENT_Region` char(2) DEFAULT NULL,
  `ENT_DateCreation` date DEFAULT NULL,
  `ENT_EtatAdministratif` char(1) DEFAULT NULL,
  `ENT_DateFermeture` date DEFAULT NULL,
  `ENT_ActivitePrincipale` varchar(10) DEFAULT NULL,
  `ENT_SectionActivite` char(1) DEFAULT NULL,
  `ENT_NatureJuridique` varchar(10) DEFAULT NULL,
  `ENT_CategorieJuridique` enum('Privee','Publique','Association') NOT NULL DEFAULT 'Privee',
  `ENT_CategorieEntreprise` varchar(5) DEFAULT NULL,
  `ENT_TrancheEffectifSalarie` varchar(3) DEFAULT NULL,
  `ENT_AnneeEffectif` year DEFAULT NULL,
  `ENT_DirigeantNom` varchar(150) DEFAULT NULL,
  `ENT_DirigeantPrenoms` varchar(150) DEFAULT NULL,
  `ENT_DirigeantQualite` varchar(255) DEFAULT NULL,
  `ENT_FinancesAnnee` year DEFAULT NULL,
  `ENT_FinancesCa` bigint DEFAULT NULL,
  `ENT_FinancesResultatNet` bigint DEFAULT NULL,
  `ENT_StatutQualification` enum('a_qualifier','qualifiee','rejetee','contactee') NOT NULL DEFAULT 'a_qualifier',
  `ENT_CanalProspection` enum('AEvaluer','Ecartee','Email','Courrier','TelephoneRobot') NOT NULL DEFAULT 'AEvaluer',
  `ENT_Phase2` varchar(1) NOT NULL DEFAULT 'N',
  `ENT_ScoreCiblage` tinyint DEFAULT NULL,
  `ENT_Notes` text,
  `ENT_DateImport` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
  `ENT_DateMajInsee` datetime DEFAULT NULL,
  `ENT_ReponseBruteJson` json DEFAULT NULL,
  PRIMARY KEY (`ENT_Siren`),
  KEY `idx_commune` (`ENT_CodeCommune`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci
  ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=8
"""

DDL_PHASE2 = """
CREATE TABLE IF NOT EXISTS `Entreprises{dep}Phase2` (
  `ENT_Siren` char(9) NOT NULL,
  `ENT_SiteWeb` varchar(255) DEFAULT NULL,
  `ENT_Mail` varchar(255) DEFAULT NULL,
  `ENT_Telephone` varchar(20) DEFAULT NULL,
  `ENT_Portable` varchar(20) DEFAULT NULL,
  PRIMARY KEY (`ENT_Siren`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci
"""

DDL_PHASE3 = """
CREATE TABLE IF NOT EXISTS `Entreprises{dep}Phase3` (
  `ENT_Siren` char(9) NOT NULL,
  `ENT_FinancesAnnee` year NOT NULL,
  `ENT_FinancesCa` bigint DEFAULT NULL,
  `ENT_FinancesResultatNet` bigint DEFAULT NULL,
  `ENT_DateMaj` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
  PRIMARY KEY (`ENT_Siren`, `ENT_FinancesAnnee`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci
"""

DDL_COMMUNES = """
CREATE TABLE IF NOT EXISTS `ENT_Communes` (
  `COM_CodeInsee` char(5) NOT NULL,
  `COM_CodePostal` varchar(100) DEFAULT NULL,
  `COM_Ville` varchar(100) NOT NULL,
  `COM_Departement` char(3) NOT NULL,
  `COM_Latitude` decimal(10,6) DEFAULT NULL,
  `COM_Longitude` decimal(10,6) DEFAULT NULL,
  PRIMARY KEY (`COM_CodeInsee`),
  KEY `idx_departement` (`COM_Departement`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
"""

DDL_DEPARTEMENTS = """
CREATE TABLE IF NOT EXISTS `ENT_Departements` (
  `DEP_Code` char(3) NOT NULL,
  `DEP_Nom` varchar(100) NOT NULL,
  `DEP_NbEntreprises` int NOT NULL DEFAULT 0,
  `DEP_DateExtractionInitiale` datetime DEFAULT NULL,
  `DEP_DateDerniereSyncFormalites` datetime DEFAULT NULL,
  PRIMARY KEY (`DEP_Code`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
"""

DDL_CONFIG = """
CREATE TABLE IF NOT EXISTS `ENT_Config` (
  `CFG_Cle` varchar(100) NOT NULL,
  `CFG_Valeur` varchar(255) DEFAULT NULL,
  PRIMARY KEY (`CFG_Cle`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
"""


def log(msg):
    print(msg, file=sys.stderr, flush=True)


def http_get_json(url, retries=8):
    # Durci le 17/09/2026 : la version precedente ne rattrapait que
    # urllib.error.URLError/TimeoutError, avec un backoff court (1-5s) --
    # insuffisant face a deux incidents reels constates le meme jour :
    # http.client.RemoteDisconnected (pas une URLError, jamais rattrapee,
    # a fait planter le departement 74) et HTTP 429 Too Many Requests
    # soutenu (deux extractions tournant par erreur en parallele ont fait
    # plafonner l'API, cascadant sur 3 departements avant plantage). Meme
    # esprit que le durcissement deja fait sur inpi_bilans_backfill.py.
    for attempt in range(retries):
        try:
            with urllib.request.urlopen(url, timeout=20) as resp:
                return json.load(resp)
        except urllib.error.HTTPError as e:
            if e.code == 429:
                if attempt == retries - 1:
                    raise
                retry_after = e.headers.get("Retry-After") if e.headers else None
                wait = int(retry_after) if retry_after and retry_after.isdigit() else min(10 * (2 ** attempt), 300)
                log(f"    !! 429 Too Many Requests -- pause de {wait}s avant nouvelle tentative "
                    f"({attempt + 1}/{retries})")
                time.sleep(wait)
                continue
            if attempt == retries - 1:
                raise
            log(f"    retry apres erreur HTTP ({e})")
            time.sleep(2 + attempt)
        except (urllib.error.URLError, TimeoutError, http.client.HTTPException,
                ConnectionError) as e:
            if attempt == retries - 1:
                raise
            log(f"    retry apres erreur reseau ({e})")
            time.sleep(2 + attempt)


# Paris, Lyon et Marseille (PLM) : dans les donnees Sirene, le siege d'une
# entreprise de ces trois villes est quasi toujours code au niveau de
# l'arrondissement (75101-75120, 69381-69389, 13201-13216), jamais au code
# commune englobant (75056, 69123, 13055) -- verifie empiriquement le 14
# septembre 2026 sur Marseille : code_commune=13055 ne renvoie qu'une dizaine
# de resultats, alors que chaque arrondissement (13201 a 13216) est a lui
# seul plafonne ou proche du plafond de 10 000. Le code englobant est donc
# remplace par la liste de ses arrondissements avant extraction.
COMMUNES_ARRONDISSEMENTS = {"75": "75056", "69": "69123", "13": "13055"}


def fetch_arrondissements_geo(dep):
    url = "https://geo.api.gouv.fr/communes?" + urllib.parse.urlencode(
        {"codeDepartement": dep, "type": "arrondissement-municipal",
         "fields": "code,nom,centre,codesPostaux,departement"}
    )
    return http_get_json(url)


def fetch_communes_geo(dep):
    url = API_GEO.format(dep=dep) + "?" + urllib.parse.urlencode(
        {"fields": "code,nom,centre,codesPostaux,departement"}
    )
    communes = http_get_json(url)
    code_englobant = COMMUNES_ARRONDISSEMENTS.get(dep)
    if code_englobant:
        communes = [c for c in communes if c["code"] != code_englobant]
        communes.extend(fetch_arrondissements_geo(dep))
    return communes


def fetch_departement_nom(dep):
    url = f"https://geo.api.gouv.fr/departements/{dep}?fields=code,nom"
    data = http_get_json(url)
    return data.get("nom", dep)


def upsert_departement(conn, dep, nom, nb_entreprises):
    with conn.cursor() as cur:
        cur.execute(
            "INSERT INTO ENT_Departements (DEP_Code, DEP_Nom, DEP_NbEntreprises, DEP_DateExtractionInitiale) "
            "VALUES (%s, %s, %s, NOW()) "
            "ON DUPLICATE KEY UPDATE DEP_Nom=VALUES(DEP_Nom), DEP_NbEntreprises=VALUES(DEP_NbEntreprises)",
            (dep, nom, nb_entreprises),
        )


NON_DIFFUSIBLE = "[NON-DIFFUSIBLE]"


def clean(v):
    """Les entites en diffusion partielle/non diffusible (RGPD) renvoient le
    sentinel textuel [NON-DIFFUSIBLE] sur de nombreux champs (adresse, code postal,
    coordonnees, nom/prenoms du dirigeant...). On le convertit en NULL plutot que
    de le stocker (et ca evite aussi de depasser la taille des colonnes typees)."""
    return None if v == NON_DIFFUSIBLE else v


def derive_categorie_juridique(nature_juridique):
    if not nature_juridique:
        return "Privee"
    first = nature_juridique[0]
    if first in ("7", "8"):
        return "Publique"
    if first == "9":
        return "Association"
    return "Privee"


def map_entreprise(r):
    siege = r.get("siege") or {}
    dirigeants = r.get("dirigeants") or []
    dirigeant = dirigeants[0] if dirigeants else {}
    finances = r.get("finances") or {}
    finances_annee = max(finances.keys()) if finances else None
    fin = finances.get(finances_annee, {}) if finances_annee else {}

    return {
        "ENT_Siren": r.get("siren"),
        "ENT_NomComplet": clean(r.get("nom_complet")),
        "ENT_NomRaisonSociale": clean(r.get("nom_raison_sociale")),
        "ENT_Sigle": clean(r.get("sigle")),
        "ENT_AdresseSiege": clean(siege.get("adresse")),
        "ENT_CodePostal": clean(siege.get("code_postal")),
        "ENT_Commune": clean(siege.get("libelle_commune")),
        "ENT_CodeCommune": clean(siege.get("commune")),
        "ENT_Departement": clean(siege.get("departement")),
        "ENT_Region": clean(siege.get("region")),
        "ENT_DateCreation": clean(r.get("date_creation")),
        "ENT_EtatAdministratif": clean(r.get("etat_administratif")),
        "ENT_DateFermeture": clean(r.get("date_fermeture")),
        "ENT_ActivitePrincipale": clean(r.get("activite_principale")),
        "ENT_SectionActivite": clean(r.get("section_activite_principale")),
        "ENT_NatureJuridique": clean(r.get("nature_juridique")),
        "ENT_CategorieJuridique": derive_categorie_juridique(clean(r.get("nature_juridique"))),
        "ENT_CategorieEntreprise": clean(r.get("categorie_entreprise")),
        "ENT_TrancheEffectifSalarie": clean(r.get("tranche_effectif_salarie")),
        "ENT_AnneeEffectif": clean(r.get("annee_tranche_effectif_salarie")),
        "ENT_DirigeantNom": clean(dirigeant.get("nom")),
        "ENT_DirigeantPrenoms": clean(dirigeant.get("prenoms")),
        "ENT_DirigeantQualite": clean(dirigeant.get("qualite")),
        "ENT_FinancesAnnee": finances_annee,
        "ENT_FinancesCa": fin.get("ca"),
        "ENT_FinancesResultatNet": fin.get("resultat_net"),
        "ENT_ReponseBruteJson": json.dumps(r, ensure_ascii=False),
    }


def probe_total(params):
    p = dict(params, per_page=1)
    data = http_get_json(API_SEARCH + "?" + urllib.parse.urlencode(p))
    time.sleep(RATE_DELAY)
    return data.get("total_results", 0)


def paginate_all(params, seen):
    p = dict(params, per_page=PAGE_SIZE)
    page = 1
    out = []
    while True:
        p["page"] = page
        data = http_get_json(API_SEARCH + "?" + urllib.parse.urlencode(p))
        time.sleep(RATE_DELAY)
        results = data.get("results", [])
        if not results:
            break
        for r in results:
            siren = r.get("siren")
            if siren and siren not in seen:
                seen.add(siren)
                out.append(r)
        total_pages = data.get("total_pages", page)
        if page >= total_pages or page >= 400:  # 400*25 = 10000, garde-fou
            break
        page += 1
    return out


def fetch_entreprises_for_filter(base, seen, label):
    """Cascade de repli generique (section NAF puis code NAF precis si
    plafonnee a CAP resultats), utilisee aussi bien pour un filtre par
    commune que, depuis le 18/09/2026, par departement entier (cas 2A/2B,
    cf. fetch_departement_entreprises_direct)."""
    total = probe_total(base)
    if total < CAP:
        return paginate_all(base, seen)

    log(f"    {label} plafonnee ({total}), repli par section NAF")
    out = []
    for section in NAF_SECTIONS:
        sec_params = dict(base, section_activite_principale=section)
        sec_total = probe_total(sec_params)
        if sec_total == 0:
            continue
        if sec_total < CAP:
            out.extend(paginate_all(sec_params, seen))
            continue

        # 3e niveau de repli : la section a elle seule reste plafonnee (cas rare,
        # ex. Rennes/section L). On decoupe par code NAF precis (732 codes possibles) --
        # un code implique deja sa section, donc on l'utilise seul, sans le combiner
        # au filtre de section.
        log(f"    {label} section {section} encore plafonnee ({sec_total}), "
            f"repli par code NAF precis")
        for code in codes_for_section(section):
            code_params = dict(base, activite_principale=code)
            code_total = probe_total(code_params)
            if code_total == 0:
                continue
            if code_total >= CAP:
                log(f"      !! {label} code {code} encore plafonnee "
                    f"({code_total}) -- a decouper manuellement (ex: par tranche_effectif_salarie)")
            out.extend(paginate_all(code_params, seen))
    return out


def fetch_commune_entreprises(code_commune, seen):
    base = {"code_commune": code_commune, "etat_administratif": "A"}
    return fetch_entreprises_for_filter(base, seen, f"commune {code_commune}")


def fetch_departement_entreprises_direct(dep, seen):
    """Repli specifique aux departements dont le code commune INSEE n'est pas
    purement numerique (2A, 2B -- Corse) : le parametre `code_commune` de
    l'API recherche-entreprises echoue silencieusement (0 resultat, HTTP 200,
    aucune erreur) sur ces codes alphanumeriques, verifie empiriquement le
    18/09/2026 (2A004/2B033 -> 0 alors que 01004 -> 6124). Le parametre
    `departement`, lui, fonctionne normalement -- on interroge donc le
    departement entier d'un coup, avec la meme cascade de repli que pour une
    commune plafonnee."""
    base = {"departement": dep, "etat_administratif": "A"}
    return fetch_entreprises_for_filter(base, seen, f"departement {dep}")


def upsert_entreprise(conn, table, row):
    cols = list(row.keys())
    placeholders = ", ".join(["%s"] * len(cols))
    colnames = ", ".join(f"`{c}`" for c in cols)
    updates = ", ".join(f"`{c}`=VALUES(`{c}`)" for c in cols if c != "ENT_Siren")
    sql = (
        f"INSERT INTO `{table}` ({colnames}) VALUES ({placeholders}) "
        f"ON DUPLICATE KEY UPDATE {updates}, `ENT_DateMajInsee`=NOW()"
    )
    with conn.cursor() as cur:
        cur.execute(sql, list(row.values()))


def upsert_finances(conn, table, row):
    if row.get("ENT_FinancesAnnee") is None:
        return
    fin = {
        "ENT_Siren": row["ENT_Siren"],
        "ENT_FinancesAnnee": row["ENT_FinancesAnnee"],
        "ENT_FinancesCa": row["ENT_FinancesCa"],
        "ENT_FinancesResultatNet": row["ENT_FinancesResultatNet"],
    }
    cols = list(fin.keys())
    placeholders = ", ".join(["%s"] * len(cols))
    colnames = ", ".join(f"`{c}`" for c in cols)
    updates = ", ".join(
        f"`{c}`=VALUES(`{c}`)" for c in cols if c not in ("ENT_Siren", "ENT_FinancesAnnee")
    )
    sql = (
        f"INSERT INTO `{table}` ({colnames}) VALUES ({placeholders}) "
        f"ON DUPLICATE KEY UPDATE {updates}"
    )
    with conn.cursor() as cur:
        cur.execute(sql, list(fin.values()))


def ensure_tables(conn, dep):
    with conn.cursor() as cur:
        cur.execute(DDL_PHASE1.format(dep=dep))
        cur.execute(DDL_PHASE2.format(dep=dep))
        cur.execute(DDL_PHASE3.format(dep=dep))
        cur.execute(DDL_COMMUNES)
        cur.execute(DDL_DEPARTEMENTS)
        cur.execute(DDL_CONFIG)


def ensure_communes(conn, dep, communes_geo):
    with conn.cursor() as cur:
        cur.execute("SELECT COUNT(*) FROM ENT_Communes WHERE COM_Departement=%s", (dep,))
        (count,) = cur.fetchone()
        if count > 0:
            log(f"  ENT_Communes deja alimentee pour le {dep} ({count} communes), on ne rejoue pas")
            return
        rows = []
        for c in communes_geo:
            lon, lat = c["centre"]["coordinates"]
            cp = ",".join(c.get("codesPostaux", []))
            dep_code = c["departement"]["code"] if isinstance(c.get("departement"), dict) else c.get("departement")
            rows.append((c["code"], cp, c["nom"], dep_code, lat, lon))
        cur.executemany(
            "INSERT INTO ENT_Communes "
            "(COM_CodeInsee, COM_CodePostal, COM_Ville, COM_Departement, COM_Latitude, COM_Longitude) "
            "VALUES (%s,%s,%s,%s,%s,%s)",
            rows,
        )
        log(f"  ENT_Communes : {len(rows)} communes inserees pour le {dep}")


def run_department(conn, dep):
    log(f"=== Departement {dep} ===")
    ensure_tables(conn, dep)
    communes_geo = fetch_communes_geo(dep)
    log(f"  {len(communes_geo)} communes (source geo.api.gouv.fr)")
    ensure_communes(conn, dep, communes_geo)

    table1 = f"Entreprises{dep}Phase1"
    table3 = f"Entreprises{dep}Phase3"
    seen = set()
    total_upserts = 0
    total_finances = 0

    def store(results):
        nonlocal total_upserts, total_finances
        for r in results:
            row = map_entreprise(r)
            if row["ENT_Siren"]:
                upsert_entreprise(conn, table1, row)
                total_upserts += 1
                if row["ENT_FinancesAnnee"] is not None:
                    upsert_finances(conn, table3, row)
                    total_finances += 1

    if not dep.isdigit():
        # Code commune INSEE alphanumerique (2A, 2B) : le filtre code_commune
        # de l'API echoue silencieusement, on interroge le departement entier
        # d'un coup (cf. fetch_departement_entreprises_direct).
        results = fetch_departement_entreprises_direct(dep, seen)
        store(results)
        log(f"  departement {dep} (requete directe) : {len(results)} entreprises")
    else:
        for c in communes_geo:
            results = fetch_commune_entreprises(c["code"], seen)
            store(results)
            if results:
                log(f"  {c['nom']} ({c['code']}) : {len(results)} entreprises")

    with conn.cursor() as cur:
        cur.execute(f"SELECT COUNT(*) FROM `{table1}`")
        (nb_total,) = cur.fetchone()
    nom_dep = fetch_departement_nom(dep)
    upsert_departement(conn, dep, nom_dep, nb_total)

    log(f"Departement {dep} termine : {total_upserts} entreprises upsertees dans {table1}, "
        f"{total_finances} lignes financieres upsertees dans {table3} (total {nb_total} -- "
        f"ENT_Departements mise a jour)")


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--departements", required=True,
                    help="Liste separee par des virgules, ex: 56,17,64")
    ap.add_argument("--db-host", default=os.environ.get("ENTREPRISES_DB_HOST", "mysql2"))
    ap.add_argument("--db-port", type=int, default=int(os.environ.get("ENTREPRISES_DB_PORT", "3306")))
    ap.add_argument("--db-user", default=os.environ.get("ENTREPRISES_DB_USER", "crm_user"))
    ap.add_argument("--db-pass", default=os.environ.get("ENTREPRISES_DB_PASS"))
    ap.add_argument("--db-name", default=os.environ.get("ENTREPRISES_DB_NAME", "ENTREPRISES"))
    args = ap.parse_args()

    if not args.db_pass:
        ap.error("mot de passe requis : --db-pass ou variable d'environnement ENTREPRISES_DB_PASS")

    departements = [d.strip() for d in args.departements.split(",") if d.strip()]

    conn = pymysql.connect(
        host=args.db_host, port=args.db_port, user=args.db_user, password=args.db_pass,
        database=args.db_name, charset="utf8mb4", autocommit=True,
    )
    try:
        for dep in departements:
            run_department(conn, dep)
    finally:
        conn.close()


if __name__ == "__main__":
    main()