#!/usr/bin/env python3
"""
Extraction Sirene/RNE (recherche-entreprises.api.gouv.fr) par departement,
avec decoupage automatique commune -> section NAF pour contourner le plafond
de 10 000 resultats par requete de l'API.

Alimente, dans la base ENTREPRISES (instance mysql2) :
  - Entreprises<DEP>Phase1  (cree si absente, jamais ecrasee si presente)
  - Entreprises<DEP>Phase2  (cree si absente)
  - ENT_Communes            (table partagee, alimentee une fois par departement)

Ne touche jamais aux colonnes de qualification/prospection (ENT_StatutQualification,
ENT_CanalProspection, ENT_Phase2, ENT_ScoreCiblage, ENT_Notes) : seules les colonnes
issues de l'API sont mises a jour sur un SIREN deja present.

Usage :
  python3 sirene_extract.py --departements 56,17,64 --db-pass '...'

  (ou export ENTREPRISES_DB_PASS=... pour ne pas passer le mot de passe en argument)
"""
import argparse
import json
import os
import sys
import time
import urllib.error
import urllib.parse
import urllib.request

import pymysql

API_SEARCH = "https://recherche-entreprises.api.gouv.fr/search"
API_GEO = "https://geo.api.gouv.fr/departements/{dep}/communes"
NAF_SECTIONS = list("ABCDEFGHIJKLMNOPQRSTU")

# Liste complete des 732 codes NAF (rev. 2) reconnus par l'API, recuperee depuis son
# propre message d'erreur de validation. Utilisee comme 3e niveau de repli (apres
# commune puis section NAF) quand une section reste plafonnee a 10000 malgre le 2e
# niveau -- rare (1 cas sur ~950 communes traitees au 12 septembre 2026 : Rennes,
# section L, tranche NN), mais le decoupage par section seule ne suffit alors plus.
NAF_CODES = [
    '01.11Z', '01.12Z', '01.13Z', '01.14Z', '01.15Z', '01.16Z', '01.19Z', '01.21Z', '01.22Z', '01.23Z',
    '01.24Z', '01.25Z', '01.26Z', '01.27Z', '01.28Z', '01.29Z', '01.30Z', '01.41Z', '01.42Z', '01.43Z',
    '01.44Z', '01.45Z', '01.46Z', '01.47Z', '01.49Z', '01.50Z', '01.61Z', '01.62Z', '01.63Z', '01.64Z',
    '01.70Z', '02.10Z', '02.20Z', '02.30Z', '02.40Z', '03.11Z', '03.12Z', '03.21Z', '03.22Z', '05.10Z',
    '05.20Z', '06.10Z', '06.20Z', '07.10Z', '07.21Z', '07.29Z', '08.11Z', '08.12Z', '08.91Z', '08.92Z',
    '08.93Z', '08.99Z', '09.10Z', '09.90Z', '10.11Z', '10.12Z', '10.13A', '10.13B', '10.20Z', '10.31Z',
    '10.32Z', '10.39A', '10.39B', '10.41A', '10.41B', '10.42Z', '10.51A', '10.51B', '10.51C', '10.51D',
    '10.52Z', '10.61A', '10.61B', '10.62Z', '10.71A', '10.71B', '10.71C', '10.71D', '10.72Z', '10.73Z',
    '10.81Z', '10.82Z', '10.83Z', '10.84Z', '10.85Z', '10.86Z', '10.89Z', '10.91Z', '10.92Z', '11.01Z',
    '11.02A', '11.02B', '11.03Z', '11.04Z', '11.05Z', '11.06Z', '11.07A', '11.07B', '12.00Z', '13.10Z',
    '13.20Z', '13.30Z', '13.91Z', '13.92Z', '13.93Z', '13.94Z', '13.95Z', '13.96Z', '13.99Z', '14.11Z',
    '14.12Z', '14.13Z', '14.14Z', '14.19Z', '14.20Z', '14.31Z', '14.39Z', '15.11Z', '15.12Z', '15.20Z',
    '16.10A', '16.10B', '16.21Z', '16.22Z', '16.23Z', '16.24Z', '16.29Z', '17.11Z', '17.12Z', '17.21A',
    '17.21B', '17.21C', '17.22Z', '17.23Z', '17.24Z', '17.29Z', '18.11Z', '18.12Z', '18.13Z', '18.14Z',
    '18.20Z', '19.10Z', '19.20Z', '20.11Z', '20.12Z', '20.13A', '20.13B', '20.14Z', '20.15Z', '20.16Z',
    '20.17Z', '20.20Z', '20.30Z', '20.41Z', '20.42Z', '20.51Z', '20.52Z', '20.53Z', '20.59Z', '20.60Z',
    '21.10Z', '21.20Z', '22.11Z', '22.19Z', '22.21Z', '22.22Z', '22.23Z', '22.29A', '22.29B', '23.11Z',
    '23.12Z', '23.13Z', '23.14Z', '23.19Z', '23.20Z', '23.31Z', '23.32Z', '23.41Z', '23.42Z', '23.43Z',
    '23.44Z', '23.49Z', '23.51Z', '23.52Z', '23.61Z', '23.62Z', '23.63Z', '23.64Z', '23.65Z', '23.69Z',
    '23.70Z', '23.91Z', '23.99Z', '24.10Z', '24.20Z', '24.31Z', '24.32Z', '24.33Z', '24.34Z', '24.41Z',
    '24.42Z', '24.43Z', '24.44Z', '24.45Z', '24.46Z', '24.51Z', '24.52Z', '24.53Z', '24.54Z', '25.11Z',
    '25.12Z', '25.21Z', '25.29Z', '25.30Z', '25.40Z', '25.50A', '25.50B', '25.61Z', '25.62A', '25.62B',
    '25.71Z', '25.72Z', '25.73A', '25.73B', '25.91Z', '25.92Z', '25.93Z', '25.94Z', '25.99A', '25.99B',
    '26.11Z', '26.12Z', '26.20Z', '26.30Z', '26.40Z', '26.51A', '26.51B', '26.52Z', '26.60Z', '26.70Z',
    '26.80Z', '27.11Z', '27.12Z', '27.20Z', '27.31Z', '27.32Z', '27.33Z', '27.40Z', '27.51Z', '27.52Z',
    '27.90Z', '28.11Z', '28.12Z', '28.13Z', '28.14Z', '28.15Z', '28.21Z', '28.22Z', '28.23Z', '28.24Z',
    '28.25Z', '28.29A', '28.29B', '28.30Z', '28.41Z', '28.49Z', '28.91Z', '28.92Z', '28.93Z', '28.94Z',
    '28.95Z', '28.96Z', '28.99A', '28.99B', '29.10Z', '29.20Z', '29.31Z', '29.32Z', '30.11Z', '30.12Z',
    '30.20Z', '30.30Z', '30.40Z', '30.91Z', '30.92Z', '30.99Z', '31.01Z', '31.02Z', '31.03Z', '31.09A',
    '31.09B', '32.11Z', '32.12Z', '32.13Z', '32.20Z', '32.30Z', '32.40Z', '32.50A', '32.50B', '32.91Z',
    '32.99Z', '33.11Z', '33.12Z', '33.13Z', '33.14Z', '33.15Z', '33.16Z', '33.17Z', '33.19Z', '33.20A',
    '33.20B', '33.20C', '33.20D', '35.11Z', '35.12Z', '35.13Z', '35.14Z', '35.21Z', '35.22Z', '35.23Z',
    '35.30Z', '36.00Z', '37.00Z', '38.11Z', '38.12Z', '38.21Z', '38.22Z', '38.31Z', '38.32Z', '39.00Z',
    '41.10A', '41.10B', '41.10C', '41.10D', '41.20A', '41.20B', '42.11Z', '42.12Z', '42.13A', '42.13B',
    '42.21Z', '42.22Z', '42.91Z', '42.99Z', '43.11Z', '43.12A', '43.12B', '43.13Z', '43.21A', '43.21B',
    '43.22A', '43.22B', '43.29A', '43.29B', '43.31Z', '43.32A', '43.32B', '43.32C', '43.33Z', '43.34Z',
    '43.39Z', '43.91A', '43.91B', '43.99A', '43.99B', '43.99C', '43.99D', '43.99E', '45.11Z', '45.19Z',
    '45.20A', '45.20B', '45.31Z', '45.32Z', '45.40Z', '46.11Z', '46.12A', '46.12B', '46.13Z', '46.14Z',
    '46.15Z', '46.16Z', '46.17A', '46.17B', '46.18Z', '46.19A', '46.19B', '46.21Z', '46.22Z', '46.23Z',
    '46.24Z', '46.31Z', '46.32A', '46.32B', '46.32C', '46.33Z', '46.34Z', '46.35Z', '46.36Z', '46.37Z',
    '46.38A', '46.38B', '46.39A', '46.39B', '46.41Z', '46.42Z', '46.43Z', '46.44Z', '46.45Z', '46.46Z',
    '46.47Z', '46.48Z', '46.49Z', '46.51Z', '46.52Z', '46.61Z', '46.62Z', '46.63Z', '46.64Z', '46.65Z',
    '46.66Z', '46.69A', '46.69B', '46.69C', '46.71Z', '46.72Z', '46.73A', '46.73B', '46.74A', '46.74B',
    '46.75Z', '46.76Z', '46.77Z', '46.90Z', '47.11A', '47.11B', '47.11C', '47.11D', '47.11E', '47.11F',
    '47.19A', '47.19B', '47.21Z', '47.22Z', '47.23Z', '47.24Z', '47.25Z', '47.26Z', '47.29Z', '47.30Z',
    '47.41Z', '47.42Z', '47.43Z', '47.51Z', '47.52A', '47.52B', '47.53Z', '47.54Z', '47.59A', '47.59B',
    '47.61Z', '47.62Z', '47.63Z', '47.64Z', '47.65Z', '47.71Z', '47.72A', '47.72B', '47.73Z', '47.74Z',
    '47.75Z', '47.76Z', '47.77Z', '47.78A', '47.78B', '47.78C', '47.79Z', '47.81Z', '47.82Z', '47.89Z',
    '47.91A', '47.91B', '47.99A', '47.99B', '49.10Z', '49.20Z', '49.31Z', '49.32Z', '49.39A', '49.39B',
    '49.39C', '49.41A', '49.41B', '49.41C', '49.42Z', '49.50Z', '50.10Z', '50.20Z', '50.30Z', '50.40Z',
    '51.10Z', '51.21Z', '51.22Z', '52.10A', '52.10B', '52.21Z', '52.22Z', '52.23Z', '52.24A', '52.24B',
    '52.29A', '52.29B', '53.10Z', '53.20Z', '55.10Z', '55.20Z', '55.30Z', '55.90Z', '56.10A', '56.10B',
    '56.10C', '56.21Z', '56.29A', '56.29B', '56.30Z', '58.11Z', '58.12Z', '58.13Z', '58.14Z', '58.19Z',
    '58.21Z', '58.29A', '58.29B', '58.29C', '59.11A', '59.11B', '59.11C', '59.12Z', '59.13A', '59.13B',
    '59.14Z', '59.20Z', '60.10Z', '60.20A', '60.20B', '61.10Z', '61.20Z', '61.30Z', '61.90Z', '62.01Z',
    '62.02A', '62.02B', '62.03Z', '62.09Z', '63.11Z', '63.12Z', '63.91Z', '63.99Z', '64.11Z', '64.19Z',
    '64.20Z', '64.30Z', '64.91Z', '64.92Z', '64.99Z', '65.11Z', '65.12Z', '65.20Z', '65.30Z', '66.11Z',
    '66.12Z', '66.19A', '66.19B', '66.21Z', '66.22Z', '66.29Z', '66.30Z', '68.10Z', '68.20A', '68.20B',
    '68.31Z', '68.32A', '68.32B', '69.10Z', '69.20Z', '70.10Z', '70.21Z', '70.22Z', '71.11Z', '71.12A',
    '71.12B', '71.20A', '71.20B', '72.11Z', '72.19Z', '72.20Z', '73.11Z', '73.12Z', '73.20Z', '74.10Z',
    '74.20Z', '74.30Z', '74.90A', '74.90B', '75.00Z', '77.11A', '77.11B', '77.12Z', '77.21Z', '77.22Z',
    '77.29Z', '77.31Z', '77.32Z', '77.33Z', '77.34Z', '77.35Z', '77.39Z', '77.40Z', '78.10Z', '78.20Z',
    '78.30Z', '79.11Z', '79.12Z', '79.90Z', '80.10Z', '80.20Z', '80.30Z', '81.10Z', '81.21Z', '81.22Z',
    '81.29A', '81.29B', '81.30Z', '82.11Z', '82.19Z', '82.20Z', '82.30Z', '82.91Z', '82.92Z', '82.99Z',
    '84.11Z', '84.12Z', '84.13Z', '84.21Z', '84.22Z', '84.23Z', '84.24Z', '84.25Z', '84.30A', '84.30B',
    '84.30C', '85.10Z', '85.20Z', '85.31Z', '85.32Z', '85.41Z', '85.42Z', '85.51Z', '85.52Z', '85.53Z',
    '85.59A', '85.59B', '85.60Z', '86.10Z', '86.21Z', '86.22A', '86.22B', '86.22C', '86.23Z', '86.90A',
    '86.90B', '86.90C', '86.90D', '86.90E', '86.90F', '87.10A', '87.10B', '87.10C', '87.20A', '87.20B',
    '87.30A', '87.30B', '87.90A', '87.90B', '88.10A', '88.10B', '88.10C', '88.91A', '88.91B', '88.99A',
    '88.99B', '90.01Z', '90.02Z', '90.03A', '90.03B', '90.04Z', '91.01Z', '91.02Z', '91.03Z', '91.04Z',
    '92.00Z', '93.11Z', '93.12Z', '93.13Z', '93.19Z', '93.21Z', '93.29Z', '94.11Z', '94.12Z', '94.20Z',
    '94.91Z', '94.92Z', '94.99Z', '95.11Z', '95.12Z', '95.21Z', '95.22Z', '95.23Z', '95.24Z', '95.25Z',
    '95.29Z', '96.01A', '96.01B', '96.02A', '96.02B', '96.03Z', '96.04Z', '96.09Z', '97.00Z', '98.10Z',
    '98.20Z', '99.00Z',
]

# Plage de divisions (2 premiers chiffres du code NAF) par section -- permet de
# limiter le repli par code NAF precis (3e niveau) aux seuls codes de la section
# en cours, au lieu de retester les 732 codes en entier a chaque section plafonnee
# (bug corrige le 13/09/2026 : provoquait un blocage de plusieurs heures sur les
# grandes villes, ex. Toulouse, ou plusieurs sections restent plafonnees).
SECTION_DIVISIONS = {
    "A": range(1, 4), "B": range(5, 10), "C": range(10, 34), "D": range(35, 36),
    "E": range(36, 40), "F": range(41, 44), "G": range(45, 48), "H": range(49, 54),
    "I": range(55, 57), "J": range(58, 64), "K": range(64, 67), "L": range(68, 69),
    "M": range(69, 76), "N": range(77, 83), "O": range(84, 85), "P": range(85, 86),
    "Q": range(86, 89), "R": range(90, 94), "S": range(94, 97), "T": range(97, 99),
    "U": range(99, 100),
}


def codes_for_section(section):
    divisions = SECTION_DIVISIONS[section]
    return [c for c in NAF_CODES if int(c[:2]) in divisions]


CAP = 10000
PAGE_SIZE = 25
RATE_DELAY = 0.15  # ~6-7 req/s, sous le quota documente de l'API

DDL_PHASE1 = """
CREATE TABLE IF NOT EXISTS `Entreprises{dep}Phase1` (
  `ENT_Siren` char(9) NOT NULL,
  `ENT_NomComplet` varchar(255) DEFAULT NULL,
  `ENT_NomRaisonSociale` varchar(255) DEFAULT NULL,
  `ENT_Sigle` varchar(50) DEFAULT NULL,
  `ENT_AdresseSiege` varchar(255) DEFAULT NULL,
  `ENT_CodePostal` varchar(20) DEFAULT NULL,
  `ENT_Commune` varchar(100) DEFAULT NULL,
  `ENT_CodeCommune` char(5) DEFAULT NULL,
  `ENT_Departement` char(3) DEFAULT NULL,
  `ENT_Region` char(2) DEFAULT NULL,
  `ENT_DateCreation` date DEFAULT NULL,
  `ENT_EtatAdministratif` char(1) DEFAULT NULL,
  `ENT_DateFermeture` date DEFAULT NULL,
  `ENT_ActivitePrincipale` varchar(10) DEFAULT NULL,
  `ENT_SectionActivite` char(1) DEFAULT NULL,
  `ENT_NatureJuridique` varchar(10) DEFAULT NULL,
  `ENT_CategorieJuridique` enum('Privee','Publique','Association') NOT NULL DEFAULT 'Privee',
  `ENT_CategorieEntreprise` varchar(5) DEFAULT NULL,
  `ENT_TrancheEffectifSalarie` varchar(3) DEFAULT NULL,
  `ENT_AnneeEffectif` year DEFAULT NULL,
  `ENT_DirigeantNom` varchar(150) DEFAULT NULL,
  `ENT_DirigeantPrenoms` varchar(150) DEFAULT NULL,
  `ENT_DirigeantQualite` varchar(255) DEFAULT NULL,
  `ENT_FinancesAnnee` year DEFAULT NULL,
  `ENT_FinancesCa` bigint DEFAULT NULL,
  `ENT_FinancesResultatNet` bigint DEFAULT NULL,
  `ENT_StatutQualification` enum('a_qualifier','qualifiee','rejetee','contactee') NOT NULL DEFAULT 'a_qualifier',
  `ENT_CanalProspection` enum('AEvaluer','Ecartee','Email','Courrier','TelephoneRobot') NOT NULL DEFAULT 'AEvaluer',
  `ENT_Phase2` varchar(1) NOT NULL DEFAULT 'N',
  `ENT_ScoreCiblage` tinyint DEFAULT NULL,
  `ENT_Notes` text,
  `ENT_DateImport` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
  `ENT_DateMajInsee` datetime DEFAULT NULL,
  `ENT_ReponseBruteJson` json DEFAULT NULL,
  PRIMARY KEY (`ENT_Siren`),
  KEY `idx_commune` (`ENT_CodeCommune`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci
  ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=8
"""

DDL_PHASE2 = """
CREATE TABLE IF NOT EXISTS `Entreprises{dep}Phase2` (
  `ENT_Siren` char(9) NOT NULL,
  `ENT_SiteWeb` varchar(255) DEFAULT NULL,
  `ENT_Mail` varchar(255) DEFAULT NULL,
  `ENT_Telephone` varchar(20) DEFAULT NULL,
  `ENT_Portable` varchar(20) DEFAULT NULL,
  PRIMARY KEY (`ENT_Siren`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci
"""

DDL_PHASE3 = """
CREATE TABLE IF NOT EXISTS `Entreprises{dep}Phase3` (
  `ENT_Siren` char(9) NOT NULL,
  `ENT_FinancesAnnee` year NOT NULL,
  `ENT_FinancesCa` bigint DEFAULT NULL,
  `ENT_FinancesResultatNet` bigint DEFAULT NULL,
  `ENT_DateMaj` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
  PRIMARY KEY (`ENT_Siren`, `ENT_FinancesAnnee`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci
"""

DDL_COMMUNES = """
CREATE TABLE IF NOT EXISTS `ENT_Communes` (
  `COM_CodeInsee` char(5) NOT NULL,
  `COM_CodePostal` varchar(100) DEFAULT NULL,
  `COM_Ville` varchar(100) NOT NULL,
  `COM_Departement` char(3) NOT NULL,
  `COM_Latitude` decimal(10,6) DEFAULT NULL,
  `COM_Longitude` decimal(10,6) DEFAULT NULL,
  PRIMARY KEY (`COM_CodeInsee`),
  KEY `idx_departement` (`COM_Departement`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
"""

DDL_DEPARTEMENTS = """
CREATE TABLE IF NOT EXISTS `ENT_Departements` (
  `DEP_Code` char(3) NOT NULL,
  `DEP_Nom` varchar(100) NOT NULL,
  `DEP_NbEntreprises` int NOT NULL DEFAULT 0,
  `DEP_DateExtractionInitiale` datetime DEFAULT NULL,
  `DEP_DateDerniereSyncFormalites` datetime DEFAULT NULL,
  PRIMARY KEY (`DEP_Code`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
"""

DDL_CONFIG = """
CREATE TABLE IF NOT EXISTS `ENT_Config` (
  `CFG_Cle` varchar(100) NOT NULL,
  `CFG_Valeur` varchar(255) DEFAULT NULL,
  PRIMARY KEY (`CFG_Cle`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
"""


def log(msg):
    print(msg, file=sys.stderr, flush=True)


def http_get_json(url, retries=5):
    for attempt in range(retries):
        try:
            with urllib.request.urlopen(url, timeout=20) as resp:
                return json.load(resp)
        except (urllib.error.URLError, TimeoutError) as e:
            if attempt == retries - 1:
                raise
            time.sleep(1 + attempt)


# Paris, Lyon et Marseille (PLM) : dans les donnees Sirene, le siege d'une
# entreprise de ces trois villes est quasi toujours code au niveau de
# l'arrondissement (75101-75120, 69381-69389, 13201-13216), jamais au code
# commune englobant (75056, 69123, 13055) -- verifie empiriquement le 14
# septembre 2026 sur Marseille : code_commune=13055 ne renvoie qu'une dizaine
# de resultats, alors que chaque arrondissement (13201 a 13216) est a lui
# seul plafonne ou proche du plafond de 10 000. Le code englobant est donc
# remplace par la liste de ses arrondissements avant extraction.
COMMUNES_ARRONDISSEMENTS = {"75": "75056", "69": "69123", "13": "13055"}


def fetch_arrondissements_geo(dep):
    url = "https://geo.api.gouv.fr/communes?" + urllib.parse.urlencode(
        {"codeDepartement": dep, "type": "arrondissement-municipal",
         "fields": "code,nom,centre,codesPostaux,departement"}
    )
    return http_get_json(url)


def fetch_communes_geo(dep):
    url = API_GEO.format(dep=dep) + "?" + urllib.parse.urlencode(
        {"fields": "code,nom,centre,codesPostaux,departement"}
    )
    communes = http_get_json(url)
    code_englobant = COMMUNES_ARRONDISSEMENTS.get(dep)
    if code_englobant:
        communes = [c for c in communes if c["code"] != code_englobant]
        communes.extend(fetch_arrondissements_geo(dep))
    return communes


def fetch_departement_nom(dep):
    url = f"https://geo.api.gouv.fr/departements/{dep}?fields=code,nom"
    data = http_get_json(url)
    return data.get("nom", dep)


def upsert_departement(conn, dep, nom, nb_entreprises):
    with conn.cursor() as cur:
        cur.execute(
            "INSERT INTO ENT_Departements (DEP_Code, DEP_Nom, DEP_NbEntreprises, DEP_DateExtractionInitiale) "
            "VALUES (%s, %s, %s, NOW()) "
            "ON DUPLICATE KEY UPDATE DEP_Nom=VALUES(DEP_Nom), DEP_NbEntreprises=VALUES(DEP_NbEntreprises)",
            (dep, nom, nb_entreprises),
        )


NON_DIFFUSIBLE = "[NON-DIFFUSIBLE]"


def clean(v):
    """Les entites en diffusion partielle/non diffusible (RGPD) renvoient le
    sentinel textuel [NON-DIFFUSIBLE] sur de nombreux champs (adresse, code postal,
    coordonnees, nom/prenoms du dirigeant...). On le convertit en NULL plutot que
    de le stocker (et ca evite aussi de depasser la taille des colonnes typees)."""
    return None if v == NON_DIFFUSIBLE else v


def derive_categorie_juridique(nature_juridique):
    if not nature_juridique:
        return "Privee"
    first = nature_juridique[0]
    if first in ("7", "8"):
        return "Publique"
    if first == "9":
        return "Association"
    return "Privee"


def map_entreprise(r):
    siege = r.get("siege") or {}
    dirigeants = r.get("dirigeants") or []
    dirigeant = dirigeants[0] if dirigeants else {}
    finances = r.get("finances") or {}
    finances_annee = max(finances.keys()) if finances else None
    fin = finances.get(finances_annee, {}) if finances_annee else {}

    return {
        "ENT_Siren": r.get("siren"),
        "ENT_NomComplet": clean(r.get("nom_complet")),
        "ENT_NomRaisonSociale": clean(r.get("nom_raison_sociale")),
        "ENT_Sigle": clean(r.get("sigle")),
        "ENT_AdresseSiege": clean(siege.get("adresse")),
        "ENT_CodePostal": clean(siege.get("code_postal")),
        "ENT_Commune": clean(siege.get("libelle_commune")),
        "ENT_CodeCommune": clean(siege.get("commune")),
        "ENT_Departement": clean(siege.get("departement")),
        "ENT_Region": clean(siege.get("region")),
        "ENT_DateCreation": clean(r.get("date_creation")),
        "ENT_EtatAdministratif": clean(r.get("etat_administratif")),
        "ENT_DateFermeture": clean(r.get("date_fermeture")),
        "ENT_ActivitePrincipale": clean(r.get("activite_principale")),
        "ENT_SectionActivite": clean(r.get("section_activite_principale")),
        "ENT_NatureJuridique": clean(r.get("nature_juridique")),
        "ENT_CategorieJuridique": derive_categorie_juridique(clean(r.get("nature_juridique"))),
        "ENT_CategorieEntreprise": clean(r.get("categorie_entreprise")),
        "ENT_TrancheEffectifSalarie": clean(r.get("tranche_effectif_salarie")),
        "ENT_AnneeEffectif": clean(r.get("annee_tranche_effectif_salarie")),
        "ENT_DirigeantNom": clean(dirigeant.get("nom")),
        "ENT_DirigeantPrenoms": clean(dirigeant.get("prenoms")),
        "ENT_DirigeantQualite": clean(dirigeant.get("qualite")),
        "ENT_FinancesAnnee": finances_annee,
        "ENT_FinancesCa": fin.get("ca"),
        "ENT_FinancesResultatNet": fin.get("resultat_net"),
        "ENT_ReponseBruteJson": json.dumps(r, ensure_ascii=False),
    }


def probe_total(params):
    p = dict(params, per_page=1)
    data = http_get_json(API_SEARCH + "?" + urllib.parse.urlencode(p))
    time.sleep(RATE_DELAY)
    return data.get("total_results", 0)


def paginate_all(params, seen):
    p = dict(params, per_page=PAGE_SIZE)
    page = 1
    out = []
    while True:
        p["page"] = page
        data = http_get_json(API_SEARCH + "?" + urllib.parse.urlencode(p))
        time.sleep(RATE_DELAY)
        results = data.get("results", [])
        if not results:
            break
        for r in results:
            siren = r.get("siren")
            if siren and siren not in seen:
                seen.add(siren)
                out.append(r)
        total_pages = data.get("total_pages", page)
        if page >= total_pages or page >= 400:  # 400*25 = 10000, garde-fou
            break
        page += 1
    return out


def fetch_commune_entreprises(code_commune, seen):
    base = {"code_commune": code_commune, "etat_administratif": "A"}
    total = probe_total(base)
    if total < CAP:
        return paginate_all(base, seen)

    log(f"    commune {code_commune} plafonnee ({total}), repli par section NAF")
    out = []
    for section in NAF_SECTIONS:
        sec_params = dict(base, section_activite_principale=section)
        sec_total = probe_total(sec_params)
        if sec_total == 0:
            continue
        if sec_total < CAP:
            out.extend(paginate_all(sec_params, seen))
            continue

        # 3e niveau de repli : la section a elle seule reste plafonnee (cas rare,
        # ex. Rennes/section L). On decoupe par code NAF precis (732 codes possibles) --
        # un code implique deja sa section, donc on l'utilise seul, sans le combiner
        # au filtre de section.
        log(f"    commune {code_commune} section {section} encore plafonnee ({sec_total}), "
            f"repli par code NAF precis")
        for code in codes_for_section(section):
            code_params = dict(base, activite_principale=code)
            code_total = probe_total(code_params)
            if code_total == 0:
                continue
            if code_total >= CAP:
                log(f"      !! commune {code_commune} code {code} encore plafonnee "
                    f"({code_total}) -- a decouper manuellement (ex: par tranche_effectif_salarie)")
            out.extend(paginate_all(code_params, seen))
    return out


def upsert_entreprise(conn, table, row):
    cols = list(row.keys())
    placeholders = ", ".join(["%s"] * len(cols))
    colnames = ", ".join(f"`{c}`" for c in cols)
    updates = ", ".join(f"`{c}`=VALUES(`{c}`)" for c in cols if c != "ENT_Siren")
    sql = (
        f"INSERT INTO `{table}` ({colnames}) VALUES ({placeholders}) "
        f"ON DUPLICATE KEY UPDATE {updates}, `ENT_DateMajInsee`=NOW()"
    )
    with conn.cursor() as cur:
        cur.execute(sql, list(row.values()))


def upsert_finances(conn, table, row):
    if row.get("ENT_FinancesAnnee") is None:
        return
    fin = {
        "ENT_Siren": row["ENT_Siren"],
        "ENT_FinancesAnnee": row["ENT_FinancesAnnee"],
        "ENT_FinancesCa": row["ENT_FinancesCa"],
        "ENT_FinancesResultatNet": row["ENT_FinancesResultatNet"],
    }
    cols = list(fin.keys())
    placeholders = ", ".join(["%s"] * len(cols))
    colnames = ", ".join(f"`{c}`" for c in cols)
    updates = ", ".join(
        f"`{c}`=VALUES(`{c}`)" for c in cols if c not in ("ENT_Siren", "ENT_FinancesAnnee")
    )
    sql = (
        f"INSERT INTO `{table}` ({colnames}) VALUES ({placeholders}) "
        f"ON DUPLICATE KEY UPDATE {updates}"
    )
    with conn.cursor() as cur:
        cur.execute(sql, list(fin.values()))


def ensure_tables(conn, dep):
    with conn.cursor() as cur:
        cur.execute(DDL_PHASE1.format(dep=dep))
        cur.execute(DDL_PHASE2.format(dep=dep))
        cur.execute(DDL_PHASE3.format(dep=dep))
        cur.execute(DDL_COMMUNES)
        cur.execute(DDL_DEPARTEMENTS)
        cur.execute(DDL_CONFIG)


def ensure_communes(conn, dep, communes_geo):
    with conn.cursor() as cur:
        cur.execute("SELECT COUNT(*) FROM ENT_Communes WHERE COM_Departement=%s", (dep,))
        (count,) = cur.fetchone()
        if count > 0:
            log(f"  ENT_Communes deja alimentee pour le {dep} ({count} communes), on ne rejoue pas")
            return
        rows = []
        for c in communes_geo:
            lon, lat = c["centre"]["coordinates"]
            cp = ",".join(c.get("codesPostaux", []))
            dep_code = c["departement"]["code"] if isinstance(c.get("departement"), dict) else c.get("departement")
            rows.append((c["code"], cp, c["nom"], dep_code, lat, lon))
        cur.executemany(
            "INSERT INTO ENT_Communes "
            "(COM_CodeInsee, COM_CodePostal, COM_Ville, COM_Departement, COM_Latitude, COM_Longitude) "
            "VALUES (%s,%s,%s,%s,%s,%s)",
            rows,
        )
        log(f"  ENT_Communes : {len(rows)} communes inserees pour le {dep}")


def run_department(conn, dep):
    log(f"=== Departement {dep} ===")
    ensure_tables(conn, dep)
    communes_geo = fetch_communes_geo(dep)
    log(f"  {len(communes_geo)} communes (source geo.api.gouv.fr)")
    ensure_communes(conn, dep, communes_geo)

    table1 = f"Entreprises{dep}Phase1"
    table3 = f"Entreprises{dep}Phase3"
    seen = set()
    total_upserts = 0
    total_finances = 0
    for c in communes_geo:
        results = fetch_commune_entreprises(c["code"], seen)
        for r in results:
            row = map_entreprise(r)
            if row["ENT_Siren"]:
                upsert_entreprise(conn, table1, row)
                total_upserts += 1
                if row["ENT_FinancesAnnee"] is not None:
                    upsert_finances(conn, table3, row)
                    total_finances += 1
        if results:
            log(f"  {c['nom']} ({c['code']}) : {len(results)} entreprises")

    with conn.cursor() as cur:
        cur.execute(f"SELECT COUNT(*) FROM `{table1}`")
        (nb_total,) = cur.fetchone()
    nom_dep = fetch_departement_nom(dep)
    upsert_departement(conn, dep, nom_dep, nb_total)

    log(f"Departement {dep} termine : {total_upserts} entreprises upsertees dans {table1}, "
        f"{total_finances} lignes financieres upsertees dans {table3} (total {nb_total} -- "
        f"ENT_Departements mise a jour)")


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--departements", required=True,
                    help="Liste separee par des virgules, ex: 56,17,64")
    ap.add_argument("--db-host", default=os.environ.get("ENTREPRISES_DB_HOST", "mysql2"))
    ap.add_argument("--db-port", type=int, default=int(os.environ.get("ENTREPRISES_DB_PORT", "3306")))
    ap.add_argument("--db-user", default=os.environ.get("ENTREPRISES_DB_USER", "crm_user"))
    ap.add_argument("--db-pass", default=os.environ.get("ENTREPRISES_DB_PASS"))
    ap.add_argument("--db-name", default=os.environ.get("ENTREPRISES_DB_NAME", "ENTREPRISES"))
    args = ap.parse_args()

    if not args.db_pass:
        ap.error("mot de passe requis : --db-pass ou variable d'environnement ENTREPRISES_DB_PASS")

    departements = [d.strip() for d in args.departements.split(",") if d.strip()]

    conn = pymysql.connect(
        host=args.db_host, port=args.db_port, user=args.db_user, password=args.db_pass,
        database=args.db_name, charset="utf8mb4", autocommit=True,
    )
    try:
        for dep in departements:
            run_department(conn, dep)
    finally:
        conn.close()


if __name__ == "__main__":
    main()