Jean-Marc/ASPIC
1
0
Fork 0
mirror of https://github.com/JMP-MS/ASPIC.git synced 2026-10-27 16:07:10 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
2825430bf41b01195b03805bf46e9bdbe2877532
ASPIC/scripts/sirene_extract.py
T

560 lines
26 KiB
Python
Raw Blame History

#!/usr/bin/env python3
"""
Extraction Sirene/RNE (recherche-entreprises.api.gouv.fr) par departement,
avec decoupage automatique commune -> section NAF pour contourner le plafond
de 10 000 resultats par requete de l'API.
Alimente, dans la base ENTREPRISES (instance mysql2) :
- Entreprises<DEP>Phase1 (cree si absente, jamais ecrasee si presente)
- Entreprises<DEP>Phase2 (cree si absente)
- ENT_Communes (table partagee, alimentee une fois par departement)
Ne touche jamais aux colonnes de qualification/prospection (ENT_StatutQualification,
ENT_CanalProspection, ENT_Phase2, ENT_ScoreCiblage, ENT_Notes) : seules les colonnes
issues de l'API sont mises a jour sur un SIREN deja present.
Usage :
python3 sirene_extract.py --departements 56,17,64 --db-pass '...'
(ou export ENTREPRISES_DB_PASS=... pour ne pas passer le mot de passe en argument)
"""
import argparse
import json
import os
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import pymysql
API_SEARCH = "https://recherche-entreprises.api.gouv.fr/search"
API_GEO = "https://geo.api.gouv.fr/departements/{dep}/communes"
NAF_SECTIONS = list("ABCDEFGHIJKLMNOPQRSTU")
# Liste complete des 732 codes NAF (rev. 2) reconnus par l'API, recuperee depuis son
# propre message d'erreur de validation. Utilisee comme 3e niveau de repli (apres
# commune puis section NAF) quand une section reste plafonnee a 10000 malgre le 2e
# niveau -- rare (1 cas sur ~950 communes traitees au 12 septembre 2026 : Rennes,
# section L, tranche NN), mais le decoupage par section seule ne suffit alors plus.
NAF_CODES = [
'01.11Z', '01.12Z', '01.13Z', '01.14Z', '01.15Z', '01.16Z', '01.19Z', '01.21Z', '01.22Z', '01.23Z',
'01.24Z', '01.25Z', '01.26Z', '01.27Z', '01.28Z', '01.29Z', '01.30Z', '01.41Z', '01.42Z', '01.43Z',
'01.44Z', '01.45Z', '01.46Z', '01.47Z', '01.49Z', '01.50Z', '01.61Z', '01.62Z', '01.63Z', '01.64Z',
'01.70Z', '02.10Z', '02.20Z', '02.30Z', '02.40Z', '03.11Z', '03.12Z', '03.21Z', '03.22Z', '05.10Z',
'05.20Z', '06.10Z', '06.20Z', '07.10Z', '07.21Z', '07.29Z', '08.11Z', '08.12Z', '08.91Z', '08.92Z',
'08.93Z', '08.99Z', '09.10Z', '09.90Z', '10.11Z', '10.12Z', '10.13A', '10.13B', '10.20Z', '10.31Z',
'10.32Z', '10.39A', '10.39B', '10.41A', '10.41B', '10.42Z', '10.51A', '10.51B', '10.51C', '10.51D',
'10.52Z', '10.61A', '10.61B', '10.62Z', '10.71A', '10.71B', '10.71C', '10.71D', '10.72Z', '10.73Z',
'10.81Z', '10.82Z', '10.83Z', '10.84Z', '10.85Z', '10.86Z', '10.89Z', '10.91Z', '10.92Z', '11.01Z',
'11.02A', '11.02B', '11.03Z', '11.04Z', '11.05Z', '11.06Z', '11.07A', '11.07B', '12.00Z', '13.10Z',
'13.20Z', '13.30Z', '13.91Z', '13.92Z', '13.93Z', '13.94Z', '13.95Z', '13.96Z', '13.99Z', '14.11Z',
'14.12Z', '14.13Z', '14.14Z', '14.19Z', '14.20Z', '14.31Z', '14.39Z', '15.11Z', '15.12Z', '15.20Z',
'16.10A', '16.10B', '16.21Z', '16.22Z', '16.23Z', '16.24Z', '16.29Z', '17.11Z', '17.12Z', '17.21A',
'17.21B', '17.21C', '17.22Z', '17.23Z', '17.24Z', '17.29Z', '18.11Z', '18.12Z', '18.13Z', '18.14Z',
'18.20Z', '19.10Z', '19.20Z', '20.11Z', '20.12Z', '20.13A', '20.13B', '20.14Z', '20.15Z', '20.16Z',
'20.17Z', '20.20Z', '20.30Z', '20.41Z', '20.42Z', '20.51Z', '20.52Z', '20.53Z', '20.59Z', '20.60Z',
'21.10Z', '21.20Z', '22.11Z', '22.19Z', '22.21Z', '22.22Z', '22.23Z', '22.29A', '22.29B', '23.11Z',
'23.12Z', '23.13Z', '23.14Z', '23.19Z', '23.20Z', '23.31Z', '23.32Z', '23.41Z', '23.42Z', '23.43Z',
'23.44Z', '23.49Z', '23.51Z', '23.52Z', '23.61Z', '23.62Z', '23.63Z', '23.64Z', '23.65Z', '23.69Z',
'23.70Z', '23.91Z', '23.99Z', '24.10Z', '24.20Z', '24.31Z', '24.32Z', '24.33Z', '24.34Z', '24.41Z',
'24.42Z', '24.43Z', '24.44Z', '24.45Z', '24.46Z', '24.51Z', '24.52Z', '24.53Z', '24.54Z', '25.11Z',
'25.12Z', '25.21Z', '25.29Z', '25.30Z', '25.40Z', '25.50A', '25.50B', '25.61Z', '25.62A', '25.62B',
'25.71Z', '25.72Z', '25.73A', '25.73B', '25.91Z', '25.92Z', '25.93Z', '25.94Z', '25.99A', '25.99B',
'26.11Z', '26.12Z', '26.20Z', '26.30Z', '26.40Z', '26.51A', '26.51B', '26.52Z', '26.60Z', '26.70Z',
'26.80Z', '27.11Z', '27.12Z', '27.20Z', '27.31Z', '27.32Z', '27.33Z', '27.40Z', '27.51Z', '27.52Z',
'27.90Z', '28.11Z', '28.12Z', '28.13Z', '28.14Z', '28.15Z', '28.21Z', '28.22Z', '28.23Z', '28.24Z',
'28.25Z', '28.29A', '28.29B', '28.30Z', '28.41Z', '28.49Z', '28.91Z', '28.92Z', '28.93Z', '28.94Z',
'28.95Z', '28.96Z', '28.99A', '28.99B', '29.10Z', '29.20Z', '29.31Z', '29.32Z', '30.11Z', '30.12Z',
'30.20Z', '30.30Z', '30.40Z', '30.91Z', '30.92Z', '30.99Z', '31.01Z', '31.02Z', '31.03Z', '31.09A',
'31.09B', '32.11Z', '32.12Z', '32.13Z', '32.20Z', '32.30Z', '32.40Z', '32.50A', '32.50B', '32.91Z',
'32.99Z', '33.11Z', '33.12Z', '33.13Z', '33.14Z', '33.15Z', '33.16Z', '33.17Z', '33.19Z', '33.20A',
'33.20B', '33.20C', '33.20D', '35.11Z', '35.12Z', '35.13Z', '35.14Z', '35.21Z', '35.22Z', '35.23Z',
'35.30Z', '36.00Z', '37.00Z', '38.11Z', '38.12Z', '38.21Z', '38.22Z', '38.31Z', '38.32Z', '39.00Z',
'41.10A', '41.10B', '41.10C', '41.10D', '41.20A', '41.20B', '42.11Z', '42.12Z', '42.13A', '42.13B',
'42.21Z', '42.22Z', '42.91Z', '42.99Z', '43.11Z', '43.12A', '43.12B', '43.13Z', '43.21A', '43.21B',
'43.22A', '43.22B', '43.29A', '43.29B', '43.31Z', '43.32A', '43.32B', '43.32C', '43.33Z', '43.34Z',
'43.39Z', '43.91A', '43.91B', '43.99A', '43.99B', '43.99C', '43.99D', '43.99E', '45.11Z', '45.19Z',
'45.20A', '45.20B', '45.31Z', '45.32Z', '45.40Z', '46.11Z', '46.12A', '46.12B', '46.13Z', '46.14Z',
'46.15Z', '46.16Z', '46.17A', '46.17B', '46.18Z', '46.19A', '46.19B', '46.21Z', '46.22Z', '46.23Z',
'46.24Z', '46.31Z', '46.32A', '46.32B', '46.32C', '46.33Z', '46.34Z', '46.35Z', '46.36Z', '46.37Z',
'46.38A', '46.38B', '46.39A', '46.39B', '46.41Z', '46.42Z', '46.43Z', '46.44Z', '46.45Z', '46.46Z',
'46.47Z', '46.48Z', '46.49Z', '46.51Z', '46.52Z', '46.61Z', '46.62Z', '46.63Z', '46.64Z', '46.65Z',
'46.66Z', '46.69A', '46.69B', '46.69C', '46.71Z', '46.72Z', '46.73A', '46.73B', '46.74A', '46.74B',
'46.75Z', '46.76Z', '46.77Z', '46.90Z', '47.11A', '47.11B', '47.11C', '47.11D', '47.11E', '47.11F',
'47.19A', '47.19B', '47.21Z', '47.22Z', '47.23Z', '47.24Z', '47.25Z', '47.26Z', '47.29Z', '47.30Z',
'47.41Z', '47.42Z', '47.43Z', '47.51Z', '47.52A', '47.52B', '47.53Z', '47.54Z', '47.59A', '47.59B',
'47.61Z', '47.62Z', '47.63Z', '47.64Z', '47.65Z', '47.71Z', '47.72A', '47.72B', '47.73Z', '47.74Z',
'47.75Z', '47.76Z', '47.77Z', '47.78A', '47.78B', '47.78C', '47.79Z', '47.81Z', '47.82Z', '47.89Z',
'47.91A', '47.91B', '47.99A', '47.99B', '49.10Z', '49.20Z', '49.31Z', '49.32Z', '49.39A', '49.39B',
'49.39C', '49.41A', '49.41B', '49.41C', '49.42Z', '49.50Z', '50.10Z', '50.20Z', '50.30Z', '50.40Z',
'51.10Z', '51.21Z', '51.22Z', '52.10A', '52.10B', '52.21Z', '52.22Z', '52.23Z', '52.24A', '52.24B',
'52.29A', '52.29B', '53.10Z', '53.20Z', '55.10Z', '55.20Z', '55.30Z', '55.90Z', '56.10A', '56.10B',
'56.10C', '56.21Z', '56.29A', '56.29B', '56.30Z', '58.11Z', '58.12Z', '58.13Z', '58.14Z', '58.19Z',
'58.21Z', '58.29A', '58.29B', '58.29C', '59.11A', '59.11B', '59.11C', '59.12Z', '59.13A', '59.13B',
'59.14Z', '59.20Z', '60.10Z', '60.20A', '60.20B', '61.10Z', '61.20Z', '61.30Z', '61.90Z', '62.01Z',
'62.02A', '62.02B', '62.03Z', '62.09Z', '63.11Z', '63.12Z', '63.91Z', '63.99Z', '64.11Z', '64.19Z',
'64.20Z', '64.30Z', '64.91Z', '64.92Z', '64.99Z', '65.11Z', '65.12Z', '65.20Z', '65.30Z', '66.11Z',
'66.12Z', '66.19A', '66.19B', '66.21Z', '66.22Z', '66.29Z', '66.30Z', '68.10Z', '68.20A', '68.20B',
'68.31Z', '68.32A', '68.32B', '69.10Z', '69.20Z', '70.10Z', '70.21Z', '70.22Z', '71.11Z', '71.12A',
'71.12B', '71.20A', '71.20B', '72.11Z', '72.19Z', '72.20Z', '73.11Z', '73.12Z', '73.20Z', '74.10Z',
'74.20Z', '74.30Z', '74.90A', '74.90B', '75.00Z', '77.11A', '77.11B', '77.12Z', '77.21Z', '77.22Z',
'77.29Z', '77.31Z', '77.32Z', '77.33Z', '77.34Z', '77.35Z', '77.39Z', '77.40Z', '78.10Z', '78.20Z',
'78.30Z', '79.11Z', '79.12Z', '79.90Z', '80.10Z', '80.20Z', '80.30Z', '81.10Z', '81.21Z', '81.22Z',
'81.29A', '81.29B', '81.30Z', '82.11Z', '82.19Z', '82.20Z', '82.30Z', '82.91Z', '82.92Z', '82.99Z',
'84.11Z', '84.12Z', '84.13Z', '84.21Z', '84.22Z', '84.23Z', '84.24Z', '84.25Z', '84.30A', '84.30B',
'84.30C', '85.10Z', '85.20Z', '85.31Z', '85.32Z', '85.41Z', '85.42Z', '85.51Z', '85.52Z', '85.53Z',
'85.59A', '85.59B', '85.60Z', '86.10Z', '86.21Z', '86.22A', '86.22B', '86.22C', '86.23Z', '86.90A',
'86.90B', '86.90C', '86.90D', '86.90E', '86.90F', '87.10A', '87.10B', '87.10C', '87.20A', '87.20B',
'87.30A', '87.30B', '87.90A', '87.90B', '88.10A', '88.10B', '88.10C', '88.91A', '88.91B', '88.99A',
'88.99B', '90.01Z', '90.02Z', '90.03A', '90.03B', '90.04Z', '91.01Z', '91.02Z', '91.03Z', '91.04Z',
'92.00Z', '93.11Z', '93.12Z', '93.13Z', '93.19Z', '93.21Z', '93.29Z', '94.11Z', '94.12Z', '94.20Z',
'94.91Z', '94.92Z', '94.99Z', '95.11Z', '95.12Z', '95.21Z', '95.22Z', '95.23Z', '95.24Z', '95.25Z',
'95.29Z', '96.01A', '96.01B', '96.02A', '96.02B', '96.03Z', '96.04Z', '96.09Z', '97.00Z', '98.10Z',
'98.20Z', '99.00Z',
]
# Plage de divisions (2 premiers chiffres du code NAF) par section -- permet de
# limiter le repli par code NAF precis (3e niveau) aux seuls codes de la section
# en cours, au lieu de retester les 732 codes en entier a chaque section plafonnee
# (bug corrige le 13/09/2026 : provoquait un blocage de plusieurs heures sur les
# grandes villes, ex. Toulouse, ou plusieurs sections restent plafonnees).
SECTION_DIVISIONS = {
"A": range(1, 4), "B": range(5, 10), "C": range(10, 34), "D": range(35, 36),
"E": range(36, 40), "F": range(41, 44), "G": range(45, 48), "H": range(49, 54),
"I": range(55, 57), "J": range(58, 64), "K": range(64, 67), "L": range(68, 69),
"M": range(69, 76), "N": range(77, 83), "O": range(84, 85), "P": range(85, 86),
"Q": range(86, 89), "R": range(90, 94), "S": range(94, 97), "T": range(97, 99),
"U": range(99, 100),
}
def codes_for_section(section):
divisions = SECTION_DIVISIONS[section]
return [c for c in NAF_CODES if int(c[:2]) in divisions]
CAP = 10000
PAGE_SIZE = 25
RATE_DELAY = 0.15 # ~6-7 req/s, sous le quota documente de l'API
DDL_PHASE1 = """
CREATE TABLE IF NOT EXISTS `Entreprises{dep}Phase1` (
`ENT_Siren` char(9) NOT NULL,
`ENT_NomComplet` varchar(255) DEFAULT NULL,
`ENT_NomRaisonSociale` varchar(255) DEFAULT NULL,
`ENT_Sigle` varchar(50) DEFAULT NULL,
`ENT_AdresseSiege` varchar(255) DEFAULT NULL,
`ENT_CodePostal` varchar(20) DEFAULT NULL,
`ENT_Commune` varchar(100) DEFAULT NULL,
`ENT_CodeCommune` char(5) DEFAULT NULL,
`ENT_Departement` char(3) DEFAULT NULL,
`ENT_Region` char(2) DEFAULT NULL,
`ENT_DateCreation` date DEFAULT NULL,
`ENT_EtatAdministratif` char(1) DEFAULT NULL,
`ENT_DateFermeture` date DEFAULT NULL,
`ENT_ActivitePrincipale` varchar(10) DEFAULT NULL,
`ENT_SectionActivite` char(1) DEFAULT NULL,
`ENT_NatureJuridique` varchar(10) DEFAULT NULL,
`ENT_CategorieJuridique` enum('Privee','Publique','Association') NOT NULL DEFAULT 'Privee',
`ENT_CategorieEntreprise` varchar(5) DEFAULT NULL,
`ENT_TrancheEffectifSalarie` varchar(3) DEFAULT NULL,
`ENT_AnneeEffectif` year DEFAULT NULL,
`ENT_DirigeantNom` varchar(150) DEFAULT NULL,
`ENT_DirigeantPrenoms` varchar(150) DEFAULT NULL,
`ENT_DirigeantQualite` varchar(255) DEFAULT NULL,
`ENT_FinancesAnnee` year DEFAULT NULL,
`ENT_FinancesCa` bigint DEFAULT NULL,
`ENT_FinancesResultatNet` bigint DEFAULT NULL,
`ENT_StatutQualification` enum('a_qualifier','qualifiee','rejetee','contactee') NOT NULL DEFAULT 'a_qualifier',
`ENT_CanalProspection` enum('AEvaluer','Ecartee','Email','Courrier','TelephoneRobot') NOT NULL DEFAULT 'AEvaluer',
`ENT_Phase2` varchar(1) NOT NULL DEFAULT 'N',
`ENT_ScoreCiblage` tinyint DEFAULT NULL,
`ENT_Notes` text,
`ENT_DateImport` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
`ENT_DateMajInsee` datetime DEFAULT NULL,
`ENT_ReponseBruteJson` json DEFAULT NULL,
PRIMARY KEY (`ENT_Siren`),
KEY `idx_commune` (`ENT_CodeCommune`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci
ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=8
"""
DDL_PHASE2 = """
CREATE TABLE IF NOT EXISTS `Entreprises{dep}Phase2` (
`ENT_Siren` char(9) NOT NULL,
`ENT_SiteWeb` varchar(255) DEFAULT NULL,
`ENT_Mail` varchar(255) DEFAULT NULL,
`ENT_Telephone` varchar(20) DEFAULT NULL,
`ENT_Portable` varchar(20) DEFAULT NULL,
PRIMARY KEY (`ENT_Siren`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci
"""
DDL_PHASE3 = """
CREATE TABLE IF NOT EXISTS `Entreprises{dep}Phase3` (
`ENT_Siren` char(9) NOT NULL,
`ENT_FinancesAnnee` year NOT NULL,
`ENT_FinancesCa` bigint DEFAULT NULL,
`ENT_FinancesResultatNet` bigint DEFAULT NULL,
`ENT_DateMaj` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`ENT_Siren`, `ENT_FinancesAnnee`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci
"""
DDL_COMMUNES = """
CREATE TABLE IF NOT EXISTS `ENT_Communes` (
`COM_CodeInsee` char(5) NOT NULL,
`COM_CodePostal` varchar(100) DEFAULT NULL,
`COM_Ville` varchar(100) NOT NULL,
`COM_Departement` char(3) NOT NULL,
`COM_Latitude` decimal(10,6) DEFAULT NULL,
`COM_Longitude` decimal(10,6) DEFAULT NULL,
PRIMARY KEY (`COM_CodeInsee`),
KEY `idx_departement` (`COM_Departement`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
"""
DDL_DEPARTEMENTS = """
CREATE TABLE IF NOT EXISTS `ENT_Departements` (
`DEP_Code` char(3) NOT NULL,
`DEP_Nom` varchar(100) NOT NULL,
`DEP_NbEntreprises` int NOT NULL DEFAULT 0,
`DEP_DateExtractionInitiale` datetime DEFAULT NULL,
`DEP_DateDerniereSyncFormalites` datetime DEFAULT NULL,
PRIMARY KEY (`DEP_Code`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
"""
DDL_CONFIG = """
CREATE TABLE IF NOT EXISTS `ENT_Config` (
`CFG_Cle` varchar(100) NOT NULL,
`CFG_Valeur` varchar(255) DEFAULT NULL,
PRIMARY KEY (`CFG_Cle`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
"""
def log(msg):
print(msg, file=sys.stderr, flush=True)
def http_get_json(url, retries=5):
for attempt in range(retries):
try:
with urllib.request.urlopen(url, timeout=20) as resp:
return json.load(resp)
except (urllib.error.URLError, TimeoutError) as e:
if attempt == retries - 1:
raise
time.sleep(1 + attempt)
# Paris, Lyon et Marseille (PLM) : dans les donnees Sirene, le siege d'une
# entreprise de ces trois villes est quasi toujours code au niveau de
# l'arrondissement (75101-75120, 69381-69389, 13201-13216), jamais au code
# commune englobant (75056, 69123, 13055) -- verifie empiriquement le 14
# septembre 2026 sur Marseille : code_commune=13055 ne renvoie qu'une dizaine
# de resultats, alors que chaque arrondissement (13201 a 13216) est a lui
# seul plafonne ou proche du plafond de 10 000. Le code englobant est donc
# remplace par la liste de ses arrondissements avant extraction.
COMMUNES_ARRONDISSEMENTS = {"75": "75056", "69": "69123", "13": "13055"}
def fetch_arrondissements_geo(dep):
url = "https://geo.api.gouv.fr/communes?" + urllib.parse.urlencode(
{"codeDepartement": dep, "type": "arrondissement-municipal",
"fields": "code,nom,centre,codesPostaux,departement"}
)
return http_get_json(url)
def fetch_communes_geo(dep):
url = API_GEO.format(dep=dep) + "?" + urllib.parse.urlencode(
{"fields": "code,nom,centre,codesPostaux,departement"}
)
communes = http_get_json(url)
code_englobant = COMMUNES_ARRONDISSEMENTS.get(dep)
if code_englobant:
communes = [c for c in communes if c["code"] != code_englobant]
communes.extend(fetch_arrondissements_geo(dep))
return communes
def fetch_departement_nom(dep):
url = f"https://geo.api.gouv.fr/departements/{dep}?fields=code,nom"
data = http_get_json(url)
return data.get("nom", dep)
def upsert_departement(conn, dep, nom, nb_entreprises):
with conn.cursor() as cur:
cur.execute(
"INSERT INTO ENT_Departements (DEP_Code, DEP_Nom, DEP_NbEntreprises, DEP_DateExtractionInitiale) "
"VALUES (%s, %s, %s, NOW()) "
"ON DUPLICATE KEY UPDATE DEP_Nom=VALUES(DEP_Nom), DEP_NbEntreprises=VALUES(DEP_NbEntreprises)",
(dep, nom, nb_entreprises),
)
NON_DIFFUSIBLE = "[NON-DIFFUSIBLE]"
def clean(v):
"""Les entites en diffusion partielle/non diffusible (RGPD) renvoient le
sentinel textuel [NON-DIFFUSIBLE] sur de nombreux champs (adresse, code postal,
coordonnees, nom/prenoms du dirigeant...). On le convertit en NULL plutot que
de le stocker (et ca evite aussi de depasser la taille des colonnes typees)."""
return None if v == NON_DIFFUSIBLE else v
def derive_categorie_juridique(nature_juridique):
if not nature_juridique:
return "Privee"
first = nature_juridique[0]
if first in ("7", "8"):
return "Publique"
if first == "9":
return "Association"
return "Privee"
def map_entreprise(r):
siege = r.get("siege") or {}
dirigeants = r.get("dirigeants") or []
dirigeant = dirigeants[0] if dirigeants else {}
finances = r.get("finances") or {}
finances_annee = max(finances.keys()) if finances else None
fin = finances.get(finances_annee, {}) if finances_annee else {}
return {
"ENT_Siren": r.get("siren"),
"ENT_NomComplet": clean(r.get("nom_complet")),
"ENT_NomRaisonSociale": clean(r.get("nom_raison_sociale")),
"ENT_Sigle": clean(r.get("sigle")),
"ENT_AdresseSiege": clean(siege.get("adresse")),
"ENT_CodePostal": clean(siege.get("code_postal")),
"ENT_Commune": clean(siege.get("libelle_commune")),
"ENT_CodeCommune": clean(siege.get("commune")),
"ENT_Departement": clean(siege.get("departement")),
"ENT_Region": clean(siege.get("region")),
"ENT_DateCreation": clean(r.get("date_creation")),
"ENT_EtatAdministratif": clean(r.get("etat_administratif")),
"ENT_DateFermeture": clean(r.get("date_fermeture")),
"ENT_ActivitePrincipale": clean(r.get("activite_principale")),
"ENT_SectionActivite": clean(r.get("section_activite_principale")),
"ENT_NatureJuridique": clean(r.get("nature_juridique")),
"ENT_CategorieJuridique": derive_categorie_juridique(clean(r.get("nature_juridique"))),
"ENT_CategorieEntreprise": clean(r.get("categorie_entreprise")),
"ENT_TrancheEffectifSalarie": clean(r.get("tranche_effectif_salarie")),
"ENT_AnneeEffectif": clean(r.get("annee_tranche_effectif_salarie")),
"ENT_DirigeantNom": clean(dirigeant.get("nom")),
"ENT_DirigeantPrenoms": clean(dirigeant.get("prenoms")),
"ENT_DirigeantQualite": clean(dirigeant.get("qualite")),
"ENT_FinancesAnnee": finances_annee,
"ENT_FinancesCa": fin.get("ca"),
"ENT_FinancesResultatNet": fin.get("resultat_net"),
"ENT_ReponseBruteJson": json.dumps(r, ensure_ascii=False),
}
def probe_total(params):
p = dict(params, per_page=1)
data = http_get_json(API_SEARCH + "?" + urllib.parse.urlencode(p))
time.sleep(RATE_DELAY)
return data.get("total_results", 0)
def paginate_all(params, seen):
p = dict(params, per_page=PAGE_SIZE)
page = 1
out = []
while True:
p["page"] = page
data = http_get_json(API_SEARCH + "?" + urllib.parse.urlencode(p))
time.sleep(RATE_DELAY)
results = data.get("results", [])
if not results:
break
for r in results:
siren = r.get("siren")
if siren and siren not in seen:
seen.add(siren)
out.append(r)
total_pages = data.get("total_pages", page)
if page >= total_pages or page >= 400: # 400*25 = 10000, garde-fou
break
page += 1
return out
def fetch_commune_entreprises(code_commune, seen):
base = {"code_commune": code_commune, "etat_administratif": "A"}
total = probe_total(base)
if total < CAP:
return paginate_all(base, seen)
log(f" commune {code_commune} plafonnee ({total}), repli par section NAF")
out = []
for section in NAF_SECTIONS:
sec_params = dict(base, section_activite_principale=section)
sec_total = probe_total(sec_params)
if sec_total == 0:
continue
if sec_total < CAP:
out.extend(paginate_all(sec_params, seen))
continue
# 3e niveau de repli : la section a elle seule reste plafonnee (cas rare,
# ex. Rennes/section L). On decoupe par code NAF precis (732 codes possibles) --
# un code implique deja sa section, donc on l'utilise seul, sans le combiner
# au filtre de section.
log(f" commune {code_commune} section {section} encore plafonnee ({sec_total}), "
f"repli par code NAF precis")
for code in codes_for_section(section):
code_params = dict(base, activite_principale=code)
code_total = probe_total(code_params)
if code_total == 0:
continue
if code_total >= CAP:
log(f" !! commune {code_commune} code {code} encore plafonnee "
f"({code_total}) -- a decouper manuellement (ex: par tranche_effectif_salarie)")
out.extend(paginate_all(code_params, seen))
return out
def upsert_entreprise(conn, table, row):
cols = list(row.keys())
placeholders = ", ".join(["%s"] * len(cols))
colnames = ", ".join(f"`{c}`" for c in cols)
updates = ", ".join(f"`{c}`=VALUES(`{c}`)" for c in cols if c != "ENT_Siren")
sql = (
f"INSERT INTO `{table}` ({colnames}) VALUES ({placeholders}) "
f"ON DUPLICATE KEY UPDATE {updates}, `ENT_DateMajInsee`=NOW()"
)
with conn.cursor() as cur:
cur.execute(sql, list(row.values()))
def upsert_finances(conn, table, row):
if row.get("ENT_FinancesAnnee") is None:
return
fin = {
"ENT_Siren": row["ENT_Siren"],
"ENT_FinancesAnnee": row["ENT_FinancesAnnee"],
"ENT_FinancesCa": row["ENT_FinancesCa"],
"ENT_FinancesResultatNet": row["ENT_FinancesResultatNet"],
}
cols = list(fin.keys())
placeholders = ", ".join(["%s"] * len(cols))
colnames = ", ".join(f"`{c}`" for c in cols)
updates = ", ".join(
f"`{c}`=VALUES(`{c}`)" for c in cols if c not in ("ENT_Siren", "ENT_FinancesAnnee")
)
sql = (
f"INSERT INTO `{table}` ({colnames}) VALUES ({placeholders}) "
f"ON DUPLICATE KEY UPDATE {updates}"
)
with conn.cursor() as cur:
cur.execute(sql, list(fin.values()))
def ensure_tables(conn, dep):
with conn.cursor() as cur:
cur.execute(DDL_PHASE1.format(dep=dep))
cur.execute(DDL_PHASE2.format(dep=dep))
cur.execute(DDL_PHASE3.format(dep=dep))
cur.execute(DDL_COMMUNES)
cur.execute(DDL_DEPARTEMENTS)
cur.execute(DDL_CONFIG)
def ensure_communes(conn, dep, communes_geo):
with conn.cursor() as cur:
cur.execute("SELECT COUNT(*) FROM ENT_Communes WHERE COM_Departement=%s", (dep,))
(count,) = cur.fetchone()
if count > 0:
log(f" ENT_Communes deja alimentee pour le {dep} ({count} communes), on ne rejoue pas")
return
rows = []
for c in communes_geo:
lon, lat = c["centre"]["coordinates"]
cp = ",".join(c.get("codesPostaux", []))
dep_code = c["departement"]["code"] if isinstance(c.get("departement"), dict) else c.get("departement")
rows.append((c["code"], cp, c["nom"], dep_code, lat, lon))
cur.executemany(
"INSERT INTO ENT_Communes "
"(COM_CodeInsee, COM_CodePostal, COM_Ville, COM_Departement, COM_Latitude, COM_Longitude) "
"VALUES (%s,%s,%s,%s,%s,%s)",
rows,
)
log(f" ENT_Communes : {len(rows)} communes inserees pour le {dep}")
def run_department(conn, dep):
log(f"=== Departement {dep} ===")
ensure_tables(conn, dep)
communes_geo = fetch_communes_geo(dep)
log(f" {len(communes_geo)} communes (source geo.api.gouv.fr)")
ensure_communes(conn, dep, communes_geo)
table1 = f"Entreprises{dep}Phase1"
table3 = f"Entreprises{dep}Phase3"
seen = set()
total_upserts = 0
total_finances = 0
for c in communes_geo:
results = fetch_commune_entreprises(c["code"], seen)
for r in results:
row = map_entreprise(r)
if row["ENT_Siren"]:
upsert_entreprise(conn, table1, row)
total_upserts += 1
if row["ENT_FinancesAnnee"] is not None:
upsert_finances(conn, table3, row)
total_finances += 1
if results:
log(f" {c['nom']} ({c['code']}) : {len(results)} entreprises")
with conn.cursor() as cur:
cur.execute(f"SELECT COUNT(*) FROM `{table1}`")
(nb_total,) = cur.fetchone()
nom_dep = fetch_departement_nom(dep)
upsert_departement(conn, dep, nom_dep, nb_total)
log(f"Departement {dep} termine : {total_upserts} entreprises upsertees dans {table1}, "
f"{total_finances} lignes financieres upsertees dans {table3} (total {nb_total} -- "
f"ENT_Departements mise a jour)")
def main():
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--departements", required=True,
help="Liste separee par des virgules, ex: 56,17,64")
ap.add_argument("--db-host", default=os.environ.get("ENTREPRISES_DB_HOST", "mysql2"))
ap.add_argument("--db-port", type=int, default=int(os.environ.get("ENTREPRISES_DB_PORT", "3306")))
ap.add_argument("--db-user", default=os.environ.get("ENTREPRISES_DB_USER", "crm_user"))
ap.add_argument("--db-pass", default=os.environ.get("ENTREPRISES_DB_PASS"))
ap.add_argument("--db-name", default=os.environ.get("ENTREPRISES_DB_NAME", "ENTREPRISES"))
args = ap.parse_args()
if not args.db_pass:
ap.error("mot de passe requis : --db-pass ou variable d'environnement ENTREPRISES_DB_PASS")
departements = [d.strip() for d in args.departements.split(",") if d.strip()]
conn = pymysql.connect(
host=args.db_host, port=args.db_port, user=args.db_user, password=args.db_pass,
database=args.db_name, charset="utf8mb4", autocommit=True,
)
try:
for dep in departements:
run_department(conn, dep)
finally:
conn.close()
if __name__ == "__main__":
main()
Reference in New Issue View Git Blame Copy Permalink