diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..86fff86 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +data +*.osm.pbf diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3f8de5e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.12-bookworm + +RUN apt-get update && \ + apt-get install -y \ + gdal-bin \ + parallel \ + postgresql-client \ + python3-virtualenv + +WORKDIR /opt/imposm +RUN wget https://github.com/omniscale/imposm3/releases/download/v0.11.1/imposm-0.11.1-linux-x86-64.tar.gz && \ + tar -xvzf imposm-0.11.1-linux-x86-64.tar.gz && \ + ln -s /opt/imposm/imposm-0.11.1-linux-x86-64/imposm /usr/bin/imposm + +WORKDIR /opt/bano + +ADD requirements.txt . +RUN pip install -r requirements.txt diff --git a/README.md b/README.md index fe1e687..7fd1092 100644 --- a/README.md +++ b/README.md @@ -2,3 +2,141 @@ bano ==== Différents outils pour la constitution de la Base Adresses Nationale Ouverte + +v3 à venir : +- unification des adresses voies & lieux-dits +- gestion des communes fusionnées +- remplacement de FANTOIR par TOPO +- etc + +## Dépendances + +Il faut pour alimenter la base OSM locale dans laquelle puise BANO : +* [imposm](https://github.com/omniscale/imposm3) pour le chargement. Par défaut la dernière version. +* [osmosis](https://github.com/openstreetmap/osmosis) pour la mise à jour incrémentale. Par défaut la dernière version. + +Autres outils : parallel. + +## Installation + +Mettre en place un environnement virtuel python : +``` +virtualenv venv +source venv/bin/activate +pip install -r requirements.txt +``` + +## Configuration + +Première étape avant de lancer les chargements de données : il faut adapter le fichier `config` à votre environnement, en déclarant différents chemins. Ce fichier est utilisé en début de plusieurs scripts pour connaître le chemin de différents répertoires. + +### Création des répertoires +Une fois le fichier `config` rempli, lancer la création des répertoires avec : +``` +arborescence.sh +``` + +### Liste des départements + +Les départements pris en compte sont listés dans `deplist.txt`. Cette liste peut être modifiée. + +### Création de la base de données + +La base de données qui accueille toutes les données BANO (les sources et les données produites) s'appelle 'bano' est doit être créée en début d'installation. C'est l'utilisateur 'postgres' qui doit exécuter les scripts de création de la base. + +``` +sudo -u postgres -s "./create_base.sh" +``` +On utilise ensuite le module python 'bano' pour terminer la configuration de la BD : +``` +bano setup_db +``` +À l'issue de cette étape toutes les tables nécessaires existent dans la BD. Elles sont toutes vides. + +## Chargement des données OSM +### Chargement initial +D'abord renseigner le fichier imposm.config, puis lancer : + +``` +./load_osm_france_db.sh +``` +À l'issue, les tables du schéma osm sont remplies. + +### Mise à jour en continu + +## Chargement des autres données sources +Chaque source a sa commande de chargement +### TOPO (ex FANTOIR) +``` +bano charge_topo +``` +### BAN +``` +bano charge_ban +bano update_bis_table +``` +### COG +``` +bano charge_cog +``` +### Lieux-dits du Cadastre +``` +bano charge_ld_cadastre +``` + +### Commande `bano` + +Activer l'environnement virtuel python (si pas déjà activé) : +``` +source venv/bin/activate +``` + +Pour connaître les commandes du module bano : +``` +python -m bano --help +``` + +Quasiment toutes les options sont utilisées dans le script `cron_bano`. + +## Docker + +### Configuration +Il ne faut pas modifier le fichier de configuration pour l'exécution avec docker. Pour changer le chemin ou sont stocké les données ajuster le volume data dans `docker-compose`. + +``` +# Créer l'espace de travail +mkdir -p data +chmod a+s data +docker-compose run --rm tools ./arborescence.sh +``` + +### Initialisation +``` +# Démarre Postgres et attend un peu avant de l'utiliser +docker-compose up -d postgres && sleep 5 +docker-compose exec -u postgres postgres psql -c "DROP schema tiger CASCADE" +docker-compose run --rm tools bash -c "source config && ./init_base.sh" +docker-compose run --rm tools bash -c "source config && python -m bano setup_db" +``` + +Si besoin de se connecter sur la base de données : +``` +docker-compose exec -u postgres postgres psql +``` + +``` +# Charger les données OSM +docker-compose run --rm tools ./load_osm_france_db.sh http://download.openstreetmap.fr/extracts/europe/france/franche_comte/territoire_de_belfort.osm.pbf + +# Charger les autres données + +docker-compose run --rm tools bash -c "source config && python -m bano charge_topo" +docker-compose run --rm tools bash -c "source config && python -m bano charge_ban" +docker-compose run --rm tools bash -c "source config && python -m bano update_bis_table" +docker-compose run --rm tools bash -c "source config && python -m bano charge_cog" +docker-compose run --rm tools bash -c "source config && python -m bano charge_ld_cadastre" +docker-compose run --rm tools bash -c "source config && python -m bano charge_cp" +### Mise à jour +``` +docker-compose run --rm tools bash -c "source config && ./cron_bano.sh" +``` diff --git a/arborescence.sh b/arborescence.sh index 0126609..03ac535 100755 --- a/arborescence.sh +++ b/arborescence.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + source config mkdir -p $LOG_DIR mkdir -p $DATA_DIR @@ -10,6 +12,7 @@ mkdir -p $BAL_CACHE_DIR mkdir -p $EXPIRE_TILES_DIR mkdir -p $DOWNLOAD_DIR mkdir -p $IMPOSM_CACHE_DIR +mkdir -p $COG_DIR mkdir -p $EXPORT_SAS_DIR mkdir -p $EXPORT_WEB_DIR -mkdir -p $TMP_DIR \ No newline at end of file +mkdir -p $TMP_DIR diff --git a/avancement_cumul.sh b/avancement_cumul.sh index d8155de..8e73c52 100755 --- a/avancement_cumul.sh +++ b/avancement_cumul.sh @@ -1,4 +1,7 @@ -psql -d cadastre -U cadastre -c "SELECT source,etape, date_debut,date_fin,dept,cadastre_com,nom_com,nombre_adresses FROM batch ORDER BY id_batch DESC LIMIT 10;" +#!/bin/bash + +set -e + +$pgsql_CADASTRE -c "SELECT source,etape, date_debut,date_fin,dept,cadastre_com,nom_com,nombre_adresses FROM batch ORDER BY id_batch DESC LIMIT 10;" ps -eaf|grep 'bano' ps -eaf|grep cumul - diff --git a/bano.yml b/bano.yml index 19c5d32..885dd95 100644 --- a/bano.yml +++ b/bano.yml @@ -19,6 +19,7 @@ tables: - {key: 'addr:housenumber', name: 'addr:housenumber', type: string} - {key: 'addr:street', name: 'addr:street', type: string} - {key: 'addr:place', name: 'addr:place', type: string} + - {key: 'addr:postcode', name: 'addr:postcode', type: string} - {key: 'ref:FR:FANTOIR', name: 'ref:FR:FANTOIR', type: string} - {key: 'highway', name: 'highway', type: string} - {key: 'amenity', name: 'amenity', type: string} @@ -67,6 +68,8 @@ tables: - {key: admin_level, name: admin_level, type: integer} - {key: 'addr:housenumber', name: 'addr:housenumber', type: string} - {key: 'addr:street', name: 'addr:street', type: string} + - {key: 'addr:place', name: 'addr:place', type: string} + - {key: 'addr:postcode', name: 'addr:postcode', type: string} - {key: 'ref:INSEE', name: 'ref:INSEE', type: string} - {key: 'ref:FR:FANTOIR', name: 'ref:FR:FANTOIR', type: string} - {key: 'ref:FR:FANTOIR:left', name: 'ref:FR:FANTOIR:left', type: string} diff --git a/bano/__init__.py b/bano/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/bano/bin.py b/bano/__main__.py similarity index 51% rename from bano/bin.py rename to bano/__main__.py index d8ceb44..95a7147 100644 --- a/bano/bin.py +++ b/bano/__main__.py @@ -1,10 +1,17 @@ +#!/usr/bin/env python +# coding: UTF-8 + import argparse import sys +from . import pre_process_suffixe +from . import setup_db +from . import rapprochement +from . import boite_a_outils +from . import export +from . import publish +from .sources import topo, ban, cog, cadastre_ld, ban2topo, datagouv_commune_summary as datagouv_cs,datagouv_cp from .constants import DEPARTEMENTS -from . import core, core_place, pre_process_suffixe, export, publish -from . import update_manager -from .sources import bal, cadastre_gouv, cadastre_json, ban def main(): @@ -14,61 +21,32 @@ def main(): subparsers = parser.add_subparsers(help="Commandes disponibles") subparser = subparsers.add_parser( - "process_commune", - help="Traite unitairement les adresses d'une commune", - description="Traite unitairement les adresses d'une commune", - ) - subparser.add_argument( - "source", - choices=["OSM", "CADASTRE", "BAL", "BAN"], - type=str, - help="Source des données à traiter", - ) - group = subparser.add_mutually_exclusive_group(required=True) - group.add_argument( - "--code_insee", type=str, help="Code INSEE de la commune à traiter" + "setup_db", + help="Initialisation de la BD BANO", ) - group.add_argument("--depts", type=str, help="Départements à traiter (toutes les communes de chaque dept sont traitées une par une)", nargs="*") - group.add_argument("--France", help="Raccourci pour tous les départements d'un coup", action="store_const", const=DEPARTEMENTS) - subparser.set_defaults(func=core.process) + subparser.set_defaults(func=setup_db.setup_bano) subparser = subparsers.add_parser( - "process_commune_lieux-dits", - help="Traite unitairement les lieux-dits d'une commune", - description="Traite unitairement les lieux-dits d'une commune" + "charge_topo", + help="Charge une version du fichier TOPO", ) - group = subparser.add_mutually_exclusive_group(required=True) - group.add_argument( - "--code_insee", type=str, help="Code INSEE de la commune à traiter" - ) - group.add_argument("--depts", type=str, help="Départements à traiter (toutes les communes de chaque dept sont traitées une par une)", nargs="*") - group.add_argument("--France", help="Raccourci pour tous les départements d'un coup", action="store_const", const=DEPARTEMENTS) - subparser.set_defaults(func=core_place.process) + subparser.set_defaults(func=topo.process_topo) subparser = subparsers.add_parser( - "download_bal", - help="Met à jour les fichiers d'adresses au format BAL", - description="Met à jour les fichiers d'adresses au format BAL", - ) - subparser.add_argument( - "source", - choices=["CADASTRE", "BAL"], - type=str, - help="Source des données à traiter", + "charge_cog", + help="Charge une version des fichiers COG", ) - subparser.add_argument( - "departements", - type=str, - help="Départements à traiter", - nargs="*", - default=DEPARTEMENTS, + subparser.set_defaults(func=cog.process_cog) + + subparser = subparsers.add_parser( + "charge_cp", + help="Charge une version du fichier Codes Postaux", ) - subparser.set_defaults(func=bal.process) + subparser.set_defaults(func=datagouv_cp.process) subparser = subparsers.add_parser( - "download_ban", - help="Met à jour les fichiers d'adresses de source BAN", - description="Met à jour les fichiers d'adresses de source BAN", + "charge_ban", + help="Charge une version des fichiers BAN", ) subparser.add_argument( "departements", @@ -77,15 +55,12 @@ def main(): nargs="*", default=DEPARTEMENTS, ) - subparser.set_defaults(func=ban.process) + subparser.set_defaults(func=ban.process_ban) subparser = subparsers.add_parser( - "download_cadastre", - help="Met à jour les fichiers du cadastre hors adresses au format JSON", - description="Met à jour les fichiers du cadastre hors adresses au format JSON", - ) - subparser.add_argument( - "prefixe", choices=["lieux_dits"], type=str, help="Source des données à traiter" + "charge_ld_cadastre", + help="Charge une version des lieux-dits du Cadastre JSON", + description="Charge une version des lieux-dits du Cadastre JSON", ) subparser.add_argument( "departements", @@ -94,26 +69,30 @@ def main(): nargs="*", default=DEPARTEMENTS, ) - subparser.set_defaults(func=cadastre_json.process) + subparser.set_defaults(func=cadastre_ld.process) subparser = subparsers.add_parser( - "update_code_cadastre", - help="Met à jour la liste des communes d'après cadastre.gouv.fr - indique le format du Cadastre", - description="Met à jour la liste des communes d'après cadastre.gouv.fr - indique le format du Cadastre", + "download_commune_summary", + help="Met à jour les stats de BAL", + description="Met à jour les stats de BAL", ) - subparser.set_defaults(func=cadastre_gouv.process) + subparser.set_defaults(func=datagouv_cs.process) subparser = subparsers.add_parser( "update_bis_table", help="Identifie les indices de répétition b,t,q assimilables à bis, ter, quater", - description="Identifie les indices de répétition b,t,q assimilables à bis, ter, quater", ) subparser.set_defaults(func=ban.update_bis_table) + subparser = subparsers.add_parser( + "update_table_communes", + help="Met à jour les polygones administratifs OSM", + ) + subparser.set_defaults(func=boite_a_outils.maj_table_communes) + subparser = subparsers.add_parser( "pre_process_suffixe", help="Détermine les zones où les noms dans le Cadastre sont suffixés", - description="Détermine les zones où les noms dans le Cadastre sont suffixés", ) subparser.add_argument( "departements", @@ -124,6 +103,44 @@ def main(): ) subparser.set_defaults(func=pre_process_suffixe.process) + subparser = subparsers.add_parser( + "rapprochement", + help="Effectue l'appariement entre sources OSM ou BAN et TOPO", + ) + group = subparser.add_mutually_exclusive_group(required=True) + group.add_argument( + "--code_insee", type=str, help="Code INSEE de la commune à traiter" + ) + group.add_argument( + "--dept", + type=str, + help="Département à traiter (toutes les communes du dept sont traitées une par une)", + ) + subparser.set_defaults(func=rapprochement.process) + + subparser = subparsers.add_parser( + "ban2topo", + help="Ajoute dans TOPO des libellés BAN", + description="Ajoute dans TOPO des libellés BAN" + ) + subparser.add_argument( + "--code_insee", type=str, help="Code INSEE de la commune à traiter" + ) + subparser.set_defaults(func=ban2topo.process) + + subparser = subparsers.add_parser( + "update_infos_communes", + help="Màj de la table infos_communes pour les exports JSON", + ) + subparser.set_defaults(func=boite_a_outils.update_infos_communes) + + subparser = subparsers.add_parser( + "prepare_export", + help="Calculs en prévision des exports", + description="Calculs en prévision des exports", + ) + subparser.set_defaults(func=export.prepare_export) + subparser = subparsers.add_parser( "export", help="Export par département dans différents formats", @@ -159,13 +176,6 @@ def main(): ) subparser.set_defaults(func=publish.process_full) - subparser = subparsers.add_parser( - "update_insee_lists", - help="Détermine les communes mises à jour grace aux tuiles impactées", - description="Détermine les communes mises à jour grace aux tuiles impactées", - ) - subparser.set_defaults(func=update_manager.update_insee_lists) - args = parser.parse_args() try: @@ -176,3 +186,7 @@ def main(): import ipdb ipdb.set_trace() + + +if __name__ == "__main__": + main() diff --git a/bano/batch.py b/bano/batch.py new file mode 100644 index 0000000..d480780 --- /dev/null +++ b/bano/batch.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# coding: UTF-8 + +import time +from .sql import sql_get_data, sql_process + + +def batch_start_log(etape, source=None, code_zone=None, nom_zone=None): + t = time.localtime() + date_debut = time.strftime("%d-%m-%Y %H:%M:%S", t) + timestamp_debut = round(time.mktime(t), 0) + + champs = "etape,timestamp_debut,date_debut" + values = f"'{etape}',{timestamp_debut},'{date_debut}'" + if source: + champs = f"{champs},source" + values = f"{values},'{source}'" + if code_zone: + champs = f"{champs},code_zone" + values = f"{values},'{code_zone}'" + if nom_zone: + champs = f"{champs},nom_zone" + values = f"{values},'{nom_zone}'" + return sql_get_data("batch_start_log", dict(etape=etape, code_zone=code_zone, champs=champs, values=values))[0][0] + + +def batch_stop_log(id_batch, status): + t = time.localtime() + date_fin = time.strftime("%d-%m-%Y %H:%M:%S", t) + timestamp_fin = round(time.mktime(t), 0) + sql_process( + "batch_stop_log", + dict( + id_batch=str(id_batch), + date_fin=str(date_fin), + timestamp_fin=str(timestamp_fin), + status=str(status), + ), + ) + if not status: + print(f"Erreur pendant le processus {id_batch}") diff --git a/bano/boite_a_outils.py b/bano/boite_a_outils.py new file mode 100755 index 0000000..e1a4b9e --- /dev/null +++ b/bano/boite_a_outils.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +# coding: UTF-8 + +from .sql import sql_process +from . import batch as b + + +def maj_table_communes(**kwargs): + batch_id = b.batch_start_log("maj_table_communes", "France", "France") + try: + sql_process("create_table_polygones_communes", dict()) + b.batch_stop_log(batch_id, True) + except: + b.batch_stop_log(batch_id, False) + +def update_infos_communes(**kwargs): + batch_id = b.batch_start_log("maj_table_infos_communes", "France", "France") + try: + sql_process("update_table_infos_communes", dict()) + b.batch_stop_log(batch_id, True) + except: + b.batch_stop_log(batch_id, False) diff --git a/bano/constants.py b/bano/constants.py index c7ef48d..1b1f2fd 100644 --- a/bano/constants.py +++ b/bano/constants.py @@ -1,3 +1,7 @@ +#!/usr/bin/env python +# coding: UTF-8 + +import json from pathlib import Path @@ -6,15 +10,21 @@ def load_pairs_from_file(basename): return [l[:-1].split("\t") for l in f.readlines() if not l.startswith("#")] -DEPARTEMENTS = [ - f"{n:>02}" - for n in [*range(1, 20), "2A", "2B", *range(21, 96), *range(971, 975), "976"] -] +def load_json_from_file(json_filename): + with (Path(__file__).parent / "data" / json_filename).open() as f: + return json.loads(f.read()) + + +def get_const_code_dir(): + return load_json_from_file("code_dir.json") -DEPARTEMENTS_3CHAR = [('0'+s)[-3:] for s in DEPARTEMENTS] + +DEPARTEMENTS = [n.strip() for n in open('deplist.txt').readlines()] + +DEPARTEMENTS_3CHAR = [("0" + s)[-3:] for s in DEPARTEMENTS] LETTRE_A_LETTRE = { - "A": ["Â", "À", "Á","Ã"], + "A": ["Â", "À", "Á", "Ã"], "C": ["Ç"], "E": ["È", "Ê", "É", "Ë"], "I": ["Ï", "Î", "Í"], @@ -71,4 +81,6 @@ def load_pairs_from_file(basename): "trunk", "motorway", ] -HIGHWAY_TYPES_INDEX = {e: 2 ** i for i, e in enumerate(HIGHWAY_TYPES)} +HIGHWAY_TYPES_INDEX = {e: 2**i for i, e in enumerate(HIGHWAY_TYPES)} + +CODE_VOIE_FANTOIR = "0123456789ABCDEFGHIJKLMNOPQRSTVWXYZ" diff --git a/bano/core.py b/bano/core.py deleted file mode 100755 index b2c89ba..0000000 --- a/bano/core.py +++ /dev/null @@ -1,329 +0,0 @@ -#!/usr/bin/env python -# coding: UTF-8 - -import os,os.path -import re -import sys -import time -import xml.etree.ElementTree as ET - -from . import constants, db -from . import helpers as hp -from . import db_helpers as dbhp -from . import log_2_file as log -from .models import Adresse, Adresses, Node, Pg_hsnr -from .outils_de_gestion import batch_start_log -from .outils_de_gestion import batch_end_log -# from .outils_de_gestion import age_etape_dept -from .sources import fantoir - -os.umask(0000) - - -def add_fantoir_to_hsnr(): - for v in adresses: - if v in fantoir.mapping.fantoir: - adresses[v]['fantoirs']['FANTOIR'] = fantoir.mapping.fantoir[v] - adresses[v]['voies']['FANTOIR'] = fantoir.mapping.code_fantoir_vers_nom_fantoir[fantoir.mapping.fantoir[v]] - else: - if 'OSM' in adresses[v]['fantoirs']: - if adresses[v]['fantoirs']['OSM'] in fantoir.mapping.code_fantoir_vers_nom_fantoir: - adresses[v]['voies']['FANTOIR'] = fantoir.mapping.code_fantoir_vers_nom_fantoir[adresses[v]['fantoirs']['OSM']] - -def append_suffixe(name,suffixe): - res = name - if suffixe: - name_norm = hp.normalize(name) - suffixe_norm = hp.normalize(suffixe) - ln = len(name_norm) - ls = len(suffixe) - if ln > ls: - if name[-ls:] != suffixe: - res = name+' '+suffixe - else: - res = name+' '+suffixe - return res - -def get_last_base_update(query_name,insee_com): - resp = 0 - str_query = "SELECT timestamp_maj FROM {} WHERE insee_com = '{}' LIMIT 1;".format(query_name,insee_com) - cur = db.bano.cursor() - cur.execute(str_query) - for l in cur : - resp = l[0] - if resp == 0 : - etape_dept = 'cache_dept_'+query_name - if dbhp.age_etape_dept(etape_dept,get_short_code_dept_from_insee(insee_com)) < 3600 : - resp = round(time.time()) - cur.close() - return resp - -def get_data_from_pg(query_name,insee_com): - cur_cache = db.bano_cache.cursor() - str_query = "DELETE FROM {} WHERE insee_com = '{}';".format(query_name,insee_com) - with open(os.path.join(os.path.dirname(os.path.abspath(__file__)),'sql/{:s}.sql'.format(query_name)),'r') as fq: - str_query+=fq.read().replace('__com__',insee_com) - - str_query+= "SELECT * FROM {} WHERE insee_com = '{}';".format(query_name,insee_com) - cur_cache.execute(str_query) - - res = [] - for l in cur_cache : - res.append(list(l)) - cur_cache.close() - return res - -def get_data_from_pg_direct(query_name,insee_com): - res = [] - with db.bano_cache.cursor() as cur_cache: - with open(os.path.join(os.path.dirname(os.path.abspath(__file__)),'sql/{:s}_nocache.sql'.format(query_name)),'r') as fq: - str_query = fq.read().replace('__com__',insee_com) - cur_cache.execute(str_query) - - for l in cur_cache : - res.append(list(l)) - return res - -def get_tags(xmlo): - dtags = {} - for tg in xmlo.iter('tag'): - dtags[tg.get('k')] = tg.get('v') - return dtags - -def has_addreses_with_suffix(insee): - res = False - str_query = 'SELECT count(*) FROM suffixe where insee_com = \'{:s}\';'.format(insee) - cur = db.bano.cursor() - cur.execute(str_query) - for c in cur: - if c[0]> 0 : - res = True - cur.close() - return res - -def load_ban_hsnr(code_insee): - dict_node_relations = {} - data = get_data_from_pg_direct('ban_hsnr',code_insee) - for id, housenumber, name, lon, lat,cp in data: - if not name or len(name) < 2 or not lon: - continue - adresses.register(name) - if not id in dict_node_relations: - dict_node_relations[id] = [] - dict_node_relations[id].append(hp.normalize(name)) - if hp.is_valid_housenumber(housenumber): - adresses.add_adresse(Adresse(Node({'id':id,'lon':lon,'lat':lat},{}),housenumber,name,'',cp), 'BAN') - -def load_bases_adresses_locales_hsnr(code_insee): - dict_node_relations = {} - with db.bano_cache.cursor() as cur: - cur.execute(f"SELECT cle_interop,TRIM (BOTH FROM (numero||' '||COALESCE(suffixe,''))), voie_nom, long, lat FROM bal_locales WHERE commune_code = '{code_insee}';") - for cle_interop, housenumber, name, lon, lat in cur: - if not name or len(name) < 2 or not lon: - continue - adresses.register(name) - if not cle_interop in dict_node_relations: - dict_node_relations[cle_interop] = [] - dict_node_relations[cle_interop].append(hp.normalize(name)) - if hp.is_valid_housenumber(housenumber): - adresses.add_adresse(Adresse(Node({'id':cle_interop,'lon':lon,'lat':lat},{}),housenumber,name,'',''), 'BAL') - -def load_hsnr_bbox_from_pg_osm(insee_com): - data = get_data_from_pg_direct('hsnr_bbox_insee',insee_com) - for x, y, provenance, osm_id, numero, voie, tags, *others in data: - for num in numero.translate(str.maketrans(',à;-/*','$$$$$$')).split('$'): - oa = Pg_hsnr([x, y, provenance, osm_id, num.rstrip().lstrip(), voie, tags], insee_com) - if oa.fantoir == '' or not oa.voie : - continue - adresses.register(oa.voie) - adresses.add_adresse(Adresse(Node({'id':oa.osm_id,'lon':oa.x,'lat':oa.y},{}),oa.numero,oa.voie,oa.fantoir,oa.code_postal), 'OSM') - -def load_hsnr_from_pg_osm(insee_com): - data = get_data_from_pg_direct('hsnr_insee', insee_com) - for x, y, provenance, osm_id, numero, voie, tags, *others in data: - for num in numero.translate(str.maketrans(',à;-/*','$$$$$$')).split('$'): - oa = Pg_hsnr([x, y, provenance, osm_id, num.rstrip().lstrip(), voie, tags], insee_com) - if not oa.voie : - continue - adresses.register(oa.voie) - adresses.add_adresse(Adresse(Node({'id':oa.osm_id,'lon':oa.x,'lat':oa.y},{}),oa.numero,oa.voie,oa.fantoir,oa.code_postal), 'OSM') - -def load_highways_bbox_from_pg_osm(insee_com): - data = get_data_from_pg_direct('highway_suffixe_insee',insee_com) - for name, fantoir_unique, fantoir_gauche, fantoir_droit, suffixe, *others in data: - if fantoir_unique and hp.is_valid_fantoir(fantoir_unique, insee_com): - code_fantoir = fantoir_unique - elif fantoir_gauche and hp.is_valid_fantoir(fantoir_gauche, insee_com): - code_fantoir = fantoir_gauche - elif fantoir_droit and hp.is_valid_fantoir(fantoir_droit, insee_com): - code_fantoir = fantoir_droit - else: - continue - if not name or len(name) < 2: - continue - name_suffixe = append_suffixe(name,suffixe) - adresses.register(name_suffixe) - cle = hp.normalize(name_suffixe) - if adresses.has_already_fantoir(cle,'OSM'): - continue - adresses.add_fantoir(cle,code_fantoir,'OSM') - adresses.add_voie(name_suffixe,'OSM',name) - -def load_highways_from_pg_osm(insee_com): - data = get_data_from_pg_direct('highway_suffixe_insee',insee_com) - for name, fantoir_unique, fantoir_gauche, fantoir_droit, suffixe, *others in data: - if not name or len(name) < 2: - continue - name_suffixe = append_suffixe(name,suffixe) - adresses.register(name_suffixe) - cle = hp.normalize(name_suffixe) - if adresses.has_already_fantoir(cle,'OSM'): - continue - if fantoir_unique and hp.is_valid_fantoir(fantoir_unique, insee_com): - code_fantoir = fantoir_unique - elif fantoir_gauche and hp.is_valid_fantoir(fantoir_gauche, insee_com): - code_fantoir = fantoir_gauche - elif fantoir_droit and hp.is_valid_fantoir(fantoir_droit, insee_com): - code_fantoir = fantoir_droit - else: - code_fantoir = '' - if code_fantoir != '': - adresses.add_fantoir(cle,code_fantoir,'OSM') - fantoir.mapping.add_fantoir_name(code_fantoir,name_suffixe,'OSM') - adresses.add_voie(name_suffixe,'OSM',name) - -def load_highways_relations_bbox_from_pg_osm(code_insee): - data = get_data_from_pg_direct('highway_relation_suffixe_insee_bbox', code_insee) - for name, tags, suffixe, insee, *others in data: - fantoir = '' - if 'ref:FR:FANTOIR' in tags and hp.is_valid_fantoir(tags['ref:FR:FANTOIR'], code_insee): - fantoir = tags['ref:FR:FANTOIR'] - else: - continue - if not name or len(name) < 2: - continue - name_suffixe = append_suffixe(name,suffixe or '') - adresses.register(name_suffixe) - cle = hp.normalize(name_suffixe) - if adresses.has_already_fantoir(cle,'OSM'): - continue - adresses.add_voie(name_suffixe,'OSM',name) - -def load_highways_relations_from_pg_osm(code_insee): - data = get_data_from_pg_direct('highway_relation_suffixe_insee', code_insee) - for name, tags, suffixe, *others in data: - if not name or len(name) < 2: - continue - name_suffixe = append_suffixe(name,suffixe or '') - adresses.register(name_suffixe) - cle = hp.normalize(name_suffixe) - if adresses.has_already_fantoir(cle,'OSM'): - continue - if tags.get('ref:FR:FANTOIR') and hp.is_valid_fantoir(tags.get('ref:FR:FANTOIR'), code_insee): - code_fantoir = tags.get('ref:FR:FANTOIR') - else: - code_fantoir = '' - if code_fantoir != '': - fantoir.mapping.add_fantoir_name(code_fantoir,name,'OSM') - adresses.add_voie(name_suffixe,'OSM',name) - -def load_point_par_rue_from_pg_osm(code_insee): - data = get_data_from_pg_direct('point_par_rue_insee',code_insee) - for lon, lat, name, *others in data: - if not name or len(name) < 2: - continue - adresses.register(name) - cle = hp.normalize(name) - adresses[cle]['point_par_rue'] = [lon, lat] - if 'OSM' not in adresses.a[cle]['voies']: - adresses.add_voie(name,'OSM') - if 'OSM' not in adresses[cle]['fantoirs']: - if cle in fantoir.mapping.fantoir: - adresses.add_fantoir(cle,fantoir.mapping.fantoir[cle],'OSM') - -def load_point_par_rue_complement_from_pg_osm(insee_com): - data = get_data_from_pg_direct('point_par_rue_complement_insee',insee_com) - for l in data: - name = l[2] - if not name or len(name) < 2: - continue - code_fantoir = l[3] - if code_fantoir and code_fantoir[0:5] != insee_com: - continue - if code_fantoir and len(code_fantoir) != 10: - continue - adresses.register(name) - cle = hp.normalize(name) - adresses[cle]['point_par_rue'] = l[0:2] - if code_fantoir: - adresses.add_fantoir(cle,code_fantoir,'OSM') - if 'OSM' not in adresses.a[cle]['voies']: - adresses.add_voie(name,'OSM') - if 'OSM' not in adresses[cle]['fantoirs']: - if cle in fantoir.mapping.fantoir: - adresses.add_fantoir(cle,fantoir.mapping.fantoir[cle],'OSM') - -def load_type_highway_from_pg_osm(insee_com): - data = get_data_from_pg('type_highway_insee',insee_com) - for name, highway_type, *_ in data: - adresses.register(name) - cle = hp.normalize(name) - if highway_type in constants.HIGHWAY_TYPES_INDEX: - adresses.add_highway_index(cle,constants.HIGHWAY_TYPES_INDEX[highway_type]) - -def addr_2_db(code_insee, source, **kwargs): - # global batch_id - global code_dept - global nodes,ways,adresses - # global schema_cible - - # schema_cible = 'public' - # if ('SCHEMA_CIBLE' in os.environ) : schema_cible = (os.environ['SCHEMA_CIBLE']) - - debut_total = time.time() - - adresses = Adresses(code_insee) - - fantoir.mapping.reset() - fantoir.mapping.load(code_insee) - - code_dept = hp.get_code_dept_from_insee(code_insee) - - batch_id = batch_start_log(source,'loadCumul',code_insee) - - if source == 'BAL': - load_bases_adresses_locales_hsnr(code_insee) - if source == 'BAN': - load_ban_hsnr(code_insee) - if source == 'CADASTRE': - adresses.load_cadastre_hsnr() - if source == 'OSM': - load_hsnr_from_pg_osm(code_insee) - load_hsnr_bbox_from_pg_osm(code_insee) - load_type_highway_from_pg_osm(code_insee) - if len(adresses.a) != 0: - load_highways_from_pg_osm(code_insee) - load_highways_relations_from_pg_osm(code_insee) - load_highways_bbox_from_pg_osm(code_insee) - load_highways_relations_bbox_from_pg_osm(code_insee) - add_fantoir_to_hsnr() - load_point_par_rue_from_pg_osm(code_insee) - load_point_par_rue_complement_from_pg_osm(code_insee) - nb_rec = adresses.save(source, code_dept) - batch_end_log(nb_rec,batch_id) - -def process(source, code_insee, depts, France, **kwargs): - liste_codes_insee = [] - if code_insee: - liste_codes_insee = dbhp.get_insee_name(code_insee) - if not liste_codes_insee: - for d in (depts or France): - liste_codes_insee += dbhp.get_insee_name_list_by_dept(d) - logfile = log.start_log_to_file(source,'process_commune','00') - for code_insee, nom in liste_codes_insee: - # print(f"{code_insee} - {nom}") - try: - addr_2_db(code_insee, source) - except: - log.write_log_to_file(logfile,f"Erreur pour {code_insee} - {nom}") - log.end_log_to_file(logfile) diff --git a/bano/core_place.py b/bano/core_place.py deleted file mode 100755 index ee71dfb..0000000 --- a/bano/core_place.py +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env python -# coding: UTF-8 - -# Place : ID = Fantoir quand présent, sinon nom normalisé -# 1- chargement Fantoir -# 2- chargement cadastre, rapprochement Fantoir, géométrie Cadastre calculée des parcelles -# 3- chargement OSM, rapprochement Fantoir, géométrie OSM -# 4- - -import sys -import time -import os - -from . import constants, db -from . import helpers as hp -from . import db_helpers as dbhp -from .sources import fantoir -from .models import Fantoir, Cadastre, Osm, Place, Places - -from .outils_de_gestion import batch_start_log, batch_end_log - -def get_data_from_pg(query_name,insee_com): - with db.bano_cache.cursor() as conn: - with open(os.path.join(os.path.dirname(os.path.abspath(__file__)),'sql/{:s}.sql'.format(query_name)),'r') as fq: - conn.execute(fq.read().replace('__com__',insee_com)) - res = [] - for l in conn : - res.append(list(l)) - return res - - -def load_fantoir(liste_fantoir): - for c in liste_fantoir: - places.add_place(Place(0,0,'',fantoir.mapping.fantoir[c]['nom'],'','',c,fantoir.mapping.fantoir[c]['ld_bati'], c[0:5])) - - -def load_cadastre(code_insee): - data = get_data_from_pg('cadastre_2_place',code_insee) - for lon, lat, name, fantoir, *others in data: - targets = places.match_name(name,'FANTOIR') - if targets: - for t in targets: - places.p[t].update_cadastre(lon,lat,name) - else: - places.add_place(Place(lon,lat,'','',name,'',fantoir,-1,code_insee)) - - - -def load_osm(code_insee): - data = get_data_from_pg('place_insee',code_insee) - for lon, lat, place, name, fantoir, ld_bati, tags, *others in data: - targets_fantoir = places.match_fantoir(fantoir) - targets_name = places.match_name(name,'FANTOIR') - if targets_fantoir: - for t in targets_fantoir: - places.p[t].update_osm(lon, lat, place, name, fantoir if hp.is_valid_fantoir(fantoir,code_insee) else '') - elif targets_name: - for t in targets_name: - places.p[t].update_osm(lon, lat, place, name, fantoir if hp.is_valid_fantoir(fantoir,code_insee) else '') - else: - places.add_place(Place(lon, lat, place,'','',name, fantoir if hp.is_valid_fantoir(fantoir,code_insee) else '',-1, code_insee)) - -def load_to_db(places, code_insee): - with db.bano.cursor() as conn: - conn.execute(f"DELETE FROM cumul_places WHERE insee_com = '{code_insee}'") - - sload = "INSERT INTO cumul_places (geometrie,libelle_cadastre,libelle_osm,libelle_fantoir,fantoir,insee_com,dept,code_postal,source,ld_bati,ld_osm) VALUES" - a_values = places.as_SQL_Cadastre_array() - nb_rec = len(a_values) - if nb_rec>0: - conn.execute(sload+','.join(a_values)) - a_values = places.as_SQL_OSM_array() - if len(a_values)>0: - conn.execute(sload+','.join(a_values)) - nb_rec+=len(a_values) - return(nb_rec) - -def place_2_db(code_insee): - global fantoir,places - format_cadastre = dbhp.get_cadastre_format(code_insee) - places = Places() - - fantoir.mapping.reset() - fantoir.mapping.load_lieux_dits(code_insee) - load_fantoir(fantoir.mapping.fantoir) - - batch_id_osm = batch_start_log('OSM','cumulPlaces',code_insee) - - if format_cadastre == 'VECT': - batch_id_cadastre = batch_start_log('CADASTRE','cumulPlaces',code_insee) - load_cadastre(code_insee) - load_osm(code_insee) - - nb_rec = load_to_db(places, code_insee) - batch_end_log(nb_rec,batch_id_osm) - if format_cadastre == 'VECT': - batch_end_log(nb_rec,batch_id_cadastre) - -def process(code_insee, depts, France, **kwargs): - liste_codes_insee = [] - if code_insee: - liste_codes_insee = dbhp.get_insee_name(code_insee) - if not liste_codes_insee: - for d in (depts or France): - liste_codes_insee += dbhp.get_insee_name_list_by_dept(d) - for code_insee, nom in liste_codes_insee: - # print(f"{code_insee} - {nom}") - place_2_db(code_insee) - diff --git a/bano/data/code_dir.json b/bano/data/code_dir.json new file mode 100644 index 0000000..341bff5 --- /dev/null +++ b/bano/data/code_dir.json @@ -0,0 +1 @@ +{"13001":"2","13002":"1","13003":"2","13004":"2","13005":"1","13006":"2","13007":"1","13008":"2","13009":"2","13010":"2","13011":"2","13012":"2","13013":"1","13014":"2","13015":"2","13016":"1","13017":"2","13018":"2","13019":"2","13020":"1","13021":"2","13022":"1","13023":"1","13024":"2","13025":"2","13026":"2","13027":"2","13028":"1","13029":"2","13030":"1","13031":"1","13032":"2","13033":"2","13034":"2","13035":"2","13036":"2","13037":"2","13038":"2","13039":"2","13040":"2","13041":"2","13042":"1","13043":"2","13044":"2","13045":"2","13046":"1","13047":"2","13048":"2","13049":"2","13050":"2","13051":"2","13052":"2","13053":"2","13054":"2","13056":"2","13057":"2","13058":"2","13059":"2","13060":"2","13061":"2","13062":"2","13063":"2","13064":"2","13065":"2","13066":"2","13067":"2","13068":"2","13069":"2","13070":"1","13071":"2","13072":"2","13073":"1","13074":"2","13075":"1","13076":"2","13077":"2","13078":"2","13079":"2","13080":"2","13081":"2","13082":"2","13083":"2","13084":"2","13085":"1","13086":"1","13087":"2","13088":"2","13089":"2","13090":"2","13091":"2","13092":"2","13093":"2","13094":"2","13095":"2","13096":"2","13097":"2","13098":"2","13099":"2","13100":"2","13101":"1","13102":"2","13103":"2","13104":"2","13105":"2","13106":"2","13107":"2","13108":"2","13109":"2","13110":"2","13111":"2","13112":"2","13113":"2","13114":"2","13115":"2","13116":"2","13117":"2","13118":"2","13119":"1","13201":"1","13202":"1","13203":"1","13204":"1","13205":"1","13206":"1","13207":"1","13208":"1","13209":"1","13210":"1","13211":"1","13212":"1","13213":"1","13214":"1","13215":"1","13216":"1","13331":"1","13332":"1","13333":"1","13334":"1","13335":"1","13336":"1","13337":"1","13338":"1","13339":"1","13340":"1","13341":"1","13342":"1","13343":"1","13344":"1","13345":"1","13346":"1","59001":"2","59002":"2","59003":"2","59004":"1","59005":"1","59006":"2","59007":"1","59008":"1","59009":"1","59010":"2","59011":"1","59012":"2","59013":"1","59014":"2","59015":"1","59016":"1","59017":"1","59018":"1","59019":"2","59021":"2","59022":"1","59023":"2","59024":"1","59025":"1","59026":"1","59027":"2","59028":"1","59029":"1","59031":"2","59032":"2","59033":"2","59034":"1","59035":"2","59036":"2","59037":"2","59038":"2","59039":"2","59041":"2","59042":"1","59043":"1","59044":"1","59045":"2","59046":"1","59047":"2","59048":"2","59049":"2","59050":"2","59051":"1","59052":"1","59053":"2","59054":"1","59055":"2","59056":"1","59057":"2","59058":"2","59059":"2","59060":"2","59061":"2","59062":"2","59063":"2","59064":"2","59065":"2","59066":"2","59067":"1","59068":"2","59069":"2","59070":"2","59071":"1","59072":"2","59073":"1","59074":"2","59075":"2","59076":"2","59077":"2","59078":"2","59079":"2","59080":"1","59081":"2","59082":"1","59083":"1","59084":"1","59085":"2","59086":"1","59087":"1","59088":"1","59089":"1","59090":"1","59091":"1","59092":"2","59093":"2","59094":"1","59096":"1","59097":"2","59098":"1","59099":"2","59100":"2","59101":"2","59102":"2","59103":"2","59104":"2","59105":"1","59106":"1","59107":"1","59108":"2","59109":"2","59110":"1","59111":"1","59112":"2","59113":"1","59114":"2","59115":"1","59116":"2","59117":"1","59118":"2","59119":"1","59120":"1","59121":"2","59122":"2","59123":"1","59124":"1","59125":"2","59126":"1","59127":"2","59128":"1","59129":"1","59130":"1","59131":"1","59132":"2","59133":"1","59134":"2","59135":"1","59136":"2","59137":"2","59138":"2","59139":"2","59140":"2","59141":"2","59142":"2","59143":"1","59144":"2","59145":"1","59146":"1","59147":"2","59148":"2","59149":"2","59150":"1","59151":"2","59152":"1","59153":"2","59154":"1","59155":"1","59156":"1","59157":"2","59158":"1","59159":"1","59160":"2","59161":"2","59162":"1","59163":"1","59164":"2","59165":"1","59166":"2","59167":"2","59168":"1","59169":"2","59170":"1","59171":"2","59172":"2","59173":"1","59174":"2","59175":"2","59176":"2","59177":"2","59178":"1","59179":"2","59180":"1","59181":"2","59182":"1","59183":"1","59184":"1","59185":"1","59186":"2","59187":"2","59188":"2","59189":"1","59190":"2","59191":"2","59192":"2","59193":"1","59194":"2","59195":"1","59196":"1","59197":"1","59198":"2","59199":"1","59200":"1","59201":"1","59202":"1","59203":"1","59204":"2","59205":"2","59206":"2","59207":"2","59208":"1","59209":"2","59210":"1","59211":"1","59212":"1","59213":"2","59214":"1","59215":"2","59216":"2","59217":"2","59218":"2","59219":"2","59220":"1","59221":"2","59222":"1","59223":"2","59224":"1","59225":"2","59226":"2","59227":"1","59228":"1","59229":"2","59230":"2","59231":"2","59232":"2","59233":"2","59234":"1","59236":"2","59237":"1","59238":"2","59239":"1","59240":"2","59241":"2","59242":"2","59243":"2","59244":"2","59246":"2","59247":"1","59248":"1","59249":"2","59250":"1","59251":"2","59252":"1","59253":"2","59254":"1","59255":"2","59256":"1","59257":"1","59258":"1","59259":"2","59260":"1","59261":"2","59262":"1","59263":"1","59264":"2","59265":"2","59266":"1","59267":"2","59268":"1","59269":"2","59270":"2","59271":"1","59272":"1","59273":"1","59274":"2","59275":"1","59276":"1","59277":"2","59278":"1","59279":"1","59280":"1","59281":"1","59282":"1","59283":"2","59284":"2","59285":"2","59286":"1","59287":"2","59288":"2","59289":"2","59290":"2","59291":"2","59292":"2","59293":"1","59294":"2","59295":"1","59296":"2","59297":"2","59299":"1","59300":"2","59301":"2","59302":"2","59303":"1","59304":"1","59305":"1","59306":"2","59307":"1","59308":"1","59309":"1","59310":"2","59311":"2","59312":"2","59313":"2","59314":"1","59315":"2","59316":"1","59317":"1","59318":"1","59319":"1","59320":"1","59321":"2","59322":"2","59323":"2","59324":"2","59325":"2","59326":"1","59327":"1","59328":"1","59329":"1","59330":"1","59331":"2","59332":"1","59333":"2","59334":"1","59335":"2","59336":"1","59337":"1","59338":"1","59339":"1","59340":"1","59341":"2","59342":"2","59343":"1","59344":"2","59345":"1","59346":"1","59347":"2","59348":"2","59349":"2","59350":"1","59351":"2","59352":"1","59353":"2","59354":"1","59355":"1","59356":"1","59357":"2","59358":"1","59359":"1","59360":"1","59361":"2","59363":"2","59364":"1","59365":"2","59366":"1","59367":"1","59368":"1","59369":"2","59370":"2","59371":"1","59372":"2","59374":"2","59375":"1","59377":"2","59378":"1","59379":"1","59381":"2","59382":"2","59383":"2","59384":"2","59385":"2","59386":"1","59387":"2","59388":"1","59389":"2","59390":"1","59391":"2","59392":"2","59393":"2","59394":"2","59395":"2","59396":"2","59397":"1","59398":"1","59399":"1","59400":"1","59401":"1","59402":"1","59403":"2","59404":"1","59405":"2","59406":"2","59407":"2","59408":"1","59409":"1","59410":"1","59411":"1","59412":"2","59413":"2","59414":"1","59415":"2","59416":"1","59418":"2","59419":"1","59420":"2","59421":"1","59422":"2","59423":"1","59424":"2","59425":"2","59426":"1","59427":"1","59428":"2","59429":"2","59430":"2","59431":"1","59432":"2","59433":"1","59434":"2","59435":"1","59436":"1","59437":"1","59438":"2","59439":"2","59440":"2","59441":"2","59442":"2","59443":"1","59444":"2","59445":"2","59446":"2","59447":"2","59448":"1","59449":"1","59450":"2","59451":"2","59452":"1","59453":"1","59454":"1","59455":"2","59456":"1","59457":"1","59458":"1","59459":"2","59461":"2","59462":"1","59463":"1","59464":"2","59465":"2","59466":"1","59467":"2","59468":"2","59469":"1","59470":"1","59471":"2","59472":"2","59473":"2","59474":"2","59475":"2","59476":"2","59477":"1","59478":"1","59479":"2","59480":"2","59481":"2","59482":"1","59483":"2","59484":"2","59485":"2","59486":"1","59487":"1","59488":"2","59489":"1","59490":"2","59491":"2","59492":"2","59493":"2","59494":"2","59495":"2","59496":"2","59497":"1","59498":"2","59499":"1","59500":"2","59501":"1","59502":"2","59503":"2","59504":"2","59505":"2","59506":"2","59507":"1","59508":"1","59509":"1","59511":"2","59512":"1","59513":"1","59514":"2","59515":"2","59516":"1","59517":"2","59518":"2","59519":"2","59520":"2","59521":"2","59522":"1","59523":"1","59524":"1","59525":"2","59526":"2","59527":"1","59528":"2","59529":"2","59530":"2","59531":"2","59532":"1","59533":"2","59534":"2","59535":"1","59536":"1","59537":"2","59538":"1","59539":"1","59540":"1","59541":"2","59542":"2","59543":"2","59544":"2","59545":"2","59546":"1","59547":"2","59548":"2","59549":"2","59550":"1","59551":"1","59552":"2","59553":"1","59554":"2","59555":"2","59556":"2","59557":"2","59558":"2","59559":"2","59560":"1","59562":"2","59563":"2","59564":"2","59565":"2","59566":"1","59567":"2","59568":"1","59569":"1","59570":"1","59571":"2","59572":"2","59573":"2","59574":"1","59575":"2","59576":"1","59577":"1","59578":"1","59579":"1","59580":"1","59581":"1","59582":"1","59583":"2","59584":"2","59585":"1","59586":"1","59587":"1","59588":"1","59589":"2","59590":"1","59591":"2","59592":"1","59593":"2","59594":"2","59595":"2","59596":"1","59597":"2","59598":"1","59599":"1","59600":"1","59601":"2","59602":"1","59603":"2","59604":"2","59605":"1","59606":"2","59607":"2","59608":"2","59609":"1","59610":"2","59611":"1","59612":"2","59613":"2","59614":"2","59615":"1","59616":"2","59617":"2","59618":"2","59619":"2","59620":"1","59622":"2","59623":"2","59624":"2","59625":"2","59626":"2","59627":"2","59628":"1","59629":"1","59630":"1","59631":"2","59632":"2","59633":"2","59634":"1","59635":"2","59636":"1","59637":"1","59638":"1","59639":"2","59640":"2","59641":"1","59642":"1","59643":"1","59645":"2","59646":"1","59647":"1","59648":"1","59649":"2","59650":"1","59651":"2","59652":"2","59653":"1","59654":"1","59655":"1","59656":"1","59657":"1","59658":"1","59659":"2","59660":"1","59661":"2","59662":"1","59663":"1","59664":"1","59665":"1","59666":"1","59667":"1","59668":"1","59669":"1","59670":"1","75001":"4","75002":"4","75003":"4","75004":"4","75005":"8","75006":"8","75007":"7","75008":"6","75009":"4","75010":"4","75011":"5","75012":"5","75013":"8","75014":"8","75015":"7","75016":"7","75017":"6","75018":"6","75019":"5","75020":"5","75101":"4","75102":"4","75103":"4","75104":"4","75105":"8","75106":"8","75107":"7","75108":"6","75109":"4","75110":"4","75111":"5","75112":"5","75113":"8","75114":"8","75115":"7","75116":"7","75117":"6","75118":"6","75119":"5","75120":"5","92002":"2","92004":"1","92007":"2","92009":"1","92012":"2","92014":"2","92019":"2","92020":"2","92022":"2","92023":"2","92024":"1","92025":"1","92026":"1","92032":"2","92033":"2","92035":"1","92036":"1","92040":"2","92044":"1","92046":"2","92047":"2","92048":"2","92049":"2","92050":"1","92051":"1","92060":"2","92062":"1","92063":"1","92064":"2","92071":"2","92072":"2","92073":"1","92075":"2","92076":"2","92077":"2","92078":"1","97101":"1","97102":"1","97103":"1","97104":"1","97105":"1","97106":"1","97107":"1","97108":"1","97109":"1","97110":"1","97111":"1","97112":"1","97113":"1","97114":"1","97115":"1","97116":"1","97117":"1","97118":"1","97119":"1","97120":"1","97121":"1","97122":"1","97123":"1","97124":"1","97125":"1","97126":"1","97127":"1","97128":"1","97129":"1","97130":"1","97131":"1","97132":"1","97133":"1","97134":"1","97201":"2","97202":"2","97203":"2","97204":"2","97205":"2","97206":"2","97207":"2","97208":"2","97209":"2","97210":"2","97211":"2","97212":"2","97213":"2","97214":"2","97215":"2","97216":"2","97217":"2","97218":"2","97219":"2","97220":"2","97221":"2","97222":"2","97223":"2","97224":"2","97225":"2","97226":"2","97227":"2","97228":"2","97229":"2","97230":"2","97231":"2","97232":"2","97233":"2","97234":"2","97301":"3","97302":"3","97303":"3","97304":"3","97305":"3","97306":"3","97307":"3","97308":"3","97309":"3","97310":"3","97311":"3","97312":"3","97313":"3","97314":"3","97352":"3","97353":"3","97356":"3","97357":"3","97358":"3","97360":"3","97361":"3","97362":"3","97401":"4","97402":"4","97403":"4","97404":"4","97405":"4","97406":"4","97407":"4","97408":"4","97409":"4","97410":"4","97411":"4","97412":"4","97413":"4","97414":"4","97415":"4","97416":"4","97417":"4","97418":"4","97419":"4","97420":"4","97421":"4","97422":"4","97423":"4","97424":"4","97601":"6","97602":"6","97603":"6","97604":"6","97605":"6","97606":"6","97607":"6","97608":"6","97609":"6","97610":"6","97611":"6","97612":"6","97613":"6","97614":"6","97615":"6","97616":"6","97617":"6","99999":"9"} \ No newline at end of file diff --git a/bano/db.py b/bano/db.py index e04e59a..9e19e1c 100644 --- a/bano/db.py +++ b/bano/db.py @@ -3,6 +3,6 @@ import psycopg2 import psycopg2.extras -bano = psycopg2.connect(os.environ.get("BANO_PG", "dbname='cadastre' user='cadastre'")) -bano_cache = psycopg2.connect(os.environ.get("BANO_PG_CACHE", "dbname='osm' user='cadastre'")) -psycopg2.extras.register_hstore(bano_cache) +bano_db = psycopg2.connect(os.environ.get("PG_BANO")) +bano_db.autocommit = True +psycopg2.extras.register_hstore(bano_db) diff --git a/bano/db_helpers.py b/bano/db_helpers.py index 37160ec..5b24db4 100644 --- a/bano/db_helpers.py +++ b/bano/db_helpers.py @@ -1,40 +1,15 @@ import time import os -from . import db +from .sql import sql_get_data -def get_insee_name_list_by_dept(dept): - with db.bano.cursor() as conn : - conn.execute(f"""SELECT com, ncc - FROM cog_commune c - LEFT OUTER JOIN (SELECT comparent FROM cog_commune WHERE dep = '{dept}' AND typecom = 'ARM') p - ON (c.com = p.comparent) - WHERE c.dep = '{dept}' AND c.typecom != 'COMD' AND p.comparent IS NULL - ORDER BY 1""") - return conn.fetchall() +def liste_communes_par_dept(dept): + return sql_get_data("liste_communes_par_dept", dict(dept=dept)) -def get_insee_name(insee_com): - with db.bano.cursor() as conn : - conn.execute(f"SELECT insee_com, nom_com FROM code_cadastre WHERE insee_com = '{insee_com}';") - return conn.fetchall() -def get_cadastre_format(insee_com): - with db.bano.cursor() as conn : - conn.execute(f"SELECT format_cadastre FROM code_cadastre WHERE insee_com = '{insee_com}';") - return conn.fetchone()[0] - -def age_etape_dept(etape,dept): - cur = db.bano.cursor() - str_query = 'SELECT timestamp_debut FROM batch WHERE etape = \'{:s}\' AND dept = \'{:s}\' UNION ALL SELECT 0 ORDER BY 1 DESC;'.format(etape,dept) - cur.execute(str_query) - c = cur.fetchone() - return round(time.mktime(time.localtime()),0) - c[0] - -def process_sql(database,query_name,dict_args): - with open(os.path.join(os.path.dirname(os.path.abspath(__file__)),'sql/{:s}.sql'.format(query_name)),'r') as fq: - str_query = fq.read() - for k,v in dict_args.items(): - str_query = str_query.replace(k,str(v)) - with database.cursor() as cur : - cur.execute(str_query) +def nom_commune(code_insee): + res = sql_get_data("nom_commune_par_code_insee", dict(code_insee=code_insee)) + if res and len(res) > 0 and len(res[0]) > 0: + return res[0][0] + return f"Commune inconnue (INSEE {code_insee})" diff --git a/bano/dictionnaires/abrev_type_voie.txt b/bano/dictionnaires/abrev_type_voie.txt index 73d554b..6078939 100644 --- a/bano/dictionnaires/abrev_type_voie.txt +++ b/bano/dictionnaires/abrev_type_voie.txt @@ -82,6 +82,7 @@ GRAND BOULEVARD GBD GRAND PLACE GPL GR GR GR GR GRANDE GR +GR GRAND GR GRAND RUE GR GRANDE RUE GR GROUPE SCOLAIRE GROUP SCOL @@ -96,6 +97,7 @@ PETITE AVENUE PAE PETITE ROUTE PRT PETITE RUE PTR ROND POINT RPT +RTE RTE RTE RUE ALEZ ALEZ RUE HENT HENT RUE STRAED STRAED diff --git a/bano/dictionnaires/expand_noms.txt b/bano/dictionnaires/expand_noms.txt index 086d4a8..56499c2 100644 --- a/bano/dictionnaires/expand_noms.txt +++ b/bano/dictionnaires/expand_noms.txt @@ -1,6 +1,7 @@ # # Format des lignes : # +& ET B.C.P BATAILLON DE CHASSEURS A PIEDS CHARLES DE GAUL CHARLES DE GAULLE CHP CHAMP diff --git a/bano/export.py b/bano/export.py index 93bd0df..eb08aaf 100644 --- a/bano/export.py +++ b/bano/export.py @@ -9,99 +9,28 @@ from pathlib import Path -from . import constants, db +from .sql import sql_get_data,sql_get_dict_data,sql_process,sql_query +from . import batch as b +from . import constants from . import helpers as hp -class Dataset: - def __init__(self, dept): - self.dept = dept - self.csv_query = self.get_csv_query() - self.csv_data = None - self.json_commune_query = self.get_json_commune_query() - self.json_commune_data = None - self.json_voies_rapprochees_query = self.get_json_voies_rapprochees_query() - self.json_voies_rapprochees_data = None - self.json_voies_rapprochees_sans_adresses_query = self.get_json_voies_rapprochees_sans_adresses_query() - self.json_voies_rapprochees_sans_adresses_data = None - self.json_voies_non_rapprochees_query = self.get_json_voies_non_rapprochees_query() - self.json_voies_non_rapprochees_data = None - self.json_lieux_dits_query = self.get_json_lieux_dits_query() - self.json_lieux_dits_data = None +def get_csv_data(dept): + return sql_get_data('export_csv_dept',dict(dept=dept)) - def get_csv_query(self): - with open(os.path.join(os.path.dirname(os.path.abspath(__file__)),'sql/export_csv_dept.sql'),'r') as fq: - return fq.read().replace('__dept__',self.dept) - - def get_csv_data(self): - with db.bano.cursor() as cur: - cur.execute(self.csv_query) - return cur.fetchall() - - def get_json_commune_query(self): - with open(os.path.join(os.path.dirname(os.path.abspath(__file__)),'sql/export_json_dept_communes.sql'),'r') as fq: - return fq.read().replace('__dept__',self.dept) - - def get_json_commune_data(self): - with db.bano.cursor() as cur: - cur.execute(self.json_commune_query) - return cur.fetchall() - - def get_json_voies_non_rapprochees_query(self): - with open(os.path.join(os.path.dirname(os.path.abspath(__file__)),'sql/export_json_dept_voies_non_rapprochees.sql'),'r') as fq: - return fq.read().replace('__dept__',self.dept) - - def get_json_voies_non_rapprochees_data(self): - with db.bano.cursor() as cur: - cur.execute(self.json_voies_non_rapprochees_query) - return cur.fetchall() - - def get_json_voies_rapprochees_query(self): - with open(os.path.join(os.path.dirname(os.path.abspath(__file__)),'sql/export_json_dept_voies_rapprochees.sql'),'r') as fq: - return fq.read().replace('__dept__',self.dept) - - def get_json_voies_rapprochees_sans_adresses_data(self): - with db.bano.cursor() as cur: - cur.execute(self.json_voies_rapprochees_sans_adresses_query) - return cur.fetchall() - - def get_json_voies_rapprochees_sans_adresses_query(self): - with open(os.path.join(os.path.dirname(os.path.abspath(__file__)),'sql/export_json_dept_voies_rapprochees_sans_adresses.sql'),'r') as fq: - return fq.read().replace('__dept__',self.dept) - - def get_json_voies_rapprochees_data(self): - with db.bano.cursor() as cur: - cur.execute(self.json_voies_rapprochees_query) - return cur.fetchall() - - def get_json_lieux_dits_query(self): - with open(os.path.join(os.path.dirname(os.path.abspath(__file__)),'sql/export_json_dept_lieux_dits.sql'),'r') as fq: - return fq.read().replace('__dept__',self.dept) - - def get_json_lieux_dits_data(self): - with db.bano.cursor() as cur: - cur.execute(self.json_lieux_dits_query) - return cur.fetchall() - - def get_target_filename(self,filetype): - return f'bano-{self.dept}.{filetype}' - - def get_sas_full_filename(self,filetype): - return Path(os.environ['EXPORT_SAS_DIR']) / self.get_target_filename(filetype) - - def get_webdir_full_filename(self,filetype): - return Path(os.environ['EXPORT_WEB_DIR']) / self.get_target_filename(filetype) - - def save_as_csv(self): - if not self.csv_data : - self.csv_data = self.get_csv_data() - with open(self.get_sas_full_filename('csv'),'w', newline='') as csvfile: +def save_as_csv(dept,csv_data): + id_batch = b.batch_start_log("export CSV", "", dept) + try : + with open(get_sas_full_filename(dept,'csv'),'w', newline='') as csvfile: writer = csv.writer(csvfile,dialect='unix',quoting=csv.QUOTE_MINIMAL) - writer.writerows([l[0:-1] for l in self.csv_data]) - - def save_as_ttl(self): - if not self.csv_data : - self.csv_data = self.get_csv_data() - with open(self.get_sas_full_filename('ttl'),'w') as ttlfile: + writer.writerows([l[0:-1] for l in csv_data]) + b.batch_stop_log(id_batch, True) + except: + b.batch_stop_log(id_batch, False) + +def save_as_ttl(dept,csv_data): + id_batch = b.batch_start_log("export TTL", "", dept) + try: + with open(get_sas_full_filename(dept,'ttl'),'w') as ttlfile: ttlfile.write(f"""@prefix xsd: . @prefix locn: . @prefix gn: . @@ -122,10 +51,10 @@ def save_as_ttl(self): \tdcterms:publisher ; #url openstreetmap France \tdcterms:issued "2014-05-14"^^xsd:date ; # data issued \tdcterms:modified "2014-08-21"^^xsd:date ; #last modification -\tdcterms:spatial , ; # region/pays (France) +\tdcterms:spatial , ; # region/pays (France) \t. """) - for id,numero,voie,cp,ville,source,lat,lon,*others in self.csv_data: + for id,numero,voie,cp,ville,source,lat,lon,*others in csv_data: ttlfile.write(f""" a locn:Address , gn:Feature ; locn:fullAddress "{numero} {voie}, {cp} {ville}, FRANCE"; locn:addressId "{id}" ; @@ -143,53 +72,70 @@ def save_as_ttl(self): locn:geometry [a geo:Point ; geo:lat "{lat}" ; geo:long "{lon}" ] ; locn:geometry [a gsp:Geometry; gsp:asWKT "POINT({lon} {lat})"^^gsp:wktLiteral ] ; .""") + b.batch_stop_log(id_batch, True) + except: + b.batch_stop_log(id_batch, False) + + +def save_as_shp(dept): + id_batch = b.batch_start_log("export SHP", "", dept) + try: + subprocess.run(['ogr2ogr', '-f',"ESRI Shapefile", '-lco', 'ENCODING=UTF-8', '-s_srs', 'EPSG:4326', '-t_srs', 'EPSG:4326', '-overwrite', get_sas_full_filename(dept,'shp'), 'PG:' + os.environ['PG_CADASTRE'], '-sql', sql_query('export_csv_dept',dict(dept=dept))]) + b.batch_stop_log(id_batch, True) + except: + b.batch_stop_log(id_batch, False) + +def save_as_json(dept): + id_batch = b.batch_start_log("export JSON", "", dept) + try: + with open(get_sas_full_filename(dept,'json'),'w') as jsonfile: + for l in sql_get_dict_data('export_json_dept_communes',dict(dept=dept)): + if ';' in l['postcode']: + l['postcode'] = l['postcode'].split(';') + jsonfile.write(f"{json.dumps(l,ensure_ascii=False,separators=(',',':'))}\n") + for l in sql_get_dict_data('export_json_dept_voies_avec_adresses',dict(dept=dept)): + if ';' in l['postcode']: + l['postcode'] = l['postcode'].split(';') + dict_hsnr = {} + for p in l['housenumbers'].split('@@@'): + numero,lat,lon = p.split('$$$') + dict_hsnr[numero] = dict(lat=float(lat),lon=float(lon)) + l['housenumbers'] = dict_hsnr + jsonfile.write(f"{json.dumps(l,ensure_ascii=False,separators=(',',':'))}\n") + for l in sql_get_dict_data('export_json_dept_voies_ld_sans_adresses',dict(dept=dept)): + if ';' in l['postcode']: + l['postcode'] = l['postcode'].split(';') + jsonfile.write(f"{json.dumps(l,ensure_ascii=False,separators=(',',':'))}\n") + b.batch_stop_log(id_batch, True) + except Exception as e: + # print(p,l,e) + b.batch_stop_log(id_batch, False) + +def get_target_filename(dept,filetype): + return f'bano-{dept}.{filetype}' + +def get_sas_full_filename(dept,filetype): + return Path(os.environ['EXPORT_SAS_DIR']) / get_target_filename(dept,filetype) + +def get_webdir_full_filename(dept,filetype): + return Path(os.environ['EXPORT_WEB_DIR']) / get_target_filename(dept,filetype) + +def prepare_export(**kwargs): + id_batch = b.batch_start_log("Preparation export", "", "") + try: + sql_process('table_polygones_postaux',dict()) + sql_process('tables_export',dict()) + b.batch_stop_log(id_batch, True) + except: + b.batch_stop_log(id_batch, False) - - def save_as_shp(self): - subprocess.run(['ogr2ogr', '-f',"ESRI Shapefile", '-lco', 'ENCODING=UTF-8', '-s_srs', 'EPSG:4326', '-t_srs', 'EPSG:4326', '-overwrite', self.get_sas_full_filename('shp'), 'PG:dbname=cadastre user=cadastre', '-sql', f'{self.csv_query}']) - - def save_as_json(self): - with open(self.get_sas_full_filename('json'),'w') as jsonfile: - if not self.json_commune_data : - self.json_commune_data = self.get_json_commune_data() - for id,type,name,postcode,lat,lon,cityname,departement,region,population,adm_weight,importance,*others in self.json_commune_data: - if ';' in postcode: - postcode = postcode.split(';') - jsonfile.write(f'{{"id":"{id}","type":"{type}", "name":"{name}", "postcode":{json.dumps(postcode)}, "lat":{lat}, "lon":{lon}, "city":"{cityname}", "departement":"{departement}", "region":"{region}", "population":{population}, "adm_weight":{adm_weight}, "importance":{importance}}}\n') - if not self.json_voies_non_rapprochees_data : - self.json_voies_non_rapprochees_data = self.get_json_voies_non_rapprochees_data() - for fantoir,citycode,type,name,postcode,lat,lon,cityname,departement,region,importance,housenumbers,*others in self.json_voies_non_rapprochees_data: - s_housenumbers = ','.join([f'"{s.split("$")[0]}":{{"lat":{s.split("$")[1]},"lon":{s.split("$")[2]}}}' for s in housenumbers.split('#') ]) - if ';' in postcode: - postcode = postcode.split(';') - jsonfile.write(f'{{"id":"{fantoir}","citycode":"{citycode}","type":"{type}","name":"{name}","postcode":{json.dumps(postcode)},"lat":"{lat}","lon":"{lon}","city":"{cityname}","departement":"{departement}","region":"{region}","importance":{importance},"housenumbers":{{{s_housenumbers}}}}}\n') - if not self.json_voies_rapprochees_data : - self.json_voies_rapprochees_data = self.get_json_voies_rapprochees_data() - for fantoir,citycode,type,name,postcode,lat,lon,cityname,departement,region,importance,housenumbers,*others in self.json_voies_rapprochees_data: - s_housenumbers = ','.join([f'"{s.split("$")[0]}":{{"lat":{s.split("$")[1]},"lon":{s.split("$")[2]}}}' for s in housenumbers.split('#') ]) - if ';' in postcode: - postcode = postcode.split(';') - jsonfile.write(f'{{"id":"{fantoir}","citycode":"{citycode}","type":"{type}","name":"{name}","postcode":{json.dumps(postcode)},"lat":"{lat}","lon":"{lon}","city":"{cityname}","departement":"{departement}","region":"{region}","importance":{importance},"housenumbers":{{{s_housenumbers}}}}}\n') - if not self.json_voies_rapprochees_sans_adresses_data : - self.json_voies_rapprochees_sans_adresses_data = self.get_json_voies_rapprochees_sans_adresses_data() - for fantoir,citycode,type,name,postcode,lat,lon,cityname,departement,region,importance in self.json_voies_rapprochees_sans_adresses_data: - if ';' in postcode: - postcode = postcode.split(';') - jsonfile.write(f'{{"id":"{fantoir}","citycode":"{citycode}","type":"{type}","name":"{name}","postcode":{json.dumps(postcode)},"lat":"{lat}","lon":"{lon}","city":"{cityname}","departement":"{departement}","region":"{region}","importance":{importance}}}\n') - if not self.json_lieux_dits_data : - self.json_lieux_dits_data = self.get_json_lieux_dits_data() - for fantoir,citycode,type,name,postcode,lat,lon,cityname,departement,region,importance,*others in self.json_lieux_dits_data: - if ';' in postcode: - postcode = postcode.split(';') - jsonfile.write(f'{{"id":"{fantoir}","citycode":"{citycode}","type":"{type}","name":"{name}","postcode":{json.dumps(postcode)},"lat":"{lat}","lon":"{lon}","city":"{cityname}","departement":"{departement}","region":"{region}","importance":{importance}}}\n') - def process(departements, **kwargs): for dept in departements: if not hp.is_valid_dept(dept): print(f"Code {dept} invalide pour un département - abandon") continue - d = Dataset(dept) - d.save_as_shp() - d.save_as_csv() - d.save_as_ttl() - d.save_as_json() + save_as_shp(dept) + csv_data = get_csv_data(dept) + save_as_csv(dept,csv_data) + save_as_ttl(dept,csv_data) + save_as_json(dept) diff --git a/bano/helpers.py b/bano/helpers.py index 821ca01..5fa089e 100644 --- a/bano/helpers.py +++ b/bano/helpers.py @@ -4,82 +4,88 @@ def find_cp_in_tags(tags): - return tags.get('addr:postcode') or tags.get('postal_code') or '' + return tags.get("addr:postcode") or tags.get("postal_code") or "" + def escape_quotes(s): - return s.replace('\'','\'\'') + return s.replace("'", "''") + def remove_quotes(s): - return s.replace('\'','') + return s.replace("'", "") + def remove_quotes_on_null(s): - return s.replace("'null'","null") - + return s.replace("'null'", "null") + + def replace_single_quotes_with_double(s): - return s.replace('\'','"') + return s.replace("'", '"') + def format_toponyme(s): - a_s = s.replace('\'',' ').split(' ') - + a_s = s.replace("'", " ").split(" ") + # Accents dic_replace_accents = {} - dic_replace_accents['DERRIERE'] = u'DERRIÈRE' - dic_replace_accents['EGLISE'] = u'ÉGLISE' - dic_replace_accents['ILE'] = u'ÎLE' - dic_replace_accents['ILOT'] = u'ÎLOT' - dic_replace_accents['PRE'] = u'PRÉ' + dic_replace_accents["DERRIERE"] = "DERRIÈRE" + dic_replace_accents["EGLISE"] = "ÉGLISE" + dic_replace_accents["ILE"] = "ÎLE" + dic_replace_accents["ILOT"] = "ÎLOT" + dic_replace_accents["PRE"] = "PRÉ" - for m in range(0,len(a_s)): + for m in range(0, len(a_s)): if a_s[m] in dic_replace_accents: a_s[m] = dic_replace_accents[a_s[m]] - + # Capitalisation a_s = [a.capitalize() for a in a_s] # Minuscules dic_replace_hors_premier_mot = {} - dic_replace_hors_premier_mot['Au'] = 'au' - dic_replace_hors_premier_mot['Aux'] = 'aux' - dic_replace_hors_premier_mot['D'] = 'd\'' - dic_replace_hors_premier_mot['De'] = 'de' - dic_replace_hors_premier_mot['Des'] = 'des' - dic_replace_hors_premier_mot['Du'] = 'du' - dic_replace_hors_premier_mot['Et'] = 'et' - dic_replace_hors_premier_mot['L'] = 'l\'' - dic_replace_hors_premier_mot['La'] = 'la' - dic_replace_hors_premier_mot['Le'] = 'le' - dic_replace_hors_premier_mot['Les'] = 'les' - dic_replace_hors_premier_mot['Un'] = 'un' - dic_replace_hors_premier_mot['Une'] = 'une' - + dic_replace_hors_premier_mot["Au"] = "au" + dic_replace_hors_premier_mot["Aux"] = "aux" + dic_replace_hors_premier_mot["D"] = "d'" + dic_replace_hors_premier_mot["De"] = "de" + dic_replace_hors_premier_mot["Des"] = "des" + dic_replace_hors_premier_mot["Du"] = "du" + dic_replace_hors_premier_mot["Et"] = "et" + dic_replace_hors_premier_mot["L"] = "l'" + dic_replace_hors_premier_mot["La"] = "la" + dic_replace_hors_premier_mot["Le"] = "le" + dic_replace_hors_premier_mot["Les"] = "les" + dic_replace_hors_premier_mot["Un"] = "un" + dic_replace_hors_premier_mot["Une"] = "une" + if len(a_s) > 1: - for m in range(1,len(a_s)): + for m in range(1, len(a_s)): if a_s[m] in dic_replace_hors_premier_mot: a_s[m] = dic_replace_hors_premier_mot[a_s[m]] - + # Appostrophes initiale dic_ajoute_apostrophe = {} - dic_ajoute_apostrophe['d'] = 'd\'' - dic_ajoute_apostrophe['D'] = 'D\'' - dic_ajoute_apostrophe['l'] = 'l\'' - dic_ajoute_apostrophe['L'] = 'L\'' + dic_ajoute_apostrophe["d"] = "d'" + dic_ajoute_apostrophe["D"] = "D'" + dic_ajoute_apostrophe["l"] = "l'" + dic_ajoute_apostrophe["L"] = "L'" if a_s[0] in dic_ajoute_apostrophe: a_s[0] = dic_ajoute_apostrophe[a_s[0]] - - s = ' '.join(a_s).replace('\' ','\'') - if len(s.strip())>1 and s.strip()[-1] == '\'': + + s = " ".join(a_s).replace("' ", "'") + if len(s.strip()) > 1 and s.strip()[-1] == "'": s = s.strip()[0:-1] return s + def get_nb_parts(s): return len(s.split()) -def get_part_debut(s,nb_parts): - resp = '' +def get_part_debut(s, nb_parts): + resp = "" if get_nb_parts(s) > nb_parts: - resp = ' '.join(s.split()[0:nb_parts]) + resp = " ".join(s.split()[0:nb_parts]) return resp @@ -88,91 +94,98 @@ def is_valid_housenumber(hsnr): return len(hsnr) <= 11 return False + def is_valid_dept(dept): return dept in constants.DEPARTEMENTS + def get_code_dept_from_insee(code_insee): code_dept = code_insee[0:2] - if code_dept == '97': + if code_dept == "97": code_dept = code_insee[0:3] return code_dept + def get_sql_like_dept_string(dept): - return (dept+'___')[0:5] + return (dept + "___")[0:5] + def normalize(s): - s = s.upper() # tout en majuscules - s = s.split(' (')[0] # parenthèses : on coupe avant - s = s.replace('-',' ') # separateur espace - s = s.replace('\'',' ') # separateur espace - s = s.replace('’',' ') # separateur espace - s = s.replace('/',' ') # separateur espace - s = s.replace(':',' ') # separateur deux points - s = ' '.join(s.split()) # separateur : 1 espace + s = s.upper() # tout en majuscules + # s = s.split(' (')[0] # parenthèses : on coupe avant + s = s.replace("(", "").replace( + ")", "" + ) # parenthèses : on supprime cf Fantoir pour les anciennes communes en suffixe + s = s.replace("-", " ") # separateur espace + s = s.replace("'", " ") # separateur espace + s = s.replace("’", " ") # separateur espace + s = s.replace("/", " ") # separateur espace + s = s.replace(":", " ") # separateur deux points + s = " ".join(s.split()) # separateur : 1 espace for l in iter(constants.LETTRE_A_LETTRE): for ll in constants.LETTRE_A_LETTRE[l]: - s = s.replace(ll,l) + s = s.replace(ll, l) - -# type de voie + # type de voie abrev_trouvee = False p = 5 while (not abrev_trouvee) and p > -1: - p-= 1 - if get_part_debut(s,p) in constants.ABREV_TYPE_VOIE: - s = replace_type_voie(s,p) + p -= 1 + if get_part_debut(s, p) in constants.ABREV_TYPE_VOIE: + s = replace_type_voie(s, p) abrev_trouvee = True -# ordinal - s = s.replace(' EME ','EME ') - s = s.replace(' 1ERE',' PREMIERE') - s = s.replace(' 1ER',' PREMIER') + # ordinal + s = s.replace(" EME ", "EME ") + s = s.replace(" 1ERE", " PREMIERE") + s = s.replace(" 1ER", " PREMIER") -# chiffres + # chiffres for c in constants.CHIFFRES: - s = s.replace(c[0],c[1]) + s = s.replace(c[0], c[1]) -# titres, etc. + # titres, etc. for r in constants.EXPAND_NOMS: - s = s.replace(' '+r[0]+' ',' '+r[1]+' ') - if s[-len(r[0]):] == r[0]: - s = s.replace(' '+r[0],' '+r[1]) + s = s.replace(" " + r[0] + " ", " " + r[1] + " ") + if s[-len(r[0]) :] == r[0]: + s = s.replace(" " + r[0], " " + r[1]) for r in constants.EXPAND_TITRES: - s = s.replace(' '+r[0]+' ',' '+r[1]+' ') - if s[-len(r[0]):] == r[0]: - s = s.replace(' '+r[0],' '+r[1]) + s = s.replace(" " + r[0] + " ", " " + r[1] + " ") + if s[-len(r[0]) :] == r[0]: + s = s.replace(" " + r[0], " " + r[1]) for r in constants.ABREV_TITRES: - s = s.replace(' '+r[0]+' ',' '+r[1]+' ') - if s[-len(r[0]):] == r[0]: - s = s.replace(' '+r[0],' '+r[1]) + s = s.replace(" " + r[0] + " ", " " + r[1] + " ") + if s[-len(r[0]) :] == r[0]: + s = s.replace(" " + r[0], " " + r[1]) -# articles + # articles for c in constants.MOT_A_BLANC: - s = s.replace(' '+c+' ',' ') + s = s.replace(" " + c + " ", " ") -# chiffres romains + # chiffres romains sp = s.split() - if len(sp)>0 and sp[-1] in constants.CHIFFRES_ROMAINS: + if len(sp) > 0 and sp[-1] in constants.CHIFFRES_ROMAINS: sp[-1] = constants.CHIFFRES_ROMAINS[sp[-1]] - s = ' '.join(sp) + s = " ".join(sp) -# substitution complete + # substitution complete if s in constants.SUBSTITUTION_COMPLETE: s = constants.SUBSTITUTION_COMPLETE[s] return s[0:30] -def replace_type_voie(s,nb): +def replace_type_voie(s, nb): sp = s.split() - spd = ' '.join(sp[0:nb]) - spf = ' '.join(sp[nb:len(sp)]) - s = constants.ABREV_TYPE_VOIE[spd]+' '+spf + spd = " ".join(sp[0:nb]) + spf = " ".join(sp[nb : len(sp)]) + s = constants.ABREV_TYPE_VOIE[spd] + " " + spf return s -def is_valid_fantoir(f, insee): - return (len(f) == 10 and f[0:5] == insee); +def fantoir_valide(f, insee): + return len(f) == 9 and f[0:5] == insee + def display_insee_commune(code_insee, nom_commune): - print(f"{code_insee} - {nom_commune}") \ No newline at end of file + print(f"{code_insee} - {nom_commune}") diff --git a/bano/log_2_file.py b/bano/log_2_file.py deleted file mode 100644 index 20b17c3..0000000 --- a/bano/log_2_file.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python -# coding: UTF-8 - -import os -import time - -def start_log_to_file(source,etape,dept): - t = time.localtime() - th = time.strftime('%d-%m-%Y %H:%M:%S',t) - t = round(time.mktime(t),0) - log_filename = '{:s}_{:s}_{:s}.log'.format(dept,etape,source) - f = open(os.path.join(os.environ['LOG_DIR'],'{:s}'.format(log_filename)),'w+') - f.write('Debut : {:s}\n'.format(th)) - f.flush() - return f - -def write_log_to_file(flog,message): - flog.write(message+'\n') - flog.flush() - -def write_sep_to_file(flog): - flog.write('\n####################\n') - flog.flush() - -def end_log_to_file(flog,display=False): - t = time.localtime() - th = time.strftime('%d-%m-%Y %H:%M:%S',t) - flog.write(u'Fin : {:s}\n'.format(th)) - flog.flush() - if display: - flog.seek(0) - print(flog.read()) - flog.close() - diff --git a/bano/models.py b/bano/models.py index 29d0a1d..df1901b 100644 --- a/bano/models.py +++ b/bano/models.py @@ -1,357 +1,831 @@ -import re -import time +#!/usr/bin/env python +# coding: UTF-8 -from . import db +import io +import json +from collections import defaultdict, OrderedDict + +from .db import bano_db from . import helpers as hp -from .sources import fantoir -from . import core as c + +from .sql import sql_get_data, sql_process + + +class Nom: + def __init__( + self, + nom, + fantoir, + nature, + source, + code_insee, + code_insee_ancienne_commune, + nom_ancienne_commune, + ): + self.code_insee = code_insee + self.code_dept = hp.get_code_dept_from_insee(code_insee) + self.code_insee_ancienne_commune = code_insee_ancienne_commune + self.nom = nom.replace("\t", " ") + self.nom_ancienne_commune = nom_ancienne_commune + self.fantoir = fantoir[0:9] if fantoir else None + self.nature = nature + self.source = source + self.nom_normalise = hp.normalize(nom) + self.niveau = ( + self.code_insee_ancienne_commune + if self.code_insee_ancienne_commune + else "RACINE" + ) + + def __eq__(self, other): + return ( + self.nom == other.nom + and self.fantoir == other.fantoir + and self.nature == other.nature + and self.source == other.source + and self.code_insee == other.code_insee + and self.code_insee_ancienne_commune == other.code_insee_ancienne_commune + ) + + def __hash__(self): + return hash( + ( + self.nom, + self.fantoir, + self.source, + self.nature, + self.code_insee, + self.code_insee_ancienne_commune, + ) + ) + + def _as_csv_format_bano(self, correspondance): + if self.source == "BAN": + fantoir = remplace_fantoir_ban(correspondance, self.niveau, self.fantoir) + else: + fantoir = self.fantoir + return f"{fantoir}\t{self.nom}\t{self.nature}\t{self.code_insee}\t{self.code_dept}\t{self.code_insee_ancienne_commune if self.code_insee_ancienne_commune else ''}\t{self.nom_ancienne_commune if self.nom_ancienne_commune else ''}\t{self.source}" + + def add_fantoir(self, topo): + if not self.fantoir: + self.fantoir = topo.topo.get(self.nom_normalise) + + +class Noms: + def __init__(self, code_insee): + self.code_insee = code_insee + self.triplets_nom_fantoir_source = [] + self.fantoir_par_nom_sous_commune = {"RACINE": defaultdict(list)} + + def __iter__(self): + return iter(self.triplets_nom_fantoir_source) + + def _print(self, correspondance, pattern=None): + for a in self: + if not pattern or pattern in a._as_csv_format_bano(correspondance): + print(a._as_csv_format_bano(correspondance)) + + # On ne charge pas les noms des numeros OSM. Ils sont ajoutés via Adresses.nom_des_adresses + def charge_noms_osm_hors_numeros(self): + data = ( + sql_get_data( + "charge_noms_voies_lieux-dits_OSM", + dict(code_insee=self.code_insee), + ) + + sql_get_data( + "charge_noms_voies_relation_bbox_OSM", + dict(code_insee=self.code_insee), + ) + + sql_get_data( + "charge_noms_voies_relation_OSM", + dict(code_insee=self.code_insee), + ) + ) + for ( + provenance, + name, + tags, + libelle_suffixe, + code_insee_ancienne_commune, + nom_ancienne_commune, + nature, + ) in data: + if provenance in (1, 2, 3, 4, 5): + self.add_nom( + Nom( + name, + tags.get("ref:FR:FANTOIR"), + nature, + "OSM", + self.code_insee, + code_insee_ancienne_commune, + nom_ancienne_commune, + ) + ) + if provenance in (6, 7) and tags.get("ref:FR:FANTOIR"): + self.add_nom( + Nom( + name, + tags["ref:FR:FANTOIR"], + nature, + "OSM", + self.code_insee, + code_insee_ancienne_commune, + nom_ancienne_commune, + ) + ) + + # On ajoute un triplet nom s'il n'a pas de FANTOIR ou si son FANTOIR appartient à la commune + def add_nom(self, nom=Nom): + if not nom.fantoir or nom.fantoir[0:5] == self.code_insee: + self.triplets_nom_fantoir_source.append(nom) + + def add_fantoir(self, topo): + for t in self.triplets_nom_fantoir_source: + t.add_fantoir(topo) + + def remplit_fantoir_par_nom_sous_commune(self): + # privilège pour la source OSM + for source in ['OSM','BAN','CADASTRE']: + for t in self.triplets_nom_fantoir_source: + if t.source != source or not t.fantoir: + continue + if t.code_insee_ancienne_commune: + if ( + not t.code_insee_ancienne_commune + in self.fantoir_par_nom_sous_commune + ): + self.fantoir_par_nom_sous_commune[ + t.code_insee_ancienne_commune + ] = {} + if not t.nom in self.fantoir_par_nom_sous_commune[t.code_insee_ancienne_commune]: + self.fantoir_par_nom_sous_commune[t.code_insee_ancienne_commune][ + t.nom + ] = t.fantoir + else: + if not t.nom in self.fantoir_par_nom_sous_commune: + self.fantoir_par_nom_sous_commune[t.nom] = t.fantoir + + def enregistre(self, correspondance): + sql_process( + "suppression_noms_commune", + dict(code_insee=self.code_insee), + ) + io_in_csv = io.StringIO() + for t in set(self.triplets_nom_fantoir_source): + if t.fantoir: + io_in_csv.write(t._as_csv_format_bano(correspondance) + "\n") + io_in_csv.seek(0) + with bano_db.cursor() as cur_insert: + cur_insert.copy_from( + io_in_csv, + "nom_fantoir", + null="", + columns=( + "fantoir", + "nom", + "nature", + "code_insee", + "code_dept", + "code_insee_ancienne_commune", + "nom_ancienne_commune", + "source", + ), + ) + + def stats_sources(self): + par_source = {"BAN": set(), "OSM": set(), "CADASTRE": set()} + for t in self: + if t.fantoir: + par_source[t.source].add(t.fantoir) + return [ + len(par_source["BAN"]), + len(par_source["CADASTRE"]), + len(par_source["OSM"]), + ] class Adresse: - def __init__(self, node, num, voie, fantoir, code_postal): - self.node = node + def __init__( + self, + code_insee, + x, + y, + num, + source, + voie=None, + place=None, + fantoir=None, + code_postal=None, + code_insee_ancienne_commune=None, + nom_ancienne_commune=None, + id_ban=None, + ): + self.code_insee = code_insee + self.code_dept = hp.get_code_dept_from_insee(code_insee) + self.x = round(x, 6) + self.y = round(y, 6) + self.source = source self.numero = num self.voie = voie - self.fantoir = fantoir + self.place = place + self.fantoir = fantoir[0:9] if fantoir else None self.code_postal = code_postal + self.code_insee_ancienne_commune = code_insee_ancienne_commune + self.nom_ancienne_commune = nom_ancienne_commune + self.voie_normalisee = hp.normalize(self.voie) if self.voie else None + self.place_normalisee = hp.format_toponyme(self.place) if self.place else None + self.niveau = ( + self.code_insee_ancienne_commune + if self.code_insee_ancienne_commune + else "RACINE" + ) + self.id_ban = id_ban + + def __hash__(self): + return hash( + ( + self.code_insee, + self.source, + self.numero, + self.voie, + self.place, + self.code_insee_ancienne_commune, + ) + ) + + def __eq__(self, other): + return ( + self.code_insee == other.code_insee + and self.source == other.source + and self.numero == other.numero + and self.voie == other.voie + and self.place == other.place + and self.code_insee_ancienne_commune == other.code_insee_ancienne_commune + ) + + def _as_csv_format_bano(self, correspondance): + if self.source == "BAN": + fantoir = remplace_fantoir_ban(correspondance, self.niveau, self.fantoir) + else: + fantoir = self.fantoir + return f"{fantoir if fantoir else ''}\t{self.x}\t{self.y}\t{self.numero}\t{self.voie if self.voie else ''}\t{self.place if self.place else ''}\t{self.code_postal}\t{self.code_insee}\t{self.code_dept}\t{self.code_insee_ancienne_commune if self.code_insee_ancienne_commune else ''}\t{self.nom_ancienne_commune if self.nom_ancienne_commune else ''}\t{self.source}\t{self.id_ban if self.id_ban else ''}" + + def _as_string(self): + return f"source : {self.source}, numero : {self.numero}, voie : {self.voie} ({self.voie_normalisee}), place : {self.place}, fantoir : {self.fantoir}, code_postal:{self.code_postal}, sous_commune : {self.code_insee_ancienne_commune} - {self.nom_ancienne_commune}" class Adresses: def __init__(self, code_insee): - self.a = {} self.code_insee = code_insee + self.liste = [] + self.index_voie = defaultdict(list) + self.noms_de_voies = set() - def __contains__(self, item): - return item in self.a + # def __contains__(self, item): + # return item in self.a def __getitem__(self, key): - return self.a[key] + return self.liste[key] - def __setitem__(self, key, value): - self.a[key] = value + # def __setitem__(self, key, value): + # self.a[key] = value def __iter__(self): - return iter(self.a) - - def register(self, voie): - cle = hp.normalize(voie) - if not cle in self: - self[cle] = {'numeros':{},'voies':{},'fantoirs':{},'point_par_rue':[],'highway_index':0} - - def add_fantoir(self,cle,fantoir,source): - self.register(cle) - if len(fantoir) == 10: - self[cle]['fantoirs'][source] = fantoir - - def add_voie(self,voie_cle,source,voie=None): - cle = hp.normalize(voie_cle) - self[cle]['voies'][source] = voie or voie_cle - - def add_adresse(self,ad,source): - """ une adresses est considérée dans la commune si sans Fantoir ou avec un Fantoir de la commune""" - if (ad.fantoir == '' or hp.is_valid_fantoir(ad.fantoir, self.code_insee)) and hp.is_valid_housenumber(ad.numero): - cle = hp.normalize(ad.voie) - self.add_voie(ad.voie,source) - self[cle]['numeros'][ad.numero] = ad - if ad.fantoir != '': - self[cle]['fantoirs'][source] = ad.fantoir - - def get_cle_by_fantoir(self,fantoir): - cle = '' - for c in self: - if 'fantoirs' in self[c]: - if 'OSM' in self[c]['fantoirs']: - if self[c]['fantoirs']['OSM'] == fantoir: - cle = c - break - if 'FANTOIR' in self[c]['fantoirs']: - if self[c]['fantoirs']['FANTOIR'] == fantoir: - cle = c - break - return cle - - def get_best_fantoir(self, cle): - return self[cle]['fantoirs'].get('OSM') or self[cle]['fantoirs'].get('FANTOIR') or '' - - def has_already_fantoir(self,cle,source): - return source in self[cle]['fantoirs'] - - def add_highway_index(self,cle,val): - self[cle]['highway_index']+=val - - def load_cadastre_hsnr(self): - dict_node_relations = {} - str_query = f"SELECT * FROM bal_cadastre WHERE commune_code = '{self.code_insee}' AND destination_principale in ('habitation','commerce','industrie','tourisme');" - with db.bano_cache.cursor() as cur: - cur.execute(str_query) - for cle_interop, ui_adresse, numero, suffixe, pseudo_adresse, name, voie_code, code_postal, libelle_acheminement, destination_principale, commune_code, commune_nom, source, lon, lat, *others in cur: - housenumber = numero+((' '+suffixe) if suffixe and suffixe.isalnum() else '') - if not name or len(name) < 2 or not lon or pseudo_adresse == 'true': - continue - self.register(name) - - if not cle_interop in dict_node_relations: - dict_node_relations[cle_interop] = [] - dict_node_relations[cle_interop].append(hp.normalize(name)) - if hp.is_valid_housenumber(housenumber): - nd = Node({'id':cle_interop,'lon':lon,'lat':lat},{}) - self.add_adresse(Adresse(nd,housenumber,name,'',code_postal), 'CADASTRE') - - def load_ban_hsnr(self): - dict_node_relations = {} - data = c.get_data_from_pg_direct('ban_hsnr',self.code_insee) - for id, housenumber, name, lon, lat in data: - if not name or len(name) < 2 or not lon: + return iter(self.liste) + + def _print(self, pattern=None): + for a in self: + if not pattern or pattern in a._as_string(): + print(a._as_string()) + + def add_adresse(self, ad): + """une adresses est considérée dans la commune si sans Fantoir ou avec un Fantoir de la commune""" + # if (ad.fantoir == None or hp.is_valid_fantoir(ad.fantoir, self.code_insee)) and hp.is_valid_housenumber(ad.numero): + self.liste.append(ad) + self.index_voie[ad.voie].append(len(self.liste) - 1) + self.noms_de_voies.add(ad.voie) + + # def add_fantoir(self): + # for nom in self.noms: + # fantoir = topo.topo.get(a.voie_normalisee) + + def charge_numeros_ban(self, topo): + data = sql_get_data("charge_ban_commune", dict(code_insee=self.code_insee)) + for ( + fantoir, + numero, + voie, + lon, + lat, + code_postal, + code_insee_ancienne_commune, + nom_ancienne_commune, + id_ban, + ) in data: + if not (fantoir and fantoir in topo.topo): + fantoir = None + + self.add_adresse( + Adresse( + self.code_insee, + lon, + lat, + numero, + "BAN", + voie=voie, + fantoir=fantoir, + code_postal=code_postal, + code_insee_ancienne_commune=code_insee_ancienne_commune, + nom_ancienne_commune=nom_ancienne_commune, + id_ban=id_ban, + ) + ) + + def charge_numeros_osm(self): + data = sql_get_data( + "charge_numeros_OSM", dict(code_insee=self.code_insee) + ) + sql_get_data("charge_numeros_bbox_OSM", dict(code_insee=self.code_insee)) + + for ( + lon, + lat, + provenance, + numero, + voie, + place, + tags, + suffixe, + code_postal, + code_insee_ancienne_commune, + nom_ancienne_commune, + ) in data: + # Fix issue #378 + if voie: + voie = voie.replace('\n',' ') + + fantoir = tags.get("ref:FR:FANTOIR") + if fantoir: + fantoir = fantoir[0:9] + if fantoir and not hp.fantoir_valide(fantoir, self.code_insee): continue - self.register(name) - if not id in dict_node_relations: - dict_node_relations[id] = [] - dict_node_relations[id].append(hp.normalize(name)) - if hp.is_valid_housenumber(housenumber): - self.add_adresse(Adresse(Node({'id':id,'lon':lon,'lat':lat},{}),housenumber,name,'',''), 'BAN') - - def save(self, source, code_dept): - with db.bano.cursor() as cur_insert : - for a in ['cumul_adresses','cumul_voies']: - cur_insert.execute(f"DELETE FROM {a} WHERE insee_com = '{self.code_insee}' AND source = '{source}';") - nb_rec = 0 - a_values = [] - a_values_voie = [] - sload = 'INSERT INTO cumul_adresses (geometrie,numero,voie_cadastre,voie_bal,voie_osm,voie_fantoir,fantoir,insee_com,dept,code_postal,source,voie_autre) VALUES' - for v in self: - code_postal = '' - cle_fantoir = self.get_best_fantoir(v) - street_name_osm = self[v]['voies'].get('OSM') or fantoir.mapping.get_fantoir_name(cle_fantoir,'OSM') or '' - street_name_fantoir = self[v]['voies'].get('FANTOIR') or '' - street_name_cadastre = self[v]['voies'].get('CADASTRE') or '' - street_name_bal = self[v]['voies'].get('BAL') or '' - if source == 'CADASTRE': - street_name_autre = self[v]['voies'].get('CADASTRE') or '' - elif source == 'BAL': - street_name_autre = self[v]['voies'].get('BAL') or '' - elif source == 'BAN': - street_name_autre = self[v]['voies'].get('BAN') or '' - else: - street_name_autre = '' - lat_point_par_rue = None - lon_point_par_rue = None - - for num in self[v]['numeros']: - numadresse = self[v]['numeros'][num] - a_values.append("(ST_GeomFromText('POINT({:6f} {:6f})', 4326),'{:s}','{:s}','{:s}','{:s}','{:s}','{:s}','{:s}','{:s}','{:s}','{:s}','{:s}')".format(numadresse.node.attribs['lon'],numadresse.node.attribs['lat'],numadresse.numero.replace("'",""),street_name_cadastre.replace("'","''"),street_name_bal.replace("'","''"),street_name_osm.replace("'","''"),street_name_fantoir.replace("'","''"),cle_fantoir,self.code_insee,code_dept,numadresse.code_postal,source,street_name_autre.replace("'","''")).replace(",''",",null").replace(",''",",null")) - if source == 'OSM': - lat_point_par_rue = numadresse.node.attribs['lat'] - lon_point_par_rue = numadresse.node.attribs['lon'] - - if source == 'OSM': - if len(self[v]['point_par_rue'])>1: - lon_point_par_rue = self[v]['point_par_rue'][0] - lat_point_par_rue = self[v]['point_par_rue'][1] - - if lat_point_par_rue and lon_point_par_rue: - a_values_voie.append(f"(ST_GeomFromText('POINT({lon_point_par_rue} {lat_point_par_rue})', 4326),'{hp.escape_quotes(street_name_cadastre)}','{hp.escape_quotes(street_name_bal)}','{hp.escape_quotes(street_name_osm)}','{hp.escape_quotes(street_name_fantoir)}','{cle_fantoir}','{self.code_insee}','{code_dept}','','{source}',{self[v]['highway_index']})".replace(",'',",",null,")) - - nb_rec +=1 - if len(a_values)>0: - cur_insert.execute(sload+','.join(a_values)+';COMMIT;') - sload_voie = 'INSERT INTO cumul_voies (geometrie,voie_cadastre,voie_bal,voie_osm,voie_fantoir,fantoir,insee_com,dept,code_postal,source,voie_index) VALUES' - if len(a_values_voie) > 0: - sload_voie = sload_voie+','.join(a_values_voie)+';COMMIT;' - cur_insert.execute(sload_voie) - return(nb_rec) - - -class Node: - def __init__(self,attribs,tags): - self.attribs = attribs - self.tags = tags - self.sent = False - self.modified = False - - def get_geom_as_text(self): - strp = 'ST_PointFromText(\'POINT('+str(self.attribs['lon'])+' '+str(self.attribs['lat'])+')\',4326)' - return strp - - def move_to(self,lon,lat): - self.attribs['lon'] = lon - self.attribs['lat'] = lat - - -class Pg_hsnr: - def __init__(self, d, code_insee): - self.x, self.y, self.provenance, self.osm_id, self.numero, self.voie, self.tags, *others = d - self.tags = self.tags or {} - self.fantoir = '' - if self.provenance == 3 or self.provenance == 4: - self.set_street_name() - self.set_fantoir(code_insee) - self.code_postal = hp.find_cp_in_tags(self.tags) - - def set_street_name(self): - if 'type' in self.tags and self.tags['type'] == 'associatedStreet' and 'name' in self.tags: - self.voie = self.tags['name'] - - def set_fantoir(self, code_insee): - if 'ref:FR:FANTOIR' in self.tags and len(self.tags['ref:FR:FANTOIR']) == 10 and self.tags['ref:FR:FANTOIR'][0:5] == code_insee: - self.fantoir = self.tags['ref:FR:FANTOIR'] - -class Fantoir: - def __init__(self,name,fantoir,bati): - self.name = name - self.name_norm = hp.normalize(name) - self.fantoir = fantoir - self.bati = bati if bati != '' else None -class Cadastre: - def __init__(self,lon,lat,name): - self.lon = lon - self.lat = lat - self.name = name - self.name_norm = hp.normalize(name) -class Osm: - def __init__(self,lon,lat,place,name,fantoir): - self.lon = lon - self.lat = lat - self.place = place - self.name = name - self.name_norm = hp.normalize(name) - self.fantoir = fantoir -class Place: - def __init__(self,lon,lat,place,name_fantoir,name_cadastre,name_osm,fantoir,bati,code_insee): - self.has_fantoir = False - self.has_osm = False - self.has_cadastre = False - self.fantoir = Fantoir(name_fantoir,fantoir,bati) - self.osm = Osm(lon,lat,place,name_osm,fantoir) - self.cadastre = Cadastre(lon,lat,name_cadastre) - self.code_insee = str(code_insee) - self.code_dept = hp.get_code_dept_from_insee(self.code_insee) - if self.fantoir.name != '': - self.has_fantoir = True - if self.osm.name != '': - self.has_osm = True - if self.cadastre.name != '': - self.has_cadastre = True - self.id = self.fantoir.fantoir or (self.cadastre.name_norm or self.osm.name_norm) - def update_fantoir(self,name,fantoir,bati): - self.fantoir = Fantoir(name,fantoir,bati) - self.has_fantoir = True - def update_osm(self,lon,lat,place,name,fantoir=''): - self.osm = Osm(lon,lat,place,name,fantoir) - self.has_osm = True - def update_cadastre(self,lon,lat,name): - self.cadastre = Cadastre(lon,lat,name) - self.has_cadastre = True - def as_string(self): - return f"ID {self.id} \n FANTOIR {self.fantoir.name} - FANTOIR {self.fantoir.fantoir} \nOSM {self.osm.lon} - {self.osm.lat} - {self.osm.place} - {self.osm.name} \nCADASTRE {self.cadastre.lon} - {self.cadastre.lat} - {self.cadastre.name}" - def as_SQL_cadastre_row(self): - if self.has_cadastre: - return hp.remove_quotes_on_null(f"(ST_PointFromText('POINT({self.cadastre.lon} {self.cadastre.lat})',4326),'{hp.escape_quotes(hp.format_toponyme(self.cadastre.name)) or 'null'}','{hp.escape_quotes(self.osm.name) or 'null'}','{hp.escape_quotes(self.fantoir.name) or 'null'}','{self.fantoir.fantoir or 'null'}','{self.code_insee}','{self.code_dept}','null','CADASTRE',{self.fantoir.bati or 'null'},'')") - def as_SQL_osm_row(self): - return hp.remove_quotes_on_null(f"(ST_PointFromText('POINT({self.osm.lon} {self.osm.lat})',4326),null,'{hp.escape_quotes(self.osm.name) or 'null'}','{hp.escape_quotes(self.fantoir.name) or 'null'}','{self.fantoir.fantoir or 'null'}','{self.code_insee}','{self.code_dept}',null,'OSM',{self.fantoir.bati or 'null'},'{self.osm.place}')") - -class Places: - def __init__(self): - self.p = {} - - def add_place(self,new_p): - self.p[new_p.id]=new_p - def match_fantoir(self,fantoir): - res = [] - for c in self.p: - if self.p[c].fantoir and self.p[c].fantoir.fantoir and self.p[c].fantoir.fantoir == fantoir: - res+=[c] - return res - def match_name(self,name,target): - res = [] - name_norm = hp.normalize(name) - if target == 'FANTOIR': - for c in self.p: - if self.p[c].fantoir and self.p[c].fantoir.name and self.p[c].fantoir.name_norm == name_norm: - res+=[c] - if target == 'CADASTRE': - for c in self.p: - if c.cadastre and c.cadastre.name and c.cadastre.name_norm == name_norm: - res+=c - if target == 'OSM': - for c in self.p: - if c.osm and c.osm.name and c.osm.name_norm == name_norm: - res+=c - return res - def _print(self): - for c in self.p: - print(f'ID : {self.p[c].id} - FANTOIR : {self.p[c].fantoir.name} - OSM : {self.p[c].osm.name}') - print(self.p[c].as_string()) - def _print_SQL_Cadastre(self): - for c in self.p: - print(self.p[c].as_SQL_cadastre_row()) - def as_SQL_Cadastre_array(self): - a = [] - for c in self.p: - if self.p[c].has_cadastre: - a.append(self.p[c].as_SQL_cadastre_row()) - return a - def as_SQL_OSM_array(self): - a = [] - for c in self.p: - if self.p[c].has_osm: - a.append(self.p[c].as_SQL_osm_row()) - return a - - -class Tile: - def __init__(self,z,x,y): - self.x = x - self.y = y - self.z = z - self.xmin = -20037508.34 + (40075016.68/(2 ** self.z)) * self.x - self.xmax = -20037508.34 + (40075016.68/(2 ** self.z)) * (self.x + 1) - self.ymin = 20037508.34 - (40075016.68/(2 ** self.z)) * (self.y + 1) - self.ymax = 20037508.34 - (40075016.68/(2 ** self.z)) * self.y - -class Tiles: - def __init__(self): - self.t = {} - - def add_tile(self,z,x,y): - self.t[f"{z}/{x}/{y}"] = Tile(z,x,y) - - def add_tiles_from_file(self,tilefile): - with open(tilefile,'r') as f: - for ligne in f.readlines(): - self.add_tile(int(ligne.split('/')[0]),int(ligne.split('/')[1]),int(ligne.split('/')[2])) - - def as_list_of_SQL_values(self): - return [f"({self.t[tile].z},{self.t[tile].x},{self.t[tile].y},ST_SetSRID(ST_MakeBox2D(ST_Point({int(self.t[tile].xmin)},{int(self.t[tile].ymin)}),ST_Point({int(self.t[tile].xmax)},{int(self.t[tile].ymax)})),3857))" for tile in self.t] - - def convert_to_insee_list(self): - with db.bano_cache.cursor() as cur: - cur.execute("TRUNCATE TABLE expire_tiles;COMMIT;") - SQL_values = self.as_list_of_SQL_values() - while len(SQL_values) > 10000: - cur.execute(f"""INSERT INTO expire_tiles VALUES {','.join(SQL_values[0:10000])};COMMIT; """) - SQL_values = SQL_values[10000:] - if SQL_values: - cur.execute(f"""INSERT INTO expire_tiles VALUES {','.join(SQL_values)};COMMIT; """) - cur.execute("""SELECT p."ref:INSEE" - FROM planet_osm_polygon p - JOIN expire_tiles e - ON ST_intersects(p.way, e.geometrie) - WHERE p.way && e.geometrie AND - p.admin_level = 8 AND - p.boundary = 'administrative' AND - p."ref:INSEE" NOT IN ('13055','69123','75056') AND - COALESCE(p."ref:INSEE",'') != '' - UNION - SELECT p."ref:INSEE" - FROM planet_osm_polygon p - JOIN expire_tiles e - ON ST_intersects(p.way, e.geometrie) - WHERE p.way && e.geometrie AND - p.admin_level = 9 AND - p.boundary = 'administrative' AND - (p."ref:INSEE" LIKE '132__' OR - p."ref:INSEE" LIKE '6938_' OR - p."ref:INSEE" LIKE '751__') - ORDER BY 1;""") - return [f[0]for f in cur] + if provenance in ( + 1, + 2, + ): + self.add_adresse( + Adresse( + self.code_insee, + lon, + lat, + numero, + "OSM", + voie=voie, + place=place, + fantoir=fantoir, + code_postal=code_postal, + code_insee_ancienne_commune=code_insee_ancienne_commune, + nom_ancienne_commune=nom_ancienne_commune, + ) + ) + if provenance in ( + 3, + 4, + ) and tags.get("name"): + self.add_adresse( + Adresse( + self.code_insee, + lon, + lat, + numero, + "OSM", + voie=tags["name"], + place=None, + fantoir=fantoir, + code_postal=code_postal, + code_insee_ancienne_commune=code_insee_ancienne_commune, + nom_ancienne_commune=nom_ancienne_commune, + ) + ) + if ( + provenance + in ( + 5, + 6, + ) + and tags.get("name") + and tags.get("ref:FR:FANTOIR") + ): + if tags["ref:FR:FANTOIR"][0:5] == self.code_insee: + self.add_adresse( + Adresse( + self.code_insee, + lon, + lat, + numero, + "OSM", + voie=tags["name"], + place=None, + fantoir=tags["ref:FR:FANTOIR"], + code_postal=code_postal, + code_insee_ancienne_commune=code_insee_ancienne_commune, + nom_ancienne_commune=nom_ancienne_commune, + ) + ) + + def noms_des_adresses(self, noms): + for a in self: + if a.voie: + noms.add_nom( + Nom( + a.voie, + a.fantoir, + "voie", + a.source, + self.code_insee, + a.code_insee_ancienne_commune, + a.nom_ancienne_commune, + ) + ) + if a.place: + noms.add_nom( + Nom( + a.place, + a.fantoir, + "place", + a.source, + self.code_insee, + a.code_insee_ancienne_commune, + a.nom_ancienne_commune, + ) + ) + + def complete_fantoir(self, noms): + for a in self: + if a.fantoir: + continue + nom = a.voie or a.place + if a.code_insee_ancienne_commune: + a.fantoir = noms.fantoir_par_nom_sous_commune.get( + a.code_insee_ancienne_commune + ).get(nom) + else: + a.fantoir = noms.fantoir_par_nom_sous_commune.get(nom) + + def enregistre(self, correspondance): + sql_process( + "suppression_adresses_commune", + dict(code_insee=self.code_insee), + ) + io_in_csv = io.StringIO() + + for a in set( + self.liste + ): # passage en set pour dedoublonner les adresses de provenances multiples + io_in_csv.write( + a._as_csv_format_bano(correspondance) + "\n" + ) # separateur $ car on trouve des virgules dans le contenu + io_in_csv.seek(0) + with bano_db.cursor() as cur_insert: + cur_insert.copy_from( + io_in_csv, + "bano_adresses", + null="", + columns=( + "fantoir", + "lon", + "lat", + "numero", + "nom_voie", + "nom_place", + "code_postal", + "code_insee", + "code_dept", + "code_insee_ancienne_commune", + "nom_ancienne_commune", + "source", + "id_ban", + ), + ) + + def stats_sources(self): + par_source = {"BAN": set(), "OSM": set()} + for t in self: + if t.fantoir: + par_source[t.source].add(f"{t.numero}{t.fantoir}") + return [len(par_source["BAN"]), len(par_source["OSM"])] + + +class Point_nomme: + def __init__( + self, + code_insee, + source, + nature, + lon, + lat, + nom, + fantoir=None, + code_insee_ancienne_commune=None, + nom_ancienne_commune=None, + ): + self.code_insee = code_insee + self.code_dept = hp.get_code_dept_from_insee(code_insee) + self.source = source + self.lon = round(lon, 6) + self.lat = round(lat, 6) + self.nature = nature + self.nom = nom.replace("\t", " ") + self.nom_normalise = hp.normalize(nom) + self.fantoir = fantoir[0:9] if fantoir else None + self.code_insee_ancienne_commune = code_insee_ancienne_commune + self.nom_ancienne_commune = nom_ancienne_commune + + def __hash__(self): + return hash( + ( + self.code_insee, + self.source, + self.nom_normalise, + self.code_insee_ancienne_commune, + ) + ) + + def __eq__(self, other): + return ( + self.code_insee == other.code_insee + and self.source == other.source + and self.nom_normalise == other.nom_normalise + and self.code_insee_ancienne_commune == other.code_insee_ancienne_commune + ) + + def _as_string(self): + return f"fantoir : {self.fantoir}, source : {self.source}, nom : {self.nom} ({self.nom_normalise}), nature : {self.nature}, sous_commune : {self.code_insee_ancienne_commune}" + + def _as_csv_format_bano(self, correspondance): + return f"{correspondance.get(self.fantoir,self.fantoir) if self.fantoir else ''}\t{self.nom}\t{self.code_insee}\t{self.code_dept}\t{self.nature}\t{self.code_insee_ancienne_commune if self.code_insee_ancienne_commune else ''}\t{self.nom_ancienne_commune if self.nom_ancienne_commune else ''}\t{self.source}\t{self.lon}\t{self.lat}" + + +class Points_nommes: + def __init__(self, code_insee): + self.code_insee = code_insee + self.liste = set() + + def __getitem__(self, key): + return self.liste[key] + + def __iter__(self): + return iter(self.liste) + + def _print(self, pattern=None): + for a in self: + if not pattern or pattern in a._as_string(): + print(a._as_string()) + + def charge_points_nommes_lieux_dits_cadastre(self): + data = sql_get_data( + "charge_points_nommes_lieux-dits_CADASTRE", + dict(code_insee=self.code_insee), + ) + for x, y, nom, code_insee_ancienne_commune, nom_ancienne_commune in data: + self.add_point_nomme( + Point_nomme( + self.code_insee, + "CADASTRE", + "lieu-dit", + x, + y, + hp.format_toponyme(nom), + code_insee_ancienne_commune=code_insee_ancienne_commune, + nom_ancienne_commune=nom_ancienne_commune, + ) + ) + + def charge_points_nommes_centroides_osm(self): + data = sql_get_data( + "charge_points_nommes_centroides_OSM", + dict(code_insee=self.code_insee), + ) + for ( + x, + y, + nom, + code_insee_ancienne_commune, + fantoir, + nom_ancienne_commune, + ) in data: + self.add_point_nomme( + Point_nomme( + self.code_insee, + "OSM", + "centroide", + x, + y, + nom, + code_insee_ancienne_commune=code_insee_ancienne_commune, + fantoir=fantoir, + nom_ancienne_commune=nom_ancienne_commune, + ) + ) + + def charge_points_nommes_place_osm(self): + data = sql_get_data( + "charge_points_nommes_places_OSM", + dict(code_insee=self.code_insee), + ) + for ( + x, + y, + nom, + code_insee_ancienne_commune, + fantoir, + nom_ancienne_commune, + ) in data: + self.add_point_nomme( + Point_nomme( + self.code_insee, + "OSM", + "place", + x, + y, + nom, + code_insee_ancienne_commune=code_insee_ancienne_commune, + fantoir=fantoir, + nom_ancienne_commune=nom_ancienne_commune, + ) + ) + + def charge_points_nommes_numeros_ban(self): + data = sql_get_data( + "charge_points_nommes_numeros_BAN", + dict(code_insee=self.code_insee), + ) + for ( + x, + y, + nom, + code_insee_ancienne_commune, + fantoir, + nom_ancienne_commune, + ) in data: + self.add_point_nomme( + Point_nomme( + self.code_insee, + "BAN", + "numero", + x, + y, + nom, + code_insee_ancienne_commune=code_insee_ancienne_commune, + fantoir=fantoir, + nom_ancienne_commune=nom_ancienne_commune, + ) + ) + + def add_point_nomme(self, ld): + self.liste.add(ld) + + def noms_des_points_nommes(self, noms): + for a in self: + if a.source == "CADASTRE": + noms.add_nom( + Nom( + a.nom, + a.fantoir, + a.nature, + a.source, + self.code_insee, + a.code_insee_ancienne_commune, + a.nom_ancienne_commune, + ) + ) + if a.source == "OSM": + noms.add_nom( + Nom( + a.nom, + a.fantoir, + a.nature, + a.source, + self.code_insee, + a.code_insee_ancienne_commune, + a.nom_ancienne_commune, + ) + ) + + def complete_fantoir(self, noms): + for a in self: + if a.fantoir and a.fantoir[0:5] == self.code_insee: + continue + if a.code_insee_ancienne_commune: + a.fantoir = noms.fantoir_par_nom_sous_commune.get( + a.code_insee_ancienne_commune + ).get(a.nom) + else: + a.fantoir = noms.fantoir_par_nom_sous_commune.get(a.nom) + + def enregistre(self, correspondance): + sql_process( + "suppression_points_nommes_commune", + dict(code_insee=self.code_insee), + ) + io_in_csv = io.StringIO() + for t in self: + io_in_csv.write(t._as_csv_format_bano(correspondance) + "\n") + io_in_csv.seek(0) + with bano_db.cursor() as cur_insert: + cur_insert.copy_from( + io_in_csv, + "bano_points_nommes", + null="", + columns=( + "fantoir", + "nom", + "code_insee", + "code_dept", + "nature", + "code_insee_ancienne_commune", + "nom_ancienne_commune", + "source", + "lon", + "lat", + ), + ) + sql_process( + "complement_points_nommes_numeros_OSM", + dict(code_insee=self.code_insee), + ) + +class Topo: + def __init__(self, code_insee): + self.code_insee = code_insee + self.topo = OrderedDict() + self.nb_fantoirs = 0 + # self.code_fantoir9_vers_fantoir10 = {} + + # self.index_by_nom_normalise = defaultdict(list) + + self.charge_topo() + + def __iter__(self): + return iter(self.topo.items()) + + # def __contains__(self, item): + # return item in self.a + + def __getitem__(self, key): + return self.topo[key] + + def _print(self, pattern=None): + for k, v in self: + if not pattern or pattern in v: + print(f"{k} : {v}") + + def charge_topo(self): + data = sql_get_data("charge_topo_commune", dict(code_insee=self.code_insee)) + for fantoir, nom in data: + nom = hp.normalize(" ".join(nom.replace("-", " ").split())) + self.topo[fantoir] = nom + self.topo[nom] = fantoir + if not "b" in fantoir: + self.nb_fantoirs += 1 + + +class Correspondance_fantoir_ban_osm: + def __init__(self, code_insee): + self.dic_fantoir = {} + self.correspondance = {} + self.code_insee = code_insee + + def process(self, noms): + niveaux = set() + for n in noms: + if n.source not in ("BAN", "OSM") or not n.fantoir: + continue + niveaux.add(n.niveau) + if n.fantoir and n.source in ("BAN", "OSM"): + if not n.niveau in self.dic_fantoir: + self.dic_fantoir[n.niveau] = {} + if not n.nom_normalise in self.dic_fantoir[n.niveau]: + self.dic_fantoir[n.niveau][n.nom_normalise] = {} + self.dic_fantoir[n.niveau][n.nom_normalise][n.source] = n.fantoir + for n in niveaux: + self.correspondance[n] = {} + + for f in self.dic_fantoir[n]: + if ( + "BAN" in self.dic_fantoir[n][f] + and "OSM" in self.dic_fantoir[n][f] + and self.dic_fantoir[n][f]["BAN"] != self.dic_fantoir[n][f]["OSM"] + ): + self.correspondance[ + f"{n} {self.dic_fantoir[n][f]['BAN']}" + ] = self.dic_fantoir[n][f]["OSM"] + + def enregistre(self): + return 0 + + +def remplace_fantoir_ban(correspondance, niveau, fantoir): + cle = f"{niveau} {fantoir}" + return correspondance.get(cle, fantoir) diff --git a/bano/outils_de_gestion.py b/bano/outils_de_gestion.py deleted file mode 100644 index 5b30c9d..0000000 --- a/bano/outils_de_gestion.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python -# coding: UTF-8 - -import time -from . import db - -def batch_start_log(source,etape,code_geo): - t = time.localtime() - th = time.strftime('%d-%m-%Y %H:%M:%S',t) - t = round(time.mktime(t),0) - if len(str(code_geo)) > 3: - code_insee = code_geo - dept = None - else: - code_insee = None - dept = code_geo - - with db.bano.cursor() as cur: - if code_insee: - whereclause = f"insee_com = '{code_insee}' AND source = '{source}' AND etape = '{etape}'" - str_query = f"INSERT INTO batch_historique (SELECT * FROM batch WHERE {whereclause});DELETE FROM batch WHERE {whereclause};INSERT INTO batch (source,etape,timestamp_debut,date_debut,dept,insee_com,nom_com,nombre_adresses) SELECT '{source}','{etape}',{t},'{th}',dept,insee_com,nom_com,0 FROM code_cadastre WHERE insee_com = '{code_insee}';COMMIT;" - if dept: - whereclause = f"dept = '{dept}' AND source = '{source}' AND etape = '{etape}'" - str_query = f"INSERT INTO batch_historique (SELECT * FROM batch WHERE {whereclause});DELETE FROM batch WHERE {whereclause};INSERT INTO batch (source,etape,timestamp_debut,date_debut,dept,nombre_adresses) VALUES ('{source}','{etape}',{t},'{th}','{dept}',0);COMMIT;" - - cur.execute(str_query) - - cur.execute(f"SELECT id_batch::integer FROM batch WHERE {whereclause};") - c = cur.fetchone() - return c[0] - -def batch_end_log(nb,batch_id): - with db.bano.cursor() as cur: - t = time.localtime() - th = time.strftime('%d-%m-%Y %H:%M:%S',t) - whereclause = 'id_batch = {:d}'.format(batch_id) - cur.execute(f"UPDATE batch SET nombre_adresses = {nb},date_fin = '{th}' WHERE {whereclause};") - -def get_cadastre_etape_timestamp_debut(code_cadastre,etape,source): - str_query = "SELECT timestamp_debut FROM batch WHERE cadastre_com = '{:s}' AND etape = '{:s}' AND source = '{:s}';".format(code_cadastre,etape,source) - cur = db.bano.cursor() - cur.execute(str_query) - for c in cur: - code_cadastre = c[0] - return code_cadastre diff --git a/bano/pre_process_suffixe.py b/bano/pre_process_suffixe.py index 9783bd6..cc8a0a5 100644 --- a/bano/pre_process_suffixe.py +++ b/bano/pre_process_suffixe.py @@ -4,98 +4,145 @@ import re import sys import time -import os,os.path - -# from .core import load_cadastre_hsnr -from .outils_de_gestion import batch_start_log -from .outils_de_gestion import batch_end_log +import os, os.path +from . import batch as b from . import db from . import helpers as hp from . import db_helpers as dh -from .models import Adresses -from .core import load_ban_hsnr +from .models import Adresses, Topo -def collect_adresses_points(selection, adresses): - kres = {} - for k in selection: - kres[k] = [] - for vv in selection[k]['liste']: - s = 0 - max = 2 - for a in adresses[vv]['numeros']: - kres[k].append('SELECT \'{:s}\' hameau,\'{:s}\' code_insee,ST_BUFFER(ST_PointFromText(\'POINT({} {})\',4326),0.0003,2) as g'.format(k.replace("'","''"),adresses.code_insee,adresses[vv]['numeros'][a].node.attribs['lon'],adresses[vv]['numeros'][a].node.attribs['lat'])) - s+=1 - if s == max: break - return kres - -def load_suffixe_2_db(adds, code_insee): - nb_res = 0 - with db.bano_cache.cursor() as cur: - str_query = f"DELETE FROM suffixe WHERE insee_com = '{code_insee}';COMMIT;" - cur.execute(str_query) - for h in adds: - # Agde (34003): detection de 'Mer' abusif, pas d'autres suffixes dans la commune - if code_insee == '34003': - continue - print(f"......... {h}") - str_query = 'INSERT INTO suffixe SELECT ST_Transform(ST_SetSRID((ST_Dump(gu)).geom,4326),3857),code_insee,hameau FROM (SELECT ST_Union(g) gu,code_insee,hameau FROM({:s})a GROUP BY 2,3)a;COMMIT;'.format(' UNION ALL '.join(adds[h])) - cur.execute(str_query) - nb_res+=len(adds[h]) - return nb_res def name_frequency(adresses): freq = {} - for v in adresses: - s = v.split() - if len(s)>4: - k = ' '.join(s[-2:]) + noms_hors_1ere_passe = set() + # for nom in ['rue du Pont - Pont Augan']: # adresses.noms_de_voies: + for nom in adresses.noms_de_voies: + s = nom.split() + # noms avec suffixe entre () quelle que soit leur longueur + if "(" in nom and nom[-1] == ")": + k = f"({nom.split('(')[1]}" + if k not in freq: + freq[k] = {"nombre": 1, "liste": {nom}} + else: + freq[k]["nombre"] += 1 + freq[k]["liste"].add(nom) + # noms avec suffixe après un trait d'union quelle que soit leur longueur + elif "-" in nom: + k = f"{nom.split('-')[-1]}" + # print(k) if k not in freq: - freq[k] = {'nombre':1,'liste':[v]} + freq[k] = {"nombre": 1, "liste": {nom}} else: - freq[k]['nombre'] +=1 - freq[k]['liste'].append(v) - if len(s)>3: - k = v.split()[-1] + freq[k]["nombre"] += 1 + freq[k]["liste"].add(nom) + # noms avec suffixe après une virgule quelle que soit leur longueur + elif "," in nom: + k = f"{nom.split(',')[-1]}" + # print(k) if k not in freq: - freq[k] = {'nombre':1,'liste':[v]} + freq[k] = {"nombre": 1, "liste": {nom}} else: - freq[k]['nombre'] +=1 - freq[k]['liste'].append(v) + freq[k]["nombre"] += 1 + freq[k]["liste"].add(nom) + elif len(s) > 4: + k = " ".join(s[-2:]) + if k not in freq: + freq[k] = {"nombre": 1, "liste": {nom}} + else: + freq[k]["nombre"] += 1 + freq[k]["liste"].add(nom) + elif len(s) > 3: + k = nom.split()[-1] + if k not in freq: + freq[k] = {"nombre": 1, "liste": {nom}} + else: + freq[k]["nombre"] += 1 + freq[k]["liste"].add(nom) + else: + noms_hors_1ere_passe.add(nom) + + # 2eme passe sur les noms courts (surtout des lieux-dits) avec un suffixe + for nom in noms_hors_1ere_passe: + s = nom.split() + if len(s) > 1 and len(s) < 4: + k = nom.split()[-1] + if k in freq: + freq[k]["nombre"] += 1 + freq[k]["liste"].add(nom) + return freq + def select_street_names_by_name(freq): sel = {} mots = {} for k in freq: ks = k.split() - if freq[k]['nombre'] > 5 and len(ks) > 1: + if freq[k]["nombre"] > 5 and len(ks) > 1: mots[ks[0]] = 1 mots[ks[1]] = 1 sel[k] = freq[k] for k in freq: ks = k.split() # un suffixe ne peut pas être un numero seul, cas dans les arrdts parisiens - if freq[k]['nombre'] > 5 and len(ks) == 1 and not k.isdigit() and not k in mots : + if freq[k]["nombre"] > 5 and len(ks) == 1 and not k.isdigit() and not k in mots: sel[k] = freq[k] return sel + +def collect_adresses_points(selection, adresses): + kres = {} + for k in selection: + kres[k] = [] + for nom_voie in selection[k]["liste"]: + s = 0 + max = 2 + for i in adresses.index_voie[nom_voie]: + add = adresses[i] + suffixe = k.replace("'", "''") + kres[k].append( + f"SELECT '{suffixe}' AS libelle_suffixe,'{adresses.code_insee}' AS code_insee,ST_BUFFER(ST_PointFromText('POINT({add.x} {add.y})',4326),0.0003,2) as g" + ) + s += 1 + if s == max: + break + return kres + + +def load_suffixe_2_db(adds, code_insee, nom_commune): + with db.bano_db.cursor() as cur: + for h in adds: + # Agde (34003): detection de 'Mer' abusif, pas d'autres suffixes dans la commune + if code_insee == "34003": + continue + print(f"......... {h}") + str_query = f"INSERT INTO suffixe SELECT ST_SetSRID((ST_Dump(gu)).geom,4326),code_insee,libelle_suffixe FROM (SELECT ST_Union(g) gu,code_insee,libelle_suffixe FROM({' UNION ALL '.join(adds[h])})a GROUP BY 2,3)a;" + cur.execute(str_query) + + def process(departements, **kwargs): - source = 'BAN' for dept in departements: if hp.is_valid_dept(dept): - for code_insee, nom_commune in dh.get_insee_name_list_by_dept(dept): + print(f"Traitement du dept {dept}") + with db.bano_db.cursor() as cur: + str_query = f"DELETE FROM suffixe WHERE code_insee LIKE '{dept}%';" + cur.execute(str_query) + for code_insee, nom_commune in dh.liste_communes_par_dept(dept): + # for code_insee, nom_commune in [['56188','Quistinic']]: + # for code_insee, nom_commune in [['49244','Mauges']]: debut_total = time.time() - hp.display_insee_commune(code_insee, nom_commune) + print(code_insee, nom_commune) adresses = Adresses(code_insee) - - batch_id = batch_start_log(source,'detectesuffixe',code_insee) - - # adresses.load_cadastre_hsnr() - adresses.load_ban_hsnr() - freq = name_frequency(adresses) - selection = select_street_names_by_name(freq) - adds = collect_adresses_points(selection, adresses) - nb_rec = load_suffixe_2_db(adds, code_insee) - - batch_end_log(nb_rec,batch_id) + topo = Topo(code_insee) + batch_id = b.batch_start_log("detecte suffixe", code_insee, nom_commune) + try: + adresses.charge_numeros_ban(topo) + freq = name_frequency(adresses) + selection = select_street_names_by_name(freq) + adds = collect_adresses_points(selection, adresses) + load_suffixe_2_db(adds, code_insee, nom_commune) + b.batch_stop_log(batch_id, True) + except (e): + print(e) + b.batch_stop_log(batch_id, False) diff --git a/bano/publish.py b/bano/publish.py index 9552cbc..cbf250e 100644 --- a/bano/publish.py +++ b/bano/publish.py @@ -11,6 +11,7 @@ from .constants import DEPARTEMENTS from . import helpers as hp +from . import batch as b def get_source_dir(): try: @@ -39,37 +40,67 @@ def get_dest_file_full(filetype,gzip=False): return Path(get_dest_dir()) / f'full.{filetype}{gz_ext}' def publish_as_shp(dept): - with tarfile.open(get_dest_file(dept, 'shp', True,True), "w:gz") as tar: - tar.add(get_source_file(dept,'shp'), arcname=f'bano-{dept}.shp') - tar.add(get_source_file(dept,'dbf'), arcname=f'bano-{dept}.dbf') - tar.add(get_source_file(dept,'shx'), arcname=f'bano-{dept}.shx') - tar.add(get_source_file(dept,'prj'), arcname=f'bano-{dept}.prj') - tar.add(get_source_file(dept,'cpg'), arcname=f'bano-{dept}.cpg') + id_batch = b.batch_start_log("publish SHP", "", dept) + try : + with tarfile.open(get_dest_file(dept, 'shp', True,True), "w:gz") as tar: + tar.add(get_source_file(dept,'shp'), arcname=f'bano-{dept}.shp') + tar.add(get_source_file(dept,'dbf'), arcname=f'bano-{dept}.dbf') + tar.add(get_source_file(dept,'shx'), arcname=f'bano-{dept}.shx') + tar.add(get_source_file(dept,'prj'), arcname=f'bano-{dept}.prj') + tar.add(get_source_file(dept,'cpg'), arcname=f'bano-{dept}.cpg') + b.batch_stop_log(id_batch, True) + except: + b.batch_stop_log(id_batch, False) def publish_as_csv(dept): - copy2(get_source_file(dept,'csv'),get_dest_dir()) + id_batch = b.batch_start_log("publish CSV", "", dept) + try : + copy2(get_source_file(dept,'csv'),get_dest_dir()) + b.batch_stop_log(id_batch, True) + except: + b.batch_stop_log(id_batch, False) def publish_as_full_csv(): - with gzip.open(get_dest_file_full('csv',True),'wb') as gz: - for infile in sorted(glob(f'{get_source_dir()}/bano-*.csv')): - with open(infile,'rb') as js: - gz.write(js.read()) + id_batch = b.batch_start_log("publish FULL CSV", "", 'France') + try: + with gzip.open(get_dest_file_full('csv',True),'wb') as gz: + for infile in sorted(glob(f'{get_source_dir()}/bano-*.csv')): + with open(infile,'rb') as js: + gz.write(js.read()) + b.batch_stop_log(id_batch, True) + except: + b.batch_stop_log(id_batch, False) def publish_as_ttl(dept): - with gzip.open(get_dest_file(dept,'ttl',True,False),'wb') as gz: - with open(get_source_file(dept,'ttl'),'rb') as ttl: - gz.write(ttl.read()) + id_batch = b.batch_start_log("publish TTL", "", dept) + try : + with gzip.open(get_dest_file(dept,'ttl',True,False),'wb') as gz: + with open(get_source_file(dept,'ttl'),'rb') as ttl: + gz.write(ttl.read()) + b.batch_stop_log(id_batch, True) + except: + b.batch_stop_log(id_batch, False) def publish_as_json(dept): - with gzip.open(get_dest_file(dept,'json',True,False),'wb') as gz: - with open(get_source_file(dept,'json'),'rb') as js: - gz.write(js.read()) + id_batch = b.batch_start_log("publish JSON", "", dept) + try : + with gzip.open(get_dest_file(dept,'json',True,False),'wb') as gz: + with open(get_source_file(dept,'json'),'rb') as js: + gz.write(js.read()) + b.batch_stop_log(id_batch, True) + except: + b.batch_stop_log(id_batch, False) def publish_as_full_json(): - with gzip.open(get_dest_file_full('sjson',True),'wb') as gz: - for infile in sorted(glob(f'{get_source_dir()}/bano-*.json')): - with open(infile,'rb') as js: - gz.write(js.read()) + id_batch = b.batch_start_log("publish FULL JSON", "", 'France') + try: + with gzip.open(get_dest_file_full('sjson',True),'wb') as gz: + for infile in sorted(glob(f'{get_source_dir()}/bano-*.json')): + with open(infile,'rb') as js: + gz.write(js.read()) + b.batch_stop_log(id_batch, True) + except: + b.batch_stop_log(id_batch, False) def process(departements, **kwargs): for dept in departements: diff --git a/bano/rapprochement.py b/bano/rapprochement.py new file mode 100644 index 0000000..8ab93ff --- /dev/null +++ b/bano/rapprochement.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +# coding: UTF-8 + +from . import db_helpers as h +from . import batch as b +from .sql import sql_process +from .models import Noms, Adresses, Topo, Points_nommes, Correspondance_fantoir_ban_osm +from .sources import ban2topo + + +def process_unitaire(code_insee): + id_batch = b.batch_start_log("rapprochement", "toutes", code_insee) + try: + ban2topo.process(code_insee) + topo = Topo(code_insee) + adresses = Adresses(code_insee) + points_nommes = Points_nommes(code_insee) + noms = Noms(code_insee) + correspondance_fantoir_ban_osm = Correspondance_fantoir_ban_osm(code_insee) + + noms.charge_noms_osm_hors_numeros() + adresses.charge_numeros_osm() + adresses.charge_numeros_ban(topo) + points_nommes.charge_points_nommes_lieux_dits_cadastre() + points_nommes.charge_points_nommes_place_osm() + # Les centroïdes viennent en dernier (fallback). Tout point déjà affecté comme lieu-dit OSM est inchangé dans l'étape charge_points_nommes_centroides_osm() + points_nommes.charge_points_nommes_centroides_osm() + points_nommes.charge_points_nommes_numeros_ban() + + + adresses.noms_des_adresses(noms) + points_nommes.noms_des_points_nommes(noms) + + noms.add_fantoir(topo) + noms.remplit_fantoir_par_nom_sous_commune() + points_nommes.complete_fantoir(noms) + + # 2e passe suite au complément Fantoir + points_nommes.noms_des_points_nommes(noms) + adresses.complete_fantoir(noms) + + correspondance_fantoir_ban_osm.process(noms) + + adresses.enregistre(correspondance_fantoir_ban_osm.correspondance) + noms.enregistre(correspondance_fantoir_ban_osm.correspondance) + points_nommes.enregistre(correspondance_fantoir_ban_osm.correspondance) + + nb_addr_ban, nb_addr_osm = adresses.stats_sources() + nb_noms_ban, nb_noms_cadastre, nb_noms_osm = noms.stats_sources() + nb_noms_topo = topo.nb_fantoirs + sql_process( + "update_table_bano_stats_communales", + dict( + code_insee=code_insee, + nb_adresses_osm=str(nb_addr_osm), + nb_adresses_ban=str(nb_addr_ban), + nb_noms_osm=str(nb_noms_osm), + nb_noms_ban=str(nb_noms_ban), + nb_noms_topo=str(nb_noms_topo), + nb_noms_cadastre=str(nb_noms_cadastre), + ), + ) + + b.batch_stop_log(id_batch, True) + except Exception as e: + # print(e) + b.batch_stop_log(id_batch, False) + + +def process(code_insee, dept, **kwargs): + if dept: + liste_insee = h.liste_communes_par_dept(dept) + else: + liste_insee = [(code_insee, h.nom_commune(code_insee))] + for code_insee, nom in liste_insee: + if dept: + print(f"{code_insee} - {nom}") + process_unitaire(code_insee) diff --git a/bano/setup_db.py b/bano/setup_db.py new file mode 100644 index 0000000..98175d3 --- /dev/null +++ b/bano/setup_db.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python +# coding: UTF-8 + +from .db import bano_db +from .sql import sql_process + + +def setup_bano(**kwargs): + sql_process("create_base", {}) + sql_process("create_table_base_bano_outils", {}) + sql_process("create_table_base_bano_sources", {}) + sql_process("create_table_base_bano_cog", {}) + sql_process("create_table_base_bano_cibles", {}) + sql_process("create_table_base_bano_pifometre", {}) diff --git a/bano/sources/ban.py b/bano/sources/ban.py index a37a623..dfeaa7a 100644 --- a/bano/sources/ban.py +++ b/bano/sources/ban.py @@ -10,67 +10,122 @@ import psycopg2 from ..constants import DEPARTEMENTS -from .. import db -from .. import db_helpers as dbh -from .. import outils_de_gestion as m -from .. import update_manager as um +from ..db import bano_db +from ..sql import sql_process +from .. import batch as b +from .. import pre_process_suffixe -def process(departements, **kwargs): - source = 'BAN' +# from .. import update_manager as um + + +def process_ban(departements, **kwargs): departements = set(departements) - depts_inconnus = departements - set(DEPARTEMENTS) + depts_inconnus = departements - set(DEPARTEMENTS) if depts_inconnus: raise ValueError(f"Départements inconnus : {depts_inconnus}") - um.set_csv_directory(um.get_directory_pathname()) + depts_en_echec = [] for dept in sorted(departements): - print(f"Processing {dept}") - status = download(source, dept) + print(f"Département {dept}") + status = download(dept) if status: - import_to_pg(source, dept) + if not (import_to_pg(dept)): + depts_en_echec.append(dept) + print("depts_en_echec", depts_en_echec) + else: + pre_process_suffixe.process(dept) + for dept in depts_en_echec: + print(f"Département {dept}") + import_to_pg_subp(dept) + pre_process_suffixe.process(dept) + -def download(source, departement): +def download(departement): destination = get_destination(departement) headers = {} if destination.exists(): - headers['If-Modified-Since'] = formatdate(destination.stat().st_mtime) + headers["If-Modified-Since"] = formatdate(destination.stat().st_mtime) - resp = requests.get(f'https://adresse.data.gouv.fr/data/ban/adresses-odbl/latest/csv/adresses-{departement}.csv.gz', headers=headers) + resp = requests.get( + f"https://adresse.data.gouv.fr/data/ban/adresses-odbl/latest/csv/adresses-{departement}.csv.gz", + headers=headers, + ) + id_batch = b.batch_start_log("download source", "BAN", departement) if resp.status_code == 200: - batch_id = m.batch_start_log(source,'downloadDeptBan',departement) - with destination.open('wb') as f: + with destination.open("wb") as f: f.write(resp.content) - mtime = parsedate_to_datetime(resp.headers['Last-Modified']).timestamp() + mtime = parsedate_to_datetime(resp.headers["Last-Modified"]).timestamp() os.utime(destination, (mtime, mtime)) - m.batch_end_log(-1,batch_id) + b.batch_stop_log(id_batch, True) return True + if resp.status_code == 304: + b.batch_stop_log(id_batch, True) + return False print(resp.status_code) + b.batch_stop_log(id_batch, False) return False -def import_to_pg(source, departement, **kwargs): - batch_id = m.batch_start_log(source,'loadDeptBal',departement) +def import_to_pg(departement, **kwargs): + id_batch = b.batch_start_log("import source", "BAN", departement) fichier_source = get_destination(departement) - with gzip.open(fichier_source, mode='rt') as f: + with gzip.open(fichier_source, mode="rt") as f: f.readline() # skip CSV headers - with db.bano_cache.cursor() as cur_insert: + with bano_db.cursor() as cur_insert: try: - cur_insert.execute(f"DELETE FROM ban_odbl WHERE code_insee LIKE '{departement+'%'}'") - cur_insert.copy_from(f, "ban_odbl", sep=';', null='') - db.bano_cache.commit() - # um.save_bal_insee_list(um.get_directory_pathname(),departement) + cur_insert.execute( + f"DELETE FROM ban WHERE code_insee LIKE '{departement}%'" + ) + cur_insert.copy_from(f, "ban", sep=";", null="") + b.batch_stop_log(id_batch, True) + return True except psycopg2.DataError as e: - db.bano_cache.reset() - m.batch_end_log(-1,batch_id) + print(f"Erreur au chargement de la BAN {departement}") + print(e) + b.batch_stop_log(id_batch, False) + return False + + +def import_to_pg_subp(departement, **kwargs): + id_batch = b.batch_start_log("import source", "BAN", departement) + print("Essai via shell") + try: + fichier_source = get_destination(departement) + ret = subprocess.run( + ["gzip", "-cd", fichier_source], capture_output=True, text=True + ) + tmp_filename = Path(os.environ["BAN_CACHE_DIR"]) / "tmp.csv" + with open(tmp_filename, "w") as tmpfile: + tmpfile.write(ret.stdout) + + subprocess.run( + [ + "psql", + "-d", + "bano", + "-U", + "cadastre", + "-1", + "-c", + f"DELETE FROM ban WHERE code_insee LIKE '{departement}%';COPY ban FROM '{tmp_filename}' WITH CSV HEADER NULL '' DELIMITER ';'", + ] + ) + tmp_filename.unlink() + b.batch_stop_log(id_batch, True) + except e: + print(f"Erreur au chargement de la BAN {departement}") + print(f"Abandon du chargement de la BAN {departement}") + b.batch_stop_log(id_batch, False) + - def get_destination(departement): try: - cwd = Path(os.environ['BAN_CACHE_DIR']) + cwd = Path(os.environ["BAN_CACHE_DIR"]) except KeyError: raise ValueError(f"La variable BAN_CACHE_DIR n'est pas définie") if not cwd.exists(): raise ValueError(f"Le répertoire {cwd} n'existe pas") - return cwd / f'adresses-{departement}.csv.gz' + return cwd / f"adresses-{departement}.csv.gz" + def update_bis_table(**kwargs): - dbh.process_sql(db.bano_cache,'update_table_rep_b_as_bis',dict()) \ No newline at end of file + sql_process("update_table_rep_b_as_bis", dict()) diff --git a/bano/sources/ban2topo.py b/bano/sources/ban2topo.py new file mode 100644 index 0000000..a2da3bf --- /dev/null +++ b/bano/sources/ban2topo.py @@ -0,0 +1,32 @@ +from bano import db +from bano import models +from ..sql import sql_process,sql_get_data +from ..helpers import normalize,get_code_dept_from_insee +# from . import topo + + +def get_noms_ban(code_insee): + return sql_get_data('noms_ban_non_rapproches_par_commune',dict(code_insee=code_insee)) + +def purge_noms_ban_dans_topo(code_insee): + sql_process('purge_noms_ban_dans_topo',dict(code_insee=code_insee)) + +def add_noms_ban_dans_topo(code_insee,noms_ban): + sql_process('add_noms_ban_dans_topo',dict(code_insee=code_insee,noms_ban=noms_ban)) + +def pseudo_fantoir(index,code_insee): + return f"{code_insee}b{str(index).ljust(3,'b')}" + +def process(code_insee,**kwargs): + purge_noms_ban_dans_topo(code_insee) + noms_ban = get_noms_ban(code_insee) + if len(noms_ban) > 0: + noms_ban_norm = set() + topo = models.Topo(code_insee) + dept = get_code_dept_from_insee(code_insee) + for i,n in enumerate(noms_ban): + nom_norm = normalize(n[0]) + if not nom_norm in topo.topo: + noms_ban_norm.add(f"'{dept}','{code_insee}','{pseudo_fantoir(i,code_insee)}',' ','{nom_norm}','B','B','0000000'") + if len(noms_ban_norm)>0: + add_noms_ban_dans_topo(code_insee, f"({'),('.join(noms_ban_norm)})") diff --git a/bano/sources/cadastre_gouv.py b/bano/sources/cadastre_gouv.py deleted file mode 100755 index c29530d..0000000 --- a/bano/sources/cadastre_gouv.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python - -import os -from pathlib import Path -import time - -from bs4 import BeautifulSoup -import requests - -from ..constants import DEPARTEMENTS, DEPARTEMENTS_3CHAR -from .. import db - - -def read_from_cadastre_gouv_to_file(): - with requests.Session() as s: - reponse = s.get('https://www.cadastre.gouv.fr/scpc/rechercherPlan.do') - token = reponse.text.split('CSRF_TOKEN=')[1].split('"')[0] - - with open(Path(os.environ['DATA_DIR']) / 'code_cadastre.csv', 'w') as output: - for index, dept in enumerate(DEPARTEMENTS_3CHAR): - time.sleep(1) - reponse = BeautifulSoup(s.get(f"https://www.cadastre.gouv.fr/scpc/listerCommune.do?CSRF_TOKEN={token}&codeDepartement={dept}&libelle=&keepVolatileSession=&offset=5000").text, "lxml") - for e in reponse.find_all(attrs={"class": "parcelles"}): - y = e.find(title="Ajouter au panier") - if y is None: - continue - - split = y.get('onclick').split("'") - code_commune = split[1] - format_type = split[3] - - commune_cp = e.strong.string - nom_commune = commune_cp[:-9] - cp = commune_cp[-7:-2] - output.write(f'{DEPARTEMENTS[index]},{dept},{nom_commune},{cp},{code_commune},{format_type}\n') - - -def update_table_in_db(): - # print((Path(__file__)).parent.parent / 'sql' / 'load_code_cadastre.sql') - with open((Path(__file__)).parent.parent / 'sql' / 'load_code_cadastre.sql', 'r') as sqlfile: - cur_update = db.bano.cursor() - cur_update.execute(sqlfile.read().replace('./code_cadastre.csv', str(Path(os.environ['DATA_DIR']) / 'code_cadastre.csv'))) - - -def process(**kwargs): - read_from_cadastre_gouv_to_file() - update_table_in_db() diff --git a/bano/sources/cadastre_json.py b/bano/sources/cadastre_json.py deleted file mode 100644 index aa17030..0000000 --- a/bano/sources/cadastre_json.py +++ /dev/null @@ -1,79 +0,0 @@ -import csv -import gzip -import json -import os -import subprocess -from datetime import datetime -from email.utils import formatdate, parsedate_to_datetime -from pathlib import Path - -import requests -import psycopg2 - -from ..constants import DEPARTEMENTS -from .. import db -from .. import helpers as hp - - -def process(prefixe, departements, **kwargs): - departements = set(departements) - depts_inconnus = departements - set(DEPARTEMENTS) - if depts_inconnus: - raise ValueError(f"Départements inconnus : {depts_inconnus}") - for dept in sorted(departements): - print(f"Processing {dept}") - status = download(prefixe, dept) - if status: - import_to_pg(prefixe, dept) - post_process(prefixe, dept) - - -def download(prefixe, departement): - destination = get_destination(prefixe, departement) - headers = {} - if destination.exists(): - headers['If-Modified-Since'] = formatdate(destination.stat().st_mtime) - - resp = requests.get(f'https://cadastre.data.gouv.fr/data/etalab-cadastre/latest/geojson/departements/{departement}/cadastre-{departement}-{prefixe}.json.gz', headers=headers) - if resp.status_code == 200: - with destination.open('wb') as f: - f.write(resp.content) - mtime = parsedate_to_datetime(resp.headers['Last-Modified']).timestamp() - os.utime(destination, (mtime, mtime)) - return True - return False - - -def import_to_pg(prefixe, departement, **kwargs): - fichier_source = get_destination(prefixe, departement) - with gzip.open(fichier_source, mode='rt') as f: - json_source = json.load(f) - with db.bano_cache.cursor() as cur_insert: - try: - cur_insert.execute(f"DELETE FROM {prefixe} WHERE insee_com LIKE '{departement+'%'}';COMMIT;") - a_values = [] - str_query = f"INSERT INTO {prefixe} VALUES " - for l in json_source['features']: - a_values.append(f"('{l['properties']['commune']}','{hp.escape_quotes(l['properties']['nom'])}','{l['properties']['created']}','{l['properties']['updated']}',ST_SetSRID(ST_GeomFromGeoJSON('{hp.replace_single_quotes_with_double(str(l['geometry']))}'),4326))") - if a_values: - cur_insert.execute(str_query+','.join(a_values)+';COMMIT;') - except psycopg2.DataError as e: - print(e) - db.bano_cache.reset() - -def post_process(prefixe, departement, **kwargs): - sqlfile = Path(__file__).parent.parent / 'sql' / f'{prefixe}_post_process.sql' - if sqlfile.exists(): - with open(sqlfile,'r') as fq: - with db.bano_cache.cursor() as cur_post_process: - str_query = fq.read().replace('__dept__',departement) - cur_post_process.execute(str_query) - -def get_destination(prefixe, departement): - try: - cwd = Path(os.environ['CADASTRE_CACHE_DIR']) - except KeyError: - raise ValueError(f"La variable CADASTRE_CACHE_DIR n'est pas définie") - if not cwd.exists(): - raise ValueError(f"Le répertoire {cwd} n'existe pas") - return cwd / f'cadastre-{departement}-{prefixe}.json.gz' diff --git a/bano/sources/cadastre_ld.py b/bano/sources/cadastre_ld.py new file mode 100644 index 0000000..1b03eaa --- /dev/null +++ b/bano/sources/cadastre_ld.py @@ -0,0 +1,88 @@ +import csv +import gzip +import json +import os +import subprocess +from datetime import datetime +from email.utils import formatdate, parsedate_to_datetime +from pathlib import Path + +import requests +import psycopg2 + +from ..constants import DEPARTEMENTS +from .. import db +from .. import helpers as hp + + +def process(departements, **kwargs): + departements = set(departements) + depts_inconnus = departements - set(DEPARTEMENTS) + if depts_inconnus: + raise ValueError(f"Départements inconnus : {depts_inconnus}") + for dept in sorted(departements): + print(f"Processing {dept}") + status = download(dept) + # if status: + import_to_pg(dept) + post_process(dept) + + +def download(departement): + destination = get_destination(departement) + headers = {} + if destination.exists(): + headers["If-Modified-Since"] = formatdate(destination.stat().st_mtime) + + resp = requests.get( + f"https://cadastre.data.gouv.fr/data/etalab-cadastre/latest/geojson/departements/{departement}/cadastre-{departement}-lieux_dits.json.gz", + headers=headers, + ) + if resp.status_code == 200: + with destination.open("wb") as f: + f.write(resp.content) + mtime = parsedate_to_datetime(resp.headers["Last-Modified"]).timestamp() + os.utime(destination, (mtime, mtime)) + return True + return False + + +def import_to_pg(departement, **kwargs): + fichier_source = get_destination(departement) + with gzip.open(fichier_source, mode="rt") as f: + json_source = json.load(f) + with db.bano_db.cursor() as cur_insert: + try: + cur_insert.execute( + f"DELETE FROM lieux_dits WHERE code_insee LIKE '{departement+'%'}';COMMIT;" + ) + a_values = [] + str_query = f"INSERT INTO lieux_dits VALUES " + for l in json_source["features"]: + a_values.append( + f"('{l['properties']['commune']}','{hp.escape_quotes(l['properties']['nom'])}','{l['properties']['created']}','{l['properties']['updated']}',ST_SetSRID(ST_GeomFromGeoJSON('{hp.replace_single_quotes_with_double(str(l['geometry']))}'),4326))" + ) + if a_values: + cur_insert.execute(str_query + ",".join(a_values) + ";COMMIT;") + except psycopg2.DataError as e: + print(e) + db.bano_db.reset() + + +def post_process(departement, **kwargs): + sqlfile = Path(__file__).parent.parent / "sql" / "lieux_dits_post_process.sql" + if sqlfile.exists(): + with open(sqlfile, "r") as fq: + with db.bano_db.cursor() as cur_post_process: + str_query = fq.read().replace("__dept__", departement) + cur_post_process.execute(str_query) + + +def get_destination(departement): + try: + cwd = Path(os.environ["CADASTRE_CACHE_DIR"]) + except KeyError: + raise ValueError(f"La variable CADASTRE_CACHE_DIR n'est pas définie") + if not cwd.exists(): + raise ValueError(f"Le répertoire {cwd} n'existe pas") + return cwd / f"cadastre-{departement}-lieux_dits.json.gz" diff --git a/bano/sources/cog.py b/bano/sources/cog.py new file mode 100644 index 0000000..860dd0b --- /dev/null +++ b/bano/sources/cog.py @@ -0,0 +1,81 @@ +import os + +from email.utils import formatdate, parsedate_to_datetime +from pathlib import Path + +import requests + +from ..db import bano_db +from ..sql import sql_process +from .. import batch as b + +DICT_COG = { + "commune": [ + "cog_commune", + "https://www.data.gouv.fr/fr/datasets/r/5d5c4384-b19a-4ddf-910b-6e9cbaa8e1d6", + ], + "canton": [ + "cog_canton", + "https://www.data.gouv.fr/fr/datasets/r/716922e8-2ab7-4806-94eb-21933c266ebf", + ], + "arrondissement": [ + "cog_arrondissement", + "https://www.data.gouv.fr/fr/datasets/r/9fdf41d6-2452-4f7c-a1e2-a7b104b38b77", + ], + "departement": [ + "cog_departement", + "https://www.data.gouv.fr/fr/datasets/r/c691a0b0-6e22-4d6f-9496-3406b017d709", + ], + "region": [ + "cog_region", + "https://www.data.gouv.fr/fr/datasets/r/6e40e6f3-7735-43a9-8f20-ce0ae53f74f7", + ], +} + + +def process_cog(**kwargs): + for k,v in DICT_COG.items(): + print(f"Chargement de la source COG {k}") + table,url = v + csv = get_destination(f"{k}.csv") + status = download(csv,url) + if status: + import_to_pg(csv,table) + sql_process('cog_pyramide_admin',dict()) + +def download(destination,url): + headers = {} + if destination.exists(): + headers["If-Modified-Since"] = formatdate(destination.stat().st_mtime) + + resp = requests.get(url, headers=headers) + id_batch = b.batch_start_log("download source", f"COG {destination}", "France") + if resp.status_code == 200: + with destination.open("wb") as f: + f.write(resp.content) + b.batch_stop_log(id_batch, True) + return True + print(resp.status_code) + b.batch_stop_log(id_batch, False) + return False + + +def import_to_pg(csv,table): + id_batch = b.batch_start_log("import source", f"COG {table}", "France") + with open(csv) as f: + with bano_db.cursor() as cur_insert: + try: + cur_insert.execute(f"TRUNCATE {table}") + cur_insert.copy_expert(f"COPY {table} FROM STDIN WITH CSV HEADER",f) + b.batch_stop_log(id_batch, True) + except psycopg2.DataError as e: + b.batch_stop_log(id_batch, False) + +def get_destination(fichier_cog): + try: + cwd = Path(os.environ["COG_DIR"]) + except KeyError: + raise ValueError(f"La variable COG_DIR n'est pas définie") + if not cwd.exists(): + raise ValueError(f"Le répertoire {cwd} n'existe pas") + return cwd / f"{fichier_cog}" diff --git a/bano/sources/datagouv_commune_summary.py b/bano/sources/datagouv_commune_summary.py new file mode 100755 index 0000000..9599c20 --- /dev/null +++ b/bano/sources/datagouv_commune_summary.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python + +import os +import requests + +from pathlib import Path +from email.utils import formatdate, parsedate_to_datetime + +from .. import batch as b +from ..db import bano_db + +def dowload(): + destination = get_destination() + headers = {} + if destination.exists(): + headers['If-Modified-Since'] = formatdate(destination.stat().st_mtime) + + resp = requests.get(f'https://plateforme.adresse.data.gouv.fr/api/communes-summary.csv', headers=headers) + if resp.status_code == 200: + id_batch = b.batch_start_log("download source", "CommuneSummary", 'FRA') + with destination.open('wb') as f: + f.write(resp.content) + b.batch_stop_log(id_batch, True) + return True + print(resp.status_code) + b.batch_stop_log(id_batch, False) + return False + + +def update_table_in_db(): + with open(get_destination()) as csv: + csv.readline() + with bano_db.cursor() as cur: + cur.execute("TRUNCATE communes_summary;") + cur.copy_from(csv,'communes_summary',sep=',',null='') + +def get_destination(): + return Path(os.environ['BAN_CACHE_DIR']) / 'communes-summary.csv' + +def process(**kwargs): + if dowload(): + update_table_in_db() diff --git a/bano/sources/datagouv_cp.py b/bano/sources/datagouv_cp.py new file mode 100644 index 0000000..8e6a020 --- /dev/null +++ b/bano/sources/datagouv_cp.py @@ -0,0 +1,70 @@ +import os + +from email.utils import formatdate, parsedate_to_datetime +from pathlib import Path + +import requests + +from ..db import bano_db +from ..sql import sql_process +from .. import batch as b + +DICT_SOURCES = { + "codes_postaux": [ + "codes_postaux", + "https://datanova.laposte.fr/data-fair/api/v1/datasets/laposte-hexasmal/raw", + "table_insee_codes_postaux", + ], +} + + +def process(**kwargs): + for k,v in DICT_SOURCES.items(): + print(f"Chargement de la source {k}") + table,url,script_post_process = v + csv = get_destination(f"{k}.csv") + status = download(csv,url) + if status: + import_to_pg(csv,table) + sql_process(script_post_process,dict()) + + +def download(destination,url): + headers = {} + if destination.exists(): + headers["If-Modified-Since"] = formatdate(destination.stat().st_mtime) + + resp = requests.get(url, headers=headers) + id_batch = b.batch_start_log("download source", url, "France") + if resp.status_code == 200: + with destination.open("w") as f: + f.write(resp.text) + b.batch_stop_log(id_batch, True) + return True + elif resp.status_code == 304: # Not Modified + return True + print(resp.status_code) + b.batch_stop_log(id_batch, False) + return False + + +def import_to_pg(csv,table): + id_batch = b.batch_start_log("import source", table, "France") + with open(csv) as f: + f.readline() + with bano_db.cursor() as cur_insert: + try: + cur_insert.execute(f"TRUNCATE {table}") + cur_insert.copy_expert(f"COPY {table} FROM STDIN WITH CSV HEADER DELIMITER ';'",f) + b.batch_stop_log(id_batch, True) + except psycopg2.DataError as e: + b.batch_stop_log(id_batch, False) + +def get_destination(fichier): + try: + cwd = Path(os.environ["DATA_DIR"]) + except KeyError: + raise ValueError(f"La variable DATA_DIR n'est pas définie") + if not cwd.exists(): + raise ValueError(f"Le répertoire {cwd} n'existe pas") + return cwd / f"{fichier}" diff --git a/bano/sources/fantoir.py b/bano/sources/fantoir.py deleted file mode 100644 index 26e2b8d..0000000 --- a/bano/sources/fantoir.py +++ /dev/null @@ -1,59 +0,0 @@ -from bano import db -from ..helpers import normalize - -class Mapping: - def __init__(self): - self.fantoir = {} - self.code_fantoir_vers_nom_fantoir = {} - self.code_fantoir_vers_noms = {} - - def reset(self): - self.fantoir = {} - self.code_fantoir_vers_nom_fantoir = {} - self.code_fantoir_vers_noms = {} - - def load(self,insee): - str_query = ("""SELECT * - FROM (SELECT code_insee||id_voie||cle_rivoli, - nature_voie||' '||libelle_voie, - rank() OVER(PARTITION BY nature_voie||' '||libelle_voie ORDER BY type_voie,id_voie,cle_rivoli) rang - FROM fantoir_voie - WHERE code_insee = '%s' AND - caractere_annul NOT IN ('O','Q')) a - WHERE rang = 1;""" % insee) - with db.bano_cache.cursor() as cur_fantoir: - cur_fantoir.execute(str_query) - for c in cur_fantoir: - self.code_fantoir_vers_nom_fantoir[c[0]] = c[1] - cle = ' '.join(c[1].replace('-',' ').split()) - cle = normalize(cle) - self.fantoir[cle] = c[0] - - def load_lieux_dits(self,insee): - str_query = f"""SELECT fantoir10, - TRIM(BOTH FROM nature_voie||' '||libelle_voie), - ld_bati - FROM fantoir_voie - WHERE code_insee = '{insee}' AND - -- type_voie = '3' AND - COALESCE(caractere_annul,'') = '';""" - with db.bano_cache.cursor() as conn: - conn.execute(str_query) - for c in conn: - self.fantoir[c[0]] = {"nom":c[1], "ld_bati":c[2].strip()} - - def add_fantoir_name(self,fantoir,name,source): - if not fantoir in self.code_fantoir_vers_noms: - self.code_fantoir_vers_noms[fantoir] = {} - if not source in self.code_fantoir_vers_noms[fantoir]: - self.code_fantoir_vers_noms[fantoir][source] = name - - def get_fantoir_name(self,fantoir,source): - res = '' - if fantoir in self.code_fantoir_vers_noms: - if source in self.code_fantoir_vers_noms[fantoir]: - res = self.code_fantoir_vers_noms[fantoir][source] - return res - - -mapping = Mapping() diff --git a/bano/sources/topo.py b/bano/sources/topo.py new file mode 100644 index 0000000..836583d --- /dev/null +++ b/bano/sources/topo.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# coding: UTF-8 + +import gzip +import io + +from ..constants import get_const_code_dir,CODE_VOIE_FANTOIR,DEPARTEMENTS + +from ..db import bano_db +from .. import helpers as h +from .. import batch as b + +CODE_DIR = get_const_code_dir() + + +def fantoir9_vers_fantoir10(fantoir): + insee = fantoir[0:5] + code_dir = CODE_DIR.get(insee,'0') + dept = fantoir[0:2] # pour les DOM le distingo se fait avec le code direction + if dept == '2A': + dept = '210' + if dept == '2B': + dept = '20' + code_dir = '1' #2B + commune = insee[2:] + code_voie = '0123456789ABCDEFGHIJKLMNOPQRSTVWXYZ'.index(fantoir[5:6]) + numero = fantoir[6:] + cle = 'ABCDEFGHJKLMNPRSTUVWXYZ'[(int(dept+code_dir+commune)*19+code_voie*11+int(numero))%23] + + # print(f"{fantoir}{cle}") + return(f"{fantoir}{cle}") + +def topo_voie_to_csv(ligne_brute): + longueurs = [5,2,9,2,4,26,40,1,1,1,1,1,1,8,8,1,8,8] + champs = [] + # champs = 'region,pays,voie,type_topo,nature,libelle,skip,caractere_voie,annulation,date_annulation,date_creation,type_voie,mot_classant,date_transition' + # pays,dept, insee,fantoir,nature,voie,caractere voie,annul,date annul,creation,mot classant + for l in longueurs: + champs.append((ligne_brute[:l]).strip()) + ligne_brute = ligne_brute[l:] + # selection + champs = [champs[2]]+champs[4:6]+champs[11:-1] + #insee + champs.insert(0,champs[0][0:5]) + # code dept + champs.insert(0,h.get_code_dept_from_insee(champs[0])) + # fantoir + # champs[2] = fantoir9_vers_fantoir10(champs[2]) + + return champs + + +def import_to_pg(): + fichier_source = '/data/download/TOPO20211101.gz' + # fichier_source = '/data/download/corse.txt.gz' + io_in_csv = io.StringIO() + with gzip.open(fichier_source, mode='rt') as f: + f.readline() # skip CSV headers + for i,line in enumerate(f): + if line[16:18] != '14': + continue + # print(line) + champs = topo_voie_to_csv(line) + if champs[0] not in DEPARTEMENTS: + continue + io_in_csv.write('$'.join(champs)+'\n') # separateur $ car on trouve des virgules dans le contenu + # if i > 20: + # break + io_in_csv.seek(0) + with bano_db.cursor() as cur_insert: + cur_insert.execute("TRUNCATE topo") + cur_insert.copy_from(io_in_csv, "topo", sep='$',null='') + bano_db.commit() + +def process_topo(**kwargs): + id_batch = b.batch_start_log('import source', 'TOPO','France','France') + try: + import_to_pg() + b.batch_stop_log(id_batch,True) + except: + b.batch_stop_log(id_batch,False) diff --git a/bano/sql.py b/bano/sql.py new file mode 100644 index 0000000..ceec38f --- /dev/null +++ b/bano/sql.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# coding: UTF-8 + +import psycopg2.extras + +from pathlib import Path + +from .db import bano_db + +SQLDIR = Path(__file__).parent / "sql" + + +def sql_process(sqlfile, args): + sqlfile = (Path(SQLDIR) / sqlfile).with_suffix(".sql") + with open(sqlfile) as s: + q = s.read() + for k, v in args.items(): + q = q.replace(f"__{k}__", v) + + with bano_db.cursor() as cur: + cur.execute(q) + + +def sql_get_data(sqlfile, args): + sqlfile = (Path(SQLDIR) / sqlfile).with_suffix(".sql") + with open(sqlfile) as s: + q = s.read() + for k, v in args.items(): + q = q.replace(f"__{k}__", v) + + with bano_db.cursor() as cur: + cur.execute(q) + + return cur.fetchall() + + return None + +def sql_get_dict_data(sqlfile, args): + sqlfile = (Path(SQLDIR) / sqlfile).with_suffix(".sql") + with open(sqlfile) as s: + q = s.read() + for k, v in args.items(): + q = q.replace(f"__{k}__", v) + + with bano_db.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: + cur.execute(q) + + return cur.fetchall() + + return None + +def sql_query(sqlfile, args): + sqlfile = (Path(SQLDIR) / sqlfile).with_suffix(".sql") + with open(sqlfile) as s: + q = s.read() + for k, v in args.items(): + q = q.replace(f"__{k}__", v) + return q \ No newline at end of file diff --git a/bano/sql/add_noms_ban_dans_topo.sql b/bano/sql/add_noms_ban_dans_topo.sql new file mode 100644 index 0000000..b282dcf --- /dev/null +++ b/bano/sql/add_noms_ban_dans_topo.sql @@ -0,0 +1,2 @@ +INSERT INTO topo(code_dep,code_insee,fantoir,nature_voie,libelle_voie,type_voie,caractere_annul,date_annul) +VALUES __noms_ban__; diff --git a/bano/sql/batch_start_log.sql b/bano/sql/batch_start_log.sql new file mode 100644 index 0000000..878456b --- /dev/null +++ b/bano/sql/batch_start_log.sql @@ -0,0 +1,11 @@ +INSERT INTO batch_historique +SELECT * +FROM batch +WHERE etape = '__etape__' AND + code_zone = '__code_zone__'; +DELETE FROM batch +WHERE etape = '__etape__' AND + code_zone = '__code_zone__'; +INSERT INTO batch(__champs__) +VALUES (__values__) +RETURNING id_batch; \ No newline at end of file diff --git a/bano/sql/batch_stop_log.sql b/bano/sql/batch_stop_log.sql new file mode 100644 index 0000000..045d56b --- /dev/null +++ b/bano/sql/batch_stop_log.sql @@ -0,0 +1 @@ +UPDATE batch SET date_fin = '__date_fin__',duree=(__timestamp_fin__ - timestamp_debut)::integer,ok = __status__ WHERE id_batch = __id_batch__; diff --git a/bano/sql/cadastre_2_place.sql b/bano/sql/cadastre_2_place.sql deleted file mode 100644 index 037cfb5..0000000 --- a/bano/sql/cadastre_2_place.sql +++ /dev/null @@ -1,7 +0,0 @@ -SELECT ST_X(geometrie), - ST_Y(geometrie), - nom, - fantoir, - insee_com -FROM cadastre_2_place -WHERE insee_com = '__com__'; \ No newline at end of file diff --git a/bano/sql/ban_hsnr_nocache.sql b/bano/sql/charge_ban_commune.sql similarity index 56% rename from bano/sql/ban_hsnr_nocache.sql rename to bano/sql/charge_ban_commune.sql index 748e93f..b93d883 100644 --- a/bano/sql/ban_hsnr_nocache.sql +++ b/bano/sql/charge_ban_commune.sql @@ -1,9 +1,9 @@ WITH j AS -(SELECT id, +(SELECT fantoir, numero, - nom_voie, + regexp_replace(nom_voie,'\t','') AS nom_voie, lon, lat, rep, @@ -17,16 +17,21 @@ AS END ELSE NULL END as rep_bis, - code_postal -FROM ban_odbl b + code_postal, + code_insee_ancienne_commune, + nom_ancienne_commune, + id +FROM ban b LEFT OUTER JOIN rep_b_as_bis r -USING (id_fantoir,numero) -WHERE code_insee = '__com__') - -SELECT id, +USING (fantoir,numero) +WHERE code_insee = '__code_insee__') +SELECT fantoir, TRIM (BOTH FROM (numero||' '||COALESCE(rep_bis,rep,''))), nom_voie, lon, lat, - code_postal + code_postal, + code_insee_ancienne_commune, + nom_ancienne_commune, + id FROM j; diff --git a/bano/sql/charge_noms_voies_lieux-dits_OSM.sql b/bano/sql/charge_noms_voies_lieux-dits_OSM.sql new file mode 100644 index 0000000..ccb04a2 --- /dev/null +++ b/bano/sql/charge_noms_voies_lieux-dits_OSM.sql @@ -0,0 +1,43 @@ +SELECT DISTINCT provenance, + name, + tags, + libelle_suffixe, + a9.code_insee, + a9.nom, + nature +FROM (SELECT 1::integer AS provenance, + pt.way, + UNNEST(ARRAY[pt.name,pt.tags->'alt_name',pt.tags->'old_name']) as name, + tags, + CASE + WHEN pt.place='' THEN 'voie'::text + ELSE 'place' + END AS nature + FROM (SELECT way FROM planet_osm_polygon WHERE "ref:INSEE" = '__code_insee__') p + JOIN (SELECT * FROM planet_osm_point WHERE ("ref:FR:FANTOIR" !='' OR place != '') AND name != '') pt + ON pt.way && p.way AND + ST_Intersects(pt.way, p.way) + UNION ALL + SELECT 2, + l.way, + UNNEST(ARRAY[l.name,l.tags->'alt_name',l.tags->'old_name']) as name, + tags, + 'voie' + FROM (SELECT way FROM planet_osm_polygon WHERE "ref:INSEE" = '__code_insee__') p + JOIN (SELECT * FROM planet_osm_line WHERE highway != '' AND name != '') l + ON p.way && l.way AND ST_Contains(p.way, l.way) + UNION ALL + SELECT 3, + pl.way, + UNNEST(ARRAY[pl.name,pl.tags->'alt_name',pl.tags->'old_name']) as name, + tags, + 'voie' + FROM (SELECT way FROM planet_osm_polygon WHERE "ref:INSEE" = '__code_insee__') p + JOIN (SELECT * FROM planet_osm_polygon WHERE (highway||"ref:FR:FANTOIR" != '' OR landuse = 'residential' OR amenity = 'parking') AND name != '') pl + ON pl.way && p.way AND + ST_Intersects(pl.way, p.way)) l +LEFT OUTER JOIN suffixe h +ON ST_Intersects(l.way, h.geometrie) +LEFT OUTER JOIN (SELECT * FROM polygones_insee_a9 where insee_a8 = '__code_insee__') a9 +ON ST_Contains(a9.geometrie,way) +WHERE l.name IS NOT NULL; \ No newline at end of file diff --git a/bano/sql/charge_noms_voies_relation_OSM.sql b/bano/sql/charge_noms_voies_relation_OSM.sql new file mode 100644 index 0000000..0f386ab --- /dev/null +++ b/bano/sql/charge_noms_voies_relation_OSM.sql @@ -0,0 +1,31 @@ +SELECT DISTINCT provenance, + name, + tags, + libelle_suffixe, + a9.code_insee, + a9.nom, + 'voie'::text +FROM (SELECT 4::integer AS provenance, + UNNEST(ARRAY[l.name,l.tags->'alt_name',l.tags->'old_name']) as name, + l.way, + r.tags + FROM (SELECT way FROM planet_osm_polygon WHERE "ref:INSEE" = '__code_insee__') p + JOIN (SELECT name,tags,osm_id,way FROM planet_osm_line WHERE highway != '' AND name != '') l + ON p.way && l.way AND ST_Contains(p.way, l.way) + JOIN planet_osm_rels r + ON r.osm_id = l.osm_id + UNION ALL + SELECT 5, + UNNEST(ARRAY[l.name,l.tags->'alt_name',l.tags->'old_name']) as name, + l.way, + r.tags + FROM (SELECT way FROM planet_osm_polygon WHERE "ref:INSEE" = '__code_insee__') p + JOIN (SELECT name,tags,osm_id,way FROM planet_osm_polygon WHERE highway != '' AND name != '') l + ON p.way && l.way AND ST_Contains(p.way, l.way) + JOIN planet_osm_rels r + ON r.osm_id = l.osm_id) l +LEFT OUTER JOIN (SELECT * FROM suffixe WHERE code_insee = '__code_insee__') h +ON ST_Intersects(way, h.geometrie) +LEFT OUTER JOIN (SELECT * FROM polygones_insee_a9 where insee_a8 = '__code_insee__') a9 +ON ST_Contains(a9.geometrie,way) +WHERE l.name IS NOT NULL; \ No newline at end of file diff --git a/bano/sql/charge_noms_voies_relation_bbox_OSM.sql b/bano/sql/charge_noms_voies_relation_bbox_OSM.sql new file mode 100644 index 0000000..09724a9 --- /dev/null +++ b/bano/sql/charge_noms_voies_relation_bbox_OSM.sql @@ -0,0 +1,30 @@ +SELECT DISTINCT provenance, + name, + tags, + libelle_suffixe, + a9.code_insee, + a9.nom, + 'voie'::text +FROM ( SELECT 6::integer AS provenance, + l.way, + l.name, + r.tags + FROM (SELECT way FROM planet_osm_polygon WHERE "ref:INSEE" = '__code_insee__') p + JOIN (SELECT name,osm_id,way FROM planet_osm_line WHERE highway != '' AND name != '') l + ON l.way && p.way + JOIN planet_osm_rels r + ON r.osm_id = l.osm_id + UNION ALL + SELECT 7, + l.way, + l.name, + r.tags + FROM (SELECT way FROM planet_osm_polygon WHERE "ref:INSEE" = '__code_insee__') p + JOIN (SELECT name,osm_id,way FROM planet_osm_polygon WHERE highway != '' AND name != '') l + ON l.way && p.way + JOIN planet_osm_rels r + ON r.osm_id = l.osm_id) l +LEFT OUTER JOIN suffixe h +ON ST_Intersects(l.way, h.geometrie) +LEFT OUTER JOIN (SELECT * FROM polygones_insee_a9 where insee_a8 = '__code_insee__') a9 +ON ST_Contains(a9.geometrie,l.way); diff --git a/bano/sql/charge_numeros_OSM.sql b/bano/sql/charge_numeros_OSM.sql new file mode 100644 index 0000000..e09da72 --- /dev/null +++ b/bano/sql/charge_numeros_OSM.sql @@ -0,0 +1,74 @@ +SELECT ST_X(way), + ST_Y(way), + provenance, + hsnr, + street_name, + place_name, + tags, + h.libelle_suffixe, + code_postal, + code_insee_a9, + nom_a9 +FROM +-- point avec addr:street ou addr:place + (SELECT 1 provenance, + pt.way, + pt."addr:housenumber" AS hsnr, + pt."addr:street" AS street_name, + pt."addr:place" AS place_name, + tags, + pt."addr:postcode" AS code_postal + FROM (SELECT geometrie FROM polygones_insee WHERE admin_level = 8 AND code_insee = '__code_insee__') p + JOIN planet_osm_point pt + ON ST_Intersects(pt.way, p.geometrie) + WHERE pt."addr:housenumber" != '' AND + pt."addr:street"||pt."addr:place" != '' + UNION ALL +-- way avec addr:street ou addr:place + SELECT 2, + ST_Centroid(w.way), + w."addr:housenumber", + w."addr:street", + w."addr:place", + tags, + w."addr:postcode" + FROM (SELECT geometrie FROM polygones_insee WHERE admin_level = 8 AND code_insee = '__code_insee__') p + JOIN planet_osm_polygon w + ON ST_Intersects(w.way, p.geometrie) + WHERE w."addr:housenumber" != '' AND + w."addr:street"||w."addr:place" != '' + UNION ALL +-- point dans relation associatedStreet + SELECT 3, + pt.way, + pt."addr:housenumber", + null, + null, + r.tags, + pt."addr:postcode" + FROM (SELECT geometrie FROM polygones_insee WHERE admin_level = 8 AND code_insee = '__code_insee__') p + JOIN planet_osm_point pt + ON ST_Intersects(pt.way, p.geometrie) + JOIN planet_osm_rels r + ON r.osm_id = pt.osm_id + WHERE pt."addr:housenumber" != '' + UNION ALL +-- way dans relation associatedStreet + SELECT 4, + ST_Centroid(w.way), + w."addr:housenumber", + null, + null, + r.tags, + w."addr:postcode" + FROM (SELECT geometrie FROM polygones_insee WHERE admin_level = 8 AND code_insee = '__code_insee__') p + JOIN planet_osm_polygon w + ON ST_Intersects(w.way, p.geometrie) + JOIN planet_osm_rels r + ON r.osm_id = w.osm_id + WHERE w."addr:housenumber" != '' +)a +LEFT OUTER JOIN (SELECT * FROM suffixe WHERE code_insee = '__code_insee__') h +ON ST_Intersects(a.way, h.geometrie) +LEFT OUTER JOIN (SELECT geometrie, code_insee code_insee_a9,nom nom_a9 FROM polygones_insee_a9 WHERE insee_a8 = '__code_insee__') a9 +ON ST_Intersects(a.way, a9.geometrie); diff --git a/bano/sql/charge_numeros_bbox_OSM.sql b/bano/sql/charge_numeros_bbox_OSM.sql new file mode 100644 index 0000000..dad3f96 --- /dev/null +++ b/bano/sql/charge_numeros_bbox_OSM.sql @@ -0,0 +1,46 @@ +select ST_X(way), + ST_Y(way), + provenance, + hsnr, + street_name, + null::text place_name, + tags, + h.libelle_suffixe, + code_postal, + code_insee_a9, + nom_a9 +FROM +-- point dans relation associatedStreet + (SELECT 5 provenance, + pt.way, + pt."addr:housenumber" hsnr, + null::text street_name, + r.tags tags, + pt."addr:postcode" AS code_postal + FROM (SELECT ST_SetSRID(ST_Extent(geometrie),4326) way, code_insee FROM polygones_insee WHERE code_insee = '__code_insee__' GROUP BY 2) p + JOIN planet_osm_point pt + ON ST_Intersects(pt.way,p.way) + JOIN planet_osm_rels r + ON r.osm_id = pt.osm_id + WHERE pt."addr:housenumber" != '' + UNION +-- way dans relation associatedStreet + SELECT 6, + ST_Centroid(w.way), + w."addr:housenumber", + null, + r.tags, + w."addr:postcode" + FROM (SELECT ST_SetSRID(ST_Extent(geometrie),4326) way, code_insee FROM polygones_insee WHERE code_insee = '__code_insee__' GROUP BY 2) p + JOIN planet_osm_polygon w + ON ST_Intersects(w.way, p.way) + JOIN planet_osm_rels r + ON r.osm_id = w.osm_id + WHERE w."addr:housenumber" != '')a +LEFT OUTER JOIN (SELECT * FROM suffixe WHERE code_insee = '__code_insee__') h +ON ST_Intersects(a.way, h.geometrie) +LEFT OUTER JOIN (SELECT geometrie, code_insee code_insee_a9,nom nom_a9 FROM polygones_insee_a9 WHERE insee_a8 = '__code_insee__') a9 +ON ST_Intersects(a.way, a9.geometrie); + + + \ No newline at end of file diff --git a/bano/sql/charge_points_nommes_centroides_OSM.sql b/bano/sql/charge_points_nommes_centroides_OSM.sql new file mode 100644 index 0000000..8ef2af6 --- /dev/null +++ b/bano/sql/charge_points_nommes_centroides_OSM.sql @@ -0,0 +1,145 @@ +WITH +lignes_brutes +AS +(SELECT l.way, + unnest(array[l.name,l.tags->'alt_name',l.tags->'old_name']) AS name, + COALESCE(a9.code_insee,'xxxxx') as insee_jointure, + a9.code_insee insee_ac, + unnest(array["ref:FR:FANTOIR","ref:FR:FANTOIR:left","ref:FR:FANTOIR:right"]) AS fantoir, + ST_Within(l.way,p.way)::integer as within, + a9.nom AS nom_ac +FROM (SELECT way FROM planet_osm_polygon WHERE "ref:INSEE" = '__code_insee__') p +JOIN planet_osm_line l +ON ST_Intersects(l.way, p.way) +LEFT OUTER JOIN (SELECT * FROM polygones_insee_a9 WHERE insee_a8 = '__code_insee__') a9 +ON ST_Intersects(l.way, a9.geometrie) +WHERE (l.highway != '' OR + l.waterway = 'dam') AND + l.highway NOT IN ('bus_stop','platform') AND + l.name != '' +UNION ALL +SELECT ST_PointOnSurface(l.way), + unnest(array[l.name,l.tags->'alt_name',l.tags->'old_name']) AS name, + COALESCE(a9.code_insee,'xxxxx') as insee_jointure, + a9.code_insee insee_ac, + "ref:FR:FANTOIR" AS fantoir, + ST_Within(l.way,p.geometrie)::integer as within, + a9.nom AS nom_ac +FROM (SELECT geometrie FROM polygones_insee WHERE code_insee = '__code_insee__') p +JOIN planet_osm_polygon l +ON ST_Intersects(l.way, p.geometrie) +LEFT OUTER JOIN (SELECT * FROM polygones_insee_a9 WHERE insee_a8 = '__code_insee__') a9 +ON ST_Intersects(l.way, a9.geometrie) +WHERE (l.highway||"ref:FR:FANTOIR" != '' OR l.landuse = 'residential' OR l.amenity = 'parking') AND + l.highway NOT IN ('bus_stop','platform') AND + l.name != '' +UNION ALL +SELECT l.way, + unnest(array[l.name,l.tags->'alt_name',l.tags->'old_name']) AS name, + COALESCE(a9.code_insee,'xxxxx') as insee_jointure, + a9.code_insee insee_ac, + "ref:FR:FANTOIR" AS fantoir, + ST_Within(l.way,p.way)::integer as within, + a9.nom AS nom_ac +FROM (SELECT way FROM planet_osm_polygon WHERE "ref:INSEE" = '__code_insee__') p +JOIN planet_osm_rels l +ON ST_Intersects(l.way, p.way) +LEFT OUTER JOIN (SELECT * FROM polygones_insee_a9 WHERE insee_a8 = '__code_insee__') a9 +ON ST_Intersects(l.way, a9.geometrie) +WHERE l.member_role = 'street' AND + l.name != ''), +lignes_noms +AS +(SELECT CASE + WHEN GeometryType(way) = 'POINT' THEN ST_MakeLine(ST_Translate(way,-0.000001,-0.000001),ST_Translate(way,0.000001,0.000001)) + WHEN GeometryType(way) LIKE '%POLYGON' THEN ST_ExteriorRing(way) + ELSE way + END AS way_line, + * +FROM lignes_brutes +WHERE name IS NOT NULL AND + (fantoir LIKE '__code_insee__%' OR fantoir = '')), +lignes_noms_rang +AS +(SELECT *, + RANK() OVER(PARTITION BY name,insee_ac ORDER BY within DESC, fantoir DESC) rang +FROM lignes_noms), +lignes_agregees +AS +(SELECT ST_LineMerge(ST_Collect(way_line)) way, + name, + insee_ac, + insee_jointure, + fantoir, + nom_ac +FROM lignes_noms_rang +WHERE rang = 1 +GROUP BY 2,3,4,5,6), +centroide_lignes_agregees +AS +(SELECT ST_Centroid(ST_LineMerge(ST_Collect(way_line))) way, + name, + insee_ac, + insee_jointure, + fantoir, + nom_ac +FROM lignes_noms_rang +WHERE rang = 1 +GROUP BY 2,3,4,5,6), +resultat +AS +(SELECT ST_SetSRID(ST_ClosestPoint(lignes_agregees.way,centroide_lignes_agregees.way),4326) point, + lignes_agregees.name, + lignes_agregees.insee_ac, + lignes_agregees.fantoir, + lignes_agregees.nom_ac +FROM lignes_agregees +JOIN centroide_lignes_agregees +USING (name,insee_jointure)), +complement +AS +(SELECT c.*, + a9.code_insee AS insee_ac, + a9.nom AS nom_ac +FROM (SELECT pl.way point, + pl.name, + pl."ref:FR:FANTOIR" fantoir + FROM (SELECT way FROM planet_osm_polygon WHERE "ref:INSEE" = '__code_insee__') p + JOIN planet_osm_point pl + ON pl.way && p.way AND + ST_Intersects(pl.way, p.way) + WHERE (pl."ref:FR:FANTOIR" != '' OR + pl.junction != '') AND + pl.highway NOT IN ('bus_stop','platform') AND + pl.name != '' + UNION + SELECT ST_Centroid(pl.way), + pl.name, + pl."ref:FR:FANTOIR" f + FROM (SELECT way FROM planet_osm_polygon WHERE "ref:INSEE" = '__code_insee__') p + JOIN planet_osm_polygon pl + ON pl.way && p.way AND + ST_Intersects(pl.way, p.way) + WHERE ( pl.highway||pl."ref:FR:FANTOIR" != '' OR + pl.landuse = 'residential' OR + pl.place = 'square' OR + pl.amenity = 'school') AND + pl.highway NOT IN ('bus_stop','platform') AND + pl.name != '')c +LEFT OUTER JOIN (SELECT * FROM polygones_insee_a9 WHERE insee_a8 = '__code_insee__') a9 +ON ST_Intersects(c.point, a9.geometrie)) +SELECT ST_x(point), + ST_y(point), + name, + insee_ac, + fantoir, + nom_ac +FROM resultat +UNION ALL +SELECT ST_x(point), + ST_y(point), + name, + insee_ac, + fantoir, + nom_ac +FROM complement; diff --git a/bano/sql/charge_points_nommes_lieux-dits_CADASTRE.sql b/bano/sql/charge_points_nommes_lieux-dits_CADASTRE.sql new file mode 100644 index 0000000..334f443 --- /dev/null +++ b/bano/sql/charge_points_nommes_lieux-dits_CADASTRE.sql @@ -0,0 +1,13 @@ +SELECT ST_X(geom_centroid), + ST_Y(geom_centroid), + regexp_replace(regexp_replace(regexp_replace(regexp_replace(ld.nom,' ',' ','g'),' ',' ','g'),' ',' ','g'),' ',' ','g') AS nom, + a9.code_insee, + a9.nom +FROM (SELECT * + FROM lieux_dits + WHERE code_insee = '__code_insee__' AND + nom IS NOT NULL) ld +LEFT OUTER JOIN suffixe h +ON ST_Intersects(ld.geom_centroid, h.geometrie) +LEFT OUTER JOIN (SELECT * FROM polygones_insee_a9 WHERE insee_a8 = '__code_insee__') a9 +ON ST_Contains(a9.geometrie,ld.geom_centroid); \ No newline at end of file diff --git a/bano/sql/charge_points_nommes_numeros_BAN.sql b/bano/sql/charge_points_nommes_numeros_BAN.sql new file mode 100644 index 0000000..2f56d81 --- /dev/null +++ b/bano/sql/charge_points_nommes_numeros_BAN.sql @@ -0,0 +1,22 @@ +WITH +duplicates +AS +(SELECT fantoir, + nom_voie, + code_insee_ancienne_commune, + nom_ancienne_commune, + lon, + lat, + RANK() OVER (PARTITION BY nom_voie ORDER BY numero) rang +FROM (SELECT fantoir,numero,nom_voie,code_insee,code_insee_ancienne_commune,nom_ancienne_commune,lon,lat + FROM ban + WHERE code_insee = '__code_insee__' + ) AS ban) +SELECT lon, + lat, + nom_voie, + code_insee_ancienne_commune, + fantoir, + nom_ancienne_commune +FROM duplicates +WHERE rang = 1; diff --git a/bano/sql/charge_points_nommes_places_OSM.sql b/bano/sql/charge_points_nommes_places_OSM.sql new file mode 100644 index 0000000..d6d9932 --- /dev/null +++ b/bano/sql/charge_points_nommes_places_OSM.sql @@ -0,0 +1,27 @@ +WITH +pts +AS +(SELECT pt.way, + UNNEST(ARRAY[pt.name,pt.tags->'alt_name',pt.tags->'old_name']) as name, + tags, + place, + a9.code_insee AS insee_ac, + "ref:FR:FANTOIR" AS fantoir, + a9.nom AS nom_ac +FROM (SELECT way FROM planet_osm_polygon WHERE "ref:INSEE" = '__code_insee__') p +JOIN (SELECT * FROM planet_osm_point WHERE place != '' AND name != '') pt +ON pt.way && p.way AND + ST_Intersects(pt.way, p.way) +LEFT OUTER JOIN (SELECT osm_id FROM planet_osm_communes_statut WHERE "ref:INSEE" = '__code_insee__' AND member_role = 'admin_centre') admin_centre +ON pt.osm_id = admin_centre.osm_id +LEFT OUTER JOIN (SELECT * FROM polygones_insee_a9 WHERE insee_a8 = '__code_insee__') a9 +ON ST_Intersects(pt.way, a9.geometrie) +WHERE admin_centre.osm_id IS NULL) +SELECT ST_x(way), + ST_y(way), + name, + insee_ac, + fantoir, + nom_ac +FROM pts +WHERE name IS NOT NULL; diff --git a/bano/sql/charge_topo_commune.sql b/bano/sql/charge_topo_commune.sql new file mode 100644 index 0000000..6666d4c --- /dev/null +++ b/bano/sql/charge_topo_commune.sql @@ -0,0 +1,19 @@ +WITH +t +AS +(SELECT fantoir, + TRIM (BOTH FROM (COALESCE(nature_voie,'')||' '||libelle_voie)) AS nom +FROM topo +WHERE code_insee = '__code_insee__' AND + COALESCE(caractere_annul,'x') NOT IN ('O','Q')), +tr +AS +(SELECT fantoir, + nom, + rank() OVER (PARTITION BY nom ORDER BY fantoir) rang +FROM t) +SELECT fantoir, + nom +FROM tr +WHERE rang = 1 +ORDER BY length(nom) DESC,nom; \ No newline at end of file diff --git a/bano/sql/cog_pyramide_admin.sql b/bano/sql/cog_pyramide_admin.sql new file mode 100644 index 0000000..2184a53 --- /dev/null +++ b/bano/sql/cog_pyramide_admin.sql @@ -0,0 +1,13 @@ +DROP TABLE IF EXISTS cog_pyramide_admin CASCADE; +CREATE TABLE cog_pyramide_admin +AS +SELECT com AS code_insee, + typecom, + c.libelle AS nom_com, + d.libelle AS nom_dep, + r.libelle AS nom_reg +FROM cog_commune c +JOIN (SELECT dep,libelle FROM cog_departement) d +USING (dep) +JOIN (SELECT reg,libelle FROM cog_region) r +USING (reg); \ No newline at end of file diff --git a/bano/sql/complement_points_nommes_numeros_OSM.sql b/bano/sql/complement_points_nommes_numeros_OSM.sql new file mode 100644 index 0000000..563dc97 --- /dev/null +++ b/bano/sql/complement_points_nommes_numeros_OSM.sql @@ -0,0 +1,22 @@ +INSERT INTO bano_points_nommes (fantoir,nom,code_insee,code_dept,nature,code_insee_ancienne_commune,nom_ancienne_commune,source,lon,lat) +SELECT fantoir, + COALESCE(nom_voie,nom_place), + code_insee, + code_dept, + 'numero', + code_insee_ancienne_commune, + nom_ancienne_commune, + 'OSM', + lon, + lat +FROM (SELECT a.*, + RANK() OVER (PARTITION BY fantoir ORDER BY numero) rang + FROM (SELECT fantoir FROM bano_adresses WHERE code_insee = '__code_insee__' AND source = 'OSM' + EXCEPT + SELECT fantoir from bano_points_nommes WHERE code_insee = '__code_insee__' AND source = 'OSM') f + JOIN (SELECT * + FROM bano_adresses + WHERE source = 'OSM' AND + code_insee = '__code_insee__')a + USING (fantoir))t +WHERE rang = 1; \ No newline at end of file diff --git a/bano/sql/create_base.sql b/bano/sql/create_base.sql index 1870633..de1c23b 100644 --- a/bano/sql/create_base.sql +++ b/bano/sql/create_base.sql @@ -1,3 +1,5 @@ -CREATE EXTENSION postgis; -CREATE EXTENSION hstore; -CREATE SCHEMA dev; +CREATE EXTENSION IF NOT EXISTS postgis; +CREATE EXTENSION IF NOT EXISTS hstore; + +-- tables Imposm dans le schema osm +ALTER ROLE cadastre IN DATABASE bano SET search_path TO public,osm; \ No newline at end of file diff --git a/bano/sql/create_table_base_bano_cibles.sql b/bano/sql/create_table_base_bano_cibles.sql new file mode 100644 index 0000000..7a0d7fc --- /dev/null +++ b/bano/sql/create_table_base_bano_cibles.sql @@ -0,0 +1,134 @@ +CREATE TABLE IF NOT EXISTS bano_adresses ( + fantoir text, + bano_id text GENERATED ALWAYS AS (fantoir||'_'|| REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REGEXP_REPLACE(UPPER(numero),'^0*',''),'BIS','B'),'TER','T'),'QUATER','Q'),'QUAT','Q'),' ',''),'à','-'),';',','),'"','')) STORED, + lon float, + lat float, + numero text, + nom_voie text, + nom_place text, + code_postal text, + code_insee text, + code_dept text, + code_insee_ancienne_commune text, + nom_ancienne_commune text, + source text, + certification_commune integer, + id_ban text, + geometrie geometry (Point, 4326) GENERATED ALWAYS AS (ST_Point(lon,lat)) STORED, + geometrie_3857 geometry (Point, 3857) GENERATED ALWAYS AS (ST_Transform(ST_SetSRID(ST_Point(lon,lat),4326),3857)) STORED); + +CREATE INDEX IF NOT EXISTS gidx_bano_adresses ON bano_adresses USING GIST(geometrie); +CREATE INDEX IF NOT EXISTS gidx_bano_adresses_3857 ON bano_adresses USING GIST(geometrie_3857); +CREATE INDEX IF NOT EXISTS idx_bano_adresses_code_insee ON bano_adresses (code_insee); +CREATE INDEX IF NOT EXISTS idx_bano_adresses_code_dept ON bano_adresses (code_dept); +CREATE INDEX IF NOT EXISTS idx_bano_adresses_fantoir ON bano_adresses (fantoir); +CREATE INDEX IF NOT EXISTS idx_bano_adresses_bano_id ON bano_adresses (bano_id); +CREATE INDEX IF NOT EXISTS idx_bano_adresses_pifo_code_insee_source ON bano_adresses (code_insee,source); + +CREATE TABLE IF NOT EXISTS bano_points_nommes ( + fantoir text, + nom text, + code_insee text, + code_dept text, + nature text, + code_insee_ancienne_commune text, + nom_ancienne_commune text, + source text, + lon float, + lat float, + geometrie geometry (Point, 4326) GENERATED ALWAYS AS (ST_Point(lon,lat)) STORED, + geometrie_3857 geometry (Point, 3857) GENERATED ALWAYS AS (ST_Transform(ST_SetSRID(ST_Point(lon,lat),4326),3857)) STORED); + +CREATE INDEX IF NOT EXISTS gidx_bano_points_nommes ON bano_points_nommes USING GIST(geometrie); +CREATE INDEX IF NOT EXISTS gidx_bano_points_nommes_3857 ON bano_points_nommes USING GIST(geometrie_3857); +CREATE INDEX IF NOT EXISTS idx_bano_points_nommes_code_insee ON bano_points_nommes (code_insee); +CREATE INDEX IF NOT EXISTS idx_bano_points_nommes_code_dept ON bano_points_nommes (code_dept); +CREATE INDEX IF NOT EXISTS idx_bano_points_nommes_fantoir ON bano_points_nommes (fantoir); +CREATE INDEX IF NOT EXISTS idx_bano_points_nommes_fantoir_source ON bano_points_nommes (fantoir,source); +CREATE INDEX IF NOT EXISTS idx_bano_points_nommes_code_insee_source ON bano_points_nommes (code_insee,source); +CREATE INDEX IF NOT EXISTS idx_bano_points_nommes_code_insee_nature ON bano_points_nommes (code_insee,nature); +CREATE INDEX IF NOT EXISTS idx_bano_points_nommes_code_dept_nature ON bano_points_nommes (code_dept,nature); + +CREATE TABLE IF NOT EXISTS nom_fantoir ( + fantoir text, + nom text, + code_insee text, + code_dept text, + nature text, + code_insee_ancienne_commune text, + nom_ancienne_commune text, + source text); + +CREATE INDEX IF NOT EXISTS idx_nom_fantoir_code_insee ON nom_fantoir (code_insee); +CREATE INDEX IF NOT EXISTS idx_nom_fantoir_code_dept ON nom_fantoir (code_dept); +CREATE INDEX IF NOT EXISTS idx_nom_fantoir_fantoir ON nom_fantoir (fantoir); +CREATE INDEX IF NOT EXISTS idx_nom_fantoir_code_insee_source ON nom_fantoir (code_insee,source); + +CREATE TABLE IF NOT EXISTS correspondance_fantoir_ban_osm( + code_insee text, + fantoir_ban text, + fantoir_osm text); + +CREATE INDEX IF NOT EXISTS idx_correspondance_fantoir_ban_osm_code_insee ON correspondance_fantoir_ban_osm (code_insee); +CREATE INDEX IF NOT EXISTS idx_correspondance_fantoir_ban_osm_fantoir_ban ON correspondance_fantoir_ban_osm (fantoir_ban); + +CREATE TABLE IF NOT EXISTS statut_fantoir ( + fantoir character varying(9), + id_statut integer, + timestamp_statut double precision, + code_insee character(5)); + +CREATE INDEX IF NOT EXISTS idx_statut_fantoir_fantoir ON statut_fantoir (fantoir); +CREATE INDEX IF NOT EXISTS idx_statut_fantoir_insee ON statut_fantoir (code_insee); + +CREATE TABLE IF NOT EXISTS statut_numero ( + numero text , + fantoir character(9) , + source text , + id_statut integer, + timestamp_statut double precision, + code_insee character(5)); + +CREATE INDEX IF NOT EXISTS idx_statut_numero_fantoir ON statut_numero (fantoir, numero); + +GRANT SELECT ON ALL TABLES IN SCHEMA public TO public; + +CREATE TABLE IF NOT EXISTS bano_stats_communales( + code_insee text, + nb_adresses_osm integer, + nb_adresses_ban integer, + nb_nom_osm integer, + nb_nom_ban integer, + nb_nom_cadastre integer, + nb_nom_topo integer, + maj timestamp DEFAULT now()); + +CREATE INDEX IF NOT EXISTS idx_bano_stats_communales_code_insee ON bano_stats_communales (code_insee); + +CREATE TABLE IF NOT EXISTS bano_stats_communales_cumul( + code_insee text, + nb_adresses_osm integer, + nb_adresses_ban integer, + nb_nom_osm integer, + nb_nom_ban integer, + nb_nom_cadastre integer, + nb_nom_topo integer, + maj timestamp); + +CREATE INDEX IF NOT EXISTS idx_bano_stats_communales_cumul_code_insee ON bano_stats_communales_cumul (code_insee); + +CREATE TABLE IF NOT EXISTS infos_communes ( + dep character varying(3), + code_insee character(5), + name text, + adm_weight integer, + population integer, + population_milliers numeric, + type text, + lon numeric, + lat numeric, + geometrie geometry(Point, 4326) +); + +CREATE INDEX IF NOT EXISTS idx_infos_communes_insee ON infos_communes(code_insee); +CREATE INDEX IF NOT EXISTS gidx_infos_communes ON infos_communes USING GIST(geometrie); diff --git a/sql/create_table_cog.sql b/bano/sql/create_table_base_bano_cog.sql similarity index 93% rename from sql/create_table_cog.sql rename to bano/sql/create_table_base_bano_cog.sql index d7eab67..cbaf363 100644 --- a/sql/create_table_cog.sql +++ b/bano/sql/create_table_base_bano_cog.sql @@ -12,6 +12,7 @@ CREATE TABLE IF NOT EXISTS cog_commune ( can character(5), comparent character(5)); CREATE INDEX IF NOT EXISTS idx_cog_commune_com ON cog_commune(com); +CREATE INDEX IF NOT EXISTS idx_cog_commune_dep ON cog_commune(dep); CREATE TABLE IF NOT EXISTS cog_canton ( can character(5), @@ -54,4 +55,6 @@ CREATE TABLE IF NOT EXISTS cog_region ( ncc text, nccenr text, libelle text); -CREATE INDEX IF NOT EXISTS idx_cog_region_reg ON cog_region(reg); \ No newline at end of file +CREATE INDEX IF NOT EXISTS idx_cog_region_reg ON cog_region(reg); + +GRANT SELECT ON ALL TABLES IN SCHEMA public TO public; \ No newline at end of file diff --git a/bano/sql/create_table_base_bano_outils.sql b/bano/sql/create_table_base_bano_outils.sql new file mode 100644 index 0000000..537bf30 --- /dev/null +++ b/bano/sql/create_table_base_bano_outils.sql @@ -0,0 +1,30 @@ + +CREATE TABLE IF NOT EXISTS batch ( + id_batch serial, + etape text, + source text, + timestamp_debut bigint, + date_debut text, + date_fin text, + duree integer, + code_zone text, + nom_zone text, + ok boolean, + CONSTRAINT batch_pkey PRIMARY KEY (id_batch)); + +CREATE INDEX IF NOT EXISTS idx_batch_zone ON batch (code_zone); +CREATE INDEX IF NOT EXISTS idx_batch_zone_etape ON batch (code_zone,etape); + +CREATE TABLE IF NOT EXISTS batch_historique( + id_batch integer, + etape text, + source text, + timestamp_debut bigint, + date_debut text, + date_fin text, + duree integer, + code_zone text, + nom_zone text, + ok boolean); + +GRANT SELECT ON ALL TABLES IN SCHEMA public TO public; \ No newline at end of file diff --git a/bano/sql/create_table_base_bano_pifometre.sql b/bano/sql/create_table_base_bano_pifometre.sql new file mode 100644 index 0000000..0db9e2b --- /dev/null +++ b/bano/sql/create_table_base_bano_pifometre.sql @@ -0,0 +1,66 @@ +CREATE TABLE IF NOT EXISTS statut_fantoir ( + fantoir character varying(10), + id_statut integer, + timestamp_statut double precision, + code_insee character(5)); + +CREATE INDEX IF NOT EXISTS idx_statut_fantoir_insee ON statut_fantoir(code_insee); +CREATE INDEX IF NOT EXISTS idx_statut_fantoir_fantoir ON statut_fantoir(fantoir); + +CREATE TABLE IF NOT EXISTS labels_statuts_fantoir( + id_statut integer primary key, + tri integer default 0, + label_statut character varying(200) +); +TRUNCATE labels_statuts_fantoir; +INSERT INTO labels_statuts_fantoir (id_statut,tri,label_statut) +VALUES (0,0,'Ok'), +(1,1,'Erreur d''orthographe'), +(2,2,'Divergence d''orthographe'), +(3,3,'Nom différent'), +(4,4,'Type de voie différent'), +(5,5,'Voie doublon et type de voie différent'), +(6,6,'Voie doublon avec orthographe différente'), +(7,8,'Répétition du type de voie'), +(8,9,'Nom introuvable sur le terrain'), +(9,10,'Ancien nom supprimé sur le terrain'), +(10,99,'Erreurs combinées'), +(11,15,'Adresses hors périmètre'), +(12,12,'Voie détruite'), +(13,13,'Voie incorporée à une autre'), +(14,14,'Voie inexistante'), +(15,7,'Voie doublon (même type et même nom)'), +(16,11,'Nom tronqué'), +(17,16,'Erreur de commune'), +(18,17,'FANTOIR annulé non remplacé'), +(19,18,'Point cardinal superflu'), +(20,19,'Voie en projet'), +(21,20,'Lieu-dit sur plusieurs communes'), +(22,21,'Chaussée hors commune'); + +CREATE TABLE IF NOT EXISTS statut_numero ( + numero text, + fantoir character (10), + source text, + id_statut integer, + timestamp_statut double precision, + code_insee character(5)); +CREATE INDEX IF NOT EXISTS idx_statut_numero_fantoir ON statut_numero(fantoir,numero); + +CREATE TABLE IF NOT EXISTS labels_statuts_numero( + id_statut integer primary key, + tri integer default 0, + label_statut character varying(200) +); +TRUNCATE labels_statuts_numero; +INSERT INTO labels_statuts_numero (id_statut,tri,label_statut) +VALUES (0,0,'Ok'), +(1,1,'Adresse fictive en 5xxx'), +(2,2,'Adresse fictive en 9xxx'), +(3,3,'Adresse invisible sur le terrain'), +(4,4,'Emplacement sur une autre voie'), +(5,5,'Adresse en dehors de la commune'), +(6,6,'Adresse doublon'), +(7,7,'Adresse de lieu-dit aussi associée à une voie'), +(8,8,'Emplacement manifestement incohérent'), +(9,9,'Ancienne numérotation plus en vigueur'); diff --git a/bano/sql/create_table_base_bano_sources.sql b/bano/sql/create_table_base_bano_sources.sql new file mode 100644 index 0000000..32cdabf --- /dev/null +++ b/bano/sql/create_table_base_bano_sources.sql @@ -0,0 +1,100 @@ +CREATE TABLE IF NOT EXISTS topo ( + code_dep character(3), + code_insee character(5), + fantoir character(9), + nature_voie text, + libelle_voie text, + caractere_voie character(1), + caractere_annul character(1), + date_annul integer, + date_creation integer, + type_voie character(1), + mot_classant character varying(8)); +CREATE INDEX IF NOT EXISTS idx_topo_dep ON topo(code_dep); +CREATE INDEX IF NOT EXISTS idx_topo_code_insee ON topo(code_insee); +CREATE INDEX IF NOT EXISTS idx_topo_fantoir ON topo(fantoir); + +CREATE TABLE IF NOT EXISTS ban ( + id text, + -- id_ban_adresse text, + -- id_ban_toponyme text, + -- id_ban_district text, + id_fantoir text, + fantoir text GENERATED ALWAYS AS (substr(id_fantoir,0,6)||substr(id_fantoir,7,10)) STORED, + numero text, + rep text, + nom_voie text, + code_postal text, + code_insee text, + nom_commune text, + code_insee_ancienne_commune text, + nom_ancienne_commune text, + x float, + y float, + lon float, + lat float, + type_position text, + alias text, + nom_ld text, + libelle_acheminement text, + nom_afnor text, + source_position text, + source_nom_voie text, + certification_commune integer, + cad_parcelles text, + geometrie geometry (Point, 4326) GENERATED ALWAYS AS (ST_Point(lon,lat)) STORED); + +CREATE INDEX IF NOT EXISTS idx_ban_code_insee ON ban(code_insee); +CREATE INDEX IF NOT EXISTS idx_ban_fantoir ON ban(fantoir); +CREATE INDEX IF NOT EXISTS gidx_ban ON ban(geometrie); + +CREATE TABLE IF NOT EXISTS lieux_dits ( + code_insee character(5), + nom text, + created date, + updated date, + geometrie geometry(Polygon,4326), + geom_centroid geometry (Point, 4326) GENERATED ALWAYS AS (ST_Centroid(geometrie)) STORED); + +CREATE INDEX IF NOT EXISTS gidx__centroid_lieux_dits ON lieux_dits USING gist(geom_centroid); +CREATE INDEX IF NOT EXISTS lieux_dits_code_insee ON lieux_dits (code_insee); + +CREATE TABLE IF NOT EXISTS suffixe ( + geometrie geometry, + code_insee character(5), + libelle_suffixe character varying(100) +); +CREATE INDEX IF NOT EXISTS gidx_suffixe ON suffixe USING GIST(geometrie); +CREATE INDEX IF NOT EXISTS idx_suffixe ON suffixe(code_insee); + +CREATE TABLE IF NOT EXISTS communes_summary ( + reg character varying(3), + dep character varying(3), + code_insee character(5), + libelle text, + population integer, + id_revision text, + date_revision text, + type_composition text, + nb_lieux_dits integer, + nb_voies integer, + nb_numeros integer, + nb_numeros_certifies integer, + analyse_adressage_nb_adresses_attendues integer, + analyse_adressage_ratio float, + analyse_adressage_deficit_adresses float, + composed_at text); + +CREATE INDEX IF NOT EXISTS communes_summary_code_insee ON communes_summary (code_insee); + +CREATE TABLE IF NOT EXISTS codes_postaux ( + insee text, + commune text, + cp text, + libelle text, + ligne_5 text); + +CREATE INDEX IF NOT EXISTS idx_codes_postaux_cp ON codes_postaux (cp); +CREATE INDEX IF NOT EXISTS idx_codes_postaux_insee ON codes_postaux (insee); + +GRANT SELECT ON ALL TABLES IN SCHEMA public TO public; \ No newline at end of file diff --git a/bano/sql/create_table_polygones_communes.sql b/bano/sql/create_table_polygones_communes.sql new file mode 100644 index 0000000..c5b48cb --- /dev/null +++ b/bano/sql/create_table_polygones_communes.sql @@ -0,0 +1,39 @@ +CREATE TABLE IF NOT EXISTS polygones_insee ( + geometrie geometry (Geometry, 4326), + code_insee character(5), + nom text, + admin_level integer, + geom_centroide_3857 geometry(Point,3857) GENERATED ALWAYS AS (ST_Transform(ST_Pointonsurface(geometrie),3857)) STORED); +CREATE INDEX IF NOT EXISTS gidx_polygones_insee ON polygones_insee USING GIST (geometrie); +CREATE INDEX IF NOT EXISTS gidx_polygones_insee_centroide_3857 ON polygones_insee USING GIST (geom_centroide_3857); +CREATE INDEX IF NOT EXISTS idx_polygones_insee_code_insee ON polygones_insee(code_insee); + +TRUNCATE TABLE polygones_insee; +INSERT INTO polygones_insee +SELECT way, + "ref:INSEE", + name, + admin_level +FROM planet_osm_polygon +WHERE boundary='administrative' AND + admin_level in (8,9) AND + name != ''; + +CREATE TABLE IF NOT EXISTS polygones_insee_a9 ( + geometrie geometry (Geometry, 4326), + code_insee character(5), + nom text, + insee_a8 character(5)); +CREATE INDEX IF NOT EXISTS gidx_polygones_insee_a9 ON polygones_insee_a9 USING GIST (geometrie); +CREATE INDEX IF NOT EXISTS idx_polygones_insee_a9_code_insee ON polygones_insee_a9(code_insee); +CREATE INDEX IF NOT EXISTS idx_polygones_insee_a9_insee_a8 ON polygones_insee_a9(insee_a8); + +TRUNCATE TABLE polygones_insee_a9; +INSERT INTO polygones_insee_a9 +SELECT a9.geometrie, + a9.code_insee, + a9.nom, + a8.code_insee +FROM (SELECT * FROM polygones_insee WHERE admin_level = 9) a9 +JOIN (SELECT * FROM polygones_insee WHERE admin_level = 8) a8 +ON ST_Contains(a8.geometrie,a9.geometrie); diff --git a/bano/sql/export_csv_dept.sql b/bano/sql/export_csv_dept.sql index 9a48468..3ed0d5f 100644 --- a/bano/sql/export_csv_dept.sql +++ b/bano/sql/export_csv_dept.sql @@ -1,106 +1,11 @@ ---SELECT * ---FROM ( -WITH -u -AS -(SELECT insee_com, - fantoir, - REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REGEXP_REPLACE(UPPER(numero),'^0*',''),'BIS','B'),'TER','T'),'QUATER','Q'),'QUAT','Q'),' ',''),'à','-'),';',','),'"','') AS num -FROM cumul_adresses -WHERE fantoir IS NOT NULL AND - dept = '__dept__' -GROUP BY 1,2,3), -lp -AS -(SELECT insee, - MIN(cp) AS cp -FROM codes_postaux -WHERE insee LIKE '__dept__%' -GROUP BY 1), -res_non_unique -AS -(SELECT CONCAT(u.fantoir,'-',u.num) AS id, - UPPER(REPLACE(COALESCE(o.numero,od.numero,c.numero),' ','')) AS numero, - REPLACE( - REPLACE( - REPLACE( - COALESCE(REPLACE(o.voie_osm,'’',CHR(39)),REPLACE(od.voie_osm,'’',CHR(39)),REPLACE(c.voie_osm,'’',CHR(39)),od.voie_autre,c.voie_autre), - '"',CHR(39)), - ', ',' '), - ',',' ') AS voie, - COALESCE(o.code_postal,c.code_postal,cp.postal_code, lp.cp) AS code_post, - cn.libelle AS ville, - CASE - WHEN u.num=o.num THEN 'OSM' - WHEN (u.num=od.num AND od.voie_osm != od.voie_autre AND od.voie_osm IS NOT NULL) THEN 'O+O' - WHEN u.num=od.num THEN 'OD' - WHEN c.voie_osm != '' THEN 'C+O' - ELSE 'CAD' - END AS SOURCE, - COALESCE(st_y(o.geometrie),st_y(od.geometrie),st_y(c.geometrie)) AS lat, - COALESCE(st_x(o.geometrie),st_x(od.geometrie),st_x(c.geometrie)) AS lon, - COALESCE(o.geometrie,od.geometrie,c.geometrie) AS geom -FROM u -LEFT JOIN lp -ON (lp.insee= u.insee_com) -LEFT JOIN -(SELECT *, - REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REGEXP_REPLACE(UPPER(numero),'^0*',''),'BIS','B'),'TER','T'),'QUATER','Q'),'QUAT','Q'),' ',''),'à','-'),';',',') AS num -FROM cumul_adresses -WHERE dept = '__dept__' AND - source = 'OSM') AS o -ON (u.num = o.num AND u.fantoir = o.fantoir) -LEFT JOIN -(SELECT *, - REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REGEXP_REPLACE(UPPER(numero),'^0*',''),'BIS','B'),'TER','T'),'QUATER','Q'),'QUAT','Q'),' ',''),'à','-'),';',',') AS num -FROM cumul_adresses -WHERE dept = '__dept__' AND - source = 'BAN' AND - st_x(geometrie)!=0 AND - st_y(geometrie)!=0) AS c -ON (c.num=u.num AND c.fantoir=u.fantoir) -LEFT JOIN -(SELECT *, - REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REGEXP_REPLACE(UPPER(numero),'^0*',''),'BIS','B'),'TER','T'),'QUATER','Q'),'QUAT','Q'),' ',''),'à','-'),';',',') AS num -FROM cumul_adresses -WHERE dept = '__dept__' AND - source = 'BAL' AND - st_x(geometrie)!=0 AND - st_y(geometrie)!=0) AS od -ON (od.num = u.num AND od.fantoir = u.fantoir) -JOIN cog_commune cn -ON (cn.com = u.insee_com) -LEFT JOIN (SELECT * FROM planet_osm_postal_code WHERE postal_code != '') cp -ON (cp."ref:INSEE" = u.insee_com AND ST_Contains(cp.way, ST_Transform(COALESCE(o.geometrie, od.geometrie, c.geometrie),3857))) -WHERE u.num>'0' AND - cn.typecom != 'COMD'), -res_avec_ordre_des_doublons -AS -(SELECT id, +SELECT id_add AS id, numero, - voie, - code_post, - ville, + nom_voie AS voie, + code_postal AS code_post, + libelle AS ville, source, lat, lon, - geom, - ROW_NUMBER() OVER(PARTITION BY id ORDER BY numero) AS sequence -FROM res_non_unique -WHERE lat IS NOT NULL AND - lon IS NOT NULL AND - numero ~ '^[0-9]{1,4}( ?[A-Z]?.*)?' AND - numero != '99999' AND - numero !~'.[0-9 .-]{9,}') -SELECT id, - numero, - voie, - code_post, - ville, - source, - lat, - lon, - geom -FROM res_avec_ordre_des_doublons -WHERE sequence = 1 -ORDER BY id + geometrie AS geom +FROM numeros_export +WHERE dep = '__dept__' \ No newline at end of file diff --git a/bano/sql/export_json_dept_communes.sql b/bano/sql/export_json_dept_communes.sql index 05204ce..9545474 100644 --- a/bano/sql/export_json_dept_communes.sql +++ b/bano/sql/export_json_dept_communes.sql @@ -1,35 +1,21 @@ -WITH -rang_cp -AS -(SELECT *, - RANK() OVER(PARTITION BY insee ORDER BY ligne_5,cp) AS rang -FROM codes_postaux), -cp -AS -(SELECT insee, - cp -FROM rang_cp ---WHERE insee in (select insee from rang where rang = 1 group by 1 having count(*) > 1) -WHERE rang = 1) ---order by 1,3 -SELECT DISTINCT insee AS id, -- id - i.type, -- type - i.name, -- name - cp.cp, -- postcode - round(lat,6) AS lat, -- lat - round(lon,6) AS lon, -- lon - i.name , -- city - cd.libelle, -- departement - cr.libelle, -- region, - population, - adm_weight, - greatest(0.075,round(log((adm_weight)+log(population+1)/3)::decimal,4)) AS importance -FROM infos_communes i -JOIN cp cp -ON insee_com=insee -JOIN cog_departement cd -USING (dep) -JOIN cog_region cr -USING (reg) -WHERE i.dep = '__dept__' -ORDER BY insee; \ No newline at end of file +SELECT DISTINCT code_insee AS id, + i.type AS type, + i.name AS name, + cp.cps AS postcode, + round(lat,6)::float AS lat, + round(lon,6)::float AS lon, + i.name AS city, + cd.libelle AS departement, + cr.libelle AS region, + population, + adm_weight, + GREATEST(0.075,ROUND(LOG((adm_weight)+LOG(population+1)/3)::decimal,4))::float AS importance +FROM infos_communes i +JOIN insee_codes_postaux cp +USING (code_insee) +JOIN cog_departement cd +USING (dep) +JOIN cog_region cr +USING (reg) +WHERE i.dep = '__dept__' +ORDER BY code_insee; \ No newline at end of file diff --git a/bano/sql/export_json_dept_lieux_dits.sql b/bano/sql/export_json_dept_lieux_dits.sql deleted file mode 100644 index 8c0b72f..0000000 --- a/bano/sql/export_json_dept_lieux_dits.sql +++ /dev/null @@ -1,66 +0,0 @@ -WITH -sort_order -AS -(SELECT 1::integer as sortnum, 'OSM' as source -UNION ALL -SELECT 2,'BAL' -UNION ALL -SELECT 3,'CADASTRE' -), -osm -AS -(SELECT geometrie, - insee_com, - fantoir, - ld_bati, - COALESCE(libelle_osm,libelle_cadastre) libelle, - code_postal, - RANK() OVER (PARTITION BY fantoir ORDER BY sortnum) rang -FROM cumul_places -JOIN sort_order USING (source) -WHERE fantoir IS NOT NULL AND - dept = '__dept__'), -osm_postal -AS -(SELECT o.geometrie, - o.insee_com, - o.libelle, - o.fantoir, - o.ld_bati, - COALESCE(o.code_postal,pp.code_postal) code_postal, - RANK() OVER(PARTITION BY o.fantoir,COALESCE(o.code_postal,pp.code_postal) ORDER BY id) rang_postal -FROM osm o -JOIN polygones_postaux pp -ON ST_Contains(pp.geometrie, o.geometrie) -WHERE o.rang = 1), -cog -AS -(SELECT cc.com AS insee, - cd.libelle AS nom_dep, - cr.libelle AS nom_reg -FROM cog_commune cc -JOIN cog_departement cd -USING (dep) -JOIN cog_region cr -ON cc.reg = cr.reg -WHERE cc.dep = '__dept__') -SELECT o.fantoir AS id, - o.insee_com AS citycode, - 'place' AS type, - replace(replace(libelle,'\"',''),'’',chr(39)) AS name, - code_postal AS postcode, - round(st_y(o.geometrie)::numeric,6) AS lat, - round(st_x(o.geometrie)::numeric,6) AS lon, - c.name AS city, - cog.nom_dep AS departement, - cog.nom_reg AS region, - least(0.05,round((log(c.adm_weight+log(c.population+1)/3)*(0.25+0.5*(1- coalesce(o.ld_bati,1))))::numeric,4)) AS importance -FROM osm_postal o -JOIN infos_communes c -ON (c.insee_com=o.insee_com) -JOIN cog -ON o.insee_com = cog.insee -WHERE o.rang_postal = 1 -ORDER BY 1; - ---least(0.05,round(log((CASE WHEN g.statut LIKE 'Capital%' THEN 6 WHEN g.statut = 'Préfecture de régi' THEN 5 WHEN g.statut='Préfecture' THEN 4 WHEN g.statut LIKE 'Sous-pr%' THEN 3 WHEN g.statut='Chef-lieu canton' THEN 2 ELSE 1 END)+log(g.population+1)/3)*(0.25+0.5*(1-('0' || coalesce(f.ld_bati,'1'))::numeric)),4)) \ No newline at end of file diff --git a/bano/sql/export_json_dept_voies_avec_adresses.sql b/bano/sql/export_json_dept_voies_avec_adresses.sql new file mode 100644 index 0000000..168680d --- /dev/null +++ b/bano/sql/export_json_dept_voies_avec_adresses.sql @@ -0,0 +1,15 @@ +SELECT id, + citycode, + type, + name, + postcode, + lat, + lon, + city, + departement, + region, + importance, + housenumbers +FROM export_voies_adresses_json +WHERE dep = '__dept__' +ORDER BY 1; \ No newline at end of file diff --git a/bano/sql/export_json_dept_voies_ld_sans_adresses.sql b/bano/sql/export_json_dept_voies_ld_sans_adresses.sql new file mode 100644 index 0000000..5c9b08b --- /dev/null +++ b/bano/sql/export_json_dept_voies_ld_sans_adresses.sql @@ -0,0 +1,14 @@ +SELECT id, + citycode, + type, + name, + postcode, + lat, + lon, + city, + departement, + region, + importance +FROM export_voies_ld_sans_adresses_json +WHERE dep = '__dept__' +ORDER BY 1; \ No newline at end of file diff --git a/bano/sql/export_json_dept_voies_non_rapprochees.sql b/bano/sql/export_json_dept_voies_non_rapprochees.sql deleted file mode 100644 index 18b0c08..0000000 --- a/bano/sql/export_json_dept_voies_non_rapprochees.sql +++ /dev/null @@ -1,95 +0,0 @@ -WITH -sort_order -AS -(SELECT 1::integer as sortnum, 'OSM' as source -UNION ALL -SELECT 2,'BAL' -UNION ALL -SELECT 3,'BAN' --- UNION ALL --- SELECT 4,'CADASTRE' -), -v -AS -(SELECT code_insee AS insee_com, - fantoir10 AS fantoir -FROM fantoir_voie f -WHERE code_dept = '__dept__' -EXCEPT -SELECT insee_com,fantoir -FROM cumul_voies v -WHERE dept = '__dept__'), -osm -AS -(SELECT c.geometrie, - c.insee_com, - c.voie_autre AS voie, - c.fantoir, - c.numero, - c.code_postal, - source, --- RANK() OVER (PARTITION BY c.fantoir,c.numero ORDER BY c.source DESC) rang - ROW_NUMBER() OVER (PARTITION BY fantoir,numero ORDER BY sortnum) rang - FROM cumul_adresses c - JOIN v - USING (fantoir) - JOIN sort_order - USING (source) - WHERE dept = '__dept__'), -osm_postal -AS -(SELECT o.geometrie, - o.insee_com, - o.voie, - o.fantoir, - o.numero, - COALESCE(o.code_postal,pp.code_postal) code_postal, - RANK() OVER(PARTITION BY o.fantoir,o.numero ORDER BY id) rang_postal -FROM osm o -JOIN polygones_postaux pp -ON ST_Contains(pp.geometrie, o.geometrie) -WHERE o.rang = 1), -cog -AS -(SELECT cc.com AS insee, - cd.libelle AS nom_dep, - cr.libelle AS nom_reg - FROM cog_commune cc - JOIN cog_departement cd - USING (dep) - JOIN cog_region cr - ON cc.reg = cr.reg - WHERE cd.dep = '__dept__') -SELECT osm_postal.fantoir, - osm_postal.insee_com as citycode, - 'street' as type, - replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(voie,'^IMP ','Impasse '),'^RTE ','Route '),'^ALL ','Allée '),'^PL ','Place '),'^PLA ','Place '),'^AV ','Avenue '),'^LOT ','Lotissement '),'^RES ','Résidence '),'^CHEM ','Chemin '),'^RLE ','Ruelle '),'^BD ','Boulevard '),'^SQ ','Square '),'^PAS ','Passage '),'^SEN ','Sentier '),'^CRS ','Cours '),'^TRA ','Traverse '),'^MTE ','Montée '),'^RPT ','Rond-point '),'^HAM ','Hameau '),'^VLA ','Villa '),'^PROM ','Promenade '),'^ESP ','Esplanade '),'^FG ','Faubourg '),'^TSSE ','Terrasse '),'^CTRE ','Centre '),'^PASS ','Passerelle '),'^FRM ','Ferme '),' GAL ',' Général '),' MAL ',' Maréchal '),' ST ',' Saint '),' STE ',' Sainte '),' PDT ',' Président '),' HT ',' Haut '),' HTE ',' Haute '),' VX ',' Vieux '),' PTE ',' Petite '),'\"',''),'’',chr(39)) AS name, - osm_postal.code_postal AS postcode, - round(st_y(st_centroid(st_convexhull(ST_Collect(osm_postal.geometrie))))::numeric,6) AS lat, - round(st_x(st_centroid(st_convexhull(ST_Collect(osm_postal.geometrie))))::numeric,6) AS lon, - c.name AS city, - cog.nom_dep AS departement, - cog.nom_reg AS region, - round(log(c.adm_weight+log(c.population+1)/3)::numeric*log(1+log(count(osm_postal.*)+1)+log(st_length(st_longestline(st_convexhull(ST_Collect(osm_postal.geometrie)),st_convexhull(ST_Collect(osm_postal.geometrie)))::geography)+1)+log(CASE WHEN voie like 'Boulevard%' THEN 4 WHEN voie LIKE 'Place%' THEN 4 WHEN voie LIKE 'Espl%' THEN 4 WHEN voie LIKE 'Av%' THEN 3 WHEN voie LIKE 'Rue %' THEN 2 ELSE 1 END))::numeric,4) AS importance, - string_agg(numero||'$'||st_y(osm_postal.geometrie)||'$'||st_x(osm_postal.geometrie),'#' ORDER BY numero)::text AS housenumbers2 -FROM v -JOIN infos_communes c -ON (c.insee_com=v.insee_com) -JOIN osm_postal -ON osm_postal.fantoir=v.fantoir -LEFT JOIN cog -ON v.insee_com = cog.insee -WHERE osm_postal.numero ~ '^[0-9]{1,4}( ?[A-Z]?.*)?' AND - osm_postal.numero !~'.[0-9 \\.\\-]{9,}' AND - osm_postal.numero != '99999' AND - osm_postal.rang_postal = 1 -GROUP BY osm_postal.fantoir, - osm_postal.insee_com, - osm_postal.voie, - osm_postal.code_postal, - c.name, - cog.nom_dep, - cog.nom_reg, - c.adm_weight, - c.population -ORDER BY osm_postal.fantoir; \ No newline at end of file diff --git a/bano/sql/export_json_dept_voies_rapprochees.sql b/bano/sql/export_json_dept_voies_rapprochees.sql deleted file mode 100644 index 0a49b2e..0000000 --- a/bano/sql/export_json_dept_voies_rapprochees.sql +++ /dev/null @@ -1,89 +0,0 @@ -WITH -sort_order -AS -(SELECT 1::integer as sortnum, 'OSM' as source -UNION ALL -SELECT 2,'BAL' -UNION ALL -SELECT 3,'BAN' -), -pre_osm -AS -(SELECT geometrie, - insee_com, - fantoir, - REPLACE(REPLACE(numero,'\',';'),'"','') AS numero, - code_postal, - source -FROM cumul_adresses -WHERE dept = '__dept__'), -osm -AS -(SELECT geometrie, - insee_com, - fantoir, - numero, - code_postal, - ROW_NUMBER() OVER (PARTITION BY fantoir,numero ORDER BY sortnum) rang -FROM pre_osm -JOIN sort_order USING (source)), -osm_postal -AS -(SELECT o.geometrie, - o.insee_com, --- o.voie_osm, - o.fantoir, - o.numero, - COALESCE(o.code_postal,pp.code_postal) code_postal, - RANK() OVER(PARTITION BY o.fantoir,o.numero ORDER BY id) rang_postal -FROM osm o -JOIN polygones_postaux pp -ON ST_Contains(pp.geometrie, o.geometrie) -WHERE o.rang = 1), -cog -AS -(SELECT cc.com AS insee, - cd.libelle AS nom_dep, - cr.libelle AS nom_reg - FROM cog_commune cc - JOIN cog_departement cd - USING (dep) - JOIN cog_region cr - ON cc.reg = cr.reg - WHERE cc.dep = '__dept__') -SELECT osm_postal.fantoir, --|| CASE WHEN coalesce(cp.postal_code, lp.cp)!=lp.cp THEN ('_' || cp.postal_code) ELSE '' END AS id, - osm_postal.insee_com AS citycode, - 'street' AS type, - REPLACE(REPLACE(REGEXP_REPLACE(v.voie_osm,'\t',' '),'"',chr(39)),'’',chr(39)) AS name, - osm_postal.code_postal AS postcode, - round(st_y(v.geometrie)::numeric,6) AS lat, - round(st_x(v.geometrie)::numeric,6) AS lon, - c.name AS city, - cog.nom_dep AS departement, - cog.nom_reg AS region, - round(log(c.adm_weight+log(c.population+1)/3)::numeric*log(1+log(count(osm_postal.*)+1)+log(st_length(st_longestline(st_convexhull(ST_Collect(osm_postal.geometrie)),st_convexhull(ST_Collect(osm_postal.geometrie)))::geography)+1)+log(CASE WHEN v.voie_osm like 'Boulevard%' THEN 4 WHEN v.voie_osm LIKE 'Place%' THEN 4 WHEN v.voie_osm LIKE 'Espl%' THEN 4 WHEN v.voie_osm LIKE 'Av%' THEN 3 WHEN v.voie_osm LIKE 'Rue %' THEN 2 ELSE 1 END))::numeric,4) AS importance, - string_agg(numero||'$'||st_y(osm_postal.geometrie)||'$'||st_x(osm_postal.geometrie),'#' ORDER BY numero)::text AS housenumbers -FROM cumul_voies v -JOIN infos_communes c -ON (c.insee_com=v.insee_com) -JOIN osm_postal -ON osm_postal.fantoir=v.fantoir -LEFT JOIN cog -ON v.insee_com = cog.insee -WHERE v.dept = '__dept__' AND - osm_postal.rang_postal = 1 AND - osm_postal.numero ~ '^[0-9]{1,4}( ?[A-Z]?.*)?' AND - osm_postal.numero != '99999' AND - osm_postal.numero !~'.[0-9 \\.\\-]{9,}' -GROUP BY osm_postal.fantoir, - osm_postal.insee_com, - osm_postal.code_postal, - v.voie_osm, - v.geometrie, - c.name, - cog.nom_dep, - cog.nom_reg, - c.adm_weight, - c.population -ORDER BY osm_postal.fantoir; - diff --git a/bano/sql/export_json_dept_voies_rapprochees_sans_adresses.sql b/bano/sql/export_json_dept_voies_rapprochees_sans_adresses.sql deleted file mode 100644 index 1d4d16b..0000000 --- a/bano/sql/export_json_dept_voies_rapprochees_sans_adresses.sql +++ /dev/null @@ -1,59 +0,0 @@ -WITH -set_fantoir -AS -(SELECT fantoir FROM cumul_voies WHERE dept = '__dept__' AND COALESCE(fantoir,'') != '' -EXCEPT -SELECT fantoir FROM cumul_adresses WHERE dept = '__dept__'), -osm_postal -AS -(SELECT v.geometrie, - insee_com, - fantoir, - REPLACE(REPLACE(REGEXP_REPLACE(v.voie_osm,'\t',' '),'"',chr(39)),'’',chr(39)) AS name, - pp.code_postal, - RANK() OVER(PARTITION BY v.fantoir ORDER BY id) rang_postal -FROM cumul_voies v -JOIN set_fantoir -USING (fantoir) -JOIN polygones_postaux pp -ON ST_Contains(pp.geometrie, v.geometrie)), -cog -AS -(SELECT cc.com AS insee, - cd.libelle AS nom_dep, - cr.libelle AS nom_reg -FROM cog_commune cc -JOIN cog_departement cd -USING (dep) -JOIN cog_region cr -ON cc.reg = cr.reg -WHERE cc.dep = '__dept__') -SELECT p.fantoir, - p.insee_com AS citycode, - 'street' AS type, - p.name, - p.code_postal AS postcode, - round(st_y(p.geometrie)::numeric,6) AS lat, - round(st_x(p.geometrie)::numeric,6) AS lon, - c.name AS city, - cog.nom_dep AS departement, - cog.nom_reg AS region, - round(log(c.adm_weight+log(c.population+1)/3)::numeric*log(1+log(count(p.*)+1)+log(CASE WHEN p.name like 'Boulevard%' THEN 4 WHEN p.name LIKE 'Place%' THEN 4 WHEN p.name LIKE 'Espl%' THEN 4 WHEN p.name LIKE 'Av%' THEN 3 WHEN p.name LIKE 'Rue %' THEN 2 ELSE 1 END))::numeric,4) AS importance -FROM osm_postal p -JOIN infos_communes c -USING (insee_com) -LEFT JOIN cog -ON p.insee_com = cog.insee -WHERE p.rang_postal = 1 -GROUP BY p.fantoir, - p.insee_com, - p.code_postal, - p.name, - p.geometrie, - c.name, - cog.nom_dep, - cog.nom_reg, - c.adm_weight, - c.population -ORDER BY p.fantoir; - diff --git a/bano/sql/geom_suffixes_insee.sql b/bano/sql/geom_suffixes_insee.sql deleted file mode 100644 index 8254799..0000000 --- a/bano/sql/geom_suffixes_insee.sql +++ /dev/null @@ -1,4 +0,0 @@ -SELECT ST_AsText(ST_Transform(ST_SetSRID(geometrie,4326),3857)) geometrie, - libelle_suffixe -FROM suffixe -WHERE insee_com ='__com__'; diff --git a/bano/sql/highway_bbox_insee.sql b/bano/sql/highway_bbox_insee.sql deleted file mode 100644 index b16931b..0000000 --- a/bano/sql/highway_bbox_insee.sql +++ /dev/null @@ -1,14 +0,0 @@ -SELECT DISTINCT l.name, - l.tags->'ref:FR:FANTOIR' f, - l.tags->'ref:FR:FANTOIR:left' fl, - l.tags->'ref:FR:FANTOIR:right' fr, - '' suffixe, - p.tags->'ref:INSEE' -FROM planet_osm_polygon p -JOIN planet_osm_line l -ON l.way && p.way -WHERE p.tags ? 'ref:INSEE' AND - p.tags->'ref:INSEE'='__com__' AND - l.highway IS NOT NULL AND - l.name IS NOT NULL -ORDER BY 6; diff --git a/bano/sql/highway_insee.sql b/bano/sql/highway_insee.sql deleted file mode 100644 index f3c3cf3..0000000 --- a/bano/sql/highway_insee.sql +++ /dev/null @@ -1,68 +0,0 @@ -WITH -f -AS -( SELECT pl.name as namecomp, - unnest(array[pl.name,pl.tags->'alt_name',pl.tags->'old_name']) as name, - pl.tags->'ref:FR:FANTOIR' f, - '' fl, - '' fr, - '' suffixe, - p.tags->'ref:INSEE' insee - FROM planet_osm_polygon p - JOIN planet_osm_point pl - ON pl.way && p.way AND - ST_Intersects(pl.way, p.way) - WHERE p.tags ? 'ref:INSEE' AND - p.tags->'ref:INSEE' {clause_insee} AND - pl.tags->'ref:FR:FANTOIR'!='' AND - pl.name IS NOT NULL - UNION - SELECT l.name, - unnest(array[l.name,l.tags->'alt_name',l.tags->'old_name']), - l.tags->'ref:FR:FANTOIR' f, - l.tags->'ref:FR:FANTOIR:left' fl, - l.tags->'ref:FR:FANTOIR:right' fr, - '' suffixe,p.tags->'ref:INSEE' - FROM planet_osm_polygon p - JOIN planet_osm_line l - ON ST_Intersects(l.way, p.way) - WHERE p.tags ? 'ref:INSEE' AND - p.tags->'ref:INSEE' {clause_insee} AND - l.highway IS NOT NULL AND - l.name IS NOT NULL - UNION - SELECT pl.name, - unnest(array[pl.name,pl.tags->'alt_name',pl.tags->'old_name']), - pl.tags->'ref:FR:FANTOIR' f, - '' fl, - '' fr, - '' suffixe, - p.tags->'ref:INSEE' - FROM planet_osm_polygon p - JOIN planet_osm_polygon pl - ON pl.way && p.way AND - ST_Intersects(pl.way, p.way) - WHERE p.tags ? 'ref:INSEE' AND - p.tags->'ref:INSEE' {clause_insee} AND - ( coalesce(pl.highway,pl.tags->'ref:FR:FANTOIR') IS NOT NULL OR - pl.landuse = 'residential' OR - pl.amenity = 'parking') AND - pl.name IS NOT NULL) -SELECT name, - CASE - WHEN name = namecomp THEN f - ELSE NULL - END f, - CASE - WHEN name = namecomp THEN fl - ELSE NULL - END fl, - CASE - WHEN name = namecomp THEN fr - ELSE NULL - END fr, - suffixe, - insee -FROM f -WHERE name IS NOT NULL -ORDER BY 6; \ No newline at end of file diff --git a/bano/sql/highway_relation_bbox_insee.sql b/bano/sql/highway_relation_bbox_insee.sql deleted file mode 100644 index 5d256e0..0000000 --- a/bano/sql/highway_relation_bbox_insee.sql +++ /dev/null @@ -1,13 +0,0 @@ -SELECT DISTINCT l.name, - %% r.tags, - p."ref:INSEE" ---FROM planet_osm_polygon p -FROM (SELECT ST_SetSRID(ST_Extent(way),3857) way, "ref:INSEE" FROM planet_osm_polygon WHERE "ref:INSEE" = '__com__' GROUP BY 2) p -JOIN planet_osm_line l -ON ST_Intersects(l.way,p.way) -JOIN planet_osm_rels r -ON r.osm_id = l.osm_id -WHERE --p."ref:INSEE" = '__com__' AND - l.highway != '' AND - l.name != '' -ORDER BY 3; diff --git a/bano/sql/highway_relation_insee.sql b/bano/sql/highway_relation_insee.sql deleted file mode 100644 index e22530d..0000000 --- a/bano/sql/highway_relation_insee.sql +++ /dev/null @@ -1,13 +0,0 @@ -SELECT DISTINCT l.name, - %% r.tags, - p."ref:INSEE" -FROM planet_osm_polygon p -JOIN planet_osm_line l -ON ST_Intersects(l.way, p.way) -JOIN planet_osm_rels r -ON r.osm_id = l.osm_id -WHERE p."ref:INSEE" = '__com__' AND - l.highway != '' AND - l.name != '' -ORDER BY 3; - diff --git a/bano/sql/highway_relation_suffixe_insee.sql b/bano/sql/highway_relation_suffixe_insee.sql deleted file mode 100644 index c871403..0000000 --- a/bano/sql/highway_relation_suffixe_insee.sql +++ /dev/null @@ -1,18 +0,0 @@ -INSERT INTO highway_relation_suffixe_insee -SELECT DISTINCT l.name, - r.tags, - h.libelle_suffixe, - p."ref:INSEE" -FROM planet_osm_polygon p -JOIN planet_osm_line l -ON ST_Intersects(l.way, p.way) -JOIN planet_osm_rels r -ON r.osm_id = l.osm_id -LEFT OUTER JOIN suffixe h -ON ST_Intersects(l.way, h.geometrie) -WHERE p."ref:INSEE" = '__com__' AND - l.highway != '' AND - l.name != '' ---ORDER BY 4 -; - diff --git a/bano/sql/highway_relation_suffixe_insee_bbox_nocache.sql b/bano/sql/highway_relation_suffixe_insee_bbox_nocache.sql deleted file mode 100644 index fededbb..0000000 --- a/bano/sql/highway_relation_suffixe_insee_bbox_nocache.sql +++ /dev/null @@ -1,30 +0,0 @@ -SELECT DISTINCT l.name, - r.tags, - h.libelle_suffixe, - p."ref:INSEE" -FROM planet_osm_polygon p -JOIN planet_osm_line l -ON l.way && p.way -JOIN planet_osm_rels r -ON r.osm_id = l.osm_id -LEFT OUTER JOIN suffixe h -ON ST_Intersects(l.way, h.geometrie) -WHERE p."ref:INSEE" = '__com__' AND - l.highway != '' AND - l.name != '' -UNION ALL -SELECT DISTINCT l.name, - r.tags, - h.libelle_suffixe, - p."ref:INSEE" -FROM planet_osm_polygon p -JOIN planet_osm_polygon l -ON l.way && p.way -JOIN planet_osm_rels r -ON r.osm_id = l.osm_id -LEFT OUTER JOIN suffixe h -ON ST_Intersects(l.way, h.geometrie) -WHERE p."ref:INSEE" = '__com__' AND - l.highway != '' AND - l.name != ''; - diff --git a/bano/sql/highway_relation_suffixe_insee_nocache.sql b/bano/sql/highway_relation_suffixe_insee_nocache.sql deleted file mode 100644 index fb1cac0..0000000 --- a/bano/sql/highway_relation_suffixe_insee_nocache.sql +++ /dev/null @@ -1,30 +0,0 @@ -SELECT DISTINCT l.name, - r.tags, - h.libelle_suffixe, - p."ref:INSEE" -FROM planet_osm_polygon p -JOIN planet_osm_line l -ON ST_Intersects(l.way, p.way) -JOIN planet_osm_rels r -ON r.osm_id = l.osm_id -LEFT OUTER JOIN suffixe h -ON ST_Intersects(l.way, h.geometrie) -WHERE p."ref:INSEE" = '__com__' AND - l.highway != '' AND - l.name != '' -UNION ALL -SELECT DISTINCT l.name, - r.tags, - h.libelle_suffixe, - p."ref:INSEE" -FROM planet_osm_polygon p -JOIN planet_osm_polygon l -ON ST_Intersects(l.way, p.way) -JOIN planet_osm_rels r -ON r.osm_id = l.osm_id -LEFT OUTER JOIN suffixe h -ON ST_Intersects(l.way, h.geometrie) -WHERE p."ref:INSEE" = '__com__' AND - l.highway != '' AND - l.name != ''; - diff --git a/bano/sql/highway_suffixe_insee.sql b/bano/sql/highway_suffixe_insee.sql deleted file mode 100644 index 5278abc..0000000 --- a/bano/sql/highway_suffixe_insee.sql +++ /dev/null @@ -1,49 +0,0 @@ -INSERT INTO highway_suffixe_insee -SELECT pl.name, - pl."ref:FR:FANTOIR" f, - '' fl, - '' fr, - h.libelle_suffixe, - p."ref:INSEE" -FROM planet_osm_polygon p -JOIN planet_osm_point pl -ON pl.way && p.way AND - ST_Intersects(pl.way, p.way) -LEFT OUTER JOIN suffixe h -ON ST_Intersects(pl.way, h.geometrie) -WHERE p."ref:INSEE" = '__com__' AND - pl."ref:FR:FANTOIR" !='' AND - pl.name != '' -UNION -SELECT l.name, - l.tags->'ref:FR:FANTOIR' f, - l.tags->'ref:FR:FANTOIR:left' fl, - l.tags->'ref:FR:FANTOIR:right' fr, - h.libelle_suffixe, - p."ref:INSEE" -FROM planet_osm_polygon p -JOIN planet_osm_line l -ON ST_Intersects(l.way, p.way) -LEFT OUTER JOIN suffixe h -ON ST_Intersects(l.way, h.geometrie) -WHERE p."ref:INSEE" = '__com__' AND - l.highway != '' AND - l.name != '' -UNION -SELECT pl.name, - pl."ref:FR:FANTOIR" f, - pl."ref:FR:FANTOIR:left" fl, - pl."ref:FR:FANTOIR:right" fr, - h.libelle_suffixe, - p."ref:INSEE" -FROM planet_osm_polygon p -JOIN planet_osm_polygon pl -ON pl.way && p.way AND - ST_Intersects(pl.way, p.way) -LEFT OUTER JOIN suffixe h -ON ST_Intersects(pl.way, h.geometrie) -WHERE p."ref:INSEE" = '__com__' AND - (pl.highway||pl."ref:FR:FANTOIR" != '' OR - pl.landuse = 'residential' OR - pl.amenity = 'parking') AND - pl.name != ''; diff --git a/bano/sql/highway_suffixe_insee_nocache.sql b/bano/sql/highway_suffixe_insee_nocache.sql deleted file mode 100644 index 6f72ec9..0000000 --- a/bano/sql/highway_suffixe_insee_nocache.sql +++ /dev/null @@ -1,48 +0,0 @@ -SELECT pl.name, - pl."ref:FR:FANTOIR" f, - '' fl, - '' fr, - h.libelle_suffixe, - p."ref:INSEE" -FROM planet_osm_polygon p -JOIN planet_osm_point pl -ON pl.way && p.way AND - ST_Intersects(pl.way, p.way) -LEFT OUTER JOIN suffixe h -ON ST_Intersects(pl.way, h.geometrie) -WHERE p."ref:INSEE" = '__com__' AND - pl."ref:FR:FANTOIR" !='' AND - pl.name != '' -UNION -SELECT l.name, - l.tags->'ref:FR:FANTOIR' f, - l.tags->'ref:FR:FANTOIR:left' fl, - l.tags->'ref:FR:FANTOIR:right' fr, - h.libelle_suffixe, - p."ref:INSEE" -FROM planet_osm_polygon p -JOIN planet_osm_line l -ON ST_Intersects(l.way, p.way) -LEFT OUTER JOIN suffixe h -ON ST_Intersects(l.way, h.geometrie) -WHERE p."ref:INSEE" = '__com__' AND - l.highway != '' AND - l.name != '' -UNION -SELECT pl.name, - pl."ref:FR:FANTOIR" f, - pl."ref:FR:FANTOIR:left" fl, - pl."ref:FR:FANTOIR:right" fr, - h.libelle_suffixe, - p."ref:INSEE" -FROM planet_osm_polygon p -JOIN planet_osm_polygon pl -ON pl.way && p.way AND - ST_Intersects(pl.way, p.way) -LEFT OUTER JOIN suffixe h -ON ST_Intersects(pl.way, h.geometrie) -WHERE p."ref:INSEE" = '__com__' AND - (pl.highway||pl."ref:FR:FANTOIR" != '' OR - pl.landuse = 'residential' OR - pl.amenity = 'parking') AND - pl.name != ''; diff --git a/bano/sql/hsnr_bbox_insee.sql b/bano/sql/hsnr_bbox_insee.sql deleted file mode 100644 index c5c1826..0000000 --- a/bano/sql/hsnr_bbox_insee.sql +++ /dev/null @@ -1,50 +0,0 @@ -INSERT INTO hsnr_bbox_insee -SELECT ST_X(pt_geo), - ST_Y(pt_geo), - provenance, - osm_id, - hsnr, - street_name, - tags, - ''::text suffixe, - insee_com -FROM --- point dans relation associatedStreet - (SELECT 3 provenance, - ST_Transform(pt.way,4326) pt_geo, - pt.osm_id, - pt."addr:housenumber" hsnr, - null::text street_name, - r.tags tags, - p."ref:INSEE" insee_com --- FROM planet_osm_polygon p - FROM (SELECT ST_SetSRID(ST_Extent(way),3857) way, "ref:INSEE" FROM planet_osm_polygon WHERE "ref:INSEE" = '__com__' GROUP BY 2) p - JOIN planet_osm_point pt - ON ST_Intersects(pt.way,p.way) - JOIN planet_osm_rels r - ON r.osm_id = pt.osm_id - WHERE p."ref:INSEE" = '__com__' AND - pt."addr:housenumber" != '' - UNION --- way dans relation associatedStreet - SELECT 4, - ST_Transform(ST_Centroid(w.way),4326), - w.osm_id, - w."addr:housenumber", - null, - r.tags, - p."ref:INSEE" --- FROM planet_osm_polygon p - FROM (SELECT ST_SetSRID(ST_Extent(way),3857) way, "ref:INSEE" FROM planet_osm_polygon WHERE "ref:INSEE" = '__com__' GROUP BY 2) p - JOIN planet_osm_polygon w - ON ST_Intersects(w.way, p.way) - JOIN planet_osm_rels r - ON r.osm_id = w.osm_id - WHERE p."ref:INSEE" = '__com__' AND - w."addr:housenumber" != '' -)a ---ORDER BY 9 --- where hsnr is not null*/ -; - - \ No newline at end of file diff --git a/bano/sql/hsnr_bbox_insee_nocache.sql b/bano/sql/hsnr_bbox_insee_nocache.sql deleted file mode 100644 index 61825ab..0000000 --- a/bano/sql/hsnr_bbox_insee_nocache.sql +++ /dev/null @@ -1,46 +0,0 @@ -SELECT ST_X(pt_geo), - ST_Y(pt_geo), - provenance, - osm_id, - hsnr, - street_name, - tags, - ''::text suffixe, - insee_com -FROM --- point dans relation associatedStreet - (SELECT 3 provenance, - ST_Transform(pt.way,4326) pt_geo, - pt.osm_id, - pt."addr:housenumber" hsnr, - null::text street_name, - r.tags tags, - p."ref:INSEE" insee_com --- FROM planet_osm_polygon p - FROM (SELECT ST_SetSRID(ST_Extent(way),3857) way, "ref:INSEE" FROM planet_osm_polygon WHERE "ref:INSEE" = '__com__' GROUP BY 2) p - JOIN planet_osm_point pt - ON ST_Intersects(pt.way,p.way) - JOIN planet_osm_rels r - ON r.osm_id = pt.osm_id - WHERE p."ref:INSEE" = '__com__' AND - pt."addr:housenumber" != '' - UNION --- way dans relation associatedStreet - SELECT 4, - ST_Transform(ST_Centroid(w.way),4326), - w.osm_id, - w."addr:housenumber", - null, - r.tags, - p."ref:INSEE" --- FROM planet_osm_polygon p - FROM (SELECT ST_SetSRID(ST_Extent(way),3857) way, "ref:INSEE" FROM planet_osm_polygon WHERE "ref:INSEE" = '__com__' GROUP BY 2) p - JOIN planet_osm_polygon w - ON ST_Intersects(w.way, p.way) - JOIN planet_osm_rels r - ON r.osm_id = w.osm_id - WHERE p."ref:INSEE" = '__com__' AND - w."addr:housenumber" != '' -)a; - - \ No newline at end of file diff --git a/bano/sql/hsnr_insee.sql b/bano/sql/hsnr_insee.sql deleted file mode 100644 index d505320..0000000 --- a/bano/sql/hsnr_insee.sql +++ /dev/null @@ -1,77 +0,0 @@ -INSERT INTO hsnr_insee -select ST_X(pt_geo), - ST_Y(pt_geo), - provenance, - osm_id, - hsnr, - street_name, - tags, - ''::text suffixe, - insee_com -FROM --- point avec addr:street - (SELECT 1 provenance, - ST_Transform(pt.way,4326) pt_geo, - pt.osm_id, - pt."addr:housenumber" hsnr, - pt."addr:street" street_name, - null::hstore tags, - p."ref:INSEE" insee_com - FROM planet_osm_polygon p - JOIN planet_osm_point pt - ON ST_Intersects(pt.way, p.way) - WHERE p."ref:INSEE" = '__com__' AND - pt."addr:housenumber" != '' AND - pt."addr:street" != '' - UNION --- way avec addr:street - SELECT 2, - ST_Transform(ST_Centroid(w.way),4326), - w.osm_id, - w."addr:housenumber", - w."addr:street", - null::hstore tags, - p."ref:INSEE" - FROM planet_osm_polygon p - JOIN planet_osm_polygon w - ON ST_Intersects(w.way, p.way) - WHERE p."ref:INSEE" = '__com__' AND - w."addr:housenumber" != '' AND - w."addr:street" != '' - UNION --- point dans relation associatedStreet - SELECT 3, - ST_Transform(pt.way,4326), - pt.osm_id, - pt."addr:housenumber", - null, - r.tags, - p."ref:INSEE" - FROM planet_osm_polygon p - JOIN planet_osm_point pt - ON ST_Intersects(pt.way, p.way) - JOIN planet_osm_rels r - ON r.osm_id = pt.osm_id - WHERE p."ref:INSEE" = '__com__' AND - pt."addr:housenumber" != '' - UNION --- way dans relation associatedStreet - SELECT 4, - ST_Transform(ST_Centroid(w.way),4326), - w.osm_id, - w."addr:housenumber", - null, - r.tags, - p."ref:INSEE" - FROM planet_osm_polygon p - JOIN planet_osm_polygon w - ON ST_Intersects(w.way, p.way) - JOIN planet_osm_rels r - ON r.osm_id = w.osm_id - WHERE p."ref:INSEE" = '__com__' AND - w."addr:housenumber" != '' -)a ---ORDER BY 9 --- where hsnr is not null*/ -; - diff --git a/bano/sql/hsnr_insee_nocache.sql b/bano/sql/hsnr_insee_nocache.sql deleted file mode 100644 index 63aa03a..0000000 --- a/bano/sql/hsnr_insee_nocache.sql +++ /dev/null @@ -1,73 +0,0 @@ -select ST_X(pt_geo), - ST_Y(pt_geo), - provenance, - osm_id, - hsnr, - street_name, - tags, - ''::text suffixe, - insee_com -FROM --- point avec addr:street - (SELECT 1 provenance, - ST_Transform(pt.way,4326) pt_geo, - pt.osm_id, - pt."addr:housenumber" hsnr, - pt."addr:street" street_name, - null::hstore tags, - p."ref:INSEE" insee_com - FROM planet_osm_polygon p - JOIN planet_osm_point pt - ON ST_Intersects(pt.way, p.way) - WHERE p."ref:INSEE" = '__com__' AND - pt."addr:housenumber" != '' AND - pt."addr:street" != '' - UNION --- way avec addr:street - SELECT 2, - ST_Transform(ST_Centroid(w.way),4326), - w.osm_id, - w."addr:housenumber", - w."addr:street", - null::hstore tags, - p."ref:INSEE" - FROM planet_osm_polygon p - JOIN planet_osm_polygon w - ON ST_Intersects(w.way, p.way) - WHERE p."ref:INSEE" = '__com__' AND - w."addr:housenumber" != '' AND - w."addr:street" != '' - UNION --- point dans relation associatedStreet - SELECT 3, - ST_Transform(pt.way,4326), - pt.osm_id, - pt."addr:housenumber", - null, - r.tags, - p."ref:INSEE" - FROM planet_osm_polygon p - JOIN planet_osm_point pt - ON ST_Intersects(pt.way, p.way) - JOIN planet_osm_rels r - ON r.osm_id = pt.osm_id - WHERE p."ref:INSEE" = '__com__' AND - pt."addr:housenumber" != '' - UNION --- way dans relation associatedStreet - SELECT 4, - ST_Transform(ST_Centroid(w.way),4326), - w.osm_id, - w."addr:housenumber", - null, - r.tags, - p."ref:INSEE" - FROM planet_osm_polygon p - JOIN planet_osm_polygon w - ON ST_Intersects(w.way, p.way) - JOIN planet_osm_rels r - ON r.osm_id = w.osm_id - WHERE p."ref:INSEE" = '__com__' AND - w."addr:housenumber" != '' -)a; - diff --git a/bano/sql/hsnr_suffixe_insee.sql b/bano/sql/hsnr_suffixe_insee.sql deleted file mode 100644 index 322734a..0000000 --- a/bano/sql/hsnr_suffixe_insee.sql +++ /dev/null @@ -1,76 +0,0 @@ -select ST_X(ST_Transform(a.pt_geo,4326))::character varying, - ST_Y(ST_Transform(a.pt_geo,4326))::character varying, - a.provenance::character varying, - a.osm_id::character varying, - a.hsnr, - a.street_name, - a.tags, - h.libelle_suffixe, - a.insee_com -FROM --- point avec addr:street - (SELECT 1 provenance, - pt.way pt_geo, - pt.osm_id::character varying, - pt."addr:housenumber" hsnr, - pt."addr:street" street_name, - ARRAY[]::character[] tags, - p."ref:INSEE" insee_com - FROM planet_osm_polygon p - JOIN planet_osm_point pt - ON ST_Intersects(pt.way, p.way) - WHERE p."ref:INSEE" = '__com__' AND - pt."addr:housenumber" != ''AND - pt."addr:street" !='' - UNION --- way avec addr:street - SELECT 2, - ST_Centroid(w.way), - w.osm_id::character varying, - w."addr:housenumber", - w."addr:street", - ARRAY[]::character[], - p."ref:INSEE" - FROM planet_osm_polygon p - JOIN planet_osm_polygon w - ON ST_Intersects(w.way, p.way) - WHERE p."ref:INSEE" = '__com__' AND - w."addr:housenumber" !='' AND - w.tags->'addr:street' != '' - UNION --- point dans relation associatedStreet - SELECT 3, - pt.way, - pt.osm_id::character varying, - pt."addr:housenumber", - null, - %% r.tags, - p."ref:INSEE" - FROM planet_osm_polygon p - JOIN planet_osm_point pt - ON ST_Intersects(pt.way, p.way) - JOIN planet_osm_rels r - ON r.osm_id = pt.osm_id - WHERE p."ref:INSEE" = '__com__' AND - pt."addr:housenumber" != '' - UNION --- way dans relation associatedStreet - SELECT 4, - ST_Centroid(w.way), - w.osm_id::character varying, - w."addr:housenumber", - null, - %% r.tags, - p."ref:INSEE" - FROM planet_osm_polygon p - JOIN planet_osm_polygon w - ON ST_Intersects(w.way, p.way) - JOIN planet_osm_rels r - ON r.osm_id = w.osm_id - WHERE p."ref:INSEE" = '__com__' AND - w."addr:housenumber" != '' - -)a -LEFT OUTER JOIN suffixe h -ON ST_Intersects(a.pt_geo, h.geometrie); - diff --git a/bano/sql/lieux_dits_post_process.sql b/bano/sql/lieux_dits_post_process.sql deleted file mode 100644 index bb66770..0000000 --- a/bano/sql/lieux_dits_post_process.sql +++ /dev/null @@ -1,32 +0,0 @@ -DELETE FROM cadastre_2_place WHERE insee_com like '__dept__%'; -WITH -fantoir -AS -(SELECT DISTINCT code_insee||id_voie||cle_rivoli AS fantoir, - TRIM(BOTH FROM nature_voie||' '||libelle_voie) AS nom, - code_insee AS insee_com -FROM fantoir_voie -WHERE code_insee LIKE '__dept__%' AND - type_voie = '3'), -lieux_dits -AS -(SELECT ST_Centroid(geometrie) AS geometrie, - regexp_replace(regexp_replace(nom,' ',' ','g'),' ',' ','g') AS nom, - insee_com -FROM lieux_dits -WHERE insee_com LIKE '__dept__%' AND - nom IS NOT NULL ), -mix -AS -(SELECT l.*, - f.fantoir -FROM lieux_dits l -LEFT OUTER JOIN fantoir f -USING (nom,insee_com)) -INSERT INTO cadastre_2_place -SELECT geometrie, - nom, - fantoir, - insee_com -FROM mix; -COMMIT; \ No newline at end of file diff --git a/bano/sql/liste_communes_par_dept.sql b/bano/sql/liste_communes_par_dept.sql new file mode 100644 index 0000000..ded6ca4 --- /dev/null +++ b/bano/sql/liste_communes_par_dept.sql @@ -0,0 +1,9 @@ +SELECT com, + ncc +FROM cog_commune c +LEFT OUTER JOIN (SELECT comparent FROM cog_commune WHERE dep = '__dept__' AND typecom = 'ARM') p +ON (c.com = p.comparent) +WHERE c.dep = '__dept__' AND + c.typecom != 'COMD' AND + p.comparent IS NULL +ORDER BY 1 diff --git a/bano/sql/load_code_cadastre.sql b/bano/sql/load_code_cadastre.sql deleted file mode 100644 index 31da241..0000000 --- a/bano/sql/load_code_cadastre.sql +++ /dev/null @@ -1,23 +0,0 @@ -TRUNCATE TABLE tmp_code_cadastre; -COPY tmp_code_cadastre (dept,cadastre_dept,nom_com,code_postal,cadastre_com,format_cadastre) FROM './code_cadastre.csv' WITH CSV DELIMITER ','; -UPDATE tmp_code_cadastre SET insee_com = dept||substr(cadastre_com,3,3) WHERE length(dept) = 2; -UPDATE tmp_code_cadastre SET insee_com = dept||substr(cadastre_com,4,2) WHERE length(dept) = 3; -UPDATE tmp_code_cadastre SET date_maj = (SELECT to_char(n,'YYMMDD')::integer FROM (SELECT now() AS n)a); - -DELETE FROM code_cadastre -WHERE insee_com IN (SELECT insee_com FROM tmp_code_cadastre WHERE format_cadastre = 'VECT' AND nom_com != '' - INTERSECT - SELECT insee_com FROM code_cadastre WHERE format_cadastre = 'IMAG'); -DELETE FROM code_cadastre -WHERE insee_com IN (SELECT insee_com - FROM (SELECT insee_com,cadastre_com FROM tmp_code_cadastre WHERE format_cadastre = 'VECT' AND nom_com != '' - EXCEPT - SELECT insee_com,cadastre_com FROM code_cadastre)i); -INSERT INTO code_cadastre -SELECT t.* -FROM tmp_code_cadastre t -LEFT OUTER JOIN code_cadastre c -ON t.insee_com = c.insee_com -WHERE c.insee_com IS NULL AND - t.nom_com != ''; -COMMIT; diff --git a/bano/sql/nom_commune_par_code_insee.sql b/bano/sql/nom_commune_par_code_insee.sql new file mode 100644 index 0000000..6f4e57a --- /dev/null +++ b/bano/sql/nom_commune_par_code_insee.sql @@ -0,0 +1,3 @@ +SELECT ncc +FROM cog_commune c +WHERE com = '__code_insee__'; diff --git a/bano/sql/noms_ban_non_rapproches_par_commune.sql b/bano/sql/noms_ban_non_rapproches_par_commune.sql new file mode 100644 index 0000000..c467473 --- /dev/null +++ b/bano/sql/noms_ban_non_rapproches_par_commune.sql @@ -0,0 +1,15 @@ +SELECT nom_voie +FROM ban +WHERE code_insee = '__code_insee__' AND + id_fantoir IS NULL +EXCEPT +SELECT nom +FROM (SELECT * + FROM nom_fantoir + WHERE code_insee = '__code_insee__' AND + source = 'BAN')n +JOIN (SELECT fantoir + FROM topo + WHERE code_insee = '__code_insee__' AND + caractere_annul = 'B') t +USING (fantoir); \ No newline at end of file diff --git a/bano/sql/place_insee.sql b/bano/sql/place_insee.sql deleted file mode 100644 index ba416fd..0000000 --- a/bano/sql/place_insee.sql +++ /dev/null @@ -1,29 +0,0 @@ ---DELETE FROM place_insee WHERE insee_com = '__com__'; -WITH -a AS -(SELECT ST_Transform(pt.way,4326) pt_geo, - pt.place, - pt.name, - pt."ref:FR:FANTOIR" fantoir, - pt.tags, - p."ref:INSEE" insee_com - FROM planet_osm_polygon p - JOIN planet_osm_point pt - ON ST_Intersects(pt.way, p.way) - WHERE p."ref:INSEE" = '__com__' AND - (pt.place != '' OR - (pt.railway !='' AND pt."ref:FR:FANTOIR" != '') OR - (pt.amenity !='' AND pt."ref:FR:FANTOIR" != '') ) AND - pt.name != '') ---INSERT INTO place_insee -SELECT ST_X(pt_geo), - ST_Y(pt_geo), - place, - name, - fantoir, - '0', --ld_bati - tags, - insee_com--, - -- round(extract(epoch from now())) -FROM a; ---COMMIT; \ No newline at end of file diff --git a/bano/sql/point_par_rue_complement_insee.sql b/bano/sql/point_par_rue_complement_insee.sql deleted file mode 100644 index df17a59..0000000 --- a/bano/sql/point_par_rue_complement_insee.sql +++ /dev/null @@ -1,34 +0,0 @@ -INSERT INTO point_par_rue_complement_insee -SELECT ST_X(ST_Transform(ST_SetSRID(way,3857),4326)), - ST_Y(ST_Transform(ST_SetSRID(way,3857),4326)), - name, - fantoir, - suffixe, - insee -FROM (SELECT pl.way, - pl.name, - pl."ref:FR:FANTOIR" fantoir, - '' suffixe, - p."ref:INSEE" insee - FROM planet_osm_polygon p - JOIN planet_osm_point pl - ON pl.way && p.way AND - ST_Intersects(pl.way, p.way) - WHERE p."ref:INSEE" = '__com__' AND - pl."ref:FR:FANTOIR" != '' AND - pl.name != '' - UNION - SELECT ST_Centroid(pl.way), - pl.name, - pl."ref:FR:FANTOIR" f, - '' suffixe, - p."ref:INSEE" - FROM planet_osm_polygon p - JOIN planet_osm_polygon pl - ON pl.way && p.way AND - ST_Intersects(pl.way, p.way) - WHERE p."ref:INSEE" = '__com__' AND - ( pl.highway||pl."ref:FR:FANTOIR" != '' OR - pl.landuse = 'residential') AND - pl.name != '')a -ORDER BY 6; \ No newline at end of file diff --git a/bano/sql/point_par_rue_complement_insee_nocache.sql b/bano/sql/point_par_rue_complement_insee_nocache.sql deleted file mode 100644 index 37b6fc1..0000000 --- a/bano/sql/point_par_rue_complement_insee_nocache.sql +++ /dev/null @@ -1,35 +0,0 @@ -SELECT ST_X(ST_Transform(ST_SetSRID(way,3857),4326)), - ST_Y(ST_Transform(ST_SetSRID(way,3857),4326)), - name, - fantoir, - suffixe, - insee -FROM (SELECT pl.way, - pl.name, - pl."ref:FR:FANTOIR" fantoir, - '' suffixe, - p."ref:INSEE" insee - FROM planet_osm_polygon p - JOIN planet_osm_point pl - ON pl.way && p.way AND - ST_Intersects(pl.way, p.way) - WHERE p."ref:INSEE" = '__com__' AND - (pl."ref:FR:FANTOIR" != '' OR - pl.junction != '') AND - pl.name != '' - UNION - SELECT ST_Centroid(pl.way), - pl.name, - pl."ref:FR:FANTOIR" f, - '' suffixe, - p."ref:INSEE" - FROM planet_osm_polygon p - JOIN planet_osm_polygon pl - ON pl.way && p.way AND - ST_Intersects(pl.way, p.way) - WHERE p."ref:INSEE" = '__com__' AND - ( pl.highway||pl."ref:FR:FANTOIR" != '' OR - pl.landuse = 'residential' OR - pl.place = 'square' OR - pl.amenity = 'school') AND - pl.name != '')a; \ No newline at end of file diff --git a/bano/sql/point_par_rue_insee.sql b/bano/sql/point_par_rue_insee.sql deleted file mode 100644 index 321b348..0000000 --- a/bano/sql/point_par_rue_insee.sql +++ /dev/null @@ -1,52 +0,0 @@ -WITH -w0 -AS -(SELECT l.way, - unnest(array[l.name,l.tags->'alt_name',l.tags->'old_name']) as name, - p."ref:INSEE" as insee, - ST_Within(l.way,p.way)::integer as within - FROM planet_osm_polygon p - JOIN planet_osm_line l - ON ST_Intersects(l.way, p.way) - WHERE p."ref:INSEE" = '__com__' AND - (l.highway != '' OR - l.waterway = 'dam') AND - l.name != ''), -w1 -AS -(SELECT * -FROM w0 -WHERE name IS NOT NULL), -w2 -AS -(SELECT *,rank() OVER(PARTITION BY name,insee ORDER BY within DESC) rang -FROM w1), -wl -AS -(SELECT ST_LineMerge(st_collect(way)) way, - name, - insee - FROM w2 - WHERE rang = 1 - GROUP BY 2,3), -wp -AS -(SELECT ST_Centroid(ST_LineMerge(ST_Collect(way))) way, - name - FROM w2 - WHERE rang = 1 - GROUP BY 2), -a -AS -(SELECT ST_Transform(ST_SetSRID(ST_ClosestPoint(wl.way,wp.way),3857),4326) point, - wl.name, - wl.insee -FROM wl -JOIN wp -ON wl.name = wp.name) -INSERT INTO point_par_rue_insee -SELECT ST_x(point), - ST_y(point), - name, - insee -FROM a; diff --git a/bano/sql/point_par_rue_insee_nocache.sql b/bano/sql/point_par_rue_insee_nocache.sql deleted file mode 100644 index efc0136..0000000 --- a/bano/sql/point_par_rue_insee_nocache.sql +++ /dev/null @@ -1,51 +0,0 @@ -WITH -w0 -AS -(SELECT l.way, - unnest(array[l.name,l.tags->'alt_name',l.tags->'old_name']) as name, - p."ref:INSEE" as insee, - ST_Within(l.way,p.way)::integer as within - FROM planet_osm_polygon p - JOIN planet_osm_line l - ON ST_Intersects(l.way, p.way) - WHERE p."ref:INSEE" = '__com__' AND - (l.highway != '' OR - l.waterway = 'dam') AND - l.name != ''), -w1 -AS -(SELECT * -FROM w0 -WHERE name IS NOT NULL), -w2 -AS -(SELECT *,rank() OVER(PARTITION BY name,insee ORDER BY within DESC) rang -FROM w1), -wl -AS -(SELECT ST_LineMerge(st_collect(way)) way, - name, - insee - FROM w2 - WHERE rang = 1 - GROUP BY 2,3), -wp -AS -(SELECT ST_Centroid(ST_LineMerge(ST_Collect(way))) way, - name - FROM w2 - WHERE rang = 1 - GROUP BY 2), -a -AS -(SELECT ST_Transform(ST_SetSRID(ST_ClosestPoint(wl.way,wp.way),3857),4326) point, - wl.name, - wl.insee -FROM wl -JOIN wp -ON wl.name = wp.name) -SELECT ST_x(point), - ST_y(point), - name, - insee -FROM a; diff --git a/bano/sql/purge_noms_ban_dans_topo.sql b/bano/sql/purge_noms_ban_dans_topo.sql new file mode 100644 index 0000000..4429f25 --- /dev/null +++ b/bano/sql/purge_noms_ban_dans_topo.sql @@ -0,0 +1,3 @@ +DELETE FROM topo +WHERE code_insee = '__code_insee__' AND + type_voie = 'B'; \ No newline at end of file diff --git a/bano/sql/replace_batiments.sql b/bano/sql/replace_batiments.sql deleted file mode 100644 index 510f4ab..0000000 --- a/bano/sql/replace_batiments.sql +++ /dev/null @@ -1,17 +0,0 @@ -SELECT :'dept'||'%' AS deptlike -\gset -DELETE FROM :schema_cible.batiments WHERE insee_com LIKE :'deptlike'; -INSERT INTO :schema_cible.batiments (insee_com, - nom, - type_batiment, - created, - updated, - geometrie) -SELECT commune, - nom, - "type", - created, - updated, - geometrie -FROM tmp_batiments:dept; -DROP TABLE tmp_batiments:dept; \ No newline at end of file diff --git a/bano/sql/replace_lieux_dits.sql b/bano/sql/replace_lieux_dits.sql deleted file mode 100644 index a9a2829..0000000 --- a/bano/sql/replace_lieux_dits.sql +++ /dev/null @@ -1,16 +0,0 @@ -SELECT :'dept'||'%' AS deptlike -\gset -DELETE FROM :schema_cible.lieux_dits WHERE insee_com LIKE :'deptlike'; -INSERT INTO :schema_cible.lieux_dits (insee_com, - nom, - created, - updated, - geometrie) -SELECT commune, - --nom, - regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(nom,' ',' ','g'),' ',' ','g'),' ',' ','g'),' ',' ','g'),' ',' ','g'), - created, - updated, - geometrie -FROM tmp_lieux_dits:dept; -DROP TABLE tmp_lieux_dits:dept; \ No newline at end of file diff --git a/bano/sql/suppression_adresses_commune.sql b/bano/sql/suppression_adresses_commune.sql new file mode 100644 index 0000000..3e94ff0 --- /dev/null +++ b/bano/sql/suppression_adresses_commune.sql @@ -0,0 +1,2 @@ +DELETE FROM bano_adresses +WHERE code_insee = '__code_insee__'; \ No newline at end of file diff --git a/bano/sql/suppression_noms_commune.sql b/bano/sql/suppression_noms_commune.sql new file mode 100644 index 0000000..e3a5023 --- /dev/null +++ b/bano/sql/suppression_noms_commune.sql @@ -0,0 +1,2 @@ +DELETE FROM nom_fantoir +WHERE code_insee = '__code_insee__'; \ No newline at end of file diff --git a/bano/sql/suppression_points_nommes_commune.sql b/bano/sql/suppression_points_nommes_commune.sql new file mode 100644 index 0000000..ac15428 --- /dev/null +++ b/bano/sql/suppression_points_nommes_commune.sql @@ -0,0 +1,2 @@ +DELETE FROM bano_points_nommes +WHERE code_insee = '__code_insee__'; \ No newline at end of file diff --git a/bano/sql/table_insee_codes_postaux.sql b/bano/sql/table_insee_codes_postaux.sql new file mode 100644 index 0000000..77dfdf5 --- /dev/null +++ b/bano/sql/table_insee_codes_postaux.sql @@ -0,0 +1,9 @@ +DROP TABLE IF EXISTS insee_codes_postaux CASCADE; +CREATE TABLE insee_codes_postaux +AS +SELECT insee AS code_insee, + STRING_AGG(cp,';') AS cps +FROM (SELECT DISTINCT insee, cp FROM codes_postaux ORDER BY 1,2) AS c +GROUP BY 1; + +CREATE INDEX idx_insee_codes_postaux_code_insee ON insee_codes_postaux(code_insee); diff --git a/sql/create_table_polygones_postaux.sql b/bano/sql/table_polygones_postaux.sql similarity index 50% rename from sql/create_table_polygones_postaux.sql rename to bano/sql/table_polygones_postaux.sql index 791e807..3715cc4 100644 --- a/sql/create_table_polygones_postaux.sql +++ b/bano/sql/table_polygones_postaux.sql @@ -1,22 +1,30 @@ +BEGIN; + DROP TABLE IF EXISTS polygones_postaux CASCADE; CREATE TABLE polygones_postaux AS -SELECT ST_Transform(way,4326) geometrie, +SELECT way AS geometrie, CASE postal_code WHEN '' THEN "addr:postcode" - ELSE postal_code END AS code_postal -FROM planet_osm_postal_code -WHERE boundary = 'postal_code' AND + ELSE postal_code + END AS code_postal +FROM planet_osm_postal_code +WHERE boundary = 'postal_code' AND "addr:postcode"||postal_code != '' ORDER BY ST_Area(way); -ALTER TABLE polygones_postaux add column id serial; -CREATE INDEX gidx_polygones_postaux ON polygones_postaux USING GIST(geometrie); +ALTER TABLE polygones_postaux ADD COLUMN id serial; + INSERT INTO polygones_postaux -SELECT ST_Transform(way,4326) geometrie, +SELECT way, CASE postal_code WHEN '' THEN "addr:postcode" - ELSE postal_code END AS code_postal -FROM planet_osm_postal_code -WHERE boundary = 'administrative' AND - "addr:postcode"||postal_code != '' + ELSE postal_code + END AS code_postal +FROM planet_osm_postal_code +WHERE boundary = 'administrative' AND + "addr:postcode"||postal_code != '' ORDER BY ST_Area(way); + +CREATE INDEX gidx_polygones_postaux ON polygones_postaux USING GIST(geometrie); + +COMMIT; \ No newline at end of file diff --git a/bano/sql/tables_export.sql b/bano/sql/tables_export.sql new file mode 100644 index 0000000..bd9e16a --- /dev/null +++ b/bano/sql/tables_export.sql @@ -0,0 +1,160 @@ +BEGIN; + +DROP TABLE IF EXISTS numeros_export CASCADE; +CREATE TABLE numeros_export +AS +WITH +cp_fantoir +AS +(SELECT fantoir, + MIN(code_postal) AS min_cp +FROM bano_adresses +GROUP BY 1), +num_norm +AS +(SELECT REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REGEXP_REPLACE(UPPER(numero), + '^0*',''),'BIS','B'),'TER','T'),'QUATER','Q'),'QUAT','Q'),' ',''),'à','-'),';',','),'"','') AS num, + * +FROM bano_adresses), +num_norm_id +AS +(SELECT fantoir||'-'||num AS id_add, + row_number() OVER (PARTITION BY fantoir||num ORDER BY CASE WHEN source = 'OSM' THEN 1 ELSE 2 END) AS rang, + * +FROM num_norm), +nom_fantoir +AS +(SELECT fantoir, + nom +FROM (SELECT fantoir, + nom, + RANK() OVER (PARTITION BY fantoir ORDER BY CASE WHEN source = 'OSM' THEN 1 ELSE 2 END, CASE nature WHEN 'lieu-dit' THEN 1 WHEN 'place' THEN 1 WHEN 'voie' THEN 2 ELSE 3 END, nom ) AS rang + FROM nom_fantoir) n +WHERE rang = 1 +GROUP BY 1,2), +resultats_multi_cp +AS +(SELECT dep, + n.code_insee, + n.fantoir, + id_add, + numero, + nf.nom AS nom_voie, + COALESCE(n.code_postal,pp.code_postal,min_cp) code_postal, + cn.libelle, + source, + lat, + lon, + n.geometrie, + RANK() OVER (PARTITION BY id_add ORDER BY pp.id) rang_postal +FROM num_norm_id n +JOIN nom_fantoir nf +USING (fantoir) +JOIN (SELECT dep, com, libelle FROM cog_commune WHERE typecom in ('ARM','COM')) cn +ON (cn.com = code_insee) +LEFT OUTER JOIN polygones_postaux pp +ON ST_Contains(pp.geometrie, n.geometrie) +LEFT OUTER JOIN cp_fantoir +USING (fantoir) +WHERE rang = 1) +SELECT * +FROM resultats_multi_cp +WHERE rang_postal = 1; + +CREATE INDEX idx_numeros_export_dep ON numeros_export(dep); + +DROP TABLE IF EXISTS numeros_export_importance CASCADE; +CREATE TABLE numeros_export_importance +AS +SELECT fantoir, + ST_Length(ST_Transform(ST_Longestline(ST_Convexhull(ST_Collect(geometrie)),ST_Convexhull(ST_Collect(geometrie))),3857)) AS longueur_max, + count(*) AS nombre_adresses +FROM numeros_export +GROUP BY fantoir; + + +DROP TABLE IF EXISTS export_voies_adresses_json CASCADE; +CREATE TABLE export_voies_adresses_json +AS +SELECT c.dep, + fantoir AS id, + ne.code_insee AS citycode, + 'street' AS type, + REPLACE(REPLACE(REGEXP_REPLACE(nom_voie,'\t',' '),'"',chr(39)),'’',chr(39)) AS name, + code_postal AS postcode, + ROUND(pn.lat::numeric,6)::float AS lat, + ROUND(pn.lon::numeric,6)::float AS lon, + nom_com AS city, + nom_dep AS departement, + nom_reg AS region, + ROUND(LOG(c.adm_weight+LOG(c.population+1)/3)::numeric*LOG(1+LOG(nombre_adresses+1)+LOG(longueur_max+1)+LOG(CASE WHEN nom_voie like 'Boulevard%' THEN 4 WHEN nom_voie LIKE 'Place%' THEN 4 WHEN nom_voie LIKE 'Espl%' THEN 4 WHEN nom_voie LIKE 'Av%' THEN 3 WHEN nom_voie LIKE 'Rue %' THEN 2 ELSE 1 END))::numeric,4)::float AS importance, + string_agg(numero||'$$$'||ROUND(ne.lat::numeric,6)::text||'$$$'||ROUND(ne.lon::numeric,6)::text,'@@@' ORDER BY numero) AS housenumbers +FROM numeros_export ne +JOIN cog_pyramide_admin AS cog +USING (code_insee) +JOIN (SELECT fantoir, + lon, + lat, + RANK() OVER (PARTITION BY fantoir ORDER BY CASE source WHEN 'OSM' THEN 1 WHEN 'BAN' THEN 3 ELSE 2 END, CASE nature WHEN 'centroide' THEN 2 ELSE 1 END) AS rang_par_fantoir + FROM bano_points_nommes + WHERE fantoir IS NOT NULL) AS pn +USING (fantoir) +JOIN infos_communes c +USING (code_insee) +JOIN numeros_export_importance +USING (fantoir) +WHERE pn.rang_par_fantoir = 1 +GROUP BY 1,2,3,4,5,6,7,8,9,10,11,12 +ORDER BY 1; + +CREATE INDEX idx_export_voies_adresses_json_dep ON export_voies_adresses_json(dep); + +DROP TABLE IF EXISTS export_voies_ld_sans_adresses_json CASCADE; +CREATE TABLE export_voies_ld_sans_adresses_json +AS +WITH +set_fantoir +AS +(SELECT fantoir FROM bano_points_nommes +EXCEPT +SELECT fantoir FROM numeros_export), +resultats_multi_cp +AS +(SELECT pn.fantoir AS id, + pn.code_insee AS citycode, + nature, + CASE + WHEN nature = 'place' THEN 'place' + WHEN nature = 'lieu-dit' THEN 'place' + ELSE 'street' + END AS type, + REPLACE(REPLACE(REGEXP_REPLACE(nom,'\t',' '),'"',chr(39)),'’',chr(39)) AS name, + code_postal AS postcode, + ROUND(pn.lat::numeric,6)::float AS lat, + ROUND(pn.lon::numeric,6)::float AS lon, + nom_com AS city, + nom_dep AS departement, + nom_reg AS region, + CASE + WHEN nature IN ('place','lieu-dit') THEN 0.05 + ELSE ROUND(LOG(c.adm_weight+LOG(c.population+1)/3)::numeric*LOG(1+LOG(CASE WHEN nom like 'Boulevard%' THEN 4 WHEN nom LIKE 'Place%' THEN 4 WHEN nom LIKE 'Espl%' THEN 4 WHEN nom LIKE 'Av%' THEN 3 WHEN nom LIKE 'Rue %' THEN 2 ELSE 1 END))::numeric,4)::float + END AS importance, + source, + RANK() OVER (PARTITION BY fantoir ORDER BY CASE source WHEN 'OSM' THEN 1 ELSE 2 END, CASE nature WHEN 'centroide' THEN 2 ELSE 1 END,pp.id) AS rang_par_fantoir, + c.dep +FROM set_fantoir +JOIN bano_points_nommes AS pn +USING (fantoir) +JOIN cog_pyramide_admin AS cog +USING (code_insee) +JOIN infos_communes c +USING (code_insee) +JOIN polygones_postaux pp +ON ST_Contains(pp.geometrie, pn.geometrie)) +SELECT * +FROM resultats_multi_cp +WHERE rang_par_fantoir = 1; + +CREATE INDEX idx_export_voies_ld_sans_adresses_json_dep ON export_voies_ld_sans_adresses_json(dep); + +COMMIT; \ No newline at end of file diff --git a/bano/sql/type_highway_insee.sql b/bano/sql/type_highway_insee.sql deleted file mode 100644 index 845c824..0000000 --- a/bano/sql/type_highway_insee.sql +++ /dev/null @@ -1,34 +0,0 @@ -INSERT INTO type_highway_insee -SELECT pl.name, - pl.highway, - p."ref:INSEE" -FROM planet_osm_polygon p -JOIN planet_osm_point pl - ON pl.way && p.way AND - ST_Intersects(pl.way, p.way) - WHERE p."ref:INSEE" = '__com__' AND - pl.highway != '' AND - pl.name != '' -UNION -SELECT l.name, - l.highway, - p."ref:INSEE" - FROM planet_osm_polygon p - JOIN planet_osm_line l - ON ST_Intersects(l.way, p.way) - WHERE p."ref:INSEE" = '__com__' AND - l.highway != '' AND - l.name != '' -UNION -SELECT pl.name, - pl.highway, - p."ref:INSEE" - FROM planet_osm_polygon p - JOIN planet_osm_polygon pl - ON pl.way && p.way AND - ST_Intersects(pl.way, p.way) - WHERE p."ref:INSEE" = '__com__' AND - pl.highway != '' AND - pl.name != '' ---ORDER BY 1 -; diff --git a/bano/sql/update_table_bano_stats_communales.sql b/bano/sql/update_table_bano_stats_communales.sql new file mode 100644 index 0000000..126b424 --- /dev/null +++ b/bano/sql/update_table_bano_stats_communales.sql @@ -0,0 +1,21 @@ +INSERT INTO bano_stats_communales_cumul +SELECT * +FROM bano_stats_communales +WHERE code_insee = '__code_insee__'; +DELETE +FROM bano_stats_communales +WHERE code_insee = '__code_insee__'; +INSERT INTO bano_stats_communales (code_insee, + nb_adresses_osm, + nb_adresses_ban, + nb_nom_osm, + nb_nom_ban, + nb_nom_cadastre, + nb_nom_topo) + VALUES ('__code_insee__', + __nb_adresses_osm__, + __nb_adresses_ban__, + __nb_noms_osm__, + __nb_noms_ban__, + __nb_noms_cadastre__, + __nb_noms_topo__); diff --git a/sql/update_table_infos_communes.sql b/bano/sql/update_table_infos_communes.sql similarity index 62% rename from sql/update_table_infos_communes.sql rename to bano/sql/update_table_infos_communes.sql index 12bf000..0ec5cfd 100644 --- a/sql/update_table_infos_communes.sql +++ b/bano/sql/update_table_infos_communes.sql @@ -1,25 +1,9 @@ -CREATE TABLE IF NOT EXISTS infos_communes ( - dep character varying(3), - insee_com character(5), - name text, - adm_weight integer, - population integer, - population_milliers numeric, - type text, - lon numeric, - lat numeric, - geometrie geometry(Point, 4326) -); - -CREATE INDEX IF NOT EXISTS idx_infos_communes_insee ON infos_communes(insee_com); -CREATE INDEX IF NOT EXISTS gidx_infos_communes ON infos_communes USING GIST(geometrie); - CREATE TEMP TABLE tmp_infos_communes AS WITH statut AS -(SELECT com insee_com, 1 AS statut FROM cog_commune +(SELECT com code_insee, 1 AS statut FROM cog_commune UNION ALL SELECT burcentral, 2 AS statut FROM cog_canton UNION ALL @@ -30,14 +14,14 @@ UNION ALL SELECT cheflieu, 5 AS statut FROM cog_region), adm_weight AS -(SELECT insee_com, max(statut) AS adm_weight +(SELECT code_insee, max(statut) AS adm_weight FROM statut GROUP BY 1), pop AS (SELECT osm_id, name, - "ref:INSEE" insee_com, + "ref:INSEE" code_insee, COALESCE(population_rel,population_member,0) AS population, RANK() OVER(PARTITION BY "ref:INSEE" ORDER BY admin_level) rang FROM planet_osm_communes_statut @@ -47,13 +31,13 @@ WHERE admin_level in (8,9) AND pp AS (SELECT osm_id, - ROUND(ST_X(ST_Transform(way,4326))::numeric,6) lon, - ROUND(ST_Y(ST_Transform(way,4326))::numeric,6) lat, - ST_Transform(way,4326) geometrie + ROUND(ST_X(way)::numeric,6) lon, + ROUND(ST_Y(way)::numeric,6) lat, + way AS geometrie FROM planet_osm_point pp WHERE place != '') SELECT cc.dep, - adm_weight.insee_com, + adm_weight.code_insee, pop.name, adm_weight.adm_weight, pop.population, @@ -68,9 +52,9 @@ SELECT cc.dep, pp.geometrie FROM adm_weight JOIN cog_commune cc -ON cc.com = insee_com +ON cc.com = code_insee LEFT OUTER JOIN pop -USING (insee_com) +USING (code_insee) JOIN pp USING (osm_id) WHERE pop.rang = 1 AND diff --git a/bano/sql/update_table_rep_b_as_bis.sql b/bano/sql/update_table_rep_b_as_bis.sql index f05bb2c..a813dd8 100644 --- a/bano/sql/update_table_rep_b_as_bis.sql +++ b/bano/sql/update_table_rep_b_as_bis.sql @@ -1,29 +1,28 @@ DROP TABLE IF EXISTS rep_b_as_bis CASCADE; CREATE TABLE rep_b_as_bis AS -SELECT id_fantoir, +SELECT fantoir, numero -FROM ban_odbl +FROM ban WHERE rep = 'b' EXCEPT -(SELECT id_fantoir, - numero -FROM ban_odbl +(SELECT fantoir, + numero +FROM ban WHERE rep = 'a' UNION -SELECT id_fantoir, +SELECT fantoir, numero -FROM ban_odbl +FROM ban WHERE rep = 'c' UNION -SELECT id_fantoir, +SELECT fantoir, numero -FROM ban_odbl +FROM ban WHERE rep = 'd'); -COMMIT; \ No newline at end of file diff --git a/config b/config index 37515e0..4cf631e 100644 --- a/config +++ b/config @@ -11,8 +11,26 @@ export CADASTRE_CACHE_DIR=$DATA_DIR/cadastre_cache export BAL_CACHE_DIR=$DATA_DIR/bal_cache export EXPIRE_TILES_DIR=$DATA_DIR/expire_tiles export CSV_DIR=$DATA_DIR/csv_insee +export COG_DIR=$DATA_DIR/cog export EXPORT_SAS_DIR=/data/sas_web export EXPORT_WEB_DIR=/data/work/bano.openstreetmap.fr/www/web/data/ export TMP_DIR=/data/tmp export SCHEMA_CIBLE=public -export LANG=fr_FR.UTF-8 +export LANG=${LANG:-fr_FR.UTF-8} + + +POSTGRES_USER=${POSTGRES_USER:-cadastre} +POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-} +POSTGRES_HOST=${POSTGRES_HOST:-localhost} +POSTGRES_PORT=${POSTGRES_PORT:-5432} + +POSTGRES_DB_BANO=${POSTGRES_DB:-bano} +POSTGRES_DB_CADASTRE=${POSTGRES_DB:-cadastre} + +export PGCON_BANO="$POSTGRES_USER${POSTGRES_PASSWORD:+:$POSTGRES_PASSWORD}@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB_BANO" +export pgsql_BANO="psql postgresql://${PGCON_BANO} -v ON_ERROR_STOP=1" +export PG_BANO="host='$POSTGRES_HOST' port=$POSTGRES_PORT dbname='$POSTGRES_DB_BANO' user='$POSTGRES_USER' ${POSTGRES_PASSWORD:+password='$POSTGRES_PASSWORD'}" + +export PGCON_CADASTRE="$POSTGRES_USER${POSTGRES_PASSWORD:+:$POSTGRES_PASSWORD}@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB_CADASTRE" +export pgsql_CADASTRE="psql postgresql://${PGCON_CADASTRE} -v ON_ERROR_STOP=1" +export PG_CADASTRE="host='$POSTGRES_HOST' port=$POSTGRES_PORT dbname='$POSTGRES_DB_CADASTRE' user='$POSTGRES_USER' ${POSTGRES_PASSWORD:+password='$POSTGRES_PASSWORD'}" diff --git a/copy_table_from_osm_to_cadastre.sh b/copy_table_from_osm_to_cadastre.sh deleted file mode 100755 index 16e757c..0000000 --- a/copy_table_from_osm_to_cadastre.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -source $SCRIPT_DIR/config - -set -e - -cd $TMP_DIR -pg_dump -t ${1} -U cadastre osm > ${1}_dump.sql -psql -d cadastre -U cadastre -c "DROP TABLE IF EXISTS ${1} CASCADE;" -psql -d cadastre -U cadastre -f ${1}_dump.sql -rm -f ${1}_dump.sql diff --git a/create_base.sh b/create_base.sh index 443ccd8..7640af3 100755 --- a/create_base.sh +++ b/create_base.sh @@ -1,8 +1,6 @@ -sudo -u postgres -s +#!/bin/bash + +set -e + createuser -s -d cadastre -createdb -E UTF-8 -T template0 -O cadastre cadastre -exit -psql -d cadastre -U cadastre -f sql/create_base.sql -psql -d cadastre -U cadastre -f sql/create_table_base_bano.sql -psql -d osm -U cadastre -f sql/create_base.sql -psql -d osm -U cadastre -f sql/create_table_base_osm.sql +createdb -E UTF-8 -T template0 -O cadastre bano diff --git a/cron_bano.sh b/cron_bano.sh index 40c4454..c1eba46 100755 --- a/cron_bano.sh +++ b/cron_bano.sh @@ -1,55 +1,52 @@ #!/bin/bash -source /data/work/vdct/bano_venv37/bin/activate +set -e -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -cd $SCRIPT_DIR +echo 'debut du cron BANO' source config ./arborescence.sh -pip install -e . -# Sources -cat deplist.txt | parallel -j 4 export LANG=fr_FR.UTF-8\; bano download_bal CADASTRE {1} -cat deplist.txt | parallel -j 4 export LANG=fr_FR.UTF-8\; bano download_bal BAL {1} -cat deplist.txt | parallel -j 4 export LANG=fr_FR.UTF-8\; bano download_ban {1} -cat deplist.txt | parallel -j 4 export LANG=fr_FR.UTF-8\; bano download_cadastre lieux_dits {1} -bano update_code_cadastre -bano update_bis_table -./copy_table_from_osm_to_cadastre.sh rep_b_as_bis +bano="python -m bano" +# Sources +cat deplist.txt | parallel -j 4 export LANG=$LANG\; $bano charge_ban {1} +cat deplist.txt | parallel -j 4 export LANG=$LANG\; $bano charge_ld_cadastre {1} +$bano update_bis_table -# BANO -bano update_insee_lists -cat insee_osm.csv | parallel -j 4 export LANG=fr_FR.UTF-8\; bano process_commune OSM --code_insee {1} -cat insee_locales.csv | parallel -j 4 export LANG=fr_FR.UTF-8\; bano process_commune BAL --code_insee {1} -cat deplist.txt | parallel -j 4 export LANG=fr_FR.UTF-8\; bano process_commune BAN --depts {1} -cat deplist.txt | parallel -j 4 export LANG=fr_FR.UTF-8\; bano process_commune_lieux-dits --depts {1} +echo 'sources ok' # Mise à jour quotidienne dans la base cadastre des couches des polygones postaux d'OSM et des statuts admin de communes en vue des exports -./copy_table_from_osm_to_cadastre.sh planet_osm_postal_code -./copy_table_from_osm_to_cadastre.sh infos_communes +$bano update_table_communes +$bano update_infos_communes + +# $pgsql_BANO -f sql/create_table_polygones_postaux.sql +# $pgsql_CADASTRE -f sql/post_copie_ban.sql +echo 'màj polygones ok' + +# BANO +cat deplist.txt | parallel -j 4 export LANG=$LANG\; $bano rapprochement --dept {1} -psql -d osm -U cadastre -f sql/create_table_polygones_communes.sql -./copy_table_from_osm_to_cadastre.sh polygones_insee -./copy_table_from_osm_to_cadastre.sh polygones_insee_geo +echo 'rapprochement ok' -psql -d osm -U cadastre -f sql/create_table_polygones_postaux.sql -./copy_table_from_osm_to_cadastre.sh polygones_postaux -./copy_table_from_osm_to_cadastre.sh ban_odbl -psql -d cadastre -U cadastre -f sql/post_copie_ban_odbl.sql +echo 'preparation export' +$bano prepare_export +echo 'preparation export finie' # exports -cat deplist.txt | parallel -j 4 bano export {1} +echo 'export' +cat deplist.txt | parallel -j 4 $bano export {1} +echo 'export fini' # copie+zip dans le dossier web -cat deplist.txt | parallel -j 4 bano publish {1} -bano publish_aggregate +cat deplist.txt | parallel -j 4 $bano publish {1} +$bano publish_aggregate # ménage PostgreSQL -psql -d cadastre -U cadastre -c "VACUUM cumul_adresses;" -psql -d cadastre -U cadastre -c "VACUUM cumul_voies;" -psql -d cadastre -U cadastre -c "VACUUM cumul_places;" -psql -d cadastre -U cadastre -c "GRANT SELECT ON ALL TABLES IN SCHEMA PUBLIC TO PUBLIC"; \ No newline at end of file +$pgsql_BANO -c "VACUUM bano_adresses;" +$pgsql_BANO -c "VACUUM bano_points_nommes;" +$pgsql_BANO -c "VACUUM nom_fantoir;" +$pgsql_BANO -c "GRANT SELECT ON ALL TABLES IN SCHEMA PUBLIC TO PUBLIC"; +echo 'fin du cron BANO' diff --git a/cron_osm.sh b/cron_osm.sh index 6eccbce..f35c6f0 100755 --- a/cron_osm.sh +++ b/cron_osm.sh @@ -1,26 +1,37 @@ #!/bin/bash -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -source $SCRIPT_DIR/config +set -e -lockfile=${SCRIPT_DIR}/imposm.lock +source /data/project/bano_v3/venv_v3/bin/activate + +pip install -e . + +source config + +lockfile=imposm.lock + +echo `date` +echo debut if test -f ${lockfile} then - echo `date`" : Process deja en cours" >> $SCRIPT_DIR/cron.log - exit 0 + diff_age=$((`date +%s` - `stat -c %Y $lockfile`)) + if [ $diff_age -gt 7200 ];then + echo "Effacement du lock" + rm ${lockfile} + else + echo `date`" : Process deja en cours" + exit 1 + fi fi -echo `date` > $SCRIPT_DIR/cron.log -echo debut >> $SCRIPT_DIR/cron.log touch ${lockfile} -osmosis --rri workingDirectory=/data/download --wxc /data/download/changes.osc.gz -imposm diff -mapping $SCRIPT_DIR/bano.yml -cachedir /data/bano_imposm_cache -dbschema-production public -diffdir /data/bano_imposm_diff -connection 'postgis://cadastre@localhost/osm'?prefix=NONE -expiretiles-dir $EXPIRE_TILES_DIR -expiretiles-zoom 16 /data/download/changes.osc.gz -$SCRIPT_DIR/update_table_infos_communes.sh +osmosis --rri workingDirectory=${DOWNLOAD_DIR} --wxc ${DOWNLOAD_DIR}/changes.osc.gz +imposm diff -config imposm.config -dbschema-production osm ${DOWNLOAD_DIR}/changes.osc.gz rm ${lockfile} -echo `date` >> $SCRIPT_DIR/cron.log -echo fin >> $SCRIPT_DIR/cron.log +echo `date` +echo fin diff --git a/detection_suffixe.py b/detection_suffixe.py deleted file mode 100755 index 16118b6..0000000 --- a/detection_suffixe.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python -# coding: UTF-8 - -import suffixe_depuis_noms_de_voies as a -import sys -import os,os.path -import subprocess - -if len(sys.argv) != 2: - print('Mauvais nombre d\'arguments') - print('USAGE : ./detection_suffixe.py ') - os._exit(0) - -clause_where = '' -if sys.argv[1].upper() != 'FRANCE': - num_dept_cadastre = ('000'+sys.argv[1])[-3:] - clause_where = 'AND c.cadastre_dept = \'{:s}\''.format(num_dept_cadastre) - -pgc = a.get_pgc() -str_query = 'SELECT DISTINCT c.insee_com,c.nom_com,c.cadastre_dept FROM code_cadastre c WHERE c.format_cadastre = \'VECT\' {:s} ORDER BY 3,2;'.format(clause_where) -cur = pgc.cursor() -cur.execute(str_query) -for c in cur: - print(c[0]+' '+c[1]) - a.main(['','{:s}'.format(c[0])]) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..79aa7de --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,34 @@ +version: '3.3' + +services: + postgres: + image: postgis/postgis:16-3.4-alpine + environment: + - POSTGRES_DB=postgres + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - POSTGRES_HOST_AUTH_METHOD=trust + ports: + - "127.0.0.1:5432:5432" + + tools: + build: . + volumes: + - .:/opt/bano + - ./data:/data + environment: + # Postgres + - POSTGRES_DB=postgres + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - POSTGRES_HOST=postgres + - POSTGRES_PORT=5432 + # BANO + - SRC_DIR=/opt/bano + - DATA_DIR=/data/bano + - DOWNLOAD_DIR=/data/download + - IMPOSM_CACHE_DIR=/data/bano_imposm_cache + - EXPORT_SAS_DIR=/data/export_sas + - EXPORT_WEB_DIR=/data/export + - TMP_DIR=/data/tmp + - LANG=C diff --git a/imposm.config b/imposm.config new file mode 100644 index 0000000..cf4d5c4 --- /dev/null +++ b/imposm.config @@ -0,0 +1,5 @@ +{ + "diffdir": "/data/download_v3", + "mapping": "bano.yml", + "srid":4326 +} diff --git a/init_base.sh b/init_base.sh new file mode 100755 index 0000000..2d224b0 --- /dev/null +++ b/init_base.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -e + +$pgsql_BANO -f bano/sql/create_base.sql diff --git a/load_COG.sh b/load_COG.sh deleted file mode 100755 index edf07c7..0000000 --- a/load_COG.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -source config -cd $DATA_DIR -mkdir -p cog -cd cog -rm *.csv -wget -NS https://www.insee.fr/fr/statistiques/fichier/5057840/cog_ensemble_2021_csv.zip -unzip -o -j cog_ensemble_2021_csv.zip - -CSV_COMMUNE=commune2021.csv -CSV_CANTON=canton2021.csv -CSV_ARRONDISSEMENT=arrondissement2021.csv -CSV_DEPARTEMENT=departement2021.csv -CSV_REGION=region2021.csv - -for d in osm cadastre -do - psql -d ${d} -U cadastre -f $BANO_DIR/sql/create_table_cog.sql - cat ${CSV_COMMUNE}| psql -d ${d} -U cadastre -c "TRUNCATE TABLE cog_commune; COPY cog_commune FROM stdin WITH csv header delimiter ',';" - cat ${CSV_CANTON}| psql -d ${d} -U cadastre -c "TRUNCATE TABLE cog_canton; COPY cog_canton FROM stdin WITH csv header delimiter ',';" - cat ${CSV_ARRONDISSEMENT}|psql -d ${d} -U cadastre -c "TRUNCATE TABLE cog_arrondissement;COPY cog_arrondissement FROM stdin WITH csv header delimiter ',';" - cat ${CSV_DEPARTEMENT}| psql -d ${d} -U cadastre -c "TRUNCATE TABLE cog_departement; COPY cog_departement FROM stdin WITH csv header delimiter ',';" - cat ${CSV_REGION}| psql -d ${d} -U cadastre -c "TRUNCATE TABLE cog_region; COPY cog_region FROM stdin WITH csv header delimiter ',';" -done diff --git a/load_code_cadastre.py b/load_code_cadastre.py deleted file mode 100755 index 9ba15ce..0000000 --- a/load_code_cadastre.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python - -import urllib.request -from http.cookiejar import CookieJar -import time -import os -from bs4 import BeautifulSoup - -a_depts = [] -for n in list(range(1, 20))+['2A', '2B']+list(range(21, 96))+list(range(971, 975)) + [976]: - a_depts.append(str(n).rjust(2, '0')) -a_0_depts = [a.rjust(3, '0') for a in a_depts] -f_output = open('code_cadastre.csv', 'w') -cookiejar = CookieJar() -opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookiejar)) -request = urllib.request.Request( - 'https://www.cadastre.gouv.fr/scpc/rechercherPlan.do') -response = opener.open(request) -token = response.read().decode().split('CSRF_TOKEN=')[1].split('"')[0] - -for i, d in enumerate(a_0_depts): - fname = 'dept_'+d+'.xml' - time.sleep(1) - opener.open(request) - request2 = urllib.request.Request( - "https://www.cadastre.gouv.fr/scpc/listerCommune.do?CSRF_TOKEN={:s}&codeDepartement={:s}&libelle=&keepVolatileSession=&offset=5000".format(token, d)) - response = opener.open(request2) - rep = response.read() - fw = open(fname, 'wb') - fw.write(rep) - fw.close() - fr = BeautifulSoup(open(fname, 'rb').read(), "lxml") - - for e in fr.find_all(attrs={"class": "parcelles"}): - y = e.find(title="Ajouter au panier") - if y is None: - continue - - # y.get('onclick') structure: "ajoutArticle('CL098','VECT','COMU');" - split = y.get('onclick').split("'") - code_commune = split[1] - format_type = split[3] - - # e.strong.string structure: "COBONNE (26400) " - commune_cp = e.strong.string - nom_commune = commune_cp[:-9] - cp = commune_cp[-7:-2] - # try: - f_output.write('{:s},{:s},{:s},{:s},{:s},{:s}\n'.format( - a_depts[i], d, nom_commune, cp, code_commune, format_type)) - # except: - # import pdb; pdb.Pdb().set_trace() - f_output.flush() - os.remove(fname) -f_output.close() diff --git a/load_code_cadastre.sh b/load_code_cadastre.sh deleted file mode 100755 index e584582..0000000 --- a/load_code_cadastre.sh +++ /dev/null @@ -1,2 +0,0 @@ -python load_code_cadastre.py -psql -d cadastre -U cadastre -f sql/load_code_cadastre.sql diff --git a/load_codes_postaux.sh b/load_codes_postaux.sh deleted file mode 100755 index e382070..0000000 --- a/load_codes_postaux.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -source config -cd $DATA_DIR -mkdir -p codes_postaux -cd codes_postaux -rm *csv* -wget https://datanova.legroupe.laposte.fr/explore/dataset/laposte_hexasmal/download/?format=csv&timezone=Europe/Berlin&use_labels_for_header=true&csv_separator=%3B -mv *csv* codes_postaux.csv -psql -d osm -U cadastre -f $BANO_DIR/sql/create_and_load_codes_postaux.sql -psql -d cadastre -U cadastre -f $BANO_DIR/sql/create_and_load_codes_postaux.sql diff --git a/load_cumul.py b/load_cumul.py deleted file mode 100755 index 1382b6f..0000000 --- a/load_cumul.py +++ /dev/null @@ -1,161 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import addr_2_db as a -import log_2_file as e -import outils_de_gestion as o -import sys -import time -import os,os.path -import codecs - -os.umask(0000) - -# Wrap sys.stdout with a writer that knows how to handle encoding -# Unicode data. -# wrapped_stdout = codecs.getwriter('UTF-8')(sys.stdout) -# Replace sys.stdout with a writer -# sys.stdout = wrapped_stdout - - -def get_code_dept_from_insee(insee): - code_dept = insee[0:2] - if insee[0:2] == '97': - code_dept = insee[0:3] - return code_dept -# def get_geom_suffixes(dept): -# fq = open('sql/geom_suffixes_insee.sql','r') -# str_query = fq.read().replace("='__com__'"," LIKE '{:s}'".format(get_sql_like_dept_string(dept))) -# cursor_bano_ro = pgc.cursor() -# cursor_bano_ro.execute(str_query) -# a_queries = [] -# for l in cursor_bano_ro : -# a_queries.append("SELECT ST_PolygonFromText('{:s}',3857) as geom,'{:s}'::text suffixe".format(l[0],l[1].replace('\'','\'\''))) -# cursor_bano_ro.close() -# return ' UNION '.join(a_queries) -# def warning_message_no_suffixe(dept,etape): -# print("Pas de commune à suffixe dans le {}. Etape {} ignorée".format(dept,etape.upper())) - -def get_data_by_dept_from_pg(query_name,dept): - current_time = round(time.time()) - etape_dept = 'cache_dept_'+query_name -# Cache gardé 1h - if o.age_etape_dept(etape_dept,dept) > 3600 : - print(u'Mise à jour du cache "{:s}"'.format(query_name.upper())) - batch_id = o.batch_start_log(source,etape_dept,dept) - fq = open('sql/{:s}.sql'.format(query_name),'r') - str_query = fq.read().replace(" = '__com__'"," LIKE '{:s}'".format(get_sql_like_dept_string(dept))) - # if suffixe_data : - # str_query = str_query.replace('__suffixe_data__',suffixe_data) - # print(str_query) - fq.close() - cur_osm_ro = pgcl.cursor() - cur_osm_ro.execute(str_query) - - list_output = list() - for lt in cur_osm_ro : - list_values = list() - for item in list(lt): - if item == None: - list_values.append('null') - elif type(item) == str : - list_values.append("'{}'".format(item.replace("'","''").replace('"',''))) - elif type(item) == list : - if (len(item)) > 0 : - list_values.append("hstore(ARRAY{})".format(str([s.replace("'","''").replace('"','') for s in item]))) - else : - list_values.append('null') - else : - list_values.append(str(item)) - list_values.append(str(current_time)) - - str_values = ','.join(list_values).replace('"',"'") - list_output.append(str_values) - cur_osm_ro.close() - cur_cache_rw = pgcl.cursor() - str_query = "DELETE FROM {} WHERE insee_com LIKE '{}';".format(query_name,get_sql_like_dept_string(dept)) - cur_cache_rw.execute(str_query) - if len(list_output) > 0 : - str_query = "INSERT INTO {} VALUES ({});COMMIT;".format(query_name,'),('.join(list_output)) - strq = open('./query.txt','w') - strq.write(str_query) - strq.close() - cur_cache_rw.execute(str_query) - cur_cache_rw.close() - o.batch_end_log(0,batch_id) - -str_usage = 'USAGE : python load_cumul.py ' -if len(sys.argv) != 3: - print("Mauvais nombre d'arguments") - print(str_usage) - os._exit(0) -source = sys.argv[2].upper() -if source != 'OSM' and source != 'CADASTRE' and source != 'BAL': - print("La source doit etre OSM, CADASTRE ou BAL") - print(str_usage) - os._exit(0) - -clause_vecteur = '' -if source == 'CADASTRE': - clause_vecteur = " AND format_cadastre = 'VECT' " - -pgc = a.get_pgc() -pgcl = a.get_pgc_osm() - -if sys.argv[1].upper() == 'FRANCE': - loop_query = 'SELECT DISTINCT dept FROM code_cadastre ORDER BY 1;' -else: - num_dept_cadastre = sys.argv[1] - loop_query = "SELECT '{:s}';".format(num_dept_cadastre) - -cur_loop = pgc.cursor() -cur_loop.execute(loop_query) -for c_loop in cur_loop: - num_dept_cadastre = c_loop[0] - global f_log - f_log = e.start_log_to_file(source,os.path.basename(sys.argv[0]).split('.')[0],num_dept_cadastre) - print('## Département {:s}'.format(num_dept_cadastre)) - # geom_suffixe = get_geom_suffixes(num_dept_cadastre) - if source == 'OSM': - get_data_by_dept_from_pg('hsnr_insee',num_dept_cadastre) - # if geom_suffixe : - get_data_by_dept_from_pg('hsnr_suffixe_insee',num_dept_cadastre) - # else : - # warning_message_no_suffixe(num_dept_cadastre,'hsnr_suffixe_insee') - get_data_by_dept_from_pg('hsnr_bbox_insee',num_dept_cadastre) - get_data_by_dept_from_pg('point_par_rue_insee',num_dept_cadastre) - get_data_by_dept_from_pg('point_par_rue_complement_insee',num_dept_cadastre) - get_data_by_dept_from_pg('type_highway_insee',num_dept_cadastre) - # os._exit(0) - - get_data_by_dept_from_pg('highway_insee',num_dept_cadastre) - # if geom_suffixe : - get_data_by_dept_from_pg('highway_suffixe_insee',num_dept_cadastre) - # else : - # warning_message_no_suffixe(num_dept_cadastre,'highway_suffixe_insee') - get_data_by_dept_from_pg('highway_bbox_insee',num_dept_cadastre) - get_data_by_dept_from_pg('highway_relation_insee',num_dept_cadastre) - # if geom_suffixe : - get_data_by_dept_from_pg('highway_relation_suffixe_insee',num_dept_cadastre) - # else : - # warning_message_no_suffixe(num_dept_cadastre,'highway_relation_suffixe_insee') - get_data_by_dept_from_pg('highway_relation_bbox_insee',num_dept_cadastre) - # os._exit(0) - - clause_vecteur = '' - if source == 'CADASTRE': - clause_vecteur = " AND format_cadastre = 'VECT' " - str_query = "SELECT insee_com,nom_com FROM code_cadastre WHERE dept = '{:s}' {:s} ORDER BY 2;".format(num_dept_cadastre,clause_vecteur) - cur = pgc.cursor() - cur.execute(str_query) - for c in cur: - print('{:s} - {:s}'.format(c[0],c[1])) - try: - a.main(['',c[0],source,True]) - except : - e.write_log_to_file(f_log,'Commune : {:s}\n'.format(c[1])) - e.write_log_to_file(f_log,str(sys.exc_info()[0])) - e.write_log_to_file(f_log,str(sys.exc_info()[1])) - e.write_sep_to_file(f_log) - e.end_log_to_file(f_log,True) - diff --git a/load_cumul_place.py b/load_cumul_place.py deleted file mode 100755 index 572ceae..0000000 --- a/load_cumul_place.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import place_2_db as a -import log_2_file as e -import outils_de_gestion as o -import sys -import os,os.path - -# Wrap sys.stdout with a writer that knows how to handle encoding -# Unicode data. -# wrapped_stdout = codecs.getwriter('UTF-8')(sys.stdout) -# Replace sys.stdout with a writer -# sys.stdout = wrapped_stdout - -def get_sql_like_dept_string(dept): - return (dept+'___')[0:5] - -def get_code_dept_from_insee(insee): - code_dept = insee[0:2] - if insee[0:2] == '97': - code_dept = insee[0:3] - return code_dept -def get_data_by_dept_from_pg(query_name,dept): - if dept[0:1] == '0': - dept = dept[1:] - etape_dept = 'cache_dept_'+query_name -# Cache gardé 1h - if o.age_etape_dept(etape_dept,dept) > 1 : #3600: - print(u'Mise à jour du cache "{:s}"'.format(query_name.upper())) - batch_id = o.batch_start_log('',etape_dept,dept) - fq = open('sql/{:s}.sql'.format(query_name),'r') - str_query = fq.read().replace(" = '__com__'"," LIKE '{:s}'".format(get_sql_like_dept_string(dept))) - fq.close() - pgc = a.get_pgc_osm() - cur = pgc.cursor() - cur.execute(str_query) - o.batch_end_log(0,batch_id) - -str_usage = 'USAGE : python load_cumul_place.py ' -if len(sys.argv) != 2: - print('Mauvais nombre d\'arguments') - print(str_usage) - os._exit(0) - -if sys.argv[1].upper() == 'FRANCE': - loop_query = 'SELECT DISTINCT dept FROM code_cadastre ORDER BY 1;' -else: - num_dept_cadastre = sys.argv[1] - loop_query = "SELECT '{:s}';".format(num_dept_cadastre) - -pgc = a.get_pgc() -cur_loop = pgc.cursor() -cur_loop.execute(loop_query) -for c_loop in cur_loop: - num_dept_cadastre = c_loop[0] - global f_log - f_log = e.start_log_to_file('',os.path.basename(sys.argv[0]).split('.')[0],num_dept_cadastre) - print('## Département {:s}'.format(num_dept_cadastre)) - - get_data_by_dept_from_pg('place_insee',num_dept_cadastre) - get_data_by_dept_from_pg('cadastre_2_place',num_dept_cadastre) - - str_query = "SELECT insee_com,nom_com FROM code_cadastre WHERE dept = '{:s}' ORDER BY 2;".format(num_dept_cadastre) - cur = pgc.cursor() - cur.execute(str_query) - for c in cur: - print('{:s} - {:s}'.format(c[0],c[1])) - try: - a.main(['',c[0],True]) - except : - e.write_log_to_file(f_log,"Commune : {:s}\n".format(c[1])) - e.write_log_to_file(f_log,str(sys.exc_info()[0])) - e.write_log_to_file(f_log,str(sys.exc_info()[1])) - e.write_sep_to_file(f_log) - e.end_log_to_file(f_log,True) - diff --git a/load_fantoir.sh b/load_fantoir.sh deleted file mode 100755 index 17fd538..0000000 --- a/load_fantoir.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -source config -cd $DATA_DIR -mkdir -p fantoir -cd fantoir -rm -rf FANTOIR* && rm *.zip -wget https://data.economie.gouv.fr/api/datasets/1.0/fichier-fantoir-des-voies-et-lieux-dits/attachments/fichier_national_fantoir_situation_avril_2022_zip/ -O fantoir.zip -unzip fantoir.zip -mv FANTOIR* fantoir.txt -psql -d osm -U cadastre -f $BANO_DIR/sql/load_fantoir.sql -psql -d cadastre -U cadastre -f $BANO_DIR/sql/load_fantoir.sql diff --git a/load_osm_france_db.sh b/load_osm_france_db.sh index 527fd46..1b3815b 100755 --- a/load_osm_france_db.sh +++ b/load_osm_france_db.sh @@ -1,24 +1,35 @@ #!/bin/bash -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -source $SCRIPT_DIR/config +set -e -lockfile=${SCRIPT_DIR}/imposm.lock +source config + +PBF_URL=${1:-http://download.openstreetmap.fr/extracts/merge/france_metro_dom_com_nc.osm.pbf} +PBF_FILE=$(basename "$PBF_URL") + +lockfile=${DATA_DIR}/imposm.lock if test -f ${lockfile} then - echo `date`" : Process deja en cours" >> $SCRIPT_DIR/cron.log - exit 0 + echo `date`" : Process deja en cours" + exit 1 fi touch ${lockfile} -cd $DOWNLOAD_DIR -wget -NS http://download.openstreetmap.fr/extracts/merge/france_metro_dom_com_nc.osm.pbf -imposm import -mapping $BANO_DIR/bano.yml -read $DOWNLOAD_DIR/france_metro_dom_com_nc.osm.pbf -overwritecache -cachedir $IMPOSM_CACHE_DIR -dbschema-import public -diff -diffdir $DOWNLOAD_DIR -imposm import -mapping $BANO_DIR/bano.yml -write -connection 'postgis://cadastre@localhost/osm'?prefix=NONE -cachedir $IMPOSM_CACHE_DIR -dbschema-import public -diff -diffdir $DOWNLOAD_DIR +mkdir -p $DOWNLOAD_DIR +wget --directory-prefix=$DOWNLOAD_DIR -NS $PBF_URL +imposm import \ + -config imposm.config \ + -read $DOWNLOAD_DIR/$PBF_FILE \ + -overwritecache \ + -cachedir $IMPOSM_CACHE_DIR \ + -diff \ + -write \ + -connection postgis://$PGCON_BANO?prefix=NONE \ + -dbschema-import osm -psql -d osm -U cadastre -f $BANO_DIR/sql/finalisation.sql +$pgsql_BANO -f $SCRIPT_DIR/sql/finalisation.sql cp $DOWNLOAD_DIR/last.state.txt $DOWNLOAD_DIR/state.txt rm ${lockfile} diff --git a/load_population_INSEE.sh b/load_population_INSEE.sh deleted file mode 100755 index 1db15b2..0000000 --- a/load_population_INSEE.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -# Après téléchargement manuel d'un CSV à https://statistiques-locales.insee.fr/#c=indicator&i=pop_legales.popmun&s=2017&view=map1 - -CSV=/data/project/bano_data/insee/population_communale_2017.csv - -for d in osm cadastre -do - psql -d ${d} -U cadastre -f $BANO_DIR/sql/create_table_population_insee.sql - cat ${CSV}| grep -v 'N/A'|psql -d ${d} -U cadastre -c "TRUNCATE TABLE population_insee; COPY population_insee FROM stdin WITH csv delimiter ';';" -done diff --git a/munin/bano_main b/munin/bano_main deleted file mode 100755 index 8afc069..0000000 --- a/munin/bano_main +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh - -# script pour graphes munin de la BANO sur le décompte global des adresses - -case $1 in - config) - cat <<'EOM' -graph_title BANO - Contenu -graph_vlabel BANO -graph_category bano -cumul.label Cumul toutes sources -dedup.label Sortie dedupliquee BANO .csv/.shp -cadastre.label Source cadastre -match.label Source cadastre rapprochee avec OSM -nomatch.label Source cadastre non rapprochee avec OSM -osm.label Source OSM -opendata.label Source opendata -EOM - exit 0;; -esac - -psql cadastre -c "select concat('cumul.value ',count(*)::text) from cumul_adresses;" -At -echo "dedup.value `head /data/work/bano.openstreetmap.fr/bano-data/comptage.txt -n 1`" -#psql cadastre -c "select concat('dedup.value ',count(*)::text) from (select fantoir,num from (select *,replace(replace(replace(replace(replace(replace(replace(regexp_replace(upper(numero),'^0*',''),'BIS','B'),'TER','T'),'QUATER','Q'),'QUAT','Q'),' ',''),'à','-'),';',',') as num from cumul_adresses where fantoir is not null and source in ('CADASTRE','OSM')) as a group by fantoir,num) as f;" -At -psql cadastre -c "select concat('cadastre.value ',count(*)::text) from cumul_adresses where source='CADASTRE';" -At -psql cadastre -c "select concat('match.value ',count(*)::text) from cumul_adresses where source='CADASTRE' and voie_osm != '';" -At -psql cadastre -c "select concat('nomatch.value ',count(*)::text) from cumul_adresses where source='CADASTRE' and (voie_osm = '' or voie_osm is null);" -At -psql cadastre -c "select concat('osm.value ',count(*)::text) from cumul_adresses where source='OSM';" -At -psql cadastre -c "select concat('opendata.value ',count(*)::text) from cumul_adresses where source like 'OD%';" -At diff --git a/munin/bano_rapproche b/munin/bano_rapproche deleted file mode 100755 index 7a7e07a..0000000 --- a/munin/bano_rapproche +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh - -case $1 in - config) - cat <<'EOM' -graph_title BANO - Rapprochements -graph_vlabel Nb de voies -graph_category bano -match.label Voies rapprochees -nomatch.label Voies non rapprochees -EOM - exit 0;; -esac - -psql cadastre -c "select concat('match.value ',count(*)::text) from (select count(*), fantoir from cumul_adresses where source='CADASTRE' and voie_osm!='' group by fantoir) as f" -At -psql cadastre -c "select concat('nomatch.value ',count(*)::text) from (select count(*), fantoir from cumul_adresses where source='CADASTRE' and (voie_osm='' or voie_osm is null) group by fantoir) as f" -At - - diff --git a/out/LICENCE.txt b/out/LICENCE.txt deleted file mode 100644 index 57e29fa..0000000 --- a/out/LICENCE.txt +++ /dev/null @@ -1,24 +0,0 @@ -Ces données sont issues du crowdsourcing effectué par les contributeurs au projet OpenStreetMap et sont sous licence ODbL et la mention d'attribution obligatoire doit être "© les contributeurs d'OpenStreetMap sous licence ODbL" conformément à http://osm.org/copyright - - -Ceci est le résumé explicatif de la licence ODbL 1.0. Merci de lire l'avertissement ci-dessous. - -Vous êtes libres : -- De partager : copier, distribuer et utiliser la base de données. -- De créer : produire des créations à partir de cette base de données. -- D'adapter : modifier, transformer et construire à partir de cette base de données. - -Aussi longtemps que : -- Vous mentionnez la paternité : vous devez mentionner la source de la base de données pour toute utilisation publique de la base de données, ou pour toute création produite à partir de la base de données, de la manière indiquée dans l'ODbL. Pour toute utilisation ou redistribution de la base de données, ou création produite à partir de cette base de données, vous devez clairement mentionner aux tiers la licence de la base de données et garder intacte toute mention légale sur la base de données originaire. - -- Vous partagez aux conditions identiques : si vous utilisez publiquement une version adaptée de cette base de données, ou que vous produisiez une création à partir d'une base de données adaptée, vous devez aussi offrir cette base de données adaptée selon les termes de la licence ODbL. - -- Gardez ouvert : si vous redistribuez la base de données, ou une version modifiée de celle-ci, alors vous ne pouvez utiliser de mesure technique restreignant la création que si vous distribuez aussi une version sans ces restrictions. - -Avertissement - -Le résumé explicatif n'est pas un contrat, mais simplement une source pratique pour faciliter la compréhension de la version complète de la licence ODbL 1.0 — il exprime en termes courants les principales notions juridiques du contrat. Ce résumé explicatif n'a pas de valeur juridique, son contenu n'apparaît pas sous cette forme dans le contrat. Seul le texte complet du contrat de licence fait loi. - -Le texte complet du contrat de licence (en anglais) est disponible sur: http://opendatacommons.org/licenses/odbl/1.0/ - -Une traduction en français est disponible sur http://www.vvlibri.org/fr/licence/odbl/10/fr/legalcode \ No newline at end of file diff --git a/out/banout-all.sh b/out/banout-all.sh deleted file mode 100755 index 09df3eb..0000000 --- a/out/banout-all.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -parallel -j 8 sh banout.sh > /dev/null -- 01 02 03 04 05 06 07 08 09 `seq 10 19` 2A 2B `seq 21 95` `seq 971 974` 976 - -# fichier global France -for d in 01 02 03 04 05 06 07 08 09 `seq 10 19` 2A 2B `seq 21 95` `seq 971 974` 976; do cat /data/project/bano.openstreetmap.fr/web/data/bano-$d.csv >> full.csv ; done -gzip -9 full.csv -mv full.csv.gz /data/project/bano.openstreetmap.fr/web/data/ - diff --git a/out/banout-json-all.sh b/out/banout-json-all.sh deleted file mode 100755 index f8c7aa0..0000000 --- a/out/banout-json-all.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -# script écrit et maintenu par cquest@openstreetmap.fr - -# export multi-thread de cumul_voie + cumul_adresse + cumul_place en json -parallel -j 8 sh banout-json.sh -- 01 02 03 04 05 06 07 08 09 `seq 10 19` 2A 2B `seq 21 95` `seq 971 974` 976 - -# fusion en un fichier unique -rm -f full.sjson* -for dep in {01..19} 2A 2B {21..95} {971..974} 976 ; do grep -v ^$ bano-$dep.json | gzip -9 >> full.sjson.gz ; gzip -9 bano-$dep.json ; done - -# compression et copie sur le site web -sudo mv *json.gz /data/project/bano.openstreetmap.fr/web/data/ - -# ménage final -rm bano-*.json diff --git a/out/banout-json.sh b/out/banout-json.sh deleted file mode 100644 index ff62cc7..0000000 --- a/out/banout-json.sh +++ /dev/null @@ -1,161 +0,0 @@ -# script écrit et maintenu par cquest@openstreetmap.fr - -OUTPUT=bano-$1.json -# dédoublement des adresses multiple OSM (séparées par ';' '-' ',' ou 'à') -psql cadastre -q -c "insert into cumul_adresses select geometrie, trim( both from regexp_split_to_table(numero,';|-|à|,')), voie_cadastre, voie_osm, fantoir, insee_com, cadastre_com, dept, code_postal, source, batch_import_id, voie_fantoir from cumul_adresses where numero ~ ';|-|à|,' and insee_com like '$1%' and source='OSM';" -psql cadastre -q -c "delete from cumul_adresses where numero ~ ';|-|à|,' and insee_com like '$1%' and source='OSM';" - -rm -f $OUTPUT - - -echo "`date +%H:%M:%S` Communes $1" -# export des communes -psql cadastre -t -A -c " \ -SELECT '{\"id\": \"' || insee || '\",\"type\": \"' || CASE WHEN population<1 THEN 'village' WHEN population<'10' THEN 'town' ELSE 'city' END || '\",\"name\": \"' || g.nom || '\",\"postcode\": \"' || ca.code_postal || '\",\"lat\": \"' || round(lat_chf,6) || '\",\"lon\": \"' || round(lon_chf,6) || '\",\"city\": \"' || g.nom|| '\",\"departement\": \"' || cog.nom_dep || '\", \"region\": \"' || cog.nom_reg || '\", \"population\": ' || population || ', \"adm_weight\": ' || CASE WHEN statut LIKE 'Capital%' THEN 6 WHEN statut = 'Préfecture de régi' THEN 5 WHEN statut='Préfecture' THEN 4 WHEN statut LIKE 'Sous-pr%' THEN 3 WHEN statut='Chef-lieu canton' THEN 2 ELSE 1 END || ', \"importance\": ' || greatest(0.075,round(log((CASE WHEN statut LIKE 'Capital%' THEN 6 WHEN statut = 'Préfecture de régi' THEN 5 WHEN statut='Préfecture' THEN 4 WHEN statut LIKE 'Sous-pr%' THEN 3 WHEN statut='Chef-lieu canton' THEN 2 ELSE 1 END)+log(population+1)/3),4)) || '}' \ -FROM geofla_plus g join code_cadastre ca on (ca.insee_com=insee) join (select dep, nom_dep, nom_reg from cog group by dep,nom_dep, nom_reg) as cog on (cog.dep=left(insee,2) or cog.dep=left(insee,3)) WHERE insee like '$1%' order by insee; \ -" | grep id >> $OUTPUT - - -echo "`date +%H:%M:%S` Voie non rapprochées $1"; -# export fantoir_voie (pour les voies non rapprochées) + cumul_adresse (ponctuel adresse) > json -for com in `psql cadastre -c "SELECT insee FROM communes where insee like '$1%';" -A -t`; do psql cadastre -t -A -c " \ -SET client_min_messages TO WARNING; WITH v as (select code_insee as insee_com, code_insee || id_voie || cle_rivoli as fantoir from fantoir_voie f left join cumul_voies v on (v.insee_com = '$com' AND f.code_insee=v.insee_com and v.fantoir = code_insee || id_voie || cle_rivoli) where v.fantoir is null and code_insee = '$com') -SELECT '{\"id\": \"' || osm.fantoir || CASE WHEN coalesce(cp.postal_cod, cad.code_postal)!=cad.code_postal THEN ('_' || cp.postal_cod) ELSE '' END || '\",\"citycode\":\"' || left(osm.fantoir,5) || '\",\"type\": \"street\",\"name\": \"' || replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(regexp_replace(max(case when osm.voie_osm !='' then osm.voie_osm else osm.voie_cadastre end),'^IMP ','Impasse '),'^RTE ','Route '),'^ALL ','Allée '),'^PL ','Place '),'^PLA ','Place '),'^AV ','Avenue '),'^LOT ','Lotissement '),'^RES ','Résidence '),'^CHEM ','Chemin '),'^RLE ','Ruelle '),'^BD ','Boulevard '),'^SQ ','Square '),'^PAS ','Passage '),'^SEN ','Sentier '),'^CRS ','Cours '),'^TRA ','Traverse '),'^MTE ','Montée '),'^RPT ','Rond-point '),'^HAM ','Hameau '),'^VLA ','Villa '),'^PROM ','Promenade '),'^ESP ','Esplanade '),'^FG ','Faubourg '),'^TSSE ','Terrasse '),'^CTRE ','Centre '),'^PASS ','Passerelle '),'^FRM ','Ferme '),' GAL ',' Général '),' MAL ',' Maréchal '),' ST ',' Saint '),' STE ',' Sainte '),' PDT ',' Président '),' HT ',' Haut '),' HTE ',' Haute '),' VX ',' Vieux '),' PTE ',' Petite '),'\"',''),'’',chr(39)) || '\",\"postcode\": \"' || coalesce(cp.postal_cod, min(lp.cp), cad.code_postal) || CASE WHEN replace(lower(cp.nom),'-',' ') != replace(lower(c.nom),'-',' ') THEN '\",\"post_office\": \"' || cp.nom ELSE '' END || '\",\"lat\": \"' || round(st_y(st_centroid(st_convexhull(ST_Collect(osm.geometrie))))::numeric,6) || '\",\"lon\": \"' || round(st_x(st_centroid(st_convexhull(ST_Collect(osm.geometrie))))::numeric,6) || '\",\"city\": \"' || c.nom || '\",\"departement\": \"' || cog.nom_dep || '\", \"region\": \"' || cog.nom_reg || '\",\"importance\": '|| round(log((CASE WHEN (cad.code_postal LIKE '75%' OR g.statut LIKE 'Capital%') THEN 6 WHEN (cad.code_postal LIKE '690%' OR cad.code_postal LIKE '130%' OR g.statut = 'Préfecture de régi') THEN 5 WHEN g.statut='Préfecture' THEN 4 WHEN g.statut LIKE 'Sous-pr%' THEN 3 WHEN g.statut='Chef-lieu canton' THEN 2 ELSE 1 END)+log(g.population+1)/3)::numeric*log(1+log(count(osm.*)+1)+log(st_length(st_longestline(st_convexhull(ST_Collect(osm.geometrie)),st_convexhull(ST_Collect(osm.geometrie)))::geography)+1)+log(CASE WHEN max(osm.voie_cadastre) like 'Boulevard%' THEN 4 WHEN max(osm.voie_cadastre) LIKE 'Place%' THEN 4 WHEN max(osm.voie_cadastre) LIKE 'Espl%' THEN 4 WHEN max(osm.voie_cadastre) LIKE 'Av%' THEN 3 WHEN max(osm.voie_cadastre) LIKE 'Rue %' THEN 2 ELSE 1 END))::numeric,4) ||' ,\"housenumbers\":' || concat('{',string_agg(DISTINCT concat('\"',replace(replace(replace(osm.numero,' ',''),'\"',''),'\\',''),'\": {\"lat\": ',round(st_y(osm.geometrie)::numeric,6),',\"lon\": ',round(st_x(osm.geometrie)::numeric,6),'}'), ','),'}}') AS sjson -FROM v -LEFT JOIN cumul_adresses osm ON (osm.fantoir=v.fantoir) -JOIN communes c ON (c.insee=v.insee_com) -JOIN code_cadastre cad ON (cad.insee_com=v.insee_com) -JOIN - (SELECT fantoir, - replace(numero,' ','') AS num, - max(SOURCE) AS src - FROM cumul_adresses - WHERE fantoir like '$com%' - GROUP BY 1, - 2) AS b ON (b.fantoir=osm.fantoir - AND osm.SOURCE=b.src - AND b.num=replace(osm.numero,' ','')) -LEFT JOIN (select dep, nom_dep, nom_reg from cog group by dep, nom_dep, nom_reg) as cog ON (cog.dep=left(v.insee_com,2) or cog.dep=left(v.insee_com,3)) -LEFT JOIN geofla_plus g ON (g.insee=v.insee_com) -LEFT JOIN laposte_cp lp ON (lp.insee=v.insee_com) -LEFT JOIN postal_code cp ON (cp.insee=v.insee_com AND ST_Contains(cp.wkb_geometry, osm.geometrie)) -WHERE osm.fantoir IS NOT NULL - AND osm.numero ~ '^[0-9]{1,4}( ?[A-Z]?.*)?' - AND osm.numero !~'.[0-9 \\.\\-]{9,}' - AND cad.insee_com = '$com' - AND c.insee = '$com' - AND g.insee = '$com' -GROUP BY osm.fantoir, - cad.code_postal, cp.postal_cod, - c.nom, cp.nom, - cog.nom_dep, - cog.nom_reg, - g.statut, - g.population -ORDER BY osm.fantoir; -" >> $OUTPUT; done - -echo "`date +%H:%M:%S` Voie rapprochées $1" -# export cumul_voie (position centre de voirie) + cumul_adresse (ponctuel adresse) > json -for com in `psql cadastre -c "SELECT insee FROM communes where insee like '$1%';" -A -t`; do psql cadastre -t -A -c " \ -SET client_min_messages TO WARNING; SELECT '{\"id\": \"' || v.fantoir || CASE WHEN coalesce(cp.postal_cod, cad.code_postal)!=cad.code_postal THEN ('_' || cp.postal_cod) ELSE '' END || '\",\"citycode\": \"' || left(v.fantoir,5) || '\",\"type\": \"street\",\"name\": \"' || replace(replace(v.voie_osm,'\"',''),'’',chr(39)) || '\",\"postcode\": \"' || coalesce(cp.postal_cod, min(lp.cp), cad.code_postal) || CASE WHEN replace(lower(cp.nom),'-',' ') != replace(lower(c.nom),'-',' ') THEN '\",\"post_office\": \"' || cp.nom ELSE '' END || '\",\"lat\": \"' || round(st_y(v.geometrie)::numeric,6) || '\",\"lon\": \"' || round(st_x(v.geometrie)::numeric,6) || '\",\"city\": \"' || c.nom || '\",\"departement\": \"' || cog.nom_dep || '\", \"region\": \"' || cog.nom_reg || '\",\"importance\": '|| round(log((CASE WHEN (cad.code_postal LIKE '75%' OR g.statut LIKE 'Capital%') THEN 6 WHEN (cad.code_postal LIKE '690%' OR cad.code_postal LIKE '130%' OR g.statut = 'Préfecture de régi') THEN 5 WHEN g.statut='Préfecture' THEN 4 WHEN g.statut LIKE 'Sous-pr%' THEN 3 WHEN g.statut='Chef-lieu canton' THEN 2 ELSE 1 END)+log(g.population+1)/3)::numeric*log(1+log(count(a.*)+1)+log(st_length(st_longestline(st_convexhull(ST_Collect(a.geometrie)),st_convexhull(ST_Collect(a.geometrie)))::geography)+1)+log(CASE WHEN v.voie_osm like 'Boulevard%' THEN 4 WHEN v.voie_osm LIKE 'Place%' THEN 4 WHEN v.voie_osm LIKE 'Espl%' THEN 4 WHEN v.voie_osm LIKE 'Av%' THEN 3 WHEN v.voie_osm LIKE 'Rue %' THEN 2 ELSE 1 END))::numeric,4) ||' ,\"housenumbers\":' || concat('{',string_agg(DISTINCT concat('\"',replace(replace(replace(a.numero,' ',''),'\"',''),'\\',''),'\": {\"lat\": ',round(st_y(a.geometrie)::numeric,6),',\"lon\": ',round(st_x(a.geometrie)::numeric,6),'}'), ','),'}}') AS sjson -FROM cumul_voies v -JOIN communes c ON (insee=insee_com) -JOIN code_cadastre cad ON (cad.insee_com=v.insee_com) -LEFT JOIN cumul_adresses a ON (a.fantoir=v.fantoir) -JOIN - (SELECT fantoir, - replace(numero,' ','') AS num, - max(SOURCE) AS src - FROM cumul_adresses - WHERE fantoir LIKE '$com%' - GROUP BY 1, - 2) AS b ON (b.fantoir=a.fantoir - AND a.SOURCE=b.src - AND b.num=replace(a.numero,' ','')) -LEFT JOIN (select dep, nom_dep, nom_reg from cog group by dep, nom_dep, nom_reg) as cog ON (cog.dep=left(v.insee_com,2) or cog.dep=left(v.insee_com,3)) -LEFT JOIN geofla_plus g ON (g.insee=v.insee_com) -LEFT JOIN laposte_cp lp ON (lp.insee=v.insee_com) -LEFT JOIN postal_code cp ON (cp.insee=v.insee_com AND ST_Contains(cp.wkb_geometry, a.geometrie)) -WHERE v.fantoir LIKE '$com%' - AND a.numero ~ '^[0-9]{1,4}( ?[A-Z]?.*)?' - AND a.numero !~'.[0-9 \\.\\-]{9,}' - AND cad.insee_com = '$com' - AND c.insee = '$com' - AND g.insee = '$com' -GROUP BY v.fantoir, - v.voie_osm, - cad.code_postal, cp.postal_cod, - v.geometrie, - c.nom, cp.nom, - cog.nom_dep, - cog.nom_reg, - g.statut, - g.population -ORDER BY v.fantoir; -" >> $OUTPUT; done - - -echo "`date +%H:%M:%S` Voie rapprochées sans adresses $1" -# export cumul_voie (position centre de voirie) > json -psql cadastre -t -A -c " \ -SET client_min_messages TO WARNING; SELECT '{\"id\": \"' || v.fantoir || CASE WHEN coalesce(cp.postal_cod, cad.code_postal)!=cad.code_postal THEN ('_' || cp.postal_cod) ELSE '' END || '\",\"citycode\": \"' || left(v.fantoir,5) || '\",\"type\": \"street\",\"name\": \"' || replace(replace(v.voie_osm,'\"',''),'’',chr(39)) || '\",\"postcode\": \"' || coalesce(cp.postal_cod, min(lp.cp), cad.code_postal) || CASE WHEN replace(lower(cp.nom),'-',' ') != replace(lower(c.nom),'-',' ') THEN '\",\"post_office\": \"' || cp.nom ELSE '' END || '\",\"lat\": \"' || round(st_y(v.geometrie)::numeric,6) || '\",\"lon\": \"' || round(st_x(v.geometrie)::numeric,6) || '\",\"city\": \"' || c.nom || '\",\"departement\": \"' || cog.nom_dep || '\", \"region\": \"' || cog.nom_reg || '\",\"importance\": '|| round(log((CASE WHEN (cad.code_postal LIKE '75%' OR g.statut LIKE 'Capital%') THEN 6 WHEN (cad.code_postal LIKE '690%' OR cad.code_postal LIKE '130%' OR g.statut = 'Préfecture de régi') THEN 5 WHEN g.statut='Préfecture' THEN 4 WHEN g.statut LIKE 'Sous-pr%' THEN 3 WHEN g.statut='Chef-lieu canton' THEN 2 ELSE 1 END)+log(g.population+1)/3)::numeric*log(1+log(count(a.*)+1)+log(CASE WHEN v.voie_osm like 'Boulevard%' THEN 4 WHEN v.voie_osm LIKE 'Place%' THEN 4 WHEN v.voie_osm LIKE 'Espl%' THEN 4 WHEN v.voie_osm LIKE 'Av%' THEN 3 WHEN v.voie_osm LIKE 'Rue %' THEN 2 ELSE 1 END))::numeric,4) ||' }' AS sjson -FROM cumul_voies v -JOIN communes c ON (insee=insee_com) -JOIN code_cadastre cad ON (cad.insee_com=v.insee_com) -LEFT JOIN cumul_adresses a ON (a.fantoir=v.fantoir) -LEFT JOIN (select dep, nom_dep, nom_reg from cog group by dep, nom_dep, nom_reg) as cog ON (cog.dep=left(v.insee_com,2) or cog.dep=left(v.insee_com,3)) -LEFT JOIN geofla_plus g ON (g.insee=v.insee_com) -LEFT JOIN laposte_cp lp ON (lp.insee=v.insee_com) -LEFT JOIN postal_code cp ON (cp.insee=v.insee_com AND ST_Contains(cp.wkb_geometry, v.geometrie)) -WHERE v.fantoir LIKE '$1%' - AND a.numero IS NULL - AND cad.insee_com like '$1%' - AND c.insee like '$1%' - AND g.insee like '$1%' -GROUP BY v.fantoir, - v.voie_osm, - cad.code_postal, cp.postal_cod, - v.geometrie, - c.nom, cp.nom, - cog.nom_dep, - cog.nom_reg, - g.statut, - g.population -ORDER BY v.fantoir; -" >> $OUTPUT - - -echo "`date +%H:%M:%S` LD $1" -# export cumul_place (lieux-dits) > json -psql cadastre -t -A -c " -SET client_min_messages TO WARNING; -WITH u AS (select fantoir as f, insee_com as insee from cumul_places where fantoir like '$1%' GROUP BY 1,2), lp as (select insee, min(cp) as cp from laposte_cp where insee like '$1%' group by 1) -select DISTINCT '{\"id\": \"' || u.f - || '\",\"citycode\": \"' || left(u.f,5) - || '\",\"type\": \"' || 'place' - || '\",\"name\": \"' || replace(replace(coalesce(o.libelle_osm, c.libelle_cadastre),'\"',''),'’',chr(39)) - || '\",\"postcode\": \"' || coalesce(cp.postal_cod, lp.cp, ca.code_postal) || CASE WHEN replace(lower(cp.nom),'-',' ') != replace(lower(coalesce(cn.nom,initcap(ca.nom_com))),'-',' ') THEN '\",\"post_office\": \"' || cp.nom ELSE '' END - || '\",\"lat\": \"' || case when o.geometrie is not null then round(st_y(o.geometrie)::numeric,6) else st_y(c.geometrie) end - || '\",\"lon\": \"' || case when o.geometrie is not null then round(st_x(o.geometrie)::numeric,6) else st_x(c.geometrie) end - || '\",\"city\": \"' || coalesce(cn.nom,initcap(ca.nom_com)) - || '\",\"departement\": \"' || cog.nom_dep - || '\", \"region\": \"' || cog.nom_reg - || '\", \"importance\": '|| least(0.05,round(log((CASE WHEN g.statut LIKE 'Capital%' THEN 6 WHEN g.statut = 'Préfecture de régi' THEN 5 WHEN g.statut='Préfecture' THEN 4 WHEN g.statut LIKE 'Sous-pr%' THEN 3 WHEN g.statut='Chef-lieu canton' THEN 2 ELSE 1 END)+log(g.population+1)/3)*(0.25+0.5*(1-('0' || coalesce(f.ld_bati,'1'))::numeric)),4)) - ||'}' -from u - LEFT JOIN fantoir_voie f on (f.code_insee=u.insee AND u.f = concat(f.code_insee,f.id_voie,f.cle_rivoli)) - LEFT JOIN cumul_places c on (c.fantoir=u.f and c.source='CADASTRE') - LEFT JOIN cumul_places o on (o.fantoir=u.f and o.source='OSM') - LEFT JOIN code_cadastre ca ON (ca.insee_com=u.insee) - LEFT JOIN communes cn ON (cn.insee=u.insee) - LEFT JOIN geofla_plus g ON (g.insee=u.insee) - LEFT JOIN lp ON (lp.insee=u.insee) - LEFT JOIN postal_code cp ON (cp.insee=u.insee AND ST_Contains(cp.wkb_geometry, o.geometrie)) - JOIN (select dep, nom_dep, nom_reg from cog group by dep, nom_dep, nom_reg) as cog ON (cog.dep=left(u.insee,2) or cog.dep=left(u.insee,3)) -where coalesce(o.libelle_osm, c.libelle_cadastre) != cn.nom ORDER BY 1; -" >> $OUTPUT - - -echo "`date +%H:%M:%S` FIN $1" - diff --git a/out/banout.sh b/out/banout.sh deleted file mode 100644 index c474b3d..0000000 --- a/out/banout.sh +++ /dev/null @@ -1,176 +0,0 @@ -# ménage préparatoire... -rm -f bano-$1* - -# dédoublement des adresses multiple OSM (séparées par ';' ',' ou 'à') -psql cadastre -q -c "insert into cumul_adresses select geometrie, trim( both from regexp_split_to_table(numero,';|à|,')), voie_cadastre, voie_osm, fantoir, insee_com, cadastre_com, dept, code_postal, source, batch_import_id, voie_fantoir from cumul_adresses where numero ~ ';|-|à|,' and insee_com like '$1%' and source='OSM';" -psql cadastre -q -c "delete from cumul_adresses where numero ~ ';|à|,' and insee_com like '$1%' and source='OSM';" - -# dédoublement des adresses multiple OSM (séparées par '-' uniquement si premier nombre inférieur au second) -psql cadastre -q -c "insert into cumul_adresses -select geometrie, trim( both from regexp_split_to_table(numero,'-')), voie_cadastre, voie_osm, fantoir, insee_com, cadastre_com, dept, code_postal, source, batch_import_id, voie_fantoir -from ( - select * from ( - select regexp_split_to_array(numero,'-') as nums,* - from cumul_adresses - where numero like '%-%' and insee_com like '$1%' and source='OSM' - ) as n - where regexp_replace('0'||nums[1],'[^0-9]','','g')::integer < regexp_replace('0'||nums[2],'[^0-9]','','g')::integer -) as n;" -psql cadastre -q -c "with n as ( - select * from ( - select regexp_split_to_array(numero,'-') as nums, fantoir as d_fantoir, numero as d_numero - from cumul_adresses - where numero like '%-%' and insee_com like '$1%' and source='OSM' - ) as n0 - where regexp_replace('0'||nums[1],'[^0-9]','','g')::integer < regexp_replace('0'||nums[2],'[^0-9]','','g')::integer -) delete from cumul_adresses where source='OSM' and fantoir||numero in (select d_fantoir||d_numero from n);" - -# export postgres > shapefile -export SHAPE_ENCODING='UTF-8' -ogr2ogr -f "ESRI Shapefile" -lco ENCODING=UTF-8 -s_srs "EPSG:4326" -t_srs "EPSG:4326" -overwrite bano-$1.shp PG:"dbname=cadastre" -sql " \ -SELECT * \ -FROM (WITH u AS \ - (SELECT fantoir, \ - replace(replace(replace(replace(replace(replace(replace(regexp_replace(upper(numero),'^0*',''),'BIS','B'),'TER','T'),'QUATER','Q'),'QUAT','Q'),' ',''),'à','-'),';',',') AS num \ - FROM cumul_adresses \ - WHERE fantoir IS NOT NULL \ - AND length(fantoir)=10 \ - AND fantoir LIKE '$1%' \ - GROUP BY fantoir, \ - num), lp as (select insee, min(cp) as cp from laposte_cp where insee like '$1%' group by 1) \ - SELECT concat(u.fantoir,'-',u.num) AS id, \ - upper(replace(CASE WHEN u.num=o.num THEN o.numero WHEN u.num=od.num THEN od.numero ELSE c.numero END,' ','')) AS numero, \ - replace(replace(regexp_replace(regexp_replace(coalesce(CASE \ - WHEN u.num=o.num THEN \ - CASE \ - WHEN o.voie_osm != '' THEN replace(o.voie_osm,'’',chr(39)) \ - ELSE o.voie_cadastre \ - END \ - WHEN u.num=od.num THEN \ - CASE \ - WHEN od.voie_osm is not null THEN replace(od.voie_osm,'’',chr(39)) \ - ELSE od.voie_cadastre \ - END \ - ELSE \ - CASE \ - WHEN c.voie_osm!='' THEN replace(c.voie_osm,'’',chr(39)) \ - ELSE c.voie_cadastre \ - END \ - END, \ - CASE \ - WHEN u.num=o.num THEN replace(o.voie_osm,'’',chr(39)) \ - ELSE \ - CASE \ - WHEN c.voie_osm !='' THEN replace(c.voie_osm,'’',chr(39)) \ - ELSE c.voie_cadastre \ - END \ - END),'([dD][eé]partementale?|Rue|[rR]urale?|[vV]icinale?|[cC]ommunale?|Cr) ([0-9]+ )?[dD]ite? ',''),'(Draille|Chemin|Sentier) [dD]ite? ','\1 '),'Voie Che ','Chemin '),'Cours Dit Che ','Chemin ') AS voie, \ - coalesce(cp.postal_cod, lp.cp, ca.code_postal) AS code_post, \ - coalesce(cn.nom,initcap(ca.nom_com)) AS ville, \ - CASE \ - WHEN u.num=o.num THEN 'OSM' \ - WHEN (u.num=od.num \ - AND od.voie_osm != od.voie_cadastre \ - AND od.voie_osm IS NOT NULL) THEN 'O+O' \ - WHEN u.num=od.num THEN 'OD' \ - WHEN c.voie_osm != '' THEN 'C+O' \ - ELSE 'CAD' \ - END AS SOURCE, \ - CASE \ - WHEN u.num=o.num THEN st_y(o.geometrie) \ - WHEN u.num=od.num THEN st_y(od.geometrie) \ - ELSE st_y(c.geometrie) \ - END AS lat, \ - CASE \ - WHEN u.num=o.num THEN st_x(o.geometrie) \ - WHEN u.num=od.num THEN st_x(od.geometrie) \ - ELSE st_x(c.geometrie) \ - END AS lon, \ - CASE \ - WHEN u.num=o.num THEN o.geometrie \ - WHEN u.num=od.num THEN od.geometrie \ - ELSE c.geometrie \ - END AS geom \ - FROM u \ - LEFT JOIN lp ON (lp.insee=left(u.fantoir,5)) \ - LEFT JOIN \ - (SELECT *, \ - replace(replace(replace(replace(replace(replace(replace(regexp_replace(upper(numero),'^0*',''),'BIS','B'),'TER','T'),'QUATER','Q'),'QUAT','Q'),' ',''),'à','-'),';',',') AS num \ - FROM cumul_adresses \ - WHERE fantoir LIKE '$1%' \ - AND SOURCE='OSM') AS o \ - ON (o.num=u.num AND o.fantoir=u.fantoir) \ - LEFT JOIN \ - (SELECT *, \ - replace(replace(replace(replace(replace(replace(replace(regexp_replace(upper(numero),'^0*',''),'BIS','B'),'TER','T'),'QUATER','Q'),'QUAT','Q'),' ',''),'à','-'),';',',') AS num \ - FROM cumul_adresses \ - WHERE fantoir LIKE '$1%' \ - AND SOURCE='CADASTRE' \ - AND st_x(geometrie)!=0 \ - AND st_y(geometrie)!=0) AS c \ - ON (c.num=u.num AND c.fantoir=u.fantoir) \ - LEFT JOIN \ - (SELECT *, \ - replace(replace(replace(replace(replace(replace(replace(regexp_replace(upper(numero),'^0*',''),'BIS','B'),'TER','T'),'QUATER','Q'),'QUAT','Q'),' ',''),'à','-'),';',',') AS num \ - FROM cumul_adresses \ - WHERE fantoir LIKE '$1%' \ - AND SOURCE LIKE 'OD%' \ - AND st_x(geometrie)!=0 \ - AND st_y(geometrie)!=0) AS od \ - ON (od.num=u.num AND od.fantoir=u.fantoir) \ - LEFT JOIN code_cadastre ca \ - ON (ca.insee_com=left(u.fantoir,5)) \ - LEFT JOIN communes cn \ - ON (cn.insee=left(u.fantoir,5)) \ - LEFT JOIN postal_code cp \ - ON (cp.insee = left(u.fantoir,5) and st_contains(cp.wkb_geometry, coalesce(o.geometrie, od.geometrie, c.geometrie))) \ - WHERE u.num>'0') AS DATA \ -WHERE lat IS NOT NULL \ - AND lon IS NOT NULL \ - AND numero ~ '^[0-9]{1,4}( ?[A-Z]?.*)?' \ - AND numero !~'.[0-9 \\.\\-]{9,}' \ -ORDER BY id \ -" - -# génération du .csv à partir du shapefile -ogr2ogr -f CSV bano-$1-tmp.csv -lco WRITE_BOM=YES bano-$1.shp -# suppression header + coupure à 6 décimale + tri -sed -e 1d bano-$1-tmp.csv | sed 's/\(\.[0-9]\{6\}\)[0-9]*/\1/g' | sort > bano-$1.csv - -#exit - -# sortie RDF "turtle" à partir du csv -python csv2ttl.py bano-$1.csv $1 > bano-$1.ttl -gzip -9 bano-$1.ttl -mv bano-$1.ttl.gz /data/project/bano.openstreetmap.fr/web/data/ - -# copie dans le dossier web -mv bano-$1.csv /data/project/bano.openstreetmap.fr/web/data/ - -# préparation du shapefile zippé -#cp -p -u /data/project/bano.openstreetmap.fr/web/data/*.txt ./ -zip -q -9 bano-$1-shp.zip bano-$1.* *.txt -chmod a+r *.zip - -# copie dans le dossier web -mv bano-$1-shp.zip /data/project/bano.openstreetmap.fr/web/data/ - - -# ménage -rm bano-$1* - - -exit - -select * from ( -with u as (select fantoir,replace(replace(replace(replace(replace(replace(replace(regexp_replace(upper(numero),'^0*',''),'BIS','B'),'TER','T'),'QUATER','Q'),'QUAT','Q'),' ',''),'à','-'),';',',') as num from cumul_adresses where fantoir like '69268%' and fantoir is not null group by fantoir,num) -select concat(u.fantoir,'-',u.num) as id, upper(replace(case when u.num=o.num then o.numero when u.num=od.num then od.num else c.numero end,' ','')) as numero,coalesce(case when u.num=o.num then case when o.voie_osm != '' then replace(o.voie_osm,'’',chr(39)) else o.voie_cadastre end when u.num=od.num then replace(od.voie_osm,'’',chr(39)) else case when c.voie_osm!='' then replace(c.voie_osm,'’',chr(39)) else c.voie_cadastre end end, case when u.num=o.num then replace(o.voie_osm,'’',chr(39)) else (case when c.voie_osm !='' then replace(c.voie_osm,'’',chr(39)) else c.voie_cadastre end) end) as voie, ca.code_postal as code_post, coalesce(cn.nom,initcap(ca.nom_com)) as ville,case when u.num=o.num then 'OSM' when (u.num=od.num and od.voie_osm != od.voie_cadastre and od.voie_osm is not null) then 'O+O' when u.num=od.num then 'OD' when c.voie_osm != '' then 'C+O' else 'CAD' end as source, case when u.num=o.num then st_y(o.geometrie) when u.num=od.num then st_y(od.geometrie) else st_y(c.geometrie) end as lat,case when u.num=o.num then st_x(o.geometrie) when u.num=od.num then st_x(od.geometrie) else st_x(c.geometrie) end as lon,case when u.num=o.num then o.geometrie when u.num=od.num then od.geometrie else c.geometrie end as geom -from u -left join (select *,replace(replace(replace(replace(replace(replace(replace(regexp_replace(upper(numero),'^0*',''),'BIS','B'),'TER','T'),'QUATER','Q'),'QUAT','Q'),' ',''),'à','-'),';',',') as num from cumul_adresses where fantoir like '69%' and source='OSM') as o on (o.num=u.num and o.fantoir=u.fantoir) -left join (select *,replace(replace(replace(replace(replace(replace(replace(regexp_replace(upper(numero),'^0*',''),'BIS','B'),'TER','T'),'QUATER','Q'),'QUAT','Q'),' ',''),'à','-'),';',',') as num from cumul_adresses where fantoir like '69%' and source='CADASTRE' and st_x(geometrie)!=0 and st_y(geometrie)!=0) as c on (c.num=u.num and c.fantoir=u.fantoir) -left join (select *,replace(replace(replace(replace(replace(replace(replace(regexp_replace(upper(numero),'^0*',''),'BIS','B'),'TER','T'),'QUATER','Q'),'QUAT','Q'),' ',''),'à','-'),';',',') as num from cumul_adresses where fantoir like '69%' and source LIKE 'OD%' and voie_osm is not null and st_x(geometrie)!=0 and st_y(geometrie)!=0) as od on (od.num=u.num and od.fantoir=u.fantoir) -left join code_cadastre ca on (ca.insee_com=left(u.fantoir,5)) -left join communes cn on (cn.insee=left(u.fantoir,5) ) -where u.num>'0' -) as data -where lat is not null and lon is not null and (upper(numero) !~ '^[0-9]{1,4}([A-Z]?.*)?' or numero ~'.[0-9 \.\-]{9,}') order by id; diff --git a/out/csv2ttl.py b/out/csv2ttl.py deleted file mode 100644 index d3594c7..0000000 --- a/out/csv2ttl.py +++ /dev/null @@ -1,48 +0,0 @@ -import sys -import csv -import string - -print('@prefix xsd: .') -print('@prefix locn: .') -print('@prefix gn: .') -print('@prefix prov: .') -print('@prefix gsp: .') -print('@prefix geo: .') -print('@prefix rdfs: .') -print('@prefix dcat: .') -print('@prefix foaf: .') -print('@prefix dcterms: .') -print -print(' a dcat:Catalog ;') -print(' dcterms:title "Donnees des adresses du projet BANO (Base Adresse Nationale Ouverte) en RDF"@fr ;') -print(' dcterms:description "Le projet BANO en RDF de Base d\'Adresses Nationale Ouverte initie par OpenStreetMap France."@fr ;') -print(' foaf:homepage ;') -print(' dcterms:language "fr" ;') -print(' dcterms:license ;') -print(' dcterms:publisher ; #url openstreetmap France') -print(' dcterms:issued "2014-05-14"^^xsd:date ; # data issued') -print(' dcterms:modified "2014-08-21"^^xsd:date ; #last modification') -print(' dcterms:spatial , ; # region/pays (France)') -print(' .') -print -with open(sys.argv[1]) as csvfile: - addr = csv.reader(csvfile, delimiter=',', quotechar='"') - for row in addr: - print(' a locn:Address , gn:Feature ;') - print(' locn:fullAddress "'+row[1]+' '+row[2]+', '+row[3]+' '+row[4]+', FRANCE";'); - print(' locn:addressId "'+row[0]+'" ;') - print(' locn:locatorDesignator "'+row[1]+'" ;') - print(' locn:thoroughfare "'+row[2]+'"@fr ;') - print(' locn:postalCode "'+row[3]+'" ;') - print(' locn:locatorName "'+row[4]+'"@fr ;') - print(' locn:adminUnitL1 "FR" ;') - # traitement des arrondissements municipaux de Paris, Lyon, Marseille - if (sys.argv[2] in ['13','69','75'] and int(row[0][0:5])) in range(13201, 13217)+range(69381, 69370)+range(75101, 75121): - print(' locn:location ;') - else: - print(' locn:location ;') - print(' locn:geometry ;') - print(' locn:geometry [a geo:Point ; geo:lat "'+row[6]+'" ; geo:long "'+row[7]+'" ] ;') - print(' locn:geometry [a gsp:Geometry; gsp:asWKT "POINT('+row[7]+' '+row[6]+')"^^gsp:wktLiteral ] ;') - print(' .') - print diff --git a/out/head.sh b/out/head.sh deleted file mode 100755 index f6819aa..0000000 --- a/out/head.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -for dep in {01..19} 2A 2B {21..95} {971..974} -do - echo $dep -# sh banout.sh $dep -done diff --git a/out/lisezmoi-bano.txt b/out/lisezmoi-bano.txt deleted file mode 100644 index e35ccd7..0000000 --- a/out/lisezmoi-bano.txt +++ /dev/null @@ -1,90 +0,0 @@ -###Descriptif du contenu des fichiers "BANO" - -**Qu'est-ce que BANO ?** - -BANO = Base d'Adresses Nationale Ouverte - -C'est un projet initié par OpenStreetMap France destiné à constituer et diffuser sous licence libre une base de données d'adresses géolocalisées sur l'ensemble du territoire français. - -Pour plus de renseignements: http://wiki.openstreetmap.org/wiki/WikiProject_France/WikiProject_Base_Adresses_Nationale_Ouverte_(BANO) - - -**Licence** - -Ces données sont sous licence libre "ODbL" (Open Database Licence). Cette licence implique: l'attribution et le partage à l'identique. - -- Pour la mention d'attribution veuillez indiquer "source: BANO" ainsi que la date du jeu de données. -- Pour le partage à l'identique, toute amélioration des données de BANO doit être repartagée sous licence identique, ceci dans le but d'une constante amélioration de BANO. - -Ceci est le résumé explicatif de la licence ODbL 1.0. Merci de lire l'avertissement ci-dessous - -Vous êtes libres : -- De partager : copier, distribuer et utiliser la base de données. -- De créer : produire des créations à partir de cette base de données. -- D'adapter : modifier, transformer et construire à partir de cette base de données. -Aussi longtemps que : -- Vous mentionnez la paternité : vous devez mentionner la source de la base de données pour toute utilisation publique de la base de données, ou pour toute création produite à partir de la base de données, de la manière indiquée dans l'ODbL. Pour toute utilisation ou redistribution de la base de données, ou création produite à partir de cette base de données, vous devez clairement mentionner aux tiers la licence de la base de données et garder intacte toute mention légale sur la base de données originaire. -- Vous partagez aux conditions identiques : si vous utilisez publiquement une version adaptée de cette base de données, ou que vous produisiez une création à partir d'une base de données adaptée, vous devez aussi offrir cette base de données adaptée selon les termes de la licence ODbL. -- Gardez ouvert : si vous redistribuez la base de données, ou une version modifiée de celle-ci, alors vous ne pouvez utiliser de mesure technique restreignant la création que si vous distribuez aussi une version sans ces restrictions. - -Avertissement - -Ce résumé explicatif n'est pas un contrat, mais simplement une source pratique pour faciliter la compréhension de la version complète de la licence ODbL 1.0 — il exprime en termes courants les principales notions juridiques du contrat. Ce résumé explicatif n'a pas de valeur juridique, son contenu n'apparaît pas sous cette forme dans le contrat. Seul le texte complet du contrat de licence fait loi. - -Version complète en français disponible sur: http://www.vvlibri.org/fr/licence/odbl/10/fr/legalcode - - -**Origine** - -BANO est une base de données composite, constituée à partir de différentes sources: -- OpenStreetMap (ODbL) -- données disponibles en opendata - - Arles Crau Camargue Montagnette (Licence Ouverte) - avril 2016 - - Ville de Montpellier (Licence Ouverte) - mai 2016 - - Toulouse Métropole (ODbL) - février 2015 - - Rennes Métropole (Licence Ouverte) - avril 2016 - - Mulhouse Alsace Agglomération (Licence Ouverte) - février 2015 - - Grand Nancy (ODbL) - avril 2016 - - Nantes Métropole (ODbL) - janvier 2015 - - Grand Lyon (Licence Ouverte) - janvier 2015 - - Bordeaux Métropole (ODbL) - février 2015 - - Strasbourg EuroMétropole (Licence Ouverte) - janvier 2014 - - Communauté Urbaine de Lille Métropole (Licence Ouverte) - juin 2014 - - Métropole Nice Côte d'Azur (Licence Ouverte) - janvier 2015 - - Ville de Limoges (ODbL) - décembre 2014 - - Ville de La Rochelle (Licence Ouverte) - octobre 2015 - - Pays de Brest (Licence Ouverte) - mars 2016 - - Mairie de Nanterre (Licence Ouverte) - février 2015 - - Ville de Grenoble (ODbL) - mars 2015 - - Ville de Paris (ODbL) - janvier 2015 - - Ville de Poitiers (Licence Ouverte) - avril 2015 - - Ville d'Angers (ODbL) - avril 2016 -- données adresses collectées sur le site du cadastre et fichier FANTOIR DGFiP (Licence Ouverte) - Octobre 2015 -- base officielle des codes postaux de La Poste (Licence Ouverte) - 2014 - - -**Format** - -Ces fichiers sont proposés au format shapefile, ainsi que sous forme de fichiers CSV (coordonnées WGS84/EPSG:4326 et textes en UTF-8). - - -**Contenu** - -Pour chaque adresse: -- id (unique) : code_insee + codefantoir + numero -- numero : numéro dans la voie avec suffixe (ex: 1, 1BIS, 1D) -- voie : nom de voie -- code_post : code postal sur 5 caractères -- nom_comm : nom de la commune -- source : OSM = donnée directement issue d'OpenStreetMap, OD = donnée provenant de source opendata locale, O+O = donnée de source opendata enrichie par OpenStreetMap, CAD = donnée directement issue du cadastre, C+O = donnée du cadastre enrichie par OSM (nom de voie par exemple) -- lat : latitude en degrés décimaux WGS84 -- lon : longitude en degrés décimaux WGS84 - - -**Mise à jour, corrections** - -Pour mettre à jour et corriger les données de BANO, il suffit de faire des améliorations directement dans OpenStreetMap, elles seront prises en compte au prochain de cycle de mise à jour. - -Un guiche unique de signalement et de correction d'erreur est en cours de co-design. Si vous voulez participer à sa mise en place, contactez-nous par mail. - -Pour toute question concernant ces exports, vous pouvez contacter bano@openstreetmap.fr diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..c4b3a43 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +beautifulsoup4==4.12.3 +psycopg2-binary==2.9.9 +requests==2.31.0 +lxml +ipdb==0.13.13 +urllib3==2.2.0 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 1803d52..0000000 --- a/setup.cfg +++ /dev/null @@ -1,18 +0,0 @@ -[metadata] -name=bano -version=2.0.0-alpha - -[options] -packages=find: -install_requires= - beautifulsoup4==4.6.3 - psycopg2-binary==2.9.3 - requests==2.21.0 - lxml - -[options.extras_require] -test= - pytest - -[options.entry_points] -console_scripts=bano=bano.bin:main diff --git a/setup.py b/setup.py deleted file mode 100644 index 8ab824c..0000000 --- a/setup.py +++ /dev/null @@ -1,2 +0,0 @@ -from setuptools import setup -setup() \ No newline at end of file diff --git a/sql/create_and_load_codes_postaux.sql b/sql/create_and_load_codes_postaux.sql deleted file mode 100644 index 0548e77..0000000 --- a/sql/create_and_load_codes_postaux.sql +++ /dev/null @@ -1,29 +0,0 @@ -CREATE TEMP TABLE tmp_codes_postaux (raw text); -\copy tmp_codes_postaux FROM 'codes_postaux.csv' WITH csv HEADER delimiter '#'; - -CREATE TABLE IF NOT EXISTS codes_postaux( - insee text, - commune text, - cp text, - libelle text, - ligne_5 text, - geometrie geometry(Point,4326)); - -TRUNCATE codes_postaux; - -WITH -a -AS -(SELECT string_to_array(raw,';') arr FROM tmp_codes_postaux) -INSERT INTO codes_postaux (insee,commune,cp,libelle,ligne_5,geometrie) -SELECT arr[1], - arr[2], - arr[3], - arr[4], - arr[5], - ST_SetSRID(ST_Point((string_to_array(arr[6],','))[2]::float,(string_to_array(arr[6],','))[1]::float),4326) -FROM a; - -CREATE INDEX IF NOT EXISTS gidx_codes_postaux ON codes_postaux USING GIST(geometrie); -CREATE INDEX IF NOT EXISTS idx_codes_postaux_insee ON codes_postaux (insee); -CREATE INDEX IF NOT EXISTS idx_codes_postaux_cp ON codes_postaux (cp); diff --git a/sql/create_table_base_bano.sql b/sql/create_table_base_bano.sql deleted file mode 100644 index 38a7192..0000000 --- a/sql/create_table_base_bano.sql +++ /dev/null @@ -1,139 +0,0 @@ -CREATE TABLE IF NOT EXISTS fantoir_voie ( - code_dept character(3), - code_dir character(1), - code_com character(3), - code_insee character(5), - id_voie character(4), - fantoir character(9), - fantoir10 character(10), - cle_rivoli character(1), - nature_voie character varying(4), - libelle_voie character varying(26), - type_commune character(1), - caractere_rur character(1), - caractere_voie character(1), - caractere_pop character(1), - pop_a_part integer, - pop_fictive integer, - caractere_annul character(1), - date_annul character varying(9), - date_creation character varying(9), - code_majic character(5), - type_voie character(1), - ld_bati character(1), - dernier_mot character varying(8)); -CREATE INDEX IF NOT EXISTS idx_fantoir_voie_dept ON fantoir_voie(code_dept); -CREATE INDEX IF NOT EXISTS idx_fantoir_code_insee ON fantoir_voie(code_insee); -CREATE INDEX IF NOT EXISTS idx_fantoir_fantoir ON fantoir_voie(fantoir); -CREATE INDEX IF NOT EXISTS idx_fantoir_fantoir10 ON fantoir_voie(fantoir10); - -CREATE TABLE IF NOT EXISTS cumul_adresses ( - geometrie geometry , - numero character varying (15), - voie_cadastre character varying (300), - voie_bal character varying (300), - voie_osm character varying (300), - voie_autre text, - voie_fantoir character varying (300), - fantoir character varying (10), - insee_com character (5), - cadastre_com character varying (10), - dept character varying (3), - code_postal text, - source character varying (100), - batch_import_id integer); - -CREATE INDEX IF NOT EXISTS cumul_adresses_fantoir ON cumul_adresses(fantoir); -CREATE INDEX IF NOT EXISTS cumul_adresses_fantoir_source_idx ON cumul_adresses(fantoir, source); -CREATE INDEX IF NOT EXISTS cumul_adresses_geo ON cumul_adresses USING gist (geometrie); -CREATE INDEX IF NOT EXISTS cumul_adresses_insee ON cumul_adresses(insee_com) WITH (fillfactor=95); -CREATE INDEX IF NOT EXISTS cumul_adresses_dept ON cumul_adresses(dept) WITH (fillfactor=95); -CREATE INDEX IF NOT EXISTS cumul_adresses_source ON cumul_adresses(source) WITH (fillfactor=95); -CREATE INDEX IF NOT EXISTS cumul_adresses_insee_source ON cumul_adresses(insee_com,source) WITH (fillfactor=95); - -CREATE TABLE IF NOT EXISTS cumul_voies -( geometrie geometry , - voie_cadastre character varying (300), - voie_bal character varying (300), - voie_osm character varying (300), - voie_autre text, - voie_fantoir character varying (300), - fantoir character varying (10), - insee_com character (5), - cadastre_com character varying (10), - dept character varying (3), - code_postal text, - source character varying (100), - voie_index integer, - batch_import_id integer); - -CREATE INDEX IF NOT EXISTS cumul_voies_fantoir ON cumul_voies(fantoir); -CREATE INDEX IF NOT EXISTS cumul_voies_fantoir_source_idx ON cumul_voies(fantoir, source); -CREATE INDEX IF NOT EXISTS cumul_voies_geo ON cumul_voies USING gist (geometrie); -CREATE INDEX IF NOT EXISTS cumul_voies_insee ON cumul_voies(insee_com) WITH (fillfactor=95); -CREATE INDEX IF NOT EXISTS cumul_voies_dept ON cumul_voies(dept) WITH (fillfactor=95); -CREATE INDEX IF NOT EXISTS cumul_voies_source ON cumul_voies(source) WITH (fillfactor=95); -CREATE INDEX IF NOT EXISTS cumul_voies_insee_source ON cumul_voies(insee_com,source) WITH (fillfactor=95); - -ALTER TABLE cumul_voies CLUSTER ON cumul_voies_insee; - -CREATE TABLE IF NOT EXISTS cumul_places -( geometrie geometry , - libelle_cadastre character varying (300), - libelle_osm character varying (300), - libelle_fantoir character varying (300), - ld_bati integer, - ld_osm character varying (30), - fantoir character varying (10), - insee_com character (5), - -- cadastre_com character varying (10), - dept character varying (3), - code_postal text, - source character varying (100), - batch_import_id integer); - -CREATE INDEX IF NOT EXISTS cumul_places_geo ON cumul_places USING GIST(geometrie); -CREATE INDEX IF NOT EXISTS cumul_places_insee_com ON cumul_places (insee_com); - -ALTER TABLE cumul_places CLUSTER ON cumul_places_insee_com; - -CREATE TABLE IF NOT EXISTS batch ( - id_batch serial, - source character varying (100), - etape character varying (100), - timestamp_debut float, - date_debut character varying (100), - date_fin character varying (100), - dept character varying (3), - cadastre_com character (5), - insee_com character (5), - nom_com character varying(250), - nombre_adresses integer); - -CREATE TABLE IF NOT EXISTS batch_historique ( - id_batch integer, - source character varying (100), - etape character varying (100), - timestamp_debut float, - date_debut character varying (100), - date_fin character varying (100), - dept character varying (3), - cadastre_com character (5), - insee_com character (5), - nom_com character varying(250), - nombre_adresses integer); - -CREATE TABLE IF NOT EXISTS code_cadastre ( - dept character varying(3), - cadastre_dept character (3), - nom_com character varying(250), - cadastre_com character(5), - insee_com character(5), - code_postal character(5), - format_cadastre character varying(10), - date_maj integer); - -CREATE TABLE IF NOT EXISTS tmp_code_cadastre -AS SELECT * FROM code_cadastre LIMIT 0; - -GRANT SELECT ON ALL TABLES IN SCHEMA public TO public; \ No newline at end of file diff --git a/sql/create_table_base_osm.sql b/sql/create_table_base_osm.sql deleted file mode 100644 index c48bc44..0000000 --- a/sql/create_table_base_osm.sql +++ /dev/null @@ -1,182 +0,0 @@ -CREATE TABLE IF NOT EXISTS fantoir_voie ( - code_dept character(3), - code_dir character(1), - code_com character(3), - code_insee character(5), - id_voie character(4), - fantoir character(9), - fantoir10 character(10), - cle_rivoli character(1), - nature_voie character varying(4), - libelle_voie character varying(26), - type_commune character(1), - caractere_rur character(1), - caractere_voie character(1), - caractere_pop character(1), - pop_a_part integer, - pop_fictive integer, - caractere_annul character(1), - date_annul character varying(9), - date_creation character varying(9), - code_majic character(5), - type_voie character(1), - ld_bati character(1), - dernier_mot character varying(8)); -CREATE INDEX IF NOT EXISTS idx_fantoir_voie_dept ON fantoir_voie(code_dept); -CREATE INDEX IF NOT EXISTS idx_fantoir_code_insee ON fantoir_voie(code_insee); -CREATE INDEX IF NOT EXISTS idx_fantoir_fantoir ON fantoir_voie(fantoir); -CREATE INDEX IF NOT EXISTS idx_fantoir_fantoir10 ON fantoir_voie(fantoir10); - -CREATE TABLE IF NOT EXISTS ban_odbl ( - id text, - id_fantoir text, - numero text, - rep text, - nom_voie text, - code_postal text, - code_insee text, - nom_commune text, - code_insee_ancienne_commune text, - nom_ancienne_commune text, - x float, - y float, - lon float, - lat float, - type_position text, - alias text, - nom_ld text, - libelle_acheminement text, - nom_afnor text, - source_position text, - source_nom_voie text, - certification_commune integer, - cad_parcelles text); --- geometrie geometry (Point, 4326) DEFAULT (ST_Point(lon,lat))); -CREATE INDEX IF NOT EXISTS idx_ban_odbl_code_insee ON ban_odbl(code_insee); - - -CREATE TABLE IF NOT EXISTS bal_cadastre ( - cle_interop text, - uid_adresse text, - numero text, - suffixe text, - pseudo_numero boolean, - voie_nom text, - voie_code text, - code_postal text, - libelle_acheminement text, - destination_principale text, - commune_code text, - commune_nom text, - source text, - long float, - lat float, - x float, - y float, - position text, - date_der_maj date); -CREATE INDEX IF NOT EXISTS idx_bal_cadastre_commune_code ON bal_cadastre(commune_code); -CREATE INDEX IF NOT EXISTS idx_bal_cadastre_commune_codedestination_principale ON bal_cadastre(commune_code,destination_principale); - -CREATE TABLE IF NOT EXISTS bal_locales ( - cle_interop text, - commune_code text, - commune_nom text, - voie_code text, - voie_nom text, - numero text, - suffixe text, - long float, - lat float, - license text); -CREATE INDEX IF NOT EXISTS idx_bal_open_data_commune_code ON bal_locales(commune_code); - -CREATE TABLE IF NOT EXISTS lieux_dits ( - insee_com character(5), - nom text, - created date, - updated date, - geometrie geometry(Polygon,4326) -); -CREATE INDEX IF NOT EXISTS lieux_dits_insee_com ON lieux_dits(insee_com); -CREATE INDEX IF NOT EXISTS gidx_lieux_dits ON lieux_dits USING gist (geometrie); - -CREATE TABLE IF NOT EXISTS parcelles_noms ( - geometrie geometry, - insee_com character(5), - libelle character varying(100), - fantoir character varying (10) -); -CREATE INDEX IF NOT EXISTS parcelles_noms_insee_com ON parcelles_noms(insee_com); - -CREATE TABLE IF NOT EXISTS type_voie ( - id_voie integer, - tag_value text, - tag_index integer); -TRUNCATE TABLE type_voie; -INSERT INTO type_voie (id_voie,tag_value) - VALUES (1,'steps'), - (2,'path'), - (3,'cycleway'), - (4,'footway'), - (5,'pedestrian'), - (6,'track'), - (7,'service'), - (8,'road'), - (9,'living_street'), - (10,'residential'), - (11,'unclassified'), - (12,'tertiary'), - (13,'secondary'), - (14,'primary'), - (15,'trunk'), - (16,'motorway'); -UPDATE type_voie SET tag_index = power(2,id_voie-1); - - -CREATE TABLE IF NOT EXISTS type_highway_insee ( - name text, - highway text, - insee_com character(5), - timestamp_maj integer DEFAULT (date_part('epoch'::text, CURRENT_TIMESTAMP))::integer -); -CREATE INDEX IF NOT EXISTS idx_type_highway_insee_insee_com ON type_highway_insee(insee_com); - -CREATE TABLE IF NOT EXISTS suffixe ( - geometrie geometry, - insee_com character(5), - libelle_suffixe character varying(100) -); -CREATE INDEX IF NOT EXISTS gidx_suffixe ON suffixe USING GIST(geometrie); - -CREATE TABLE IF NOT EXISTS place_insee ( - long float, - lat float, - place text, - name text, - fantoir text, - ld_bati text, - tags hstore, - insee_com character(5), - timestamp_maj integer DEFAULT (date_part('epoch'::text, CURRENT_TIMESTAMP))::integer -); -CREATE INDEX IF NOT EXISTS idx_place_insee_insee_com ON place_insee(insee_com); - -CREATE TABLE IF NOT EXISTS cadastre_2_place ( - geometrie geometry(Point, 4326), - nom text, - fantoir text, - insee_com character(5) -); -CREATE INDEX IF NOT EXISTS gidx_cadastre_2_place ON cadastre_2_place USING GIST(geometrie); -CREATE INDEX IF NOT EXISTS idx_cadastre_2_place_insee_com ON cadastre_2_place(insee_com); - -CREATE TABLE IF NOT EXISTS expire_tiles ( - z integer, - x integer, - y integer, - geometrie geometry(Polygon, 3857) -); -CREATE INDEX IF NOT EXISTS gidx_expire_tiles ON expire_tiles(geometrie); - -GRANT SELECT ON ALL TABLES IN SCHEMA public TO public; \ No newline at end of file diff --git a/sql/create_table_polygones_communes.sql b/sql/create_table_polygones_communes.sql deleted file mode 100644 index 0322724..0000000 --- a/sql/create_table_polygones_communes.sql +++ /dev/null @@ -1,37 +0,0 @@ -CREATE TABLE IF NOT EXISTS polygones_insee ( - geometrie geometry (Geometry, 3857), - insee_com character(5), - nom text, - admin_level integer); -CREATE INDEX IF NOT EXISTS gidx_polygones_insee ON polygones_insee USING GIST (geometrie); -CREATE INDEX IF NOT EXISTS idx_population_insee_insee_com ON polygones_insee(insee_com); - -TRUNCATE TABLE polygones_insee; -INSERT INTO polygones_insee -SELECT way, - "ref:INSEE", - name, - admin_level -FROM planet_osm_polygon -WHERE boundary='administrative' AND - admin_level in (8,9) AND - name != ''; - -CREATE TABLE IF NOT EXISTS polygones_insee_geo ( - geometrie geometry (Geometry, 4326), - insee_com character(5), - nom text, - admin_level integer); -CREATE INDEX IF NOT EXISTS gidx_polygones_insee_geo ON polygones_insee_geo USING GIST (geometrie); -CREATE INDEX IF NOT EXISTS idx_polygones_insee_geo_insee_com ON polygones_insee_geo(insee_com); - -TRUNCATE TABLE polygones_insee_geo; -INSERT INTO polygones_insee_geo -SELECT ST_Transform(way,4326), - "ref:INSEE", - name, - admin_level -FROM planet_osm_polygon -WHERE boundary='administrative' AND - admin_level in (8,9) AND - name != ''; diff --git a/sql/create_table_population_insee.sql b/sql/create_table_population_insee.sql deleted file mode 100644 index 21467b4..0000000 --- a/sql/create_table_population_insee.sql +++ /dev/null @@ -1,6 +0,0 @@ -CREATE TABLE IF NOT EXISTS population_insee ( - insee_com character(5), - nom text, - population integer); -CREATE INDEX IF NOT EXISTS idx_population_insee_insee_com ON population_insee(insee_com); - diff --git a/sql/finalisation.sql b/sql/finalisation.sql index a970592..3d6d705 100644 --- a/sql/finalisation.sql +++ b/sql/finalisation.sql @@ -1,6 +1,8 @@ -CREATE INDEX idx_planet_osm_point_fantoir ON planet_osm_point("ref:FR:FANTOIR"); -CREATE INDEX idx_planet_osm_line_fantoir ON planet_osm_line("ref:FR:FANTOIR"); -CREATE INDEX idx_planet_osm_polygon_fantoir ON planet_osm_polygon("ref:FR:FANTOIR"); -CREATE INDEX idx_planet_osm_polygon_ref_insee ON planet_osm_polygon("ref:INSEE"); -CREATE INDEX idx_planet_osm_rels_id ON planet_osm_rels(osm_id); +CREATE INDEX idx_planet_osm_point_fantoir ON planet_osm_point("ref:FR:FANTOIR"); +CREATE INDEX idx_planet_osm_point_housenumber ON planet_osm_point("addr:housenumber"); +CREATE INDEX idx_planet_osm_line_fantoir ON planet_osm_line("ref:FR:FANTOIR"); +CREATE INDEX idx_planet_osm_polygon_fantoir ON planet_osm_polygon("ref:FR:FANTOIR"); +CREATE INDEX idx_planet_osm_polygon_housenumber ON planet_osm_polygon("addr:housenumber"); +CREATE INDEX idx_planet_osm_polygon_ref_insee ON planet_osm_polygon("ref:INSEE"); +CREATE INDEX idx_planet_osm_rels_id ON planet_osm_rels(osm_id); CREATE INDEX idx_planet_osm_communes_statut_ref_insee ON planet_osm_communes_statut("ref:INSEE"); diff --git a/sql/load_expire_tiles.sql b/sql/load_expire_tiles.sql deleted file mode 100644 index db3ee7f..0000000 --- a/sql/load_expire_tiles.sql +++ /dev/null @@ -1,6 +0,0 @@ -TRUNCATE expire_tiles; -COPY expire_tiles(z,x,y) FROM '/data/project/bano_data/expire_tiles/20190930/211505.316.tiles' WITH CSV DELIMITER '/'; - -UPDATE expire_tiles -SET geometrie = ST_SetSRID(ST_MakeBox2D(ST_Point(-20037508.34 + (40075016.68/(2 ^ :zoom)) * x, 20037508.34 - (40075016.68/(2 ^ :zoom)) * y), - ST_Point( -20037508.34 + (40075016.68/(2 ^ :zoom)) * (x + 1) ,20037508.34 - (40075016.68/(2 ^ :zoom)) * (y + 1))),3857) diff --git a/sql/load_fantoir.sql b/sql/load_fantoir.sql deleted file mode 100644 index c9012ca..0000000 --- a/sql/load_fantoir.sql +++ /dev/null @@ -1,34 +0,0 @@ -CREATE TEMP TABLE load_fantoir (raw text); -\copy load_fantoir FROM 'fantoir.txt' WITH csv delimiter '#' quote '>'; - -TRUNCATE fantoir_voie; -INSERT INTO fantoir_voie (SELECT CASE - WHEN substr(raw,1,2) = '97' - THEN substr(raw,1,2)||substr(raw,4,1) - ELSE substr(raw,1,2) - END AS code_dept, - substr(raw,3,1) as code_dir, - substr(raw,4,3) as code_com, - substr(raw,1,2)||substr(raw,4,3) as code_insee, - substr(raw,7,4) as id_voie, - substr(raw,1,2)||substr(raw,4,3)||substr(raw,7,4) as fantoir, - substr(raw,1,2)||substr(raw,4,3)||substr(raw,7,4)||substr(raw,11,1) as fantoir10, - substr(raw,11,1) as cle_rivoli, - rtrim(substr(raw,12,4)) as nature_voie, - rtrim(substr(raw,16,26)) as libelle_voie, - substr(raw,49,1) as type_commune, - substr(raw,50,1) as caractere_rur, - substr(raw,51,1) as caractere_voie, - substr(raw,52,1) as caractere_pop, - substr(raw,60,7)::integer as pop_a_part, - substr(raw,67,7)::integer as pop_fictive, - substr(raw,74,1) as caractere_annul, - substr(raw,75,7) as date_annul, - substr(raw,82,7) as date_creation, - substr(raw,104,5) as code_majic, - substr(raw,109,1) as type_voie, - substr(raw,110,1) as ld_bati, - substr(raw,113,8) as dernier_mot - FROM load_fantoir - WHERE raw NOT LIKE '______ %' AND - raw NOT LIKE '___ %'); diff --git a/sql/post_copie_ban_odbl.sql b/sql/post_copie_ban_odbl.sql deleted file mode 100644 index 97c57cf..0000000 --- a/sql/post_copie_ban_odbl.sql +++ /dev/null @@ -1,3 +0,0 @@ -ALTER TABLE ban_odbl ADD COLUMN geometrie geometry(Point, 4326); -UPDATE ban_odbl SET geometrie = ST_SetSRID(ST_Point(lon,lat),4326); -CREATE INDEX gidx_ban_odbl ON ban_odbl USING GIST(geometrie); \ No newline at end of file diff --git a/stats.sh b/stats.sh index 2ebb25d..189b1f5 100755 --- a/stats.sh +++ b/stats.sh @@ -1 +1,5 @@ -psql -d cadastre -f stats.sql -v dept=$1 +#!/bin/bash + +set -e + +$pgsql_CADASTRE -f stats.sql -v dept=$1 diff --git a/stats_export.sh b/stats_export.sh index d0fd81a..d9187b5 100755 --- a/stats_export.sh +++ b/stats_export.sh @@ -1,3 +1,7 @@ +#!/bin/bash + +set -e + source config echo $EXPORT_SAS_DIR @@ -18,4 +22,4 @@ do echo $b $csvcount $csvcountunique $shpcount $status_csv $status_shp #|awk 'BEGIN{print $1 $2 $3 $4 $5};FS=" ";OFS="\t"' #awk 'BEGIN{print $1 $2 $3 $4 $5};FS=" ";OFS="\t"' "$b $csvcount $csvcountunique $shpcount $status_csv $status_shp" # wc -l $f; -done \ No newline at end of file +done diff --git a/update_bases_adresses_locales.sh b/update_bases_adresses_locales.sh deleted file mode 100755 index 8dc27d2..0000000 --- a/update_bases_adresses_locales.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -source config - -cd $BAL_CACHE_DIR -for dep in {01..19} 2A 2B {21..95} {971..974} 976 -# for dep in {01..06} 2A -#for dep in {24..95} {971..974} 976 -do - echo 'Département '$dep - wget -NS https://adresse.data.gouv.fr/data/adresses-locales/latest/csv/adresses-locales-$dep.csv.gz - quoted_dep=\'$dep'%'\' - psql -d osm -U cadastre -c "DELETE FROM bal_locales WHERE commune_code LIKE $quoted_dep;" - gunzip -c $BAL_CACHE_DIR/adresses-locales-$dep.csv.gz|psql -d osm -U cadastre -c "\copy bal_locales from stdin with csv header delimiter ';'" - sleep 1 -done diff --git a/update_cadastre_adresses.sh b/update_cadastre_adresses.sh deleted file mode 100755 index e7f9a14..0000000 --- a/update_cadastre_adresses.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -source config - -cd $CADASTRE_CACHE_DIR -for dep in {01..19} 2A 2B {21..95} {971..974} 976 -# for dep in {01..03} 2A -#for dep in {24..95} {971..974} 976 -do - echo 'Département '$dep - wget -NS https://adresse.data.gouv.fr/data/adresses-cadastre/latest/csv/adresses-cadastre-$dep.csv.gz - quoted_dep=\'$dep'%'\' - psql -d osm -U cadastre -c "DELETE FROM bal_cadastre WHERE commune_code LIKE $quoted_dep;" - gunzip -c $CADASTRE_CACHE_DIR/adresses-cadastre-$dep.csv.gz|psql -d osm -U cadastre -c "\copy bal_cadastre from stdin with csv header delimiter ';'" - sleep 1 -done diff --git a/update_cadastre_batiments.sh b/update_cadastre_batiments.sh deleted file mode 100755 index 7c73915..0000000 --- a/update_cadastre_batiments.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -source config - -cd $DATA_DIR/bano_cache - -# for dep in {01..19} 2A 2B {21..95} {971..974} 976 -# for dep in {02..03} 2A -for dep in {04..95} {971..974} 976 2B -do - mkdir $dep - cd $dep - full_dep=`pwd` - wget https://cadastre.data.gouv.fr/data/etalab-cadastre/latest/shp/departements/$dep/cadastre-$dep-batiments-shp.zip -O $CADASTRE_CACHE_DIR/cadastre-$dep-batiments-shp.zip - unzip $CADASTRE_CACHE_DIR/cadastre-$dep-batiments-shp.zip - # reconstruction du shapefile avec ogr2ogr car corrompu pour shp2pgsql - ogr2ogr -overwrite -f 'ESRI Shapefile' batiments_ok.shp batiments.shp - shp2pgsql -s 2154:4326 -g geometrie -W LATIN1 batiments_ok.shp public.tmp_batiments$dep | psql -d cadastre -q - psql -d cadastre -f $BANO_DIR/sql/replace_batiments.sql -v schema_cible=$SCHEMA_CIBLE -v dept=$dep - zip -mT $CADASTRE_CACHE_DIR/cadastre-$dep-batiments-shp.zip batiments.* - sleep 1 - cd .. -done diff --git a/update_cadastre_lieux-dits.sh b/update_cadastre_lieux-dits.sh deleted file mode 100755 index b78ac8c..0000000 --- a/update_cadastre_lieux-dits.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -source config - -cd $CADASTRE_CACHE_DIR -echo "Mise à jour des lieux-dits" > ld.log -date >> ld.log - -for DEPT in {01..19} 2A 2B {21..95} {971..974} 976 -# for DEPT in {01..03} 2A -do - URL="https://cadastre.data.gouv.fr/data/etalab-cadastre/latest/geojson/departements/${DEPT}/cadastre-${DEPT}-lieux_dits.json.gz" - wget -NS ${URL} -o wget.log - HTTP200=`grep '200 OK' wget.log|wc -l` - if (( ${HTTP200} )) - then - echo "Téléchargement des lieux-dits pour le département ${DEPT} OK" >> ld.log - ZIPFILE=`basename ${URL}` - JSONFILE=`basename ${ZIPFILE} .gz` - # echo $URL - # echo $ZIPFILE - # echo $JSONFILE - gzip -dfk ${ZIPFILE} - mv ${JSONFILE} lieux_dits.json - # ls -al - psql -d osm -U cadastre -c "DELETE FROM lieux_dits WHERE insee_com LIKE '${DEPT}%';" - ogr2ogr -append -nln lieux_dits -fieldmap 1,0,2,3 -f PostgreSQL PG:'user=cadastre dbname=osm' lieux_dits.json - rm lieux_dits.json - else - echo "Pas de source plus à jour pour ${DEPT}" >> ld.log - fi -done diff --git a/update_table_infos_communes.sh b/update_table_infos_communes.sh deleted file mode 100755 index 39e59ae..0000000 --- a/update_table_infos_communes.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -cd $SCRIPT_DIR - -psql -d osm -U cadastre -f ./sql/update_table_infos_communes.sql - diff --git a/utils/fantoir2topo.sh b/utils/fantoir2topo.sh new file mode 100755 index 0000000..f4e1df8 --- /dev/null +++ b/utils/fantoir2topo.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e + +$pgsql_CADASTRE --csv -f fantoir2topo.sql > fantoir_202304.csv +$pgsql_BANO -v ON_ERROR_STOP=1 -c "TRUNCATE topo; COPY topo FROM STDIN WITH CSV HEADER" +rm fantoir_202304.csv diff --git a/utils/fantoir2topo.sql b/utils/fantoir2topo.sql new file mode 100644 index 0000000..6ba8622 --- /dev/null +++ b/utils/fantoir2topo.sql @@ -0,0 +1,15 @@ +SELECT code_dept, + code_insee, + fantoir, + nature_voie, + libelle_voie, + caractere_voie, + TRIM (both FROM caractere_annul), + CASE date_annul + WHEN '0000000' then 0 + ELSE to_char(to_timestamp(date_annul::text, 'YYYYDDD'),'YYYYMMDD')::integer + END, + TO_CHAR(TO_TIMESTAMP(date_creation::text,'YYYYDDD'),'YYYYMMDD')::integer, + type_voie, + dernier_mot +FROM fantoir_voie; diff --git a/utils/load_csv_export.sh b/utils/load_csv_export.sh index f32702a..9e96841 100755 --- a/utils/load_csv_export.sh +++ b/utils/load_csv_export.sh @@ -1,17 +1,20 @@ #!/bin/bash -psql -d cadastre -U cadastre -c "DROP TABLE IF EXISTS export_csv CASCADE; - CREATE TABLE export_csv( - id text, - numero text, - voie text, - cp text, - ville text, - source text, - lat decimal, - lon decimal);" +set -e -for dep in {01..19} 2A 2B {21..95} {971..974} 976 +$pgsql_CADASTRE -c " + DROP TABLE IF EXISTS export_csv CASCADE; \ + CREATE TABLE export_csv( + id text, + numero text, + voie text, + cp text, + ville text, + source text, + lat decimal, + lon decimal);" + +for dep in `cat $BANO_DIR/deplist.txt` do - cat /data/sas_web/bano-${dep}.csv|psql -d cadastre -U cadastre -c "COPY export_csv FROM STDIN WITH CSV" -done \ No newline at end of file + cat /data/sas_web/bano-${dep}.csv| $pgsql_CADASTRE -c "COPY export_csv FROM STDIN WITH CSV" +done