From 7894da32e2a243e74b26c9ed4c446b642040ca94 Mon Sep 17 00:00:00 2001 From: jeremy Date: Mon, 28 Mar 2022 10:13:39 +0200 Subject: [PATCH] classification1st_version --- .../Launch_ChainUrbanTypo_MaPuce_BDTopo_v2.sh | 6 +- .../UrbanTypo_MaPuce_BDTopo_v2/cols2Keep.csv | 2 +- .../test_cities.csv | 2 +- .../Launch_ChainUrbanTypo_MaPuce_OSM.sh | 4 +- .../UrbanTypo_MaPuce_OSM/cols2Keep.csv | 2 +- .../UrbanTypo_MaPuce_OSM/configFile.json | 1 + .../UrbanTypo_MaPuce_OSM/test_cities.csv | 2 +- ...aunch_ChainBuildingHeight_BDTOPO_V2_OSM.sh | 2 +- .../calculateIndependentVariables.groovy | 7 ++- .../classification_investigation.py | 55 ++++++++++-------- classification/createFinalDataset.groovy | 6 +- classification/createTrueValues.groovy | 6 +- classification/groovy_generic_function.groovy | 6 +- .../python_model_optimization/AnalyseData.py | 4 +- .../__pycache__/AnalyseData.cpython-36.pyc | Bin 0 -> 4225 bytes .../__pycache__/__init__.cpython-36.pyc | Bin 0 -> 206 bytes examplesDemat.groovy | 26 +++++++++ 17 files changed, 82 insertions(+), 49 deletions(-) create mode 100644 classification/UrbanTypo_MaPuce_OSM/configFile.json create mode 100644 classification/python_model_optimization/__pycache__/AnalyseData.cpython-36.pyc create mode 100644 classification/python_model_optimization/__pycache__/__init__.cpython-36.pyc create mode 100644 examplesDemat.groovy diff --git a/classification/UrbanTypo_MaPuce_BDTopo_v2/Launch_ChainUrbanTypo_MaPuce_BDTopo_v2.sh b/classification/UrbanTypo_MaPuce_BDTopo_v2/Launch_ChainUrbanTypo_MaPuce_BDTopo_v2.sh index 47a1921..3326337 100644 --- a/classification/UrbanTypo_MaPuce_BDTopo_v2/Launch_ChainUrbanTypo_MaPuce_BDTopo_v2.sh +++ b/classification/UrbanTypo_MaPuce_BDTopo_v2/Launch_ChainUrbanTypo_MaPuce_BDTopo_v2.sh @@ -4,10 +4,10 @@ # Path of the data location ############################################################# # File where are stored the list of cities to process (note that the cities to be processed should be String - even for insee codes - separated by comma and the file should be located in the same folder as the current file) -nameFileCities="allCities.csv" +nameFileCities="test_cities.csv" outputFolder="/home/decide/Data/URBIO/Donnees_brutes/LCZ/TrainingDataSets/Indicators/" dependentVariablePath="/home/decide/Documents/CloudS/LABSTICC/ClassificationSupervisee/Data/data_apprentissage.shp" -pathToSaveTrainingDataSet="/home/decide/Data/URBIO/Donnees_brutes/UrbanTypo/BDTOPO_V2/MaPuce/TrainingDataset/" +pathToSaveTrainingDataSet="/home/decide/Data/URBIO/Donnees_brutes/UrbanTypo/BDTOPO_V2/MaPuce/TrainingDataset2/" ##############################################################" # Parameters to set @@ -43,7 +43,7 @@ pathToSaveResultSensit="/home/decide/Data/URBIO/Donnees_brutes/UrbanTypo/BDTOPO_ # Whether of not the filename containing the dataset by city should contain the datasetName ('BDTOPO_V2' or 'OSM') datasetByCityContainsDataset=0 # File path to save the resulting dataset WITHOUT THE EXTENSION !! -pathToSaveFinalDataset="/home/decide/Code/Intel/geoclimate/models/TRAINING_DATA_URBAN_TYPOLOGY_BDTOPO_V2_RF_1_0" +pathToSaveFinalDataset="/home/decide/Code/Intel/geoclimate/models/TRAINING_DATA_UTRF_BDTOPO_V2_RF_2_2" thresholdCol="UNIQUENESS_VALUE:0.3" # File where are saved all columns to use as independent variables for the training fileNameCol2keep="cols2Keep.csv" diff --git a/classification/UrbanTypo_MaPuce_BDTopo_v2/cols2Keep.csv b/classification/UrbanTypo_MaPuce_BDTopo_v2/cols2Keep.csv index c16e1ca..5daf984 100644 --- a/classification/UrbanTypo_MaPuce_BDTopo_v2/cols2Keep.csv +++ b/classification/UrbanTypo_MaPuce_BDTopo_v2/cols2Keep.csv @@ -1 +1 @@ -BLOCK_AREA,BLOCK_AVG_HEIGHT_ROOF_AREA_WEIGHTED,BLOCK_BUILDING_DIRECTION_EQUALITY,BLOCK_BUILDING_DIRECTION_UNIQUENESS,BLOCK_CLOSINGNESS,BLOCK_FLOOR_AREA,BLOCK_HOLE_AREA_DENSITY,BLOCK_NET_COMPACTNESS,BLOCK_STD_HEIGHT_ROOF_AREA_WEIGHTED,BLOCK_VOLUME,BUILD_AREA,BUILD_AREA_CONCAVITY,BUILD_COMMON_WALL_FRACTION,BUILD_CONTIGUITY,BUILD_FLOOR_AREA,BUILD_FORM_FACTOR,BUILD_HEIGHT_ROOF,BUILD_HEIGHT_WALL,BUILD_LIKELIHOOD_LARGE_BUILDING,BUILD_MAIN_USE,BUILD_MINIMUM_BUILDING_SPACING,BUILD_NB_LEV,BUILD_NUMBER_BUILDING_NEIGHBOR,BUILD_PERIMETER,BUILD_PERIMETER_CONVEXITY,BUILD_RAW_COMPACTNESS,BUILD_ROAD_DISTANCE,BUILD_TOTAL_FACADE_LENGTH,BUILD_TYPE,BUILD_VOLUME,RSU_AREA,RSU_ASPECT_RATIO,RSU_AVG_HEIGHT_ROOF_AREA_WEIGHTED,RSU_AVG_MINIMUM_BUILDING_SPACING,RSU_AVG_NUMBER_BUILDING_NEIGHBOR,RSU_AVG_VOLUME,RSU_BUILDING_DIRECTION_EQUALITY,RSU_BUILDING_DIRECTION_UNIQUENESS,RSU_BUILDING_FLOOR_AREA_DENSITY,RSU_BUILDING_FRACTION,RSU_BUILDING_NUMBER_DENSITY,RSU_BUILDING_TOTAL_FRACTION,RSU_BUILDING_VOLUME_DENSITY,RSU_FREE_EXTERNAL_FACADE_DENSITY,RSU_GEOM_AVG_HEIGHT_ROOF,RSU_GROUND_LINEAR_ROAD_DENSITY,RSU_HIGH_VEGETATION_BUILDING_FRACTION,RSU_HIGH_VEGETATION_FRACTION,RSU_HIGH_VEGETATION_IMPERVIOUS_FRACTION,RSU_HIGH_VEGETATION_IMPERVIOUS_FRACTION_URB,RSU_HIGH_VEGETATION_LOW_VEGETATION_FRACTION,RSU_HIGH_VEGETATION_PERVIOUS_FRACTION_URB,RSU_HIGH_VEGETATION_ROAD_FRACTION,RSU_HIGH_VEGETATION_WATER_FRACTION,RSU_IMPERVIOUS_FRACTION,RSU_IMPERVIOUS_FRACTION_URB,RSU_LOW_VEGETATION_FRACTION,RSU_LOW_VEGETATION_FRACTION_URB,RSU_ROAD_FRACTION,RSU_ROAD_FRACTION_URB,RSU_STD_HEIGHT_ROOF_AREA_WEIGHTED,RSU_VEGETATION_FRACTION_URB,RSU_WATER_FRACTION +BLOCK_AREA,BLOCK_AVG_HEIGHT_ROOF_AREA_WEIGHTED,BLOCK_BUILDING_DIRECTION_EQUALITY,BLOCK_BUILDING_DIRECTION_UNIQUENESS,BLOCK_CLOSINGNESS,BLOCK_FLOOR_AREA,BLOCK_HOLE_AREA_DENSITY,BLOCK_NET_COMPACTNESS,BLOCK_STD_HEIGHT_ROOF_AREA_WEIGHTED,BLOCK_VOLUME,BUILD_AREA,BUILD_AREA_CONCAVITY,BUILD_COMMON_WALL_FRACTION,BUILD_CONTIGUITY,BUILD_FLOOR_AREA,BUILD_FORM_FACTOR,BUILD_HEIGHT_ROOF,BUILD_HEIGHT_WALL,BUILD_LIKELIHOOD_LARGE_BUILDING,BUILD_MAIN_USE,BUILD_MINIMUM_BUILDING_SPACING,BUILD_NB_LEV,BUILD_NUMBER_BUILDING_NEIGHBOR,BUILD_PERIMETER,BUILD_PERIMETER_CONVEXITY,BUILD_RAW_COMPACTNESS,BUILD_ROAD_DISTANCE,BUILD_TOTAL_FACADE_LENGTH,BUILD_TYPE,BUILD_VOLUME,RSU_AREA,RSU_ASPECT_RATIO,RSU_AVG_HEIGHT_ROOF_AREA_WEIGHTED,RSU_AVG_MINIMUM_BUILDING_SPACING,RSU_AVG_NUMBER_BUILDING_NEIGHBOR,RSU_AVG_VOLUME,RSU_BUILDING_DIRECTION_EQUALITY,RSU_BUILDING_DIRECTION_UNIQUENESS,RSU_BUILDING_FLOOR_AREA_DENSITY,RSU_BUILDING_FRACTION,RSU_BUILDING_NUMBER_DENSITY,RSU_BUILDING_TOTAL_FRACTION,RSU_BUILDING_VOLUME_DENSITY,RSU_FREE_EXTERNAL_FACADE_DENSITY,RSU_GEOM_AVG_HEIGHT_ROOF,RSU_GROUND_LINEAR_ROAD_DENSITY,RSU_HIGH_VEGETATION_BUILDING_FRACTION,RSU_HIGH_VEGETATION_FRACTION,RSU_HIGH_VEGETATION_IMPERVIOUS_FRACTION,RSU_HIGH_VEGETATION_IMPERVIOUS_FRACTION_UTRF,RSU_HIGH_VEGETATION_LOW_VEGETATION_FRACTION,RSU_HIGH_VEGETATION_PERVIOUS_FRACTION_UTRF,RSU_HIGH_VEGETATION_ROAD_FRACTION,RSU_HIGH_VEGETATION_WATER_FRACTION,RSU_IMPERVIOUS_FRACTION,RSU_IMPERVIOUS_FRACTION_UTRF,RSU_LOW_VEGETATION_FRACTION,RSU_LOW_VEGETATION_FRACTION_UTRF,RSU_ROAD_FRACTION,RSU_ROAD_FRACTION_UTRF,RSU_STD_HEIGHT_ROOF_AREA_WEIGHTED,RSU_VEGETATION_FRACTION_UTRF,RSU_WATER_FRACTION diff --git a/classification/UrbanTypo_MaPuce_BDTopo_v2/test_cities.csv b/classification/UrbanTypo_MaPuce_BDTopo_v2/test_cities.csv index 9a32cb3..4b504d8 100644 --- a/classification/UrbanTypo_MaPuce_BDTopo_v2/test_cities.csv +++ b/classification/UrbanTypo_MaPuce_BDTopo_v2/test_cities.csv @@ -1 +1 @@ -"13010","13027","13066","13083","17010","17028","17032","17059","17094","17136","17193","17194","17274","17291","17300","17391","17407","17414","17420","17443","30351","31044","31052","31069","31091","31113","31116","31149","31150","31165","31187","31203","31230","31341","31395","31420","31424","31433","31445","31446","31458","31490","31506","31547","31555","31557","31588","44014","44018","44020","44026","44029","44037","44047","44070","44094","44101","44109","44130","44143","44162","44172","44190","44215","68015","68043","68056","68154","68166","68218","68224","68256","68271","68278","68300","68321","68375","68376","74010","74011","74019","74026","74067","74093","74104","74108","74112","74147","74152","74182","74213","74217","74242","74267","74268","74272","75101","75104","75105","75106","75107","75108","75109","75110","75111","75112","75113","75114","75115","75116","75117","75118","75119","75120","84007","84016","84034","84035","84036","84039","84054","84081","84087","84092","84129","84141","84142","92004","92009","92012","92025","92026","92033","92036","92040","92046","92047","92048","92049","92050","92062","92063","92064","92072","92073","92075","92076","92077","92078" +"13010","13027","13066","13083","17010","17028","17059","17094","17136","17193","17274","17291","17300","17391","17407","17414","17420","17443","30351","31044","31052","31069","31091","31113","31116","31149","31150","31160","31165","31187","31203","31230","31341","31395","31420","31424","31433","31445","31446","31458","31490","31506","31547","31555","31557","31588","44014","44018","44020","44026","44029","44037","44047","44070","44094","44101","44109","44130","44143","44162","44172","44190","44215","68015","68043","68056","68154","68166","68218","68224","68256","68271","68278","68300","68321","68375","68376","74010","74011","74019","74026","74067","74093","74104","74108","74112","74147","74152","74182","74213","74217","74242","74267","74268","74272","75101","75104","75105","75106","75107","75108","75109","75110","75111","75112","75113","75114","75115","75116","75117","75118","75119","75120","84007","84016","84034","84035","84036","84039","84054","84081","84087","84092","84129","84141","84142","92004","92009","92012","92025","92026","92033","92036","92040","92046","92047","92048","92049","92050","92062","92063","92064","92072","92073","92075","92076","92077","92078" diff --git a/classification/UrbanTypo_MaPuce_OSM/Launch_ChainUrbanTypo_MaPuce_OSM.sh b/classification/UrbanTypo_MaPuce_OSM/Launch_ChainUrbanTypo_MaPuce_OSM.sh index 4a8bd49..82a01d7 100644 --- a/classification/UrbanTypo_MaPuce_OSM/Launch_ChainUrbanTypo_MaPuce_OSM.sh +++ b/classification/UrbanTypo_MaPuce_OSM/Launch_ChainUrbanTypo_MaPuce_OSM.sh @@ -7,7 +7,7 @@ nameFileCities="test_cities.csv" outputFolder="/home/decide/Data/URBIO/Donnees_brutes/LCZ/TrainingDataSets/Indicators/" dependentVariablePath="/home/decide/Documents/CloudS/LABSTICC/ClassificationSupervisee/Data/data_apprentissage" -pathToSaveTrainingDataSet="/home/decide/Data/URBIO/Donnees_brutes/UrbanTypo/OSM/MaPuce/TrainingDataset/" +pathToSaveTrainingDataSet="/home/decide/Data/URBIO/Donnees_brutes/UrbanTypo/OSM/MaPuce/TrainingDataset2/" ##############################################################" # Parameters to set @@ -43,7 +43,7 @@ pathToSaveResultSensit="/home/decide/Data/URBIO/Donnees_brutes/UrbanTypo/BDTOPO_ # Whether of not the filename containing the dataset by city should contain the datasetName ('BDTOPO_V2' or 'OSM') datasetByCityContainsDataset=0 # File path to save the resulting dataset WITHOUT THE EXTENSION !! -pathToSaveFinalDataset="/home/decide/Code/Intel/geoclimate/models/TRAINING_DATA_URBAN_TYPOLOGY_OSM_RF_2_0" +pathToSaveFinalDataset="/home/decide/Code/Intel/geoclimate/models/TRAINING_DATA_UTRF_OSM_RF_2_2" thresholdCol="UNIQUENESS_VALUE:0.3" # File where are saved all columns to use as independent variables for the training fileNameCol2keep="cols2Keep.csv" diff --git a/classification/UrbanTypo_MaPuce_OSM/cols2Keep.csv b/classification/UrbanTypo_MaPuce_OSM/cols2Keep.csv index 839048c..8633b03 100644 --- a/classification/UrbanTypo_MaPuce_OSM/cols2Keep.csv +++ b/classification/UrbanTypo_MaPuce_OSM/cols2Keep.csv @@ -1 +1 @@ -BUILD_AVG_PERIMETER,BUILD_STD_PERIMETER,BUILD_AVG_AREA,BUILD_STD_AREA,BUILD_AVG_TOTAL_FACADE_LENGTH,BUILD_STD_TOTAL_FACADE_LENGTH,BUILD_STD_COMMON_WALL_FRACTION,BUILD_STD_NUMBER_BUILDING_NEIGHBOR,BUILD_AVG_AREA_CONCAVITY,BUILD_STD_AREA_CONCAVITY,BUILD_AVG_FORM_FACTOR,BUILD_STD_FORM_FACTOR,BUILD_AVG_PERIMETER_CONVEXITY,BUILD_STD_PERIMETER_CONVEXITY,BUILD_STD_MINIMUM_BUILDING_SPACING,BUILD_AVG_ROAD_DISTANCE,BUILD_STD_ROAD_DISTANCE,BUILD_AVG_LIKELIHOOD_LARGE_BUILDING,BUILD_STD_LIKELIHOOD_LARGE_BUILDING,RSU_HIGH_VEGETATION_FRACTION,RSU_HIGH_VEGETATION_WATER_FRACTION,RSU_HIGH_VEGETATION_BUILDING_FRACTION,RSU_HIGH_VEGETATION_LOW_VEGETATION_FRACTION,RSU_HIGH_VEGETATION_ROAD_FRACTION,RSU_HIGH_VEGETATION_IMPERVIOUS_FRACTION,RSU_WATER_FRACTION,RSU_BUILDING_FRACTION,RSU_LOW_VEGETATION_FRACTION,RSU_ROAD_FRACTION,RSU_IMPERVIOUS_FRACTION,RSU_VEGETATION_FRACTION_URB,RSU_LOW_VEGETATION_FRACTION_URB,RSU_HIGH_VEGETATION_IMPERVIOUS_FRACTION_URB,RSU_HIGH_VEGETATION_PERVIOUS_FRACTION_URB,RSU_ROAD_FRACTION_URB,RSU_IMPERVIOUS_FRACTION_URB,RSU_AREA,RSU_GROUND_LINEAR_ROAD_DENSITY,RSU_AVG_NUMBER_BUILDING_NEIGHBOR,RSU_AVG_MINIMUM_BUILDING_SPACING,RSU_BUILDING_NUMBER_DENSITY,RSU_BUILDING_TOTAL_FRACTION,RSU_BUILDING_DIRECTION_EQUALITY,RSU_BUILDING_DIRECTION_UNIQUENESS,BUILD_TYPE,BUILD_MAIN_USE,BUILD_PERIMETER,BUILD_AREA,BUILD_TOTAL_FACADE_LENGTH,BUILD_COMMON_WALL_FRACTION,BUILD_NUMBER_BUILDING_NEIGHBOR,BUILD_AREA_CONCAVITY,BUILD_FORM_FACTOR,BUILD_PERIMETER_CONVEXITY,BUILD_MINIMUM_BUILDING_SPACING,BUILD_ROAD_DISTANCE,BUILD_LIKELIHOOD_LARGE_BUILDING,BLOCK_AREA,BLOCK_HOLE_AREA_DENSITY,BLOCK_BUILDING_DIRECTION_EQUALITY,BLOCK_BUILDING_DIRECTION_UNIQUENESS,BLOCK_CLOSINGNESS +BUILD_AVG_PERIMETER,BUILD_STD_PERIMETER,BUILD_AVG_AREA,BUILD_STD_AREA,BUILD_AVG_TOTAL_FACADE_LENGTH,BUILD_STD_TOTAL_FACADE_LENGTH,BUILD_STD_COMMON_WALL_FRACTION,BUILD_STD_NUMBER_BUILDING_NEIGHBOR,BUILD_AVG_AREA_CONCAVITY,BUILD_STD_AREA_CONCAVITY,BUILD_AVG_FORM_FACTOR,BUILD_STD_FORM_FACTOR,BUILD_AVG_PERIMETER_CONVEXITY,BUILD_STD_PERIMETER_CONVEXITY,BUILD_STD_MINIMUM_BUILDING_SPACING,BUILD_AVG_ROAD_DISTANCE,BUILD_STD_ROAD_DISTANCE,BUILD_AVG_LIKELIHOOD_LARGE_BUILDING,BUILD_STD_LIKELIHOOD_LARGE_BUILDING,RSU_HIGH_VEGETATION_FRACTION,RSU_HIGH_VEGETATION_WATER_FRACTION,RSU_HIGH_VEGETATION_BUILDING_FRACTION,RSU_HIGH_VEGETATION_LOW_VEGETATION_FRACTION,RSU_HIGH_VEGETATION_ROAD_FRACTION,RSU_HIGH_VEGETATION_IMPERVIOUS_FRACTION,RSU_WATER_FRACTION,RSU_BUILDING_FRACTION,RSU_LOW_VEGETATION_FRACTION,RSU_ROAD_FRACTION,RSU_IMPERVIOUS_FRACTION,RSU_VEGETATION_FRACTION_UTRF,RSU_LOW_VEGETATION_FRACTION_UTRF,RSU_HIGH_VEGETATION_IMPERVIOUS_FRACTION_UTRF,RSU_HIGH_VEGETATION_PERVIOUS_FRACTION_UTRF,RSU_ROAD_FRACTION_UTRF,RSU_IMPERVIOUS_FRACTION_UTRF,RSU_AREA,RSU_GROUND_LINEAR_ROAD_DENSITY,RSU_AVG_NUMBER_BUILDING_NEIGHBOR,RSU_AVG_MINIMUM_BUILDING_SPACING,RSU_BUILDING_NUMBER_DENSITY,RSU_BUILDING_TOTAL_FRACTION,RSU_BUILDING_DIRECTION_EQUALITY,RSU_BUILDING_DIRECTION_UNIQUENESS,BUILD_TYPE,BUILD_MAIN_USE,BUILD_PERIMETER,BUILD_AREA,BUILD_TOTAL_FACADE_LENGTH,BUILD_COMMON_WALL_FRACTION,BUILD_NUMBER_BUILDING_NEIGHBOR,BUILD_AREA_CONCAVITY,BUILD_FORM_FACTOR,BUILD_PERIMETER_CONVEXITY,BUILD_MINIMUM_BUILDING_SPACING,BUILD_ROAD_DISTANCE,BUILD_LIKELIHOOD_LARGE_BUILDING,BLOCK_AREA,BLOCK_HOLE_AREA_DENSITY,BLOCK_BUILDING_DIRECTION_EQUALITY,BLOCK_BUILDING_DIRECTION_UNIQUENESS,BLOCK_CLOSINGNESS diff --git a/classification/UrbanTypo_MaPuce_OSM/configFile.json b/classification/UrbanTypo_MaPuce_OSM/configFile.json new file mode 100644 index 0000000..ae7f7fb --- /dev/null +++ b/classification/UrbanTypo_MaPuce_OSM/configFile.json @@ -0,0 +1 @@ +{"description":"Apply the OSM workflow","geoclimatedb":{"path":"/tmp/geoclimate_db;AUTO_SERVER=TRUE","delete":true},"input":{"osm":["ORANGE"]},"output":{"folder":"/home/decide/Data/URBIO/Donnees_brutes/LCZ/TrainingDataSets/Indicators/OSM"},"parameters":{"indicatorUse":["URBAN_TYPOLOGY"],"svfSimplified":false,"prefixName":""}} \ No newline at end of file diff --git a/classification/UrbanTypo_MaPuce_OSM/test_cities.csv b/classification/UrbanTypo_MaPuce_OSM/test_cities.csv index 587a0a3..3f75c98 100644 --- a/classification/UrbanTypo_MaPuce_OSM/test_cities.csv +++ b/classification/UrbanTypo_MaPuce_OSM/test_cities.csv @@ -1 +1 @@ -"BARBENTANE","CHATEAURENARD","NOVES","ROGNONAS","ANGOULINS","AYTRE","BALLON","BOURGNEUF","CHATELAILLON-PLAGE","CROIX-CHAPEAU","LA JARNE","LA JARRIE","PERIGNY","PUILBOREAU","LA ROCHELLE","SAINT-ROGATIEN","SAINTE-SOULLE","SAINT-XANDRE","SALLES-SUR-MER","THAIRE","VILLENEUVE-LES-AVIGNON","BALMA","BEAUMONT-SUR-LEZE","BLAGNAC","BRUGUIERES","CASTANET-TOLOSAN","CASTELGINEST","COLOMIERS","CORNEBARRIEU","DAUX","EAUNES","FONSORBES","FROUZINS","GRATENTOUR","MERVILLE","MURET","PINSAGUEL","PLAISANCE-DU-TOUCH","PORTET-SUR-GARONNE","QUINT-FONSEGRIVES","RAMONVILLE-SAINT-AGNE","ROQUES","SAINT-JORY","SAINT-ORENS-DE-GAMEVILLE","SEYSSES","TOULOUSE","TOURNEFEUILLE","VILLENEUVE-TOLOSANE","LE BIGNON","BOUAYE","BOUGUENAIS","CARQUEFOU","LA CHAPELLE-BASSE-MER","CHATEAU-THEBAUD","COUERON","LA HAIE-FOUASSIERE","MAUVES-SUR-LOIRE","LA MONTAGNE","NANTES","PONT-SAINT-MARTIN","REZE","SAINT-HERBLAIN","SAINTE-LUCE-SUR-LOIRE","SAINT-SEBASTIEN-SUR-LOIRE","VERTOU","BALDERSHEIM","BOLLWILLER","BRUNSTATT","DIDENHEIM","ILLZACH","KINGERSHEIM","MORSCHWILLER-LE-BAS","MULHOUSE","PFASTATT","RIEDISHEIM","RIXHEIM","SAUSHEIM","STAFFELFELDEN","WITTELSHEIM","WITTENHEIM","ANNECY","ANNECY-LE-VIEUX","ARGONAY","LA BALME-DE-SILLINGY","CHAVANOD","CRAN-GEVRIER","DOUSSARD","DUINGT","EPAGNY","LATHUILE","LOVAGNY","METZ-TESSY","MEYTHET","POISY","PRINGY","SAINT-JORIOZ","SEVRIER","SEYNOD","SILLINGY","PARIS","AVIGNON","BEDARRIDES","CAUMONT-SUR-DURANCE","CAVAILLON","CHATEAUNEUF-DE-GADAGNE","COURTHEZON","L'ISLE-SUR-LA-SORGUE","MORIERES-LES-AVIGNON","LE PONTET","SORGUES","VEDENE","VELLERON","ASNIERES-SUR-SEINE","BOIS-COLOMBES","BOULOGNE-BILLANCOURT","COLOMBES","COURBEVOIE","GARCHES","GENNEVILLIERS","ISSY-LES-MOULINEAUX","MALAKOFF","MARNES-LA-COQUETTE","MEUDON","MONTROUGE","NANTERRE","PUTEAUX","RUEIL-MALMAISON","SAINT-CLOUD","SEVRES","SURESNES","VANVES","VAUCRESSON","VILLE-D'AVRAY","VILLENEUVE-LA-GARENNE" +"BARBENTANE","CHATEAURENARD","NOVES","ROGNONAS","ANGOULINS","AYTRE","BOURGNEUF","CHATELAILLON-PLAGE","CROIX-CHAPEAU","LA JARNE","PUILBOREAU","LA ROCHELLE","SAINT-ROGATIEN","SAINTE-SOULLE","SAINT-XANDRE","SALLES-SUR-MER","THAIRE","VILLENEUVE-LES-AVIGNON","BALMA","BEAUMONT-SUR-LEZE","BLAGNAC","BRUGUIERES","CASTANET-TOLOSAN","CASTELGINEST","COLOMIERS","CORNEBARRIEU","DAUX","FONSORBES","FROUZINS","GRATENTOUR","MURET","PINSAGUEL","PLAISANCE-DU-TOUCH","PORTET-SUR-GARONNE","QUINT-FONSEGRIVES","RAMONVILLE-SAINT-AGNE","ROQUES","SAINT-JORY","SAINT-ORENS-DE-GAMEVILLE","SEYSSES","TOULOUSE","TOURNEFEUILLE","VILLENEUVE-TOLOSANE","LE BIGNON","BOUAYE","BOUGUENAIS","CARQUEFOU","LA CHAPELLE-BASSE-MER","CHATEAU-THEBAUD","COUERON","LA HAIE-FOUASSIERE","MAUVES-SUR-LOIRE","NANTES","PONT-SAINT-MARTIN","REZE","SAINT-HERBLAIN","SAINTE-LUCE-SUR-LOIRE","SAINT-SEBASTIEN-SUR-LOIRE","VERTOU","BALDERSHEIM","BOLLWILLER","BRUNSTATT","DIDENHEIM","ILLZACH","KINGERSHEIM","MORSCHWILLER-LE-BAS","MULHOUSE","PFASTATT","RIEDISHEIM","RIXHEIM","SAUSHEIM","STAFFELFELDEN","WITTELSHEIM","WITTENHEIM","ANNECY","ANNECY-LE-VIEUX","ARGONAY","LA BALME-DE-SILLINGY","CHAVANOD","CRAN-GEVRIER","DOUSSARD","DUINGT","EPAGNY","LATHUILE","LOVAGNY","METZ-TESSY","MEYTHET","POISY","SAINT-JORIOZ","SEVRIER","SEYNOD","SILLINGY","PARIS","AVIGNON","BEDARRIDES","CAUMONT-SUR-DURANCE","CAVAILLON","CHATEAUNEUF-DE-GADAGNE","COURTHEZON","L'ISLE-SUR-LA-SORGUE","MORIERES-LES-AVIGNON","SORGUES","VEDENE","ASNIERES-SUR-SEINE","BOIS-COLOMBES","BOULOGNE-BILLANCOURT","COLOMBES","COURBEVOIE","GARCHES","GENNEVILLIERS","ISSY-LES-MOULINEAUX","MALAKOFF","MARNES-LA-COQUETTE","MEUDON","MONTROUGE","NANTERRE","PUTEAUX","RUEIL-MALMAISON","SAINT-CLOUD","SEVRES","SURESNES","VANVES","VAUCRESSON","VILLE-D'AVRAY","VILLENEUVE-LA-GARENNE" diff --git a/classification/buildingHeight_BDTopo_v2_OSM/Launch_ChainBuildingHeight_BDTOPO_V2_OSM.sh b/classification/buildingHeight_BDTopo_v2_OSM/Launch_ChainBuildingHeight_BDTOPO_V2_OSM.sh index 29a985d..37ea8fb 100644 --- a/classification/buildingHeight_BDTopo_v2_OSM/Launch_ChainBuildingHeight_BDTOPO_V2_OSM.sh +++ b/classification/buildingHeight_BDTopo_v2_OSM/Launch_ChainBuildingHeight_BDTOPO_V2_OSM.sh @@ -75,7 +75,7 @@ pathToSaveResultSensit="/home/decide/Data/URBIO/Donnees_brutes/BuildingHeight/BD datasetByCityContainsDataset=0 # File path to save the resulting dataset WITHOUT THE EXTENSION !! pathToSaveFinalDataset="/home/decide/Code/Intel/geoclimate/models/TRAINING_DATA_BUILDINGHEIGHT_OSM_RF_1_0" -thresholdCol="UNIQUENESS_VALUE:0.95" +thresholdCol="UNIQUENESS_VALUE:0.7" # File where are saved all columns to use as independent variables for the training fileNameCol2keep="cols2Keep.csv" optionalinputFileSuffix=".geojson" diff --git a/classification/calculateIndependentVariables.groovy b/classification/calculateIndependentVariables.groovy index c1b4d4d..0007f79 100644 --- a/classification/calculateIndependentVariables.groovy +++ b/classification/calculateIndependentVariables.groovy @@ -1,14 +1,15 @@ // Declaration of the maven repository @GrabResolver(name='orbisgis', root='https://nexus.orbisgis.org/repository/orbisgis/') + // Declaration of our Nexus repository, where the geoclimate project is stored -@Grab(group='org.orbisgis.orbisprocess', module='geoclimate', version='1.0.0-SNAPSHOT', classifier='jar-with-dependencies', transitive=false) +@Grab(group='org.orbisgis.geoclimate', module='geoclimate', version='1.0.0-SNAPSHOT') //JSON lib @Grab(group='org.codehaus.groovy', module='groovy-json', version='3.0.4') -import org.orbisgis.orbisprocess.geoclimate.Geoclimate -import org.orbisgis.orbisprocess.geoclimate.geoindicators.Geoindicators +import org.orbisgis.geoclimate.Geoclimate +import org.orbisgis.geoclimate.Geoindicators import org.orbisgis.orbisdata.datamanager.jdbc.h2gis.H2GIS import org.orbisgis.orbisdata.datamanager.jdbc.* import groovy.json.JsonOutput diff --git a/classification/classification_investigation.py b/classification/classification_investigation.py index 6634bb9..9ad9d94 100644 --- a/classification/classification_investigation.py +++ b/classification/classification_investigation.py @@ -226,12 +226,12 @@ if df.drop(colNotIndic, axis = 1).columns.difference(df_default_val.index).empty: df.fillna(df_default_val["default_val"], axis = 0, inplace=True) else: - print "You should define a default value for the following columns of your input data: {0}".format(df.drop(colNotIndic, axis = 1).columns.difference(df_default_val.index)) + print("You should define a default value for the following columns of your input data: {0}".format(df.drop(colNotIndic, axis = 1).columns.difference(df_default_val.index))) list_df = pd.Series(df.drop(colNotIndic, axis = 1).columns.difference(df_default_val.index)) list_df.to_csv("/home/decide/Bureau/list_indic.csv") # Minimum number of classified objects (within the entire dataset) -nb_min_default = df.index.size*0.05 +nb_min_default = round(df.index.size*0.05) # DEFINED SENSITIVITY ANALYSIS #Constants @@ -241,13 +241,13 @@ #Parameters for the default case classification scenari = {"default" : pd.Series({"perc_bu_calib" : [0.7],\ "algo2use" : "randomForest",\ - "model_stat_params" : [{"ntree" : 500, "min_size_node" : 1,\ - "nb_var_tree" : 9, "norm_indic" : False, - "max_depth" : 80, "max_leaf_nodes" : 400}], - "nb_test" : 5, "indic2use" : {"default" : initial_indic},\ + "model_stat_params" : [{"ntree" : 500, "min_size_node" : 0.00001,\ + "nb_var_tree" : 0.35, "norm_indic" : False, + "max_depth" : None, "max_leaf_nodes" : 500}], + "nb_test" : 10, "indic2use" : {"default" : initial_indic},\ "dist_class_typ" : ["REPRESENTATIVE"],\ "nb_min" : nb_min_default, - "threshold_uniqueness" : [0.7]})} + "threshold_uniqueness" : [0.9]})} #We initialize the default parameters for all the classification cases and then #we set the variations of each case @@ -259,17 +259,18 @@ scenari["distrib_class"] = scenari["default"].copy() scenari["distrib_class"]["dist_class_typ"] = ["RANDOM"] #scenari["distrib_class"]["dist_class_typ"] = ["EQUALLY", "REPRESENTATIVE"] -scenari["distrib_class"]["nb_min"] = nb_min_class * nb_class +if classif == "true": + scenari["distrib_class"]["nb_min"] = nb_min_class * nb_class #3.3 in the report scenari["uniqueness"] = scenari["default"].copy() -scenari["uniqueness"]["threshold_uniqueness"] = [0.3, 0.5, 0.7, 1.0] +scenari["uniqueness"]["threshold_uniqueness"] = [0.3, 0.7, 0.9, 0.95] #3.5 in the report scenari["param_RF"] = scenari["default"].copy() scenari["param_RF"]["model_stat_params"] = [{"ntree" : nt, "min_size_node" : msn,\ "nb_var_tree" : nvt, "norm_indic" : ni, "max_depth": md, - "max_leaf_nodes": mln} for nt in [100, 500, 1000]\ -for msn in [1, 3, 5, 7] for nvt in [3,7,9,15] for ni in [False] -for md in [20, 40, 60, 80] for mln in [100, 200, 300, 400]] + "max_leaf_nodes": mln} for nt in [100, 350, 500, 650]\ +for msn in [0.0001, 0.00005, 0.00001, 0.000001] for nvt in [0.2, 0.35, 0.5] for ni in [False] +for md in [None] for mln in [300, 500, 800, 1100]] scenari["param_RF"]["model_stat_params"].remove(scenari["default"]["model_stat_params"][0]) #3.6 in the report #scenari["classif_algo"] = scenari["default"].copy() @@ -301,7 +302,7 @@ fig, ax = plt.subplots() df2hist.hist(bins = bin_nb, ax = ax) """ -print df +print(df) # ========================================================== #CLASSIFICATION TESTS #Scenari that are calculated @@ -310,7 +311,7 @@ "min_size_node", "nb_var_tree", "max_depth", "max_leaf_nodes", "norm_indic",\ "algo2use", "score_inter", "nb_min", "threshold_uniqueness"]) for s in scenari: - print "\n\nScenario : " + s + print("\n\nScenario : " + s) scenario = scenari[s].copy() #For each indicators combination of the scenario for ind_case in scenario["indic2use"]: @@ -359,14 +360,14 @@ for thresh_uniqueness in scenario["threshold_uniqueness"]: - print u"Uniqueness threshold :"+str(thresh_uniqueness*100)+"%" + print(u"Uniqueness threshold :"+str(thresh_uniqueness*100)+"%") data2use_cal = df_transf.copy() for dist_class_typ in scenario["dist_class_typ"]: for perc_bu_calib in scenario["perc_bu_calib"]: - print u"Percentage of objects used for calibration :"+str(perc_bu_calib*100)+"%" + print(u"Percentage of objects used for calibration :"+str(perc_bu_calib*100)+"%") #Calibration and verification are performed 'nb_test' times for i in range(0, nb_test): - print u"Calibration N° :" + str(i + 1) + print(u"Calibration N° :" + str(i + 1)) # Select the data having the uniqueness value higher than a certain value data2use_i=data2use_cal[data2use_cal[uniqueness_val] >= thresh_uniqueness] #Select data in order to have the same number of items @@ -421,23 +422,27 @@ del df_transf del df_indic -result.to_csv(pathResults+"result2.csv") +result.to_csv(pathResults+"result3.csv") # ============================================================= # ========== PRINT THE BEST COMBINATION ======================= # ============================================================= # Identify the 'percentileToKeep' % of best values -percentileToKeep = 0.02 -best = result[result.score_inter>result.score_inter.quantile(1-percentileToKeep)][["score_inter", "ntree","min_size_node","nb_var_tree","max_depth","max_leaf_nodes"]] +percentileToKeep = 1.0 +#col2keep = ["score_inter", "ntree","min_size_node","nb_var_tree","max_depth","max_leaf_nodes"] +col2keep = ["score_inter", "ntree","min_size_node","nb_var_tree","max_leaf_nodes"] +col2keepWscore = col2keep.copy() +col2keepWscore.remove("score_inter") +best = result[result.score_inter<8_M$hU>)ft_ugQoZ6c>Su_wR&Pbw` zBIy~*@w!w{VBN--pl%O6=9G(XJ@p?5&_ho>&$XBO)`Jg44sCyrlw@gD7)dXsz|nB# z&3ljU^T&&ei_SlP`@H#R)iC~MOkE|+-@&bD;Kr`OO>PC|u9f{4c8l46X}81+y!go2 zEsJ8WLN_n*GOx5t+~(Cs=B~}HJ4W3Z5F6*36mB9oi$b>39k51ppKV@b7k+f6PEa?iy zK~EYbW3^|_jaE0^I1cu60e6J-MXp)SHy^J@k_p%AvZ$T&Z9`Hp=!fTEhY52xvqZ8V z3#Pi!AmFSeSjWFF!Wpr|@@(zqt)JG;vD%Hshs}>ZrrFIKjT&s2E!FCCCngKGMKX}# zcd_2i#1dmO2wpdJ$q%`BpbOCC8GLK#a;{7Vb6kIYTNkkK?BBR#1q*dYmsKQ@6}q5A zqOF+g>ck_GVqIHdtP4TZ0mY!D3w=M-#a`rx^|H2jTQAO-T-yOcsM+=sU6w)(f<&8b zT^Z3lHRK7f{ATxjH|mS?TzEbg=dVQ=BH8j)$5(GCPx^79&U=BYl;8Hf%mwG;eWYdB z?1MIFMsecz{fGI^)zA(0l_0G*;(fg|)+Z0uCCF2!Fc`}V){^CzE0%4ZGHv`$;(i5d zat*t%*|lB0bbI^#+uL`x8#`BbZhW|{i-{Zvk5Zj3;zjcUZnXvDBu3ACZ1hl-kmhA> z4J$~(!d^l4(h?=D`T_n+NXqvd;B6+{T$dBDR!*aAt&*k-;TE_+(8%Mz?7TRcOX*el5Ogv|22&+Z!G?_q&snY2GY_)Ti#?YD-`f0H}SJGEZM_lbEg1?RUipV52#3%<=tHNDl`<3Lb#S`S-Is(S0}Y&ht*P>`|M8u$T6O{_DdS&Wn8`j{1! z^|p_!Xik^&^^DbMrHNA43{iB!bIggM%0x${^tBtK&KdTL=^f_eG8Sdnu9css8#{3Q zu!8F&2r>^9Nc4-q7WW270|X*P{&&}ZZ1IueFNzAi^^_QeCL6eNVl6iaa@BlE*-zmk&pFhWq46?0w^0CWz=o{rJiUcvhrI*?4#+f5#@(^{m}(Tn-`+ zX1#*R_<$t!Ws!zgG3X*$S)RspgmC}y_6`hD&9NWKc0*T2(XJOq?W37tWKflXX90Ta ztCH*`kI?WMaV;StIzrmCP^E>cK#mBBuSZ?SB2zD9&POXnbSi?#9x!j#yumu_ z-Z_9SClS4XF5{y_gj|Gh0+x%as`)iQ&vWOLod!W5cz#l zY;mT*A^>vGEb%>U=!LPDR5NUi_Joq(qdm(Dw9^{2+j%?Ekg`ebP1_V>$436l zgk|2-%ZJ^x33lpT4906qW)a|e*<$80MnG+oW~%_&D;QVJSF_NGOg=+8Ni6sPyx}+b literal 0 HcmV?d00001 diff --git a/examplesDemat.groovy b/examplesDemat.groovy new file mode 100644 index 0000000..c89c4e4 --- /dev/null +++ b/examplesDemat.groovy @@ -0,0 +1,26 @@ +@GrabResolver(name='orbisgis', root='https://nexus.orbisgis.org/repository/orbisgis/') +@Grab(group='org.orbisgis', module='demat', version='0.0.6') +@Grab(group='org.orbisgis.orbisdata.datamanager', module='jdbc', version='1.0.1-SNAPSHOT') +@Grab(group='org.orbisgis.geoclimate', module='geoclimate', version='0.0.1') + + +import static org.orbisgis.demat.Plot.* +import org.orbisgis.orbisdata.datamanager.jdbc.h2gis.H2GIS + +H2GIS h2GIS = H2GIS.open("/tmp/demo_db") + +// To make a graphic +h2GIS.execute("""DROP TABLE IF EXISTS TEST; CREATE TABLE TEST(ID int, TOTO int); + INSERT INTO TEST VALUES (1,2), (2,15), (3,25);""") + +Chart(Data(h2GIS.rows("SELECT * FROM TEST"))).mark_bar() +.encode(X("ID").nominal(), Y("TOTO").quantitative()).show() + +// To make a map +h2GIS.execute("""DROP TABLE IF EXISTS TEST; CREATE TABLE TESTS AS SELECT st_makepoint(-60 + x*random()/500.00, 30 + x*random()/500.00) FROM GENERATE_SERIES(1, 200)""") +h2GIS.save("TEST", "/tmp/data.geojson") + +Chart(Data(h2GIS.rows("SELECT * FROM TEST"))).mark_bar() +.encode(X("ID").nominal(), Y("TOTO").quantitative()).show() + +