Merge pull request #79 from the-scouts/naming

application.py

@@ -5,6 +5,7 @@
 import dash_html_components as html
 
 dashboard = dash.Dash(__name__, external_stylesheets=["https://codepen.io/chriddyp/pen/bWLwgP.css"])
+app = dashboard.server
 
 dashboard.layout = html.Div(
     [
@@ -22,5 +23,3 @@
 
 if __name__ == "__main__":
     dashboard.run_server(debug=False)
-else:
-    app = dashboard.server

incognita-config.toml

@@ -18,8 +18,99 @@ boundaries = "data/Boundary shape files/"
 # Folder for generated files
 output = "Output/"
 
-[custom_boundaries."District"]
-name = "D_ID"
+[ons2019."Local Authority"]
+key = "oslaua"  # Local Authority Districts
+
+codes.path = "LA_UA names and codes UK as at 12_19.csv"
+codes.key = "LAD19CD"
+codes.key_type = "string"
+codes.name = "LAD19NM"
+
+shapefile.path = "Local_Authority_Districts__December_2019__Boundaries_UK_BUC/Local_Authority_Districts__December_2019__Boundaries_UK_BUC.shp"
+shapefile.key = "lad19cd"
+shapefile.name = "lad19nm"
+
+age_profile.path = "lad_by_age.csv"
+age_profile.key = "Code"
+
+[ons2019."County (LA)"]
+key = "oslaua"  # Counties
+
+codes.path = "LA_UA names and codes UK as at 12_19.csv"
+codes.key = "LAD19CD"
+codes.key_type = "string"
+codes.name = "LAD19NM"
+
+shapefile.path = "Counties_and_Unitary_Authorities_December_2017_Generalised_Clipped_Boundaries_in_UK/Counties_and_Unitary_Authorities_December_2017_Generalised_Clipped_Boundaries_in_UK.shp"
+shapefile.key = "ctyua17cd"
+shapefile.name = "ctyua17nm"
+
+[ons2019."Ward"]
+key = "osward"  # Council Wards
+
+codes.path = "Ward names and codes UK as at 05_19_NSPD.csv"
+codes.key = "WD19CD"
+codes.key_type = "string"
+codes.name = "WD19NM"
+
+shapefile.path = "Wards_December_2019_Boundaries_UK_BGC/Wards_December_2019_Boundaries_UK_BGC.shp"
+shapefile.key = "WD19CD"
+shapefile.name = "WD19NM"
+
+age_profile.path = "osward_by_age_mid_2018_population_may_2019_wards.csv"
+age_profile.key = "Ward Code"
+
+[ons2019."Constituency"]
+key = "pcon"  # Parliamentary Constituencies
+
+codes.path = "Westminster Parliamentary Constituency names and codes UK as at 12_14.csv"
+codes.key = "PCON14CD"
+codes.key_type = "string"
+codes.name = "PCON14NM"
+
+shapefile.path = "Westminster_PCON_Dec_2017_Generalised_Clipped_UK/Westminster_Parliamentary_Constituencies_December_2017_Generalised_Clipped_Boundaries_in_the_UK.shp"
+shapefile.key = "pcon17cd"
+shapefile.name = "pcon17nm"
+
+age_profile.path = "pcon_by_age.csv"
+age_profile.key = "PCON11CD"
+
+[ons2019."LSOA"]
+key = "lsoa11"  # Lower Level Super Output Areas
+
+codes.path = "LSOA (2011) names and codes UK as at 12_12.csv"
+codes.key = "LSOA11CD"
+codes.key_type = "string"
+codes.name = "LSOA11NM"
+
+shapefile.path = "Lower_Layer_Super_Output_Areas_December_2011_Boundaries_EW_BSC/Lower_Layer_Super_Output_Areas_December_2011_Boundaries_EW_BSC.shp"
+shapefile.key = "LSOA11CD"
+shapefile.name = "LSOA11NM"
+
+[ons2019."MSOA"]
+key = "msoa11"  # Middle Layer Super Output Areas
+
+codes.path = "MSOA (2011) names and codes UK as at 12_12.csv"
+codes.key = "MSOA11CD"
+codes.key_type = "string"
+codes.name = "MSOA11NM"
+
+shapefile.path = "Middle_Layer_Super_Output_Areas_December_2011_Full_Clipped_Boundaries_in_England_and_Wales/Middle_Layer_Super_Output_Areas_December_2011_Full_Clipped_Boundaries_in_England_and_Wales.shp"
+shapefile.key = "msoa11cd"
+
+[ons2019."Intermediate Zone"]
+key = "msoa11"  # Intermediate Zones (codepages identical to MSOA but different shapefiles)
+
+codes.path = "MSOA (2011) names and codes UK as at 12_12.csv"
+codes.key = "MSOA11CD"
+codes.key_type = "string"
+codes.name = "MSOA11NM"
+
+shapefile.path = "SG_IntermediateZoneBdry_2011/SG_IntermediateZone_Bdry_2011.shp"
+shapefile.key = "InterZone"
+
+[custom_boundaries."Scout District"]
+key = "D_ID"
 
 codes.path = "data/Scout Census Data/district_id_mapping.csv"
 codes.key = "D_ID"
@@ -30,16 +121,16 @@ shapefile.path = "../../scripts/districts_buffered.geojson"
 shapefile.key = "id"
 shapefile.name = "name"
 
-[custom_boundaries."imd_decile"]
-name = "imd_decile"
+[custom_boundaries."IMD Decile"]
+key = "imd_decile"
 
 codes.path = "data/Scout Census Data/imd_names_and_codes.csv"
 codes.key = "IMD Decile"
 codes.key_type = "Int32"
 codes.name = "Name"
 
-[custom_boundaries."NYS"]
-name = "nys_districts"
+[custom_boundaries."District (NYS)"]
+key = "nys_districts"
 
 codes.path = "data/Scout Census Data/North_Yorkshire_Districts.csv"
 codes.key = "ID"
@@ -55,7 +146,7 @@ age_profile.key = "Area Codes"
 age_profile.pivot_key = "lsoa11"
 
 [custom_boundaries."County"]
-name = "ctyua19cd"
+key = "ctyua19cd"
 
 codes.path = "Scout Census Data/Counties_and_Unitary_Authorities__December_2019__Boundaries_UK.csv"
 codes.key = "ctyua19cd"

scripts/map_add_custom_data.py

@@ -14,23 +14,22 @@
     scout_data.filter_records("postcode_is_valid", {1}, exclusion_analysis=True)
 
     # Generate boundary report
-    reports = Reports("lsoa", scout_data)
+    reports = Reports("LSOA", scout_data)
     reports.filter_boundaries("oslaua", {la_code})  # Leeds LA code
-    reports.create_boundary_report(["Section numbers"], report_name="leeds_sections")
+    reports.create_boundary_report({"Section numbers"}, report_name="leeds_sections")
 
     # Create map object
-    mapper = Map(scout_data, map_name="Leeds")
+    mapper = Map(map_name="Leeds")
 
     # Plot
-    dimension = {"column": f"Beavers-{year}", "tooltip": f"Beavers {year}", "legend": "# Beavers"}
-    mapper.add_areas(dimension, reports, show=True)
+    mapper.add_areas(f"Beavers-{year}", f"Beavers {year}", "# Beavers", reports, show=True)
     mapper.add_custom_data(
         config.SETTINGS.folders.national_statistical / "leeds_primary_schools.csv",
         "Primary Schools",
         location_cols="Postcodes",
         marker_data=["EstablishmentName"],
     )
-    mapper.add_sections_to_map(scout_data, mapper.district_colour_mapping(), ["youth membership"], single_section="Beavers")
+    mapper.add_sections_to_map(scout_data, mapper.district_colour_mapping(scout_data), {"youth membership"}, single_section="Beavers")
 
     # Save the map and display
     mapper.save_map()

scripts/map_awards_script.py

@@ -21,16 +21,15 @@
     scout_data.filter_records("C_name", {"Bailiwick of Guernsey", "Isle of Man", "Jersey"}, mask=True)
 
     # Generate boundary report
-    reports = Reports("lad", scout_data)
-    reports.create_boundary_report(["awards"], report_name="laua_awards_report")
+    reports = Reports("Local Authority", scout_data)
+    reports.create_boundary_report({"awards"}, report_name="laua_awards_report")
 
     # Create map object
-    mapper = Map(scout_data, map_name="UK_QSA_awards")
+    mapper = Map(map_name="UK_QSA_awards")
 
     # Plot
-    dimension = {"column": "%-QSA", "tooltip": "QSA %", "legend": "QSA %"}
-    mapper.add_areas(dimension, reports, show=True)
-    mapper.add_sections_to_map(scout_data, mapper.district_colour_mapping(), ["youth membership", "awards"], single_section="Explorers", cluster_markers=True)
+    mapper.add_areas("%-QSA", "QSA %", "QSA %", reports, show=True)
+    mapper.add_sections_to_map(scout_data, mapper.district_colour_mapping(scout_data), {"youth membership", "awards"}, single_section="Explorers", cluster_markers=True)
 
     # Save the map and display
     mapper.save_map()

scripts/map_heatmap_county_growth.py

@@ -62,7 +62,7 @@
         "Number of Sections",
     ]
 
-    # lad_reports = Reports("lad", scout_data)
+    # lad_reports = Reports("Local Authority", scout_data)
     # lad_reports.filter_boundaries("X_name", country_names, "oslaua")
     # lad_reports.create_boundary_report(opts, historical=True, report_name=f"{location_name} - LADs")
     # for i in range(offset):
@@ -72,9 +72,9 @@
     # lad_reports.data[f"Sections_change"] = (lad_reports.data[['Colonys-2020', 'Packs-2020', 'Troops-2020', 'Units-2020']].sum(axis=1) / lad_reports.data[['Colonys-2019', 'Packs-2019', 'Troops-2019', 'Units-2019']].sum(axis=1) - 1) * 100
 
     cty_reports = Reports("County", scout_data)
-    cty_reports.ons_pd.fields.append(cty_reports.geography.metadata.name)
+    cty_reports.ons_pd.fields.add(cty_reports.geography.metadata.key)
     cty_reports.filter_boundaries("ctry", {"W92000004"})
-    cty_reports.geography.region_ids_mapping = cty_reports.geography.region_ids_mapping[cty_reports.geography.region_ids_mapping["ctyua19cd"].str.startswith("W")]
+    cty_reports.geography.boundary_codes = cty_reports.geography.boundary_codes[cty_reports.geography.boundary_codes["codes"].str.startswith("W")]
     cty_reports.add_shapefile_data()
     cty_reports.create_boundary_report(opts, historical=True, report_name=f"{location_name} - Counties")
     cty_reports.create_uptake_report(report_name=f"{location_name} - Counties (uptake)")
@@ -90,63 +90,48 @@
         / cty_reports.data[["Colonys-2019", "Packs-2019", "Troops-2019", "Units-2019"]].sum(axis=1)
         - 1
     ) * 100
-    cty_reports._save_report(cty_reports.data, f"{location_name} - Counties with change")
+    utility.save_report(cty_reports.data, f"{location_name} - Counties with change")
 
     # Create map object
-    mapper = Map(scout_data, map_name=f"{location_name} uptake map")
+    mapper = Map(map_name=f"{location_name} uptake map")
 
-    # dimension = {"column": "All_change", "tooltip": "% Change 6-18", "legend": "% Change 6-18 (LADs)"}
-    # mapper.add_areas(dimension, lad_reports)
+    # mapper.add_areas("All_change", "% Change 6-18", "% Change 6-18 (LADs)", lad_reports)
 
     # TODO BUG only last add areas has correct colour mapping for same reports instance
 
-    dimension = {"column": "All_change", "tooltip": "% Change 6-18", "legend": "% Change 6-18 (Counties)"}
-    mapper.add_areas(dimension, cty_reports, show=True)
+    mapper.add_areas("All_change", "% Change 6-18", "% Change 6-18 (Counties)", cty_reports, show=True)
 
-    dimension = {"column": "Beavers_change", "tooltip": "% Change Beavers", "legend": "% Change Beavers (Counties)"}
-    mapper.add_areas(dimension, cty_reports)
+    mapper.add_areas("Beavers_change", "% Change Beavers", "% Change Beavers (Counties)", cty_reports)
 
-    dimension = {"column": "Cubs_change", "tooltip": "% Change Cubs", "legend": "% Change Cubs (Counties)"}
-    mapper.add_areas(dimension, cty_reports)
+    mapper.add_areas("Cubs_change", "% Change Cubs", "% Change Cubs (Counties)", cty_reports)
 
-    dimension = {"column": "Scouts_change", "tooltip": "% Change Scouts", "legend": "% Change Scouts (Counties)"}
-    mapper.add_areas(dimension, cty_reports)
+    mapper.add_areas("Scouts_change", "% Change Scouts", "% Change Scouts (Counties)", cty_reports)
 
-    dimension = {"column": "Explorers_change", "tooltip": "% Change Explorers", "legend": "% Change Explorers (Counties)"}
-    mapper.add_areas(dimension, cty_reports)
+    mapper.add_areas("Explorers_change", "% Change Explorers", "% Change Explorers (Counties)", cty_reports)
 
-    dimension = {"column": "Adults_change", "tooltip": "% Change Adults", "legend": "% Change Adults (Counties)"}
-    mapper.add_areas(dimension, cty_reports)
+    mapper.add_areas("Adults_change", "% Change Adults", "% Change Adults (Counties)", cty_reports)
 
-    dimension = {"column": "Sections_change", "tooltip": "% Change # Sections", "legend": "% Change # Sections (Counties)"}
-    mapper.add_areas(dimension, cty_reports)
+    mapper.add_areas("Sections_change", "% Change # Sections", "% Change # Sections (Counties)", cty_reports)
 
-    # dimension = {"column": "%-All-2020", "tooltip": "% Uptake 6-18", "legend": "% Uptake 6-18 (Counties)"}
-    # mapper.add_areas(dimension, cty_reports, significance_threshold=0)
+    # mapper.add_areas("%-All-2020", "% Uptake 6-18", "% Uptake 6-18 (Counties)", cty_reports, significance_threshold=0)
     #
-    # dimension = {"column": "%-Beavers-2020", "tooltip": "% Uptake Beavers", "legend": "% Uptake Beavers (Counties)"}
-    # mapper.add_areas(dimension, cty_reports, significance_threshold=0)
+    # mapper.add_areas("%-Beavers-2020", "% Uptake Beavers", "% Uptake Beavers (Counties)", cty_reports, significance_threshold=0)
     #
-    # dimension = {"column": "%-Cubs-2020", "tooltip": "% Uptake Cubs", "legend": "% Uptake Cubs (Counties)"}
-    # mapper.add_areas(dimension, cty_reports, significance_threshold=0)
+    # mapper.add_areas("%-Cubs-2020", "% Uptake Cubs", "% Uptake Cubs (Counties)", cty_reports, significance_threshold=0)
     #
-    # dimension = {"column": "%-Scouts-2020", "tooltip": "% Uptake Scouts", "legend": "% Uptake Scouts (Counties)"}
-    # mapper.add_areas(dimension, cty_reports, significance_threshold=0)
+    # mapper.add_areas("%-Scouts-2020", "% Uptake Scouts", "% Uptake Scouts (Counties)", cty_reports, significance_threshold=0)
     #
-    # dimension = {"column": "%-Explorers-2020", "tooltip": "% Uptake Explorers", "legend": "% Uptake Explorers (Counties)"}
-    # mapper.add_areas(dimension, cty_reports, significance_threshold=0)
+    # mapper.add_areas("%-Explorers-2020", "% Uptake Explorers", "% Uptake Explorers (Counties)", cty_reports, significance_threshold=0)
 
     # Save the map and display
     mapper.save_map()
     mapper.show_map()
 
     # create_section_maps
-    # static_scale = {"index": [0, 8, 20], "min": 0, "max": 20, "boundaries": [0, 3, 4, 6, 8, 11]}
     # for section_label in Reports.SECTION_AGES.keys():
-    #     dimension = {"column": f"%-{section_label}-{year}", "tooltip": section_label, "legend": f"{year} {section_label} uptake (%)"}
-    #     section_map = Map(scout_data, map_name=f"pcon_uptake_report_{section_label}")
-    #     section_map.add_areas(dimension, pcon_reports, scale=static_scale)
-    #     section_map.add_sections_to_map(scout_data, section_map.district_colour_mapping(), ["youth membership"], single_section=section_label)
+    #     section_map = Map(map_name=f"pcon_uptake_report_{section_label}")
+    #     section_map.add_areas(f"%-{section_label}-{year}", section_label, f"{year} {section_label} uptake (%)", pcon_reports, scale_index=[0, 8, 20], scale_step_boundaries=[0, 3, 4, 6, 8, 11])
+    #     section_map.add_sections_to_map(scout_data, section_map.district_colour_mapping(scout_data), {"youth membership"}, single_section=section_label)
     #     section_map.save_map()
 
     # get script execution time etc.

scripts/map_imd_uk.py

@@ -21,27 +21,30 @@
     scout_data.filter_records("ctry", country_codes)
     scout_data.filter_records("postcode_is_valid", {1}, exclusion_analysis=True)
 
-    lsoa = Reports("lsoa", scout_data)
+    lsoa = Reports("LSOA", scout_data)
     lsoa.filter_boundaries("ctry", country_codes)
-    lsoa.create_boundary_report(["Section numbers", "6 to 17 numbers"], report_name="lsoa_ew")
+    lsoa.create_boundary_report({"Section numbers", "6 to 17 numbers"}, report_name="lsoa_ew")
 
-    # iz = Reports("iz", scout_data)
+    # iz = Reports("Intermediate Zone", scout_data)
     # iz.filter_boundaries(field="ctry", value_list={"S92000003"})
-    # iz.create_boundary_report(["Section numbers", "6 to 17 numbers"], report_name="iz_all")
+    # iz.create_boundary_report({"Section numbers", "6 to 17 numbers"}, report_name="iz_all")
 
     # Create map object
-    mapper = Map(scout_data, map_name="lsoa_ew_map 6")
+    mapper = Map(map_name="lsoa_ew_map 6")
 
     # Create 6 to 17 map - IMD deciles
-    dimension = {"column": "imd_decile", "tooltip": "IMD", "legend": "Index of Multiple Deprivation Decile"}
-    scale = {"min": 1, "max": 10, "index": [1, 3, 7, 10], "boundaries": [1, 3, 7, 10]}
-    mapper.add_areas(dimension, lsoa, show=True, scale=scale)
-    # map.add_areas(dimension, iz, show=True)
+    mapper.add_areas("imd_decile", "IMD", "Index of Multiple Deprivation Decile", lsoa, show=True, scale_index=[1, 3, 7, 10], scale_step_boundaries=[1, 3, 7, 10])
+    # mapper.add_areas("imd_decile", "IMD", "Index of Multiple Deprivation Decile", iz, show=True)
 
     # Plot sections
-    mapper.set_region_of_colour("X_name", countries)
-    sections = dict(name="Sections", markers_clustered=True, show=True)
-    mapper.add_meeting_places_to_map(sections=scout_data.data, colour=mapper.district_colour_mapping(), marker_data=["youth membership"], layer=sections)
+    mapper.add_meeting_places_to_map(
+        sections=scout_data.census_data,
+        colour=mapper.district_colour_mapping(scout_data),
+        marker_data={"youth membership"},
+        cluster_markers=True,
+        coloured_region=countries,
+        coloured_region_key="X_name",
+    )
 
     # Save the map and display
     mapper.save_map()