bugfix, ended bulshit bulkiness , gitgnore prevention, started fields…

… to keep

.gitignore

@@ -174,9 +174,10 @@ svg-inkscape/
 osmnx_routing/cache/
 cache/
 
-# the tests
-tests/
-tests.py
-
 # we shall start to log:
-*.log
+*.log
+
+# development stuff
+.vscode
+tests/
+tests.py

constants.py

@@ -77,6 +77,26 @@
     "other_footways": OTHER_FOOTWAY_RULES,
 }
 
+# columns to keep in the parquet files
+# TODO: complete and apply it
+in_all = [
+    "surface",
+    "smoothness",
+    "width",
+    "incline",
+    "tactile_paving",
+    "incline:across",
+    "osm_id",
+    "last_update",
+]
+in_linear = in_all + ["highway"]
+
+columns_to_keep = {}
+
+for k, v in layer_tags_dict.items():
+    columns_to_keep[k] = in_all.copy() if k == "kerbs" else in_linear.copy()
+    columns_to_keep[k].extend(list(v.keys()))
+
 layer_exclusion_tags = {
     "kerbs": {},
     "sidewalks": {},

dashboard/statistics_funcs.py

@@ -140,9 +140,20 @@ def double_scatter_bar(
     yh2="smoothness",
     hcolor=None,
     fontsize=24,
-    tooltip_fields=["element_type", "id"],
+    tooltip_fields: list = ["element_type", "id"],
 ):
 
+    # preselect only the needed columns:
+    needed_columns = tooltip_fields + [xs, ys, xh, yh1, yh2, hcolor, scolor]
+
+    # remove None, and any one with "()":
+    needed_columns = [column for column in needed_columns if column]
+    needed_columns = [column for column in needed_columns if "()" not in column]
+    needed_columns = list(set(needed_columns))
+
+    # only keep the needed columns:
+    input_df = input_df[needed_columns].copy()
+
     interval = alt.selection_interval()
 
     default_color = alt.value("lightseagreen")

dashboard/statistics_generation.py

@@ -131,11 +131,11 @@
 
 category_bars = {}
 
-full_url_dict = {}
-
 for category in generated_list_dict:
     # url_list = [get_url(rel_path) for rel_path in generated_list_dict[category]]
 
+    # this dict is meant to be temporary and merely classwise:
+    full_url_dict = {}
     for rel_path in generated_list_dict[category]:
         full_url_dict[rel_path] = get_url(rel_path)
 

filtering_adapting_data.py

@@ -128,8 +128,8 @@
     #         # also creating a default note
     #         gdf_dict[category][f'{req_col}_score'] = default_score
 
-    # gdf_dict[category].fillna('?',inplace=True)
-    # TODO: the '?' should only be set only at generate_webmap.py
+    # # replacing missing values with '?', again:
+    gdf_dict[category].fillna("?", inplace=True)
 
     # replacing wrong values with "?" (unknown) or misspelled with the nearest valid:
     # TODO: check if this is the better approach to handle invalid values