[Custom DC] Fix Disappearing Stat Var bug on Map and Scatter tool (#4251

)

This PR fixes some bugs in the map and scatter tool on Custom DC

## Bug 1: Disappearing stat vars

When a user selects a place + child place type in the dropdowns, the
custom variables sometimes disappear from the sidebar. For example, when
using the default OECD preview data, the OECD variables disappear from
the sidebar when selecting "Asia" and "Country" in the dropdowns.

This happens because the custom variables don't have enough geographic
coverage to meet the 10 place minimum set in PR #2314.

This PR sets this minimum to 1 place for custom DC (so that variables
with no places are still filtered out) and leaves the 10 place minimum
unchanged on main DC.

![Screenshot 2024-05-17 at 4 06
45 PM](https://github.com/datacommonsorg/website/assets/4034366/25b70c84-0e9d-44ce-9d12-cc05175641a0)

![Screenshot 2024-05-28 at 3 03
55 PM](https://github.com/datacommonsorg/website/assets/4034366/c4f83423-eca1-4126-87bc-c31d6ee460c8)

![Screenshot 2024-05-28 at 3 04
25 PM](https://github.com/datacommonsorg/website/assets/4034366/4e7b417e-6642-4b30-9758-a37e19a0894d)

![Screenshot 2024-05-28 at 3 04
50 PM](https://github.com/datacommonsorg/website/assets/4034366/ed98cb71-3f23-4c22-96be-ef053bb2aae2)


## Bug 2: ResizeObserver error

This PR also fixes a console error on the scatter tool encountered while
debugging:
![Screenshot 2024-05-17 at 4 31
53 PM](https://github.com/datacommonsorg/website/assets/4034366/931de621-891f-4a78-a1a6-fed7d54e9f29)

server/__init__.py

@@ -398,6 +398,10 @@ def create_app(nl_root=DEFAULT_NL_ROOT):
     blocklist_svg = ["dc/g/Uncategorized", "oecd/g/OECD"]
   app.config['BLOCKLIST_SVG'] = blocklist_svg
 
+  # Set whether to filter stat vars with low geographic coverage in the
+  # map and scatter tools.
+  app.config['MIN_STAT_VAR_GEO_COVERAGE'] = cfg.MIN_STAT_VAR_GEO_COVERAGE
+
   if not cfg.TEST:
     urls = get_health_check_urls()
     libutil.check_backend_ready(urls)

server/app_env/_base.py

@@ -81,4 +81,9 @@ class Config:
   # Optional: custom dc template folder name:
   # /server/templates/custom_dc/<CUSTOM_DC_TEMPLATE_FOLDER>/
   # Defaults to the custom DC application environment name (Config.ENV value)
-  CUSTOM_DC_TEMPLATE_FOLDER = ''
+  CUSTOM_DC_TEMPLATE_FOLDER = ''
+  # Optional: Minimum number of entities a stat var needs to have data for it to
+  # be included in the map and scatter plot tools. Setting a value of 1 shows
+  # all stat vars available for a given entity. Setting a value > 1 prevents
+  # users from encountering almost-empty maps and sparse scatter plots.
+  MIN_STAT_VAR_GEO_COVERAGE = 10

server/app_env/custom.py

@@ -21,6 +21,7 @@ class Config(_base.Config):
   NAME = "Custom Data Commons"
   OVERRIDE_CSS_PATH = '/custom_dc/custom/overrides.css'
   LOGO_PATH = "/custom_dc/custom/logo.png"
+  MIN_STAT_VAR_GEO_COVERAGE = 1
   SHOW_DISASTER = False
   USE_LLM = False
   USE_MEMCACHE = False
@@ -31,4 +32,4 @@ class LocalConfig(Config, local.Config):
 
 
 class ComposeConfig(Config, local.Config):
-  pass
+  pass

server/templates/tools/map.html

@@ -35,6 +35,7 @@
   <div id="main-pane"></div>
   <script>
     infoConfig = {{info_json|tojson|safe}};
+    globalThis.minStatVarGeoCoverage = {{ config['MIN_STAT_VAR_GEO_COVERAGE'] | int(1) }};
   </script>
 {% endblock %}
 

server/templates/tools/scatter.html

@@ -31,6 +31,7 @@
 <div id="main-pane"></div>
 <script>
   infoConfig = {{info_json|tojson|safe}};
+  globalThis.minStatVarGeoCoverage = {{ config['MIN_STAT_VAR_GEO_COVERAGE'] | int(1) }};
 </script>
 {% endblock %}
 

server/templates/tools/visualization.html

@@ -29,6 +29,7 @@
   <div id="main-pane"></div>
   <script>
     infoConfig = {{info_json|tojson|safe}};
+    globalThis.minStatVarGeoCoverage = {{ config['MIN_STAT_VAR_GEO_COVERAGE'] | int(1) }};
   </script>
 {% endblock %}
 

static/js/apps/visualization/stat_var_selector.tsx

@@ -35,7 +35,10 @@ import {
 import { Spinner } from "../../components/spinner";
 import { getStatVarInfo } from "../../shared/stat_var";
 import { StatVarHierarchy } from "../../stat_var_hierarchy/stat_var_hierarchy";
-import { getFilteredStatVarPromise } from "../../utils/app/visualization_utils";
+import {
+  getFilteredStatVarPromise,
+  getNumEntitiesExistence,
+} from "../../utils/app/visualization_utils";
 import { AppContext } from "./app_context";
 import { VIS_TYPE_CONFIG } from "./vis_type_configs";
 
@@ -104,9 +107,9 @@ export function StatVarSelector(props: StatVarSelectorPropType): JSX.Element {
             selectSV={addSv}
             searchLabel={""}
             deselectSV={removeSv}
-            numEntitiesExistence={Math.min(
-              Math.max(samplePlaces.length, 1),
-              visTypeConfig.svHierarchyNumExistence || 1
+            numEntitiesExistence={getNumEntitiesExistence(
+              samplePlaces,
+              visTypeConfig
             )}
           />
         )}

static/js/apps/visualization/vis_type_configs/map_config.tsx

@@ -217,7 +217,7 @@ function getFooter(): string {
 export const MAP_CONFIG = {
   displayName: "Map Explorer",
   svHierarchyType: StatVarHierarchyType.MAP,
-  svHierarchyNumExistence: 10,
+  svHierarchyNumExistence: globalThis.minStatVarGeoCoverage,
   singlePlace: true,
   getChildTypesFn: getAllChildPlaceTypes,
   numSv: 1,

static/js/apps/visualization/vis_type_configs/scatter_config.tsx

@@ -224,7 +224,7 @@ function getInfoContent(): JSX.Element {
 export const SCATTER_CONFIG = {
   displayName: "Scatter Plot",
   svHierarchyType: StatVarHierarchyType.SCATTER,
-  svHierarchyNumExistence: 10,
+  svHierarchyNumExistence: globalThis.minStatVarGeoCoverage,
   singlePlace: true,
   numSv: 2,
   getChartArea,

static/js/shared/ga_events.test.tsx

@@ -89,6 +89,7 @@ import { ScatterChartType } from "../tools/scatter/util";
 import { Chart as TimelineToolChart } from "../tools/timeline/chart";
 import * as dataFetcher from "../tools/timeline/data_fetcher";
 import { axiosMock } from "../tools/timeline/mock_functions";
+import { getNumEntitiesExistence } from "../utils/app/visualization_utils";
 import {
   GA_EVENT_PLACE_CATEGORY_CLICK,
   GA_EVENT_PLACE_CHART_CLICK,

static/js/tools/download/mock_functions.ts

@@ -114,7 +114,7 @@ export function axiosMock(): void {
     .calledWith("/api/variable-group/info", {
       dcid: "dc/g/Root",
       entities: [],
-      numEntitiesExistence: undefined,
+      numEntitiesExistence: 0,
     })
     .mockResolvedValue(rootGroupsData);
 
@@ -123,15 +123,15 @@ export function axiosMock(): void {
     .calledWith("/api/variable-group/info", {
       dcid: "dc/g/Root",
       entities: ["geoId/06001", "geoId/06002"],
-      numEntitiesExistence: undefined,
+      numEntitiesExistence: 1,
     })
     .mockResolvedValue(rootGroupsData);
 
   when(axios.post)
     .calledWith("/api/variable-group/info", {
       dcid: "dc/g/Root",
       entities: ["geoId/06002", "geoId/06001"],
-      numEntitiesExistence: undefined,
+      numEntitiesExistence: 1,
     })
     .mockResolvedValue(rootGroupsData);
 
@@ -140,15 +140,15 @@ export function axiosMock(): void {
     .calledWith("/api/variable-group/info", {
       dcid: "dc/g/Demographics",
       entities: ["geoId/06001", "geoId/06002"],
-      numEntitiesExistence: undefined,
+      numEntitiesExistence: 1,
     })
     .mockResolvedValue(demographicsGroupsData);
 
   when(axios.post)
     .calledWith("/api/variable-group/info", {
       dcid: "dc/g/Demographics",
       entities: ["geoId/06002", "geoId/06001"],
-      numEntitiesExistence: undefined,
+      numEntitiesExistence: 1,
     })
     .mockResolvedValue(demographicsGroupsData);
 

static/js/tools/map/stat_var_chooser.tsx

@@ -38,8 +38,6 @@ import {
 } from "./context";
 import { DEFAULT_DISPLAY_OPTIONS, getMapPointPlaceType } from "./util";
 
-const NUM_ENTITIES_EXISTENCE = 10;
-
 interface StatVarChooserProps {
   openSvHierarchyModalCallback: () => void;
   openSvHierarchyModal: boolean;
@@ -114,10 +112,6 @@ export function StatVarChooser(props: StatVarChooserProps): JSX.Element {
       selectSV={(svDcid) =>
         selectStatVar(dateCtx, statVar, display, placeInfo, svDcid)
       }
-      numEntitiesExistence={Math.min(
-        NUM_ENTITIES_EXISTENCE,
-        samplePlaces.length
-      )}
     />
   );
 }

static/js/tools/scatter/app.test.tsx

@@ -494,95 +494,95 @@ function mockAxios(): void {
     .calledWith("/api/variable-group/info", {
       dcid: "dc/g/Root",
       entities: ["geoId/10001", "geoId/10003", "geoId/10005"],
-      numEntitiesExistence: 3,
+      numEntitiesExistence: 1,
     })
     .mockResolvedValue(rootGroupsData);
 
   when(axios.post)
     .calledWith("/api/variable-group/info", {
       dcid: "dc/g/Root",
       entities: ["geoId/10001", "geoId/10005", "geoId/10003"],
-      numEntitiesExistence: 3,
+      numEntitiesExistence: 1,
     })
     .mockResolvedValue(rootGroupsData);
 
   when(axios.post)
     .calledWith("/api/variable-group/info", {
       dcid: "dc/g/Root",
       entities: ["geoId/10003", "geoId/10001", "geoId/10005"],
-      numEntitiesExistence: 3,
+      numEntitiesExistence: 1,
     })
     .mockResolvedValue(rootGroupsData);
 
   when(axios.post)
     .calledWith("/api/variable-group/info", {
       dcid: "dc/g/Root",
       entities: ["geoId/10003", "geoId/10005", "geoId/10001"],
-      numEntitiesExistence: 3,
+      numEntitiesExistence: 1,
     })
     .mockResolvedValue(rootGroupsData);
 
   when(axios.post)
     .calledWith("/api/variable-group/info", {
       dcid: "dc/g/Root",
       entities: ["geoId/10005", "geoId/10003", "geoId/10001"],
-      numEntitiesExistence: 3,
+      numEntitiesExistence: 1,
     })
     .mockResolvedValue(rootGroupsData);
 
   when(axios.post)
     .calledWith("/api/variable-group/info", {
       dcid: "dc/g/Root",
       entities: ["geoId/10005", "geoId/10001", "geoId/10003"],
-      numEntitiesExistence: 3,
+      numEntitiesExistence: 1,
     })
     .mockResolvedValue(rootGroupsData);
 
   when(axios.post)
     .calledWith("/api/variable-group/info", {
       dcid: "dc/g/Demographics",
       entities: ["geoId/10001", "geoId/10003", "geoId/10005"],
-      numEntitiesExistence: 3,
+      numEntitiesExistence: 1,
     })
     .mockResolvedValue(demographicsGroupsData);
 
   when(axios.post)
     .calledWith("/api/variable-group/info", {
       dcid: "dc/g/Demographics",
       entities: ["geoId/10001", "geoId/10005", "geoId/10003"],
-      numEntitiesExistence: 3,
+      numEntitiesExistence: 1,
     })
     .mockResolvedValue(demographicsGroupsData);
 
   when(axios.post)
     .calledWith("/api/variable-group/info", {
       dcid: "dc/g/Demographics",
       entities: ["geoId/10003", "geoId/10001", "geoId/10005"],
-      numEntitiesExistence: 3,
+      numEntitiesExistence: 1,
     })
     .mockResolvedValue(demographicsGroupsData);
 
   when(axios.post)
     .calledWith("/api/variable-group/info", {
       dcid: "dc/g/Demographics",
       entities: ["geoId/10003", "geoId/10005", "geoId/10001"],
-      numEntitiesExistence: 3,
+      numEntitiesExistence: 1,
     })
     .mockResolvedValue(demographicsGroupsData);
 
   when(axios.post)
     .calledWith("/api/variable-group/info", {
       dcid: "dc/g/Demographics",
       entities: ["geoId/10005", "geoId/10003", "geoId/10001"],
-      numEntitiesExistence: 3,
+      numEntitiesExistence: 1,
     })
     .mockResolvedValue(demographicsGroupsData);
 
   when(axios.post)
     .calledWith("/api/variable-group/info", {
       dcid: "dc/g/Demographics",
       entities: ["geoId/10005", "geoId/10001", "geoId/10003"],
-      numEntitiesExistence: 3,
+      numEntitiesExistence: 1,
     })
     .mockResolvedValue(demographicsGroupsData);
 

static/js/tools/scatter/chart.tsx

@@ -139,11 +139,15 @@ export function Chart(props: ChartPropsType): JSX.Element {
       }
     }, DEBOUNCE_INTERVAL_MS);
     const resizeObserver = new ResizeObserver(debouncedHandler);
-    if (chartContainerRef.current) {
-      resizeObserver.observe(chartContainerRef.current);
+    // The value of chartContainerRef.current may change between setting the
+    // observe and unobserving during cleanup, so we store the current value
+    // in a variable.
+    const currentChartContainerElement = chartContainerRef.current;
+    if (currentChartContainerElement) {
+      resizeObserver.observe(currentChartContainerElement);
     }
     return () => {
-      resizeObserver.unobserve(chartContainerRef.current);
+      resizeObserver.unobserve(currentChartContainerElement);
       debouncedHandler.cancel();
     };
   }, [props, chartContainerRef, geoJsonFetched]);

static/js/tools/scatter/statvar.tsx

@@ -62,8 +62,6 @@ const defaultModalSelected: ModalSelected = Object.freeze({
   y: false,
 });
 
-const NUM_ENTITIES_EXISTENCE = 10;
-
 interface StatVarChooserProps {
   openSvHierarchyModalCallback: () => void;
   openSvHierarchyModal: boolean;
@@ -164,10 +162,6 @@ export function StatVarChooser(props: StatVarChooserProps): JSX.Element {
         }
         selectedSVs={selectedSvs}
         selectSV={(sv) => addStatVar(x, y, sv, setThirdStatVar, setModalOpen)}
-        numEntitiesExistence={Math.min(
-          NUM_ENTITIES_EXISTENCE,
-          samplePlaces.length
-        )}
       />
       {/* Modal for selecting 2 stat vars when a third is selected */}
       <Modal isOpen={modalOpen} backdrop="static" id="statvar-modal">

static/js/tools/shared/stat_var_widget.tsx

@@ -50,8 +50,6 @@ interface StatVarWidgetPropsType {
   selectSV?: (sv: string) => void;
   // Whether to disable the alert when there are unavailable SVs.
   disableAlert?: boolean;
-  // Number of entities that should have data for each stat var (group) shown
-  numEntitiesExistence?: number;
 }
 
 export function StatVarWidget(props: StatVarWidgetPropsType): JSX.Element {
@@ -130,7 +128,7 @@ export function StatVarWidget(props: StatVarWidgetPropsType): JSX.Element {
             selectSV={props.selectSV}
             searchLabel={"Statistical variables"}
             deselectSV={(sv) => props.deselectSVs([sv])}
-            numEntitiesExistence={props.numEntitiesExistence}
+            numEntitiesExistence={getNumEntitiesExistence()}
           />
         </div>
         <DrawerResize
@@ -160,7 +158,7 @@ export function StatVarWidget(props: StatVarWidgetPropsType): JSX.Element {
             selectSV={props.selectSV}
             searchLabel={"Statistical variables"}
             deselectSV={(sv) => props.deselectSVs([sv])}
-            numEntitiesExistence={props.numEntitiesExistence}
+            numEntitiesExistence={getNumEntitiesExistence()}
           />
         </ModalBody>
         <ModalFooter>
@@ -171,4 +169,23 @@ export function StatVarWidget(props: StatVarWidgetPropsType): JSX.Element {
       </Modal>
     </>
   );
+
+  /**
+   * Get number of required entities for stat var filtering.
+   *
+   * NumEntitiesExistence is a parameter that sets the number of entities that
+   * should have data for each stat var (group) shown in the widget. For
+   * example, setting a value of 10 means that at least 10 entities must have
+   * data for a stat var for that stat var to show in the widget. This prevents
+   * showing users stat vars with low geographic coverage that lead to sparse
+   * charts.
+   *
+   * @returns minimum number of entities to use for stat var filtering
+   */
+  function getNumEntitiesExistence(): number {
+    return Math.min(
+      globalThis.minStatVarGeoCoverage || 1,
+      props.sampleEntities.length
+    );
+  }
 }