2024-12-17 Custom DC stable release

build/ci/cloudbuild.py.yaml

@@ -14,9 +14,20 @@
 
 # Run Python tests, lint, etc.
 steps:
+  # Download the files needed for nl embeddings tests.
+  - id: download_nl_files
+    name: python:3.11.3
+    entrypoint: /bin/sh
+    args:
+      - -c
+      - |
+        cd tools/nl/download_nl_files
+        ./run.sh
+
   - id: flask_test
     name: python:3.11.3
     entrypoint: /bin/sh
+    waitFor: ["download_nl_files"]
     args:
       - -c
       - |

deploy/helm_charts/envs/dev.yaml

@@ -87,4 +87,4 @@ cronTesting:
   enabled: true
   screenshotDomain: "dev.datacommons.org"
   nodePool: "default-pool"
-  schedule: "* * 31 2 *"
+  schedule: "* * 31 2 *" # invalid day for month -> never run automatically

deploy/helm_charts/envs/staging.yaml

@@ -73,3 +73,9 @@ serviceGroups:
 
 svg:
   blocklistFile: ["dc/g/Uncategorized", "oecd/g/OECD"]
+
+cronTesting:
+  enabled: true
+  screenshotDomain: "staging.datacommons.org"
+  nodePool: "default-pool"
+  schedule: "* * 31 2 *" # invalid day for month -> never run automatically

packages/client/src/data_commons_web_client.ts

@@ -21,7 +21,9 @@
 import {
   ApiNodePropvalOutResponse,
   ObservationDatesApiResponse,
+  PlaceChartsApiResponse,
   PointApiResponse,
+  RelatedPlacesApiResponse,
   SeriesApiResponse,
 } from "./data_commons_web_client_types";
 import { parseWebsiteApiRoot, toURLSearchParams } from "./utils";
@@ -203,6 +205,38 @@ class DataCommonsWebClient {
     const response = await fetch(url);
     return (await response.json()) as ObservationDatesApiResponse;
   }
+
+  /**
+   * Gets place charts for the given category
+   * Uses /api/dev-place/charts/<placeDcid> endpoint
+   * @param params.category [optional] place category
+   * @param params.placeDcid place dcid to fetch data for
+   */
+  async getPlaceCharts(params: {
+    category?: string;
+    placeDcid: string;
+  }): Promise<PlaceChartsApiResponse> {
+    const url = `${this.apiRoot || ""}/api/dev-place/charts/${
+      params.placeDcid
+    }${params.category ? "?category=" + params.category : ""}`;
+    const response = await fetch(url);
+    return (await response.json()) as PlaceChartsApiResponse;
+  }
+
+  /**
+   * Gets related place info charts for the given place
+   * Uses /api/dev-place/related-places/<placeDcid> endpoint
+   * @param params.placeDcid place dcid to fetch data for
+   */
+  async getRelatedPLaces(params: {
+    placeDcid: string;
+  }): Promise<RelatedPlacesApiResponse> {
+    const url = `${this.apiRoot || ""}/api/dev-place/related-places/${
+      params.placeDcid
+    }`;
+    const response = await fetch(url);
+    return (await response.json()) as RelatedPlacesApiResponse;
+  }
 }
 
 export { DataCommonsWebClient };

packages/client/src/data_commons_web_client_types.ts

@@ -144,3 +144,47 @@ export type ObservationDatesApiResponse = {
     };
   };
 };
+
+type ChartType = "BAR" | "LINE" | "MAP" | "RANKING";
+export interface Chart {
+  type: ChartType;
+  title: string;
+  category: string;
+  description: string;
+  statisticalVariableDcids: string[];
+  topicDcids: string[];
+  denominator?: string; // Optional
+  unit?: string; // Optional
+  scaling?: number; // Optional
+}
+
+export interface Place {
+  dcid: string;
+  name: string;
+  types: string[];
+}
+
+/**
+ * Website API response for /api/dev-place/charts/<place_dcid>
+ */
+export interface PlaceChartsApiResponse {
+  charts: Chart[];
+  childPlaceType: string;
+  childPlaces: Place[];
+  nearbyPlaces: Place[];
+  place: Place;
+  similarPlaces: Place[];
+  translatedCategoryStrings: Record<string, string>;
+}
+
+/**
+ * Website API response for /api/dev-place/related-places/<place_dcid>
+ */
+export interface RelatedPlacesApiResponse {
+  childPlaceType: string;
+  childPlaces: Place[];
+  nearbyPlaces: Place[];
+  place: Place;
+  similarPlaces: Place[];
+  parentPlaces: Place[];
+}

run_website_sanity.sh

@@ -17,8 +17,22 @@ set -e
 
 # DC Instance domain like: "dev.datacommons.org", "datacommons.org"
 domain=$1
+# Check if the domain is set
+if [[ -z "$domain" ]]; then
+  echo "Error: Domain is not set."
+  echo "Usage: $0 <domain> [NO_PIP]"
+  echo "Example: $0 autopush.datacommons.org"
+  exit 1
+fi
+
+# Prepend https:// if no protocol is specified
+if [[ ! "$domain" =~ ^http:// && ! "$domain" =~ ^https:// ]]; then
+  domain="https://$domain"
+fi
+
 echo "Domain: $domain"
 
+# Set to true if you don't want to install the dependencies
 NO_PIP=$2
 
 export FLASK_ENV=webdriver
@@ -32,6 +46,6 @@ if [[ $NO_PIP != "true" ]]; then
 fi
 
 date_str=$(TZ="America/Los_Angeles" date +"%Y_%m_%d_%H_%M_%S")
-python3 server/webdriver/tests/standalone/sanity.py --mode=home --url="https://$domain"
+python3 server/webdriver/tests/standalone/sanity.py --mode=home --url="$domain"
 gsutil cp ./output/*.csv gs://datcom-website-sanity/$domain/$date_str/
 rm ./output/*.csv

scripts/get_commits_label.sh

@@ -18,12 +18,12 @@
 
 # Optionally use the commit before HEAD for the main repo.
 if [[ "$1" == "--head-is-temporary" ]]; then
-  website_rev="$(git rev-parse --short HEAD~1)"
+  website_rev="$(git rev-parse --short=7 HEAD~1)"
 else
-  website_rev="$(git rev-parse --short HEAD)"
+  website_rev="$(git rev-parse --short=7 HEAD)"
 fi
-mixer_rev="$(git rev-parse --short HEAD:mixer)"
-import_rev="$(git rev-parse --short HEAD:import)"
+mixer_rev="$(git rev-parse --short=7 HEAD:mixer)"
+import_rev="$(git rev-parse --short=7 HEAD:import)"
 image_label="${website_rev}-${mixer_rev}-${import_rev}"
 
 echo "$image_label"