merge from master

.dockerignore

@@ -8,4 +8,11 @@
 .git
 .env
 
-pytest_log.txt
+pytest_log.txt
+
+**/eval/**/result
+
+import/simple/sample/
+**/.data
+**/testdata
+**/test_data

server/.gcloudignore → .gcloudignore

@@ -12,17 +12,21 @@
 # below:
 .git
 .gitignore
+#!include:.gitignore
+#!include:.dockerignore
+!mcf_parser.py
 
 # Python pycache:
 **/__pycache__/**
-.pytest_cache
+.pytest_cache/
 
 # Ignored by the build system
 /setup.cfg
 
-**/node_modules/**
+static/node_modules/
 
-.env/
+server/dist/
 
-# Ignore tests
-tests/
+**/.env/
+
+screenshots/*.png

.github/workflows/codeql-analysis.yml

@@ -35,11 +35,11 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v3
         with:
           languages: ${{ matrix.language }}
           # If you wish to specify custom queries, you can do so here or in a config file.
@@ -50,7 +50,7 @@ jobs:
       # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below)
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v2
+        uses: github/codeql-action/autobuild@v3
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 https://git.io/JvXDl
@@ -64,4 +64,4 @@ jobs:
       #   make release
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v3

.github/workflows/release-branch-checks.yml

@@ -0,0 +1,50 @@
+name: Release branch checks
+
+on:
+  pull_request:
+    branches: [ "customdc_stable" ]
+  # Required for merge queue to work: https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/configuring-pull-request-merges/managing-a-merge-queue#triggering-merge-group-checks-with-github-actions
+  merge_group:
+    branches: [ "customdc_stable" ]
+
+jobs:
+  verify_all_commits_are_already_in_master:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # Fetch all history for accurate comparison
+          fetch-depth: 0
+          # Check out the PR branch
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+
+      - name: Verify that all commits are already in the master branch
+        run: |
+          git remote add dc https://github.com/datacommonsorg/website.git
+          git fetch dc
+          MASTER_BRANCH="dc/master"
+
+          # Get the list of commits in the source branch that are not in the master branch.
+          # Exclude merge commits only if this is the final run in the merge queue.
+          # This way the only merge commits that end up in the final commit history
+          # are the ones added by GitHub when merging PRs.
+          if [[ ${{ github.event_name }}  == 'merge_group' ]]; then
+            MISSING_COMMITS=$(git log --pretty="%H - %s" --no-merges $MASTER_BRANCH..HEAD --)
+          else
+            MISSING_COMMITS=$(git log --pretty="%H - %s" $MASTER_BRANCH..HEAD --)
+          fi
+
+          if [[ -n "$MISSING_COMMITS" ]]; then
+            echo ""
+            echo "ERROR: The following commits are not present in $MASTER_BRANCH:"
+            echo ""
+            echo "$MISSING_COMMITS"
+            echo ""
+            echo "PRs to release branches should only contain commits that are already in master."
+            echo "To fix this PR, reset its branch locally to a commit at or behind https://github.com/datacommonsorg/website/commits/master/ and then force-push it."
+            echo "Note that a release branch PR should be based on master and not the previous version of the release branch, which contains merge commits."
+            exit 1
+          fi
+
+          echo "All commits are present in $MASTER_BRANCH"

.gitignore

@@ -3,18 +3,21 @@ __pycache__/
 .pytest_cache
 *.pyc
 .env/
-test_screenshots/
+screenshots/*.png
+screenshots/screenshot_url.json
 
 # Production static files
-server/dist/**
+**/dist/**
 !server/dist/.empty
 
 **/node_modules/**
 
 # Local env.
 *.swp
 .vscode
+.idea
 !.vscode/launch.json
+.env.list
 
 # Mac
 .DS_Store
@@ -24,9 +27,11 @@ tmp/
 .air.toml
 
 # Kubernetes
-gke/config.yaml
 gke/endpoints.yaml
 gke/mci.yaml
+endpoints.yaml
+gke/*.crt
+gke/*.key
 
 mixer-grpc.*.pb
 deploy/overlays/**/endpoints.yaml
@@ -37,6 +42,8 @@ deploy/overlays/kustomization.yaml
 deploy/git/mixer_hash.txt
 deploy/git/website_hash.txt
 
+deploy/apigee/**/*.zip
+
 # Helm
 build/helm/**/*.tgz
 
@@ -49,4 +56,51 @@ build/helm/**/*.tgz
 *.tfstate.*
 pylist.json
 
-pytest_log.txt
+pytest_log.txt
+
+
+tools/sdg/sdg.textproto
+tools/sdg/subject_page_pb2.py
+
+*diff_report.html
+
+# Experimental - ignore symbolic links
+experimental/sdg-static/datacommons/datacommons.js
+experimental/sdg-static/datacommons/ranking.min.css
+experimental/sdg-static/datacommons/nl_interface.min.css
+
+# Custom DC data
+dc-data/
+custom_dc/env.list
+
+# Topic cache
+gen_ordered_list_for_topics.mcf
+mcf_parser.py
+
+# Test output files
+output/
+
+# wireit files
+.wireit
+
+# finetuned model
+ft_final_v20230717230459.all-MiniLM-L6-v2/
+
+# model eval output
+
+**/eval/**/result
+
+# CDC ignores
+custom_dc/sample/datacommons/**
+custom_dc/data/*.db
+
+.cache/
+
+**/*.egg-info
+*.egg-info
+
+# Custom DC dev and terraform ignores
+build/cdc/dev/.env
+*.tfvars
+deploy/terraform-custom-datacommons/modules/backend.tf
+

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "mixer"]
 	path = mixer
 	url = https://github.com/datacommonsorg/mixer.git
+[submodule "import"]
+	path = import
+	url = https://github.com/datacommonsorg/import.git

.run_cdc_dev.env

@@ -0,0 +1,63 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Env variables for running run_cdc_dev.sh
+
+# The 3 keys / passwords below must be specified before the script can be run.
+# Either specify them here or better still is to 
+# copy this file, specify the values in the copy and set the env variable RUN_CDC_DEV_ENV_FILE to that file's path before running the script.
+# e.g. if the copied file is at tmp/cdc.env, you can run the script as follows:
+# RUN_CDC_DEV_ENV_FILE=tmp/cdc.env ./run_cdc_dev.sh
+DC_API_KEY=
+MAPS_API_KEY=
+DB_PASS=
+
+# If your DC_API_KEY is for autopush, change this to https://autopush.api.datacommons.org
+DC_API_ROOT=https://api.datacommons.org
+
+# Developers will typically change the Cloud SQL and GCS folders below when developing against various DB instances and embeddings files.
+# So keep them at the top of the file.
+
+# Cloud SQL settings.
+# Set to false is not using Cloud SQL.
+USE_CLOUDSQL=true
+CLOUDSQL_INSTANCE=datcom-website-dev:us-central1:dc-dev
+DB_USER=root
+
+# GCS or local folder to "user data" folder.
+# This is primarily for NL embeddings.
+# The code looks for it under $USER_DATA_PATH/datacommons/nl/embeddings/custom_catalog.yaml
+USER_DATA_PATH=gs://customdc-data/dc-dev
+GOOGLE_CLOUD_PROJECT=datcom-website-dev
+ADDITIONAL_CATALOG_PATH=$USER_DATA_PATH/datacommons/nl/embeddings/custom_catalog.yaml
+
+# SQLite settings.
+# If using sqlite, set USE_CLOUDSQL to false and USE_SQLITE to true.
+USE_SQLITE=false
+SQLITE_PATH=
+
+# Set this to false if you don't want to start the NL server.
+ENABLE_MODEL=true
+
+# You will typically not change these.
+IS_CUSTOM_DC=true
+FLASK_ENV=custom_dev
+WEBSITE_MIXER_API_ROOT=http://127.0.0.1:8081
+ENV_PREFIX=Compose
+ENABLE_ADMIN=true
+DEBUG=true
+# https://stackoverflow.com/a/62703850
+TOKENIZERS_PARALLELISM=false
+# https://github.com/UKPLab/sentence-transformers/issues/1318#issuecomment-1084731111
+OMP_NUM_THREADS=1