Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reorganize custom DC Terraform script for readability #4786

Merged
merged 21 commits into from
Dec 20, 2024
Merged
Show file tree
Hide file tree
Changes from 20 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
8373289
2024-11-06 Custom DC stable release (#4710)
hqpho Nov 6, 2024
5ce8052
Merge branch 'master' of https://github.com/datacommonsorg/website in…
kmoscoe Nov 25, 2024
a46e43f
Merge branch 'master' of https://github.com/datacommonsorg/website
kmoscoe Nov 26, 2024
0eb7a0c
Merge branch 'master' of https://github.com/kmoscoe/website
kmoscoe Dec 11, 2024
982f3d2
reorganize entries for readability
kmoscoe Dec 11, 2024
8a8df54
reorganize for readability
kmoscoe Dec 11, 2024
9ad9dbc
Merge branch 'master' into terraform
kmoscoe Dec 12, 2024
3425dd0
Merge branch 'datacommonsorg:master' into terraform
kmoscoe Dec 17, 2024
8370e57
Merge branch 'master' of https://github.com/kmoscoe/website into terr…
kmoscoe Dec 17, 2024
9ab2bd9
make API host inline
kmoscoe Dec 17, 2024
f97d1e4
Merge branch 'terraform' of https://github.com/kmoscoe/website into t…
kmoscoe Dec 17, 2024
8f3f6d2
Merge branch 'master' into terraform
dwnoble Dec 17, 2024
179fd64
Merge branch 'master' into terraform
kmoscoe Dec 19, 2024
16c02fc
fix locals file
kmoscoe Dec 19, 2024
c7290d5
Merge branch 'terraform' of https://github.com/kmoscoe/website into t…
kmoscoe Dec 19, 2024
9fa2c2d
fix broken variable
kmoscoe Dec 19, 2024
9722322
Merge branch 'master' into terraform
kmoscoe Dec 20, 2024
a747ff4
revert these files
kmoscoe Dec 20, 2024
1e39092
Merge branch 'terraform' of https://github.com/kmoscoe/website into t…
kmoscoe Dec 20, 2024
ab07722
revert files
kmoscoe Dec 20, 2024
bb3944e
revert again'
kmoscoe Dec 20, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions deploy/terraform-custom-datacommons/modules/locals.tf
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,16 @@ locals {

# Use var.billing_project_id if set, otherwise use project_id for billing
billing_project_id = var.billing_project_id != null ? var.billing_project_id : var.project_id

# Data Commons API hostname
dc_api_hostname = "api.datacommons.org"

# Data Commons API root url
dc_api_root = "${var.dc_api_protocol}://${var.dc_api_hostname}"
# Data Commons API protocol
dc_api_protocol = "https"

# Data Commons API root URL
dc_api_root = "${local.dc_api_protocol}://${local.dc_api_hostname}"

# Shared environment variables used by the Data Commons web service and the Data
# Commons data loading job
cloud_run_shared_env_variables = [
Expand Down
158 changes: 79 additions & 79 deletions deploy/terraform-custom-datacommons/modules/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
# Variable definitions

# Required variables

variable "project_id" {
description = "The GCP project ID where the solution will be deployed"
type = string
Expand All @@ -31,15 +32,8 @@ variable "dc_api_key" {
}

# Optional variables
# Set to your Google Analytics Tag ID to enable Google Analytics tracking.
# Reference: https://support.google.com/analytics/answer/9539598
variable "google_analytics_tag_id" {
description = "Google Analytics Tag ID"
type = string
default = null
}

# Optional: If blank will use generated maps API key
# Optional: If blank, will generate a Maps API key.
variable "maps_api_key" {
description = "Google Maps API Key"
type = string
Expand All @@ -64,71 +58,41 @@ variable billing_project_id {
default = null
}

variable "vpc_network_name" {
description = "VPC network name to use"
type = string
default = "default"
}

variable "vpc_network_subnet_name" {
description = "VPC network subnet name to use"
type = string
default = "default"
}

variable "vpc_base_cidr_block" {
description = "Base CIDR block to be subdivided for VPC connectors"
type = string
default = "10.8.0.0/24"
}

variable "enable_redis" {
description = "Enable redis instance in this deployment"
type = bool
default = false
}

variable "redis_instance_name" {
description = "Name of the redis instance"
variable "google_analytics_tag_id" {
description = "Google Analytics Tag ID"
type = string
default = "datacommons-redis-instance"
}
variable "redis_memory_size_gb" {
description = "The memory size for the Redis instance in GB"
type = number
default = 2
default = null
}

variable "redis_tier" {
description = "The service tier for the Redis instance"
type = string
default = "STANDARD_HA"
}
# Data Commons Cloud Storage bucket variables

variable "redis_location_id" {
description = "Redis location id (zone)"
# If not set, the default is <namespace>-datacommons-data-<project_id>
variable "dc_gcs_data_bucket_path_override" {
description = "Custom GCS data bucket path."
type = string
default = "us-central1-a"
default = ""
}

variable "redis_alternative_location_id" {
description = "Redis alternate location id (alternate zone)"
variable "dc_gcs_data_bucket_location" {
description = "Data Commons GCS data bucket location"
type = string
default = "us-central1-b"
default = "US"
}

variable "redis_replica_count" {
description = "Redis reserved IP range"
type = number
default = 1
}
# Data Commons Cloud SQL instance variables

variable "mysql_instance_name" {
description = "The name of the MySQL instance"
type = string
default = "datacommons-mysql-instance"
}

variable mysql_database_name {
description = "MySQL database name"
type = string
default = "datacommons"
}

variable "mysql_database_version" {
description = "The version of MySQL"
type = string
Expand All @@ -155,12 +119,6 @@ variable "mysql_storage_size_gb" {
default = 20
}

variable mysql_database_name {
description = "MySQL database name"
type = string
default = "datacommons"
}

variable "mysql_user" {
description = "The username for the MySQL instance"
type = string
Expand All @@ -173,7 +131,8 @@ variable "mysql_deletion_protection" {
default = false
}

## Data Commons Cloud Run service variables
# Data Commons Cloud Run service variables

variable "dc_web_service_image" {
description = "Container image for Cloud Run service"
type = string
Expand All @@ -192,6 +151,14 @@ variable "dc_web_service_memory" {
default = "16G"
}

variable "make_dc_web_service_public" {
description = "Whether to make the Data Commons Cloud Run service publicly accessible"
type = bool
default = true
}

# Data Commons Cloud Run job variables

variable "dc_data_job_image" {
description = "The container image for the data job"
type = string
Expand All @@ -210,32 +177,65 @@ variable "dc_data_job_memory" {
default = "8G"
}

variable "dc_api_hostname" {
description = "Data Commons API hostname"
# Data Commons Cloud VPC Network variables

variable "vpc_network_name" {
description = "VPC network name to use"
type = string
default = "api.datacommons.org"
default = "default"
}

variable "dc_api_protocol" {
description = "Data Commons API protocol"
variable "vpc_network_subnet_name" {
description = "VPC network subnet name to use"
type = string
default = "https"
default = "default"
}

variable "dc_gcs_data_bucket_path_override" {
description = "[Optional] Set a custom GCS data bucket path"
variable "vpc_base_cidr_block" {
description = "Base CIDR block to be subdivided for VPC connectors"
type = string
default = ""
default = "10.8.0.0/24"
}

variable "dc_gcs_data_bucket_location" {
description = "Data Commons data bucket GCS location"
# Data Commons Cloud Redis Memorystore instance variables

variable "enable_redis" {
description = "Enable redis instance in this deployment"
type = bool
default = false
}

variable "redis_instance_name" {
description = "Name of the redis instance"
type = string
default = "US"
default = "datacommons-redis-instance"
}
variable "redis_memory_size_gb" {
description = "The memory size for the Redis instance in GB"
type = number
default = 2
}

variable "make_dc_web_service_public" {
description = "Whether to make the Data Commons Cloud Run service publicly accessible"
type = bool
default = true
variable "redis_tier" {
description = "The service tier for the Redis instance"
type = string
default = "STANDARD_HA"
}

variable "redis_location_id" {
description = "Redis location id (zone)"
type = string
default = "us-central1-a"
}

variable "redis_alternative_location_id" {
description = "Redis alternate location id (alternate zone)"
type = string
default = "us-central1-b"
}

variable "redis_replica_count" {
description = "Redis reserved IP range"
type = number
default = 1
}
2 changes: 1 addition & 1 deletion import
Submodule import updated 47 files
+1 −1 run_test.sh
+1 −3 simple/requirements.txt
+57 −69 simple/stats/config.py
+41 −47 simple/stats/db.py
+14 −13 simple/stats/entities_importer.py
+22 −19 simple/stats/events_importer.py
+3 −3 simple/stats/main.py
+8 −7 simple/stats/mcf_importer.py
+23 −22 simple/stats/nl.py
+14 −13 simple/stats/nodes.py
+17 −14 simple/stats/observations_importer.py
+19 −22 simple/stats/reporter.py
+130 −154 simple/stats/runner.py
+11 −10 simple/stats/variable_per_row_importer.py
+34 −106 simple/tests/stats/config_test.py
+26 −40 simple/tests/stats/db_test.py
+13 −23 simple/tests/stats/entities_importer_test.py
+19 −23 simple/tests/stats/events_importer_test.py
+19 −26 simple/tests/stats/mcf_importer_test.py
+9 −12 simple/tests/stats/nl_test.py
+16 −40 simple/tests/stats/nodes_test.py
+10 −20 simple/tests/stats/observations_importer_test.py
+24 −37 simple/tests/stats/runner_test.py
+2 −3 simple/tests/stats/schema_test.py
+0 −48 simple/tests/stats/test_data/runner/config/config_exclude_subdirs.json
+0 −48 simple/tests/stats/test_data/runner/config/config_include_subdirs.json
+0 −2 simple/tests/stats/test_data/runner/expected/with_subdirs_excluded/key_value_store.db.csv
+0 −4 simple/tests/stats/test_data/runner/expected/with_subdirs_excluded/nl/sentences.csv
+0 −9 simple/tests/stats/test_data/runner/expected/with_subdirs_excluded/observations.db.csv
+0 −65 simple/tests/stats/test_data/runner/expected/with_subdirs_excluded/triples.db.csv
+0 −2 simple/tests/stats/test_data/runner/expected/with_subdirs_included/key_value_store.db.csv
+0 −5 simple/tests/stats/test_data/runner/expected/with_subdirs_included/nl/sentences.csv
+0 −31 simple/tests/stats/test_data/runner/expected/with_subdirs_included/observations.db.csv
+0 −109 simple/tests/stats/test_data/runner/expected/with_subdirs_included/triples.db.csv
+0 −4 simple/tests/stats/test_data/runner/input/with_subdirs/author_entities.csv
+0 −4 simple/tests/stats/test_data/runner/input/with_subdirs/subdir1/article_entities.csv
+0 −5 simple/tests/stats/test_data/runner/input/with_subdirs/subdir1/variables.mcf
+0 −15 simple/tests/stats/test_data/runner/input/with_subdirs/subdir2/subsubdir/countries.csv
+0 −5 simple/tests/stats/test_data/runner/input/with_subdirs/variable_per_row.csv
+0 −6 simple/tests/stats/test_data/runner/input/with_subdirs/variables.mcf
+0 −3 simple/tests/stats/test_data/runner/input/with_subdirs/wikidataids.csv
+7 −10 simple/tests/stats/variable_per_row_importer_test.py
+0 −155 simple/tests/util/file_match_test.py
+0 −115 simple/tests/util/filesystem_test.py
+0 −139 simple/util/file_match.py
+188 −0 simple/util/filehandler.py
+0 −256 simple/util/filesystem.py
2 changes: 1 addition & 1 deletion mixer
Submodule mixer updated 172 files
Loading