Merge pull request #109 from uche-madu/develop

partition bigquery tables

argocd-app/my-airflow/values-dev.yaml

@@ -17,12 +17,11 @@ airflow:
     airflow:
       # To be replaced automatically by the Github Actions CICD workflow
       # in the DEB Application repository
-      repository: us-central1-docker.pkg.dev/wizeline-deb/deb-capstone-airflow-gke/airflow2.7.1-custom
-      tag: 771dbead
+      repository: ~
+      tag: ~
       # Specifying digest takes precedence over tag.
       digest: ~
       pullPolicy: IfNotPresent
-      #useDefaultImageForMigration: true
 
   # Add common labels to all objects and pods defined in this chart.
   labels:
@@ -66,15 +65,11 @@ airflow:
   enableBuiltInSecretEnvVars:
     AIRFLOW__CORE__FERNET_KEY: false
     AIRFLOW__WEBSERVER__SECRET_KEY: false
-    #AIRFLOW__CORE__SQL_ALCHEMY_CONN: false
-    #AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: false
-    #AIRFLOW_CONN_AIRFLOW_DB: false
 
   fernetKeySecretName: "fernet-key-secret"
 
   webserverSecretKeySecretName: "airflow-webserver-secret"
 
-
   # Airflow webserver settings
   webserver:
     service:
@@ -104,11 +99,9 @@ airflow:
       size: 1Gi
     serviceAccount:
       name: airflow
-
-  # Airflow Triggerer Config
+
   triggerer:
     persistence:
-      # Volume size for triggerer StatefulSet
       size: 1Gi
 
   # logs:
@@ -129,4 +122,4 @@ airflow:
     useHelmHooks: false
     applyCustomEnv: false
     jobAnnotations:
-          "argocd.argoproj.io/hook": Sync
+      "argocd.argoproj.io/hook": Sync

terraform/bigquery.tf

@@ -4,20 +4,30 @@ resource "google_bigquery_dataset" "default" {
   location    = "US"
 }
 
-resource "google_bigquery_table" "review_logs" {
+resource "google_bigquery_table" "user_purchase" {
   dataset_id          = google_bigquery_dataset.default.dataset_id
-  table_id            = "review_logs"
+  table_id            = "user_purchase"
   deletion_protection = false
 }
 
-resource "google_bigquery_table" "user_purchase" {
+resource "google_bigquery_table" "review_logs" {
   dataset_id          = google_bigquery_dataset.default.dataset_id
-  table_id            = "user_purchase"
+  table_id            = "review_logs"
   deletion_protection = false
+
+  time_partitioning {
+    type  = "DAY"
+    field = "day"
+  }
 }
 
 resource "google_bigquery_table" "classified_movie_review" {
   dataset_id          = google_bigquery_dataset.default.dataset_id
   table_id            = "classified_movie_review"
   deletion_protection = false
-}
+
+  time_partitioning {
+    type  = "DAY"
+    field = "insert_date"
+  }
+}

terraform/gke.tf

@@ -26,13 +26,14 @@ module "gke" {
   ip_range_pods         = "deb-sub1-secondary-gke-pods"
   ip_range_services     = "deb-sub1-secondary-gke-services"
   grant_registry_access = true
+
   cluster_autoscaling = {
     "auto_repair" : true,
     "auto_upgrade" : true,
     "disk_size" : 32,
     "disk_type" : "pd-standard",
     "enabled" : true,
-    "max_cpu_cores" : 4,
+    "max_cpu_cores" : 12,
     "min_cpu_cores" : 1,
     "gpu_resources" : [],
     "max_memory_gb" : 15,