Add configuration parsing specifications.

We make sure all default, minimal and reference configuration are parsed correctly for Azure/GCP/AWS.
snowplow-incubator · Mar 6, 2024 · 7ae3f3c · 7ae3f3c
1 parent 016b844
commit 7ae3f3c
Show file tree

Hide file tree

Showing 16 changed files with 652 additions and 25 deletions.
diff --git a/config/config.azure.minimal.hocon b/config/config.azure.minimal.hocon
@@ -12,7 +12,7 @@
 
     "good": {
       "project": "my-project"
-      "dataset": "snowplow"
+      "dataset": "my-dataset"
     }
 
     "bad": {

diff --git a/config/config.azure.reference.hocon b/config/config.azure.reference.hocon
@@ -27,7 +27,7 @@
       "project": "my-project"
 
       # -- the BigQuery dataset to which events will be loaded
-      "dataset": "my-project"
+      "dataset": "my-dataset"
 
       # -- the table within the BigQuery dataset to which events will be loaded
       "table": "events"
@@ -84,7 +84,7 @@
     # -- Configures backoff when waiting for the BigQuery Writer to detect that we have altered the
     # -- table by adding new columns
     "alterTableWait": {
-      "delay": 1 second"
+      "delay": "1 second"
     }
   } 
 
@@ -105,7 +105,7 @@
         "period": "1 minute"
 
         # -- Prefix used for the metric name when sending to statsd
-        "prefix": "snowplow.bigquery-loader"
+        "prefix": "snowplow.bigquery.loader"
       }
     }
 
@@ -118,6 +118,16 @@
         "myTag": "xyz"
       }
     }
+
+    # -- Report alerts to the webhook 
+    "webhook": {
+      # An actual HTTP endpoint
+      "endpoint": "https://webhook.acme.com",
+      # Set of arbitrary key-value pairs attached to the payload
+      "tags": {
+        "pipeline": "production"
+      }
+    }
   }
 
   # -- Optional, configure telemetry

diff --git a/config/config.kinesis.minimal.hocon b/config/config.kinesis.minimal.hocon
@@ -11,7 +11,7 @@
 
     "good": {
       "project": "my-project"
-      "dataset": "snowplow"
+      "dataset": "my-dataset"
     }
 
     "bad": {

diff --git a/config/config.kinesis.reference.hocon b/config/config.kinesis.reference.hocon
@@ -45,7 +45,7 @@
       "project": "my-project"
 
       # -- the BigQuery dataset to which events will be loaded
-      "dataset": "my-project"
+      "dataset": "my-dataset"
 
       # -- the table within the BigQuery dataset to which events will be loaded
       "table": "events"
@@ -106,7 +106,7 @@
     # -- Configures backoff when waiting for the BigQuery Writer to detect that we have altered the
     # -- table by adding new columns
     "alterTableWait": {
-      "delay": 1 second"
+      "delay": "1 second"
     }
   } 
 
@@ -127,7 +127,7 @@
         "period": "1 minute"
 
         # -- Prefix used for the metric name when sending to statsd
-        "prefix": "snowplow.bigquery-loader"
+        "prefix": "snowplow.bigquery.loader"
       }
     }
 
@@ -140,6 +140,16 @@
         "myTag": "xyz"
       }
     }
+
+    # -- Report alerts to the webhook 
+    "webhook": {
+      # An actual HTTP endpoint
+      "endpoint": "https://webhook.acme.com",
+      # Set of arbitrary key-value pairs attached to the payload
+      "tags": {
+        "pipeline": "production"
+      }
+    }
   }
 
   # -- Optional, configure telemetry

diff --git a/config/config.pubsub.minimal.hocon b/config/config.pubsub.minimal.hocon
@@ -11,11 +11,11 @@
 
     "good": {
       "project": "my-project"
-      "dataset": "snowplow"
+      "dataset": "my-dataset"
     }
 
     "bad": {
-      "topic": "projects/myproject/topics/snowplow-bad"
+      "topic": "projects/my-project/topics/snowplow-bad"
     }
 
   }

diff --git a/config/config.pubsub.reference.hocon b/config/config.pubsub.reference.hocon
@@ -6,7 +6,7 @@
 
   "input": {
     # -- pubsub subscription for the source of enriched events
-    "subscription": "projects/myproject/subscriptions/snowplow-enriched"
+    "subscription": "projects/my-project/subscriptions/snowplow-enriched"
 
     # -- How many threads are used by the pubsub client library for fetching events
     "parallelPullCount": 3
@@ -32,7 +32,7 @@
       "project": "my-project"
 
       # -- the BigQuery dataset to which events will be loaded
-      "dataset": "my-project"
+      "dataset": "my-dataset"
 
       # -- the table within the BigQuery dataset to which events will be loaded
       "table": "events"
@@ -44,7 +44,7 @@
 
     "bad": {
       # -- output pubsub topic for emitting failed events that could not be processed
-      "topic": "projects/myproject/topics/snowplow-bad"
+      "topic": "projects/my-project/topics/snowplow-bad"
 
       # -- Failed sends events to pubsub in batches not exceeding this size.
       "batchSize": 100
@@ -86,7 +86,7 @@
     # -- Configures backoff when waiting for the BigQuery Writer to detect that we have altered the
     # -- table by adding new columns
     "alterTableWait": {
-      "delay": 1 second"
+      "delay": "1 second"
     }
   } 
 
@@ -107,7 +107,7 @@
         "period": "1 minute"
 
         # -- Prefix used for the metric name when sending to statsd
-        "prefix": "snowplow.bigquery-loader"
+        "prefix": "snowplow.bigquery.loader"
       }
     }
 
@@ -120,6 +120,16 @@
         "myTag": "xyz"
       }
     }
+
+    # -- Report alerts to the webhook 
+    "webhook": {
+      # An actual HTTP endpoint
+      "endpoint": "https://webhook.acme.com",
+      # Set of arbitrary key-value pairs attached to the payload
+      "tags": {
+        "pipeline": "production"
+      }
+    }    
   }
 
   # -- Optional, configure telemetry

diff --git a/modules/core/src/main/resources/reference.conf b/modules/core/src/main/resources/reference.conf
@@ -3,7 +3,7 @@
   "gcpUserAgent": {
     "productName": "Snowplow OSS"
     "productName": ${?GCP_USER_AGENT_PRODUCT_NAME}
-    "productVersion": "lake-loader"
+    "productVersion": "bigquery-loader"
   }
 
   "output": {

diff --git a/...re/src/main/scala/com.snowplowanalytics.snowplow.bigquery/processing/BigQueryCaster.scala b/...re/src/main/scala/com.snowplowanalytics.snowplow.bigquery/processing/BigQueryCaster.scala
@@ -18,7 +18,7 @@ import com.snowplowanalytics.iglu.schemaddl.parquet.Caster
 
 import scala.jdk.CollectionConverters._
 
-/** Converts schema-ddl values into objects which are compatible with the snowflake ingest sdk */
+/** Converts schema-ddl values into objects which are compatible with the BigQuery sdk */
 private[processing] object BigQueryCaster extends Caster[AnyRef] {
 
   override def nullValue: Null                             = null

diff --git a/...s/core/src/main/scala/com.snowplowanalytics.snowplow.bigquery/processing/Processing.scala b/...s/core/src/main/scala/com.snowplowanalytics.snowplow.bigquery/processing/Processing.scala
@@ -135,7 +135,7 @@ object Processing {
       } yield ParseResult(events, badRows, numBytes, token)
     }
 
-  /** Transform the Event into values compatible with the snowflake ingest sdk */
+  /** Transform the Event into values compatible with the BigQuery sdk */
   private def transform[F[_]: Sync: RegistryLookup](
     env: Environment[F],
     badProcessor: BadRowProcessor

diff --git a/modules/core/src/test/scala/com.snowplowanalytics.snowplow.bigquery/MockEnvironment.scala b/modules/core/src/test/scala/com.snowplowanalytics.snowplow.bigquery/MockEnvironment.scala
@@ -104,9 +104,9 @@ object MockEnvironment {
   }
 
   val appInfo = new AppInfo {
-    def name        = "snowflake-loader-test"
+    def name        = "bigquery-loader-test"
     def version     = "0.0.0"
-    def dockerAlias = "snowplow/snowflake-loader-test:0.0.0"
+    def dockerAlias = "snowplow/bigquery-loader-test:0.0.0"
     def cloud       = "OnPrem"
   }