Merge pull request #5 from immutable/d1155

DATA-1181:Add config for imtbl zkevm mainnet

Dockerfile

@@ -6,4 +6,4 @@ ENV FLEX_TEMPLATE_JAVA_CLASSPATH=/template/*
 ENV FLEX_TEMPLATE_JAVA_MAIN_CLASS=io.blockchainetl.ethereum.EthereumPubSubToBigQueryPipeline
 
 COPY target/blockchain-etl-dataflow-bundled-0.1.jar /template/
-COPY chain-config/blockchain_zkevm_imtbl_testnet_${env}.json /template/
+COPY chain-config/blockchain_*_${env}.json /template/

chain-config/blockchain_zkevm_imtbl_mainnet_dev.json

@@ -0,0 +1,8 @@
+[
+ {
+ "transformNamePrefix": "blockchain_zkevm_imtbl_mainnet_",
+ "pubSubSubscriptionPrefix": "projects/dev-im-data/subscriptions/blockchain-zkevm-imtbl-mainnet-dataflow",
+ "bigQueryDataset": "raw_blockchain_zkevm_imtbl_mainnet",
+ "startTimestamp": "2019-03-02T00:00:00Z"
+ }
+]

chain-config/blockchain_zkevm_imtbl_mainnet_prod.json

@@ -0,0 +1,8 @@
+[
+ {
+ "transformNamePrefix": "blockchain_zkevm_imtbl_mainnet_",
+ "pubSubSubscriptionPrefix": "projects/prod-im-data/subscriptions/blockchain-zkevm-imtbl-mainnet-dataflow",
+ "bigQueryDataset": "raw_blockchain_zkevm_imtbl_mainnet",
+ "startTimestamp": "2019-03-02T00:00:00Z"
+ }
+]

terraform/main.tf

@@ -97,4 +97,31 @@ resource "google_dataflow_flex_template_job" "flex_template_job" {
  region=local.region
  zone=local.zone
  }
+}
+
+
+resource "google_dataflow_flex_template_job" "zkevm_imtbl_mainnet_job" {
+ provider = google-beta
+ project = "${terraform.workspace}-im-data"
+ name = "zkevm-imtbl-mainnet-etl-dataflow-${formatdate("YYYYMMDD-hhmmss", timestamp())}"
+ region = local.region
+ skip_wait_on_job_termination = true
+ container_spec_gcs_path = "gs://${google_storage_bucket_object.blockchain_etl_dataflow.bucket}/${google_storage_bucket_object.blockchain_etl_dataflow.name}"
+ on_delete = "drain"
+
+
+ parameters = {
+ chainConfigFile = "/template/blockchain_zkevm_imtbl_mainnet_${terraform.workspace}.json"
+ allowedTimestampSkewSeconds = "5184000"
+ gcpTempLocation = "gs://${terraform.workspace}-im-data-imx-resource/ethereum-etl/zkevm-imtbl-mainnet-streaming/temp"
+ tempLocation = "gs://${terraform.workspace}-im-data-imx-resource/ethereum-etl/zkevm-imtbl-mainnet-streaming/temp"
+ project = "${terraform.workspace}-im-data"
+ runner = "DataflowRunner"
+ workerMachineType = "n1-standard-1"
+ maxNumWorkers = 1
+ diskSizeGb=30
+ enableStreamingEngine=true
+ region=local.region
+ zone=local.zone
+ }
 }

terraform/version.tf

@@ -9,7 +9,11 @@ terraform {
  required_providers {
  google = {
  source = "hashicorp/google"
- version = ">= 4.0.0"
+ version = "~> 4.82.0, < 5.0.0"
+ }
+ google-beta = {
+ source = "hashicorp/google-beta"
+ version = ">= 4.82.0, < 5.0.0"
  }
  }
  provider_meta "google" {