chore: update environment

JasperHG90 · Jan 14, 2024 · 44b6198 · 44b6198
1 parent 7b4c1fb
commit 44b6198
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 7 deletions.
diff --git a/dags/luchtmeetnet_ingestion/src/luchtmeetnet_ingestion/__init__.py b/dags/luchtmeetnet_ingestion/src/luchtmeetnet_ingestion/__init__.py
@@ -1,7 +1,9 @@
+import os
 from importlib import metadata
 
-from dagster import Definitions, FilesystemIOManager
+from dagster import Definitions, EnvVar
 from luchtmeetnet_ingestion.assets import air_quality_data
+from luchtmeetnet_ingestion.IO.duckdb_io_manager import duckdb_parquet_io_manager
 from luchtmeetnet_ingestion.IO.resources import LuchtMeetNetResource
 from luchtmeetnet_ingestion.jobs import ingestion_job
 
@@ -15,15 +17,26 @@
 
 env_resources = {
  "dev": shared_resources
+ | {"landing_zone": duckdb_parquet_io_manager.configured({"path": ".tmp/landing_zone"})},
+ "prd": shared_resources
  | {
- "landing_zone": FilesystemIOManager()
- } # {"landing_zone": duckdb_parquet_io_manager.configured({"path": ".tmp/landing_zone"})}
+ "landing_zone": duckdb_parquet_io_manager.configured(
+ {
+ "path": "s3://inge-cst-euw4-jgdag-prd",
+ "aws_access_key": EnvVar("GCS_ACCESS_KEY_ID"),
+ "aws_secret_key": EnvVar("GCS_SECRET_ACCESS_KEY"),
+ "aws_endpoint": "https://storage.googleapis.com",
+ }
+ )
+ },
 }
 
+environment = os.getenv("ENVIRONMENT", "dev")
+
 definition = Definitions(
  assets=[
  air_quality_data,
  ],
- resources=env_resources["dev"],
+ resources=env_resources[environment],
  jobs=[ingestion_job],
 )
diff --git a/dags/luchtmeetnet_ingestion/src/luchtmeetnet_ingestion/assets/__init__.py b/dags/luchtmeetnet_ingestion/src/luchtmeetnet_ingestion/assets/__init__.py
@@ -20,7 +20,7 @@ def air_quality_data(context, luchtmeetnet_api: LuchtMeetNetResource):
  "station_number": "NL01494",
  }
  df = pd.DataFrame(luchtmeetnet_api.request("measurements", request_params=rp))
- return df.to_json()
+ return df
 
 
 # @asset(

diff --git a/deployment/values.yaml.j2 b/deployment/values.yaml.j2
@@ -87,7 +87,9 @@ deployments:
  # value: "one"
  # - name: ENV_TWO
  # value: "two"
- env: []
+ env:
+ - name: ENVIRONMENT
+ value: "prd"
 
  # Additional environment variables can be retrieved and set from ConfigMaps. See:
  # https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#configure-all-key-value-pairs-in-a-configmap-as-container-environment-variables
@@ -105,7 +107,9 @@ deployments:
  #
  # envSecrets:
  # - name: secret
- envSecrets: []
+ envSecrets:
+ - gcs-access-key-id
+ - gcs-secret-access-key
 
  annotations: {}
  nodeSelector: {}