Skip to content

Commit

Permalink
first attempt to point dev HCA dagster @ prod TDR (we are a client)
Browse files Browse the repository at this point in the history
  • Loading branch information
bahill committed May 14, 2024
1 parent 13ecc6e commit 817af27
Showing 1 changed file with 6 additions and 5 deletions.
11 changes: 6 additions & 5 deletions orchestration/hca_orchestration/repositories/dev_repository.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
"""
Pipelines here are intended to be run in the DEV HCA GCP project
They, however, run against the production data repo
"""

from dagster import PipelineDefinition, in_process_executor, repository
Expand Down Expand Up @@ -61,7 +62,7 @@ def load_hca_job() -> PipelineDefinition:
resource_defs={
"beam_runner": preconfigure_resource_for_mode(k8s_dataflow_beam_runner, "dev"),
"bigquery_client": bigquery_client,
"data_repo_client": preconfigure_resource_for_mode(jade_data_repo_client, "dev"),
"data_repo_client": preconfigure_resource_for_mode(jade_data_repo_client, "prod"),
"gcs": google_storage_client,
"io_manager": preconfigure_resource_for_mode(gcs_pickle_io_manager, "dev"),
"load_tag": load_tag,
Expand All @@ -83,7 +84,7 @@ def project_load_hca_job() -> PipelineDefinition:
"bigquery_client": bigquery_client,
"bigquery_service": bigquery_service,
"dagit_config": preconfigure_resource_for_mode(dagit_config, "dev"),
"data_repo_client": preconfigure_resource_for_mode(jade_data_repo_client, "dev"),
"data_repo_client": preconfigure_resource_for_mode(jade_data_repo_client, "prod"),
"data_repo_service": data_repo_service,
"gcs": google_storage_client,
"io_manager": preconfigure_resource_for_mode(gcs_pickle_io_manager, "dev"),
Expand All @@ -103,7 +104,7 @@ def per_project_load_hca() -> PipelineDefinition:
resource_defs={
"beam_runner": preconfigure_resource_for_mode(k8s_dataflow_beam_runner, "dev"),
"bigquery_client": bigquery_client,
"data_repo_client": preconfigure_resource_for_mode(jade_data_repo_client, "dev"),
"data_repo_client": preconfigure_resource_for_mode(jade_data_repo_client, "prod"),
"gcs": google_storage_client,
"io_manager": preconfigure_resource_for_mode(gcs_pickle_io_manager, "dev"),
"load_tag": load_tag,
Expand All @@ -123,8 +124,8 @@ def per_project_load_hca() -> PipelineDefinition:
@repository
def all_jobs() -> list[PipelineDefinition]:
jobs = [
make_snapshot_public_job("dev", "dev"),
cut_project_snapshot_job("dev", "dev", "[email protected]"),
make_snapshot_public_job("dev", "prod"),
cut_project_snapshot_job("dev", "prod", "[email protected]"),
legacy_cut_snapshot_job("dev", "[email protected]"),
per_project_load_hca(),
validate_ingress_job()
Expand Down

0 comments on commit 817af27

Please sign in to comment.