update keys

TEAMSchools · Sep 21, 2023 · d19fec4 · d19fec4
1 parent 727f8c4
commit d19fec4
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 15 deletions.
diff --git a/src/teamster/core/adp/assets.py b/src/teamster/core/adp/assets.py
@@ -26,12 +26,12 @@ def build_wfm_asset(
     date_partitions_def: DynamicPartitionsDefinition,
     symbolic_ids,
     hyperfind,
+    auto_materialize_policy,
     op_tags={},
     **kwargs,
 ) -> AssetsDefinition:
     @asset(
-        name=asset_name,
-        key_prefix=[code_location, source_system],
+        key=[code_location, source_system, asset_name],
         metadata={"hyperfind": hyperfind},
         partitions_def=MultiPartitionsDefinition(
             {
@@ -41,6 +41,7 @@ def build_wfm_asset(
         ),
         io_manager_key="io_manager_gcs_avro",
         op_tags=op_tags,
+        auto_materialize_policy=auto_materialize_policy,
     )
     def _asset(context: OpExecutionContext, adp_wfm: AdpWorkforceManagerResource):
         asset = context.assets_def

diff --git a/src/teamster/kipptaf/adp/assets.py b/src/teamster/kipptaf/adp/assets.py
@@ -1,4 +1,5 @@
 from dagster import (
+    AutoMaterializePolicy,
     DailyPartitionsDefinition,
     DynamicPartitionsDefinition,
     config_from_files,
@@ -10,14 +11,12 @@
 
 from .. import CODE_LOCATION, LOCAL_TIMEZONE
 
-SOURCE_SYSTEM = "adp"
-
 config_dir = f"src/teamster/{CODE_LOCATION}/adp/config"
 
 sftp_assets = [
     build_sftp_asset(
         code_location=CODE_LOCATION,
-        source_system=SOURCE_SYSTEM,
+        source_system="adp_workforce_now",
         asset_fields=ASSET_FIELDS,
         **a,
     )
@@ -27,13 +26,14 @@
 wfm_assets_daily = [
     build_wfm_asset(
         code_location=CODE_LOCATION,
-        source_system=SOURCE_SYSTEM,
+        source_system="adp_workforce_manager",
         date_partitions_def=DailyPartitionsDefinition(
             start_date=a["partition_start_date"],
             timezone=LOCAL_TIMEZONE.name,
             fmt="%Y-%m-%d",
             end_offset=1,
         ),
+        auto_materialize_policy=AutoMaterializePolicy.eager(),
         **a,
     )
     for a in config_from_files([f"{config_dir}/wfm-assets-daily.yaml"])["assets"]
@@ -42,10 +42,11 @@
 wfm_assets_dynamic = [
     build_wfm_asset(
         code_location=CODE_LOCATION,
-        source_system=SOURCE_SYSTEM,
+        source_system="adp_workforce_manager",
         date_partitions_def=DynamicPartitionsDefinition(
-            name=f"{CODE_LOCATION}__{SOURCE_SYSTEM}__{a['asset_name']}_date"
+            name=f"{CODE_LOCATION}__adp_workforce_manager__{a['asset_name']}_date"
         ),
+        auto_materialize_policy=AutoMaterializePolicy.eager(),
         **a,
     )
     for a in config_from_files([f"{config_dir}/wfm-assets-dynamic.yaml"])["assets"]

diff --git a/src/teamster/kipptaf/adp/config/sftp-assets.yaml b/src/teamster/kipptaf/adp/config/sftp-assets.yaml
@@ -1,13 +1,10 @@
 assets:
   - asset_name: pension_and_benefits_enrollments
     remote_filepath: .
-    # trunk-ignore(yamllint/quoted-strings)
-    remote_file_regex: 'pension_and_benefits_enrollments\.csv'
+    remote_file_regex: pension_and_benefits_enrollments\.csv
   - asset_name: comprehensive_benefits_report
     remote_filepath: .
-    # trunk-ignore(yamllint/quoted-strings)
-    remote_file_regex: 'comprehensive_benefits_report\.csv'
+    remote_file_regex: comprehensive_benefits_report\.csv
   - asset_name: additional_earnings_report
     remote_filepath: .
-    # trunk-ignore(yamllint/quoted-strings)
-    remote_file_regex: 'additional_earnings_report\.csv'
+    remote_file_regex: additional_earnings_report\.csv
diff --git a/src/teamster/kipptaf/definitions.py b/src/teamster/kipptaf/definitions.py
@@ -184,7 +184,7 @@
             username=EnvVar("ACHIEVE3K_SFTP_USERNAME"),
             password=EnvVar("ACHIEVE3K_SFTP_PASSWORD"),
         ),
-        "ssh_adp": SSHConfigurableResource(
+        "ssh_adp_workforce_now": SSHConfigurableResource(
             remote_host="sftp.kippnj.org",
             username=EnvVar("ADP_SFTP_USERNAME"),
             password=EnvVar("ADP_SFTP_PASSWORD"),