diff --git a/elementary/clients/dbt/base_dbt_runner.py b/elementary/clients/dbt/base_dbt_runner.py index 181aae60e..d048187a8 100644 --- a/elementary/clients/dbt/base_dbt_runner.py +++ b/elementary/clients/dbt/base_dbt_runner.py @@ -10,12 +10,14 @@ def __init__( target: Optional[str] = None, vars: Optional[dict] = None, secret_vars: Optional[dict] = None, + allow_macros_without_package_prefix: bool = False, ) -> None: self.project_dir = project_dir self.profiles_dir = profiles_dir self.target = target self.vars = vars or {} self.secret_vars = secret_vars or {} + self.allow_macros_without_package_prefix = allow_macros_without_package_prefix @abstractmethod def deps(self, *args, **kwargs): diff --git a/elementary/clients/dbt/dbt_runner.py b/elementary/clients/dbt/dbt_runner.py index ca092a81b..1b9034abd 100644 --- a/elementary/clients/dbt/dbt_runner.py +++ b/elementary/clients/dbt/dbt_runner.py @@ -27,8 +27,16 @@ def __init__( env_vars: Optional[Dict[str, str]] = None, vars: Optional[Dict[str, Any]] = None, secret_vars: Optional[Dict[str, Any]] = None, + allow_macros_without_package_prefix: bool = False, ) -> None: - super().__init__(project_dir, profiles_dir, target, vars, secret_vars) + super().__init__( + project_dir, + profiles_dir, + target, + vars, + secret_vars, + allow_macros_without_package_prefix, + ) self.raise_on_failure = raise_on_failure self.env_vars = env_vars self._run_deps_if_needed() @@ -126,6 +134,11 @@ def run_operation( should_log: bool = True, log_output: bool = False, ) -> list: + if "." not in macro_name and not self.allow_macros_without_package_prefix: + raise ValueError( + f"Macro name '{macro_name}' is missing package prefix. " + f"Please use the following format: ." + ) macro_to_run = macro_name macro_to_run_args = macro_args if macro_args else dict() if should_log: diff --git a/elementary/clients/dbt/slim_dbt_runner.py b/elementary/clients/dbt/slim_dbt_runner.py index 5462576e7..10aad54cf 100644 --- a/elementary/clients/dbt/slim_dbt_runner.py +++ b/elementary/clients/dbt/slim_dbt_runner.py @@ -80,9 +80,17 @@ def __init__( target: Optional[str] = None, vars: Optional[dict] = None, secret_vars: Optional[dict] = None, + allow_macros_without_package_prefix: bool = False, **kwargs, ): - super().__init__(project_dir, profiles_dir, target, vars, secret_vars) + super().__init__( + project_dir, + profiles_dir, + target, + vars, + secret_vars, + allow_macros_without_package_prefix, + ) self.config: Optional[RuntimeConfig] = None self.adapter: Optional[BaseAdapter] = None @@ -194,6 +202,12 @@ def run_operation( if self.profiles_dir is None: raise Exception("profiles_dir must be passed to SlimDbtRunner") + if "." not in macro_name and not self.allow_macros_without_package_prefix: + raise ValueError( + f"Macro name '{macro_name}' is missing package prefix. " + f"Please use the following format: ." + ) + macro_args = macro_args or {} all_vars = self._get_all_vars(vars) diff --git a/elementary/monitor/data_monitoring/data_monitoring.py b/elementary/monitor/data_monitoring/data_monitoring.py index d23bebb9f..b5b6a8110 100644 --- a/elementary/monitor/data_monitoring/data_monitoring.py +++ b/elementary/monitor/data_monitoring/data_monitoring.py @@ -116,7 +116,7 @@ def properties(self): def get_elementary_database_and_schema(self): try: relation = self.internal_dbt_runner.run_operation( - "get_elementary_database_and_schema", quiet=True + "elementary_cli.get_elementary_database_and_schema", quiet=True )[0] logger.info(f"Elementary's database and schema: '{relation}'") return relation @@ -129,7 +129,7 @@ def get_elementary_database_and_schema(self): def get_latest_invocation(self) -> Dict[str, Any]: try: latest_invocation = self.internal_dbt_runner.run_operation( - "get_latest_invocation", quiet=True + "elementary_cli.get_latest_invocation", quiet=True )[0] return json.loads(latest_invocation)[0] if latest_invocation else {} except Exception as err: @@ -177,7 +177,7 @@ def _get_warehouse_info(self, hash_id: bool = False) -> Optional[WarehouseInfo]: try: warehouse_type, warehouse_unique_id = json.loads( self.internal_dbt_runner.run_operation( - "get_adapter_type_and_unique_id", quiet=True + "elementary_cli.get_adapter_type_and_unique_id", quiet=True )[0] ) return WarehouseInfo( diff --git a/elementary/monitor/data_monitoring/data_monitoring_alerts.py b/elementary/monitor/data_monitoring/data_monitoring_alerts.py index 29525bbe0..ba6d938b4 100644 --- a/elementary/monitor/data_monitoring/data_monitoring_alerts.py +++ b/elementary/monitor/data_monitoring/data_monitoring_alerts.py @@ -239,7 +239,7 @@ def run_alerts( ) -> bool: logger.info("Running internal dbt run to aggregate alerts") success = self.internal_dbt_runner.run( - models="alerts", full_refresh=dbt_full_refresh, vars=dbt_vars + models="elementary_cli.alerts", full_refresh=dbt_full_refresh, vars=dbt_vars ) self.execution_properties["alerts_run_success"] = success if not success: diff --git a/elementary/monitor/dbt_project/dbt_project.yml b/elementary/monitor/dbt_project/dbt_project.yml index ee4ee490b..71a16879e 100644 --- a/elementary/monitor/dbt_project/dbt_project.yml +++ b/elementary/monitor/dbt_project/dbt_project.yml @@ -1,7 +1,7 @@ # Name your project! Project names should contain only lowercase characters # and underscores. A good package name should reflect your organization's # name or the intended use of these models -name: "elementary_internal" +name: "elementary_cli" version: "1.0.0" config-version: 2 diff --git a/elementary/monitor/dbt_project/macros/alerts/get_model_alerts.sql b/elementary/monitor/dbt_project/macros/alerts/get_model_alerts.sql index 99f0f8cd9..d501faaa9 100644 --- a/elementary/monitor/dbt_project/macros/alerts/get_model_alerts.sql +++ b/elementary/monitor/dbt_project/macros/alerts/get_model_alerts.sql @@ -8,7 +8,7 @@ with alerts_in_time_limit as ( select * from {{ ref('alerts_models') }} - where {{ elementary.edr_cast_as_timestamp('detected_at') }} >= {{ get_alerts_time_limit(days_back) }} + where {{ elementary.edr_cast_as_timestamp('detected_at') }} >= {{ elementary_cli.get_alerts_time_limit(days_back) }} ), models as ( @@ -107,7 +107,7 @@ end as suppression_status, sent_at from {{ ref('alerts_models') }} - where {{ elementary.edr_cast_as_timestamp('detected_at') }} >= {{ get_alerts_time_limit(days_back) }} + where {{ elementary.edr_cast_as_timestamp('detected_at') }} >= {{ elementary_cli.get_alerts_time_limit(days_back) }} ) select diff --git a/elementary/monitor/dbt_project/macros/alerts/get_source_freshness_alerts.sql b/elementary/monitor/dbt_project/macros/alerts/get_source_freshness_alerts.sql index a70fda8df..c525ab1a3 100644 --- a/elementary/monitor/dbt_project/macros/alerts/get_source_freshness_alerts.sql +++ b/elementary/monitor/dbt_project/macros/alerts/get_source_freshness_alerts.sql @@ -3,7 +3,7 @@ {% set select_pending_alerts_query %} with alerts_in_time_limit as ( select * from {{ ref('alerts_source_freshness') }} - where {{ elementary.edr_cast_as_timestamp('detected_at') }} >= {{ get_alerts_time_limit(days_back) }} + where {{ elementary.edr_cast_as_timestamp('detected_at') }} >= {{ elementary_cli.get_alerts_time_limit(days_back) }} ), models as ( @@ -107,7 +107,7 @@ end as suppression_status, sent_at from {{ ref('alerts_source_freshness') }} - where {{ elementary.edr_cast_as_timestamp('detected_at') }} >= {{ get_alerts_time_limit(days_back) }} + where {{ elementary.edr_cast_as_timestamp('detected_at') }} >= {{ elementary_cli.get_alerts_time_limit(days_back) }} ) select diff --git a/elementary/monitor/dbt_project/macros/alerts/get_test_alerts.sql b/elementary/monitor/dbt_project/macros/alerts/get_test_alerts.sql index 767895d69..6b14a651b 100644 --- a/elementary/monitor/dbt_project/macros/alerts/get_test_alerts.sql +++ b/elementary/monitor/dbt_project/macros/alerts/get_test_alerts.sql @@ -3,7 +3,7 @@ {% set select_pending_alerts_query %} with alerts_in_time_limit as ( select * from {{ ref('alerts') }} - where {{ elementary.edr_cast_as_timestamp('detected_at') }} >= {{ get_alerts_time_limit(days_back) }} + where {{ elementary.edr_cast_as_timestamp('detected_at') }} >= {{ elementary_cli.get_alerts_time_limit(days_back) }} ), models as ( @@ -77,7 +77,7 @@ {% endset %} {% set alerts_agate = elementary.run_query(select_pending_alerts_query) %} - {% set test_result_rows_agate = elementary_internal.get_result_rows_agate(days_back) %} + {% set test_result_rows_agate = elementary_cli.get_result_rows_agate(days_back) %} {% set test_result_alert_dicts = elementary.agate_to_dicts(alerts_agate) %} {% set pending_alerts = [] %} {% for alert in test_result_alert_dicts %} @@ -86,7 +86,7 @@ {% set test_rows_sample = none %} {%- if not disable_samples and ((test_type == 'dbt_test' and status in ['fail', 'warn']) or (test_type != 'dbt_test' and status != 'error')) -%} - {% set test_rows_sample = elementary_internal.get_test_rows_sample(alert.result_rows, test_result_rows_agate.get(alert.alert_id)) %} + {% set test_rows_sample = elementary_cli.get_test_rows_sample(alert.result_rows, test_result_rows_agate.get(alert.alert_id)) %} {%- endif -%} {% set pending_alert_dict = {'id': alert.alert_id, 'alert_class_id': alert.alert_class_id, @@ -135,7 +135,7 @@ end as suppression_status, sent_at from {{ ref('alerts') }} - where {{ elementary.edr_cast_as_timestamp('detected_at') }} >= {{ get_alerts_time_limit(days_back) }} + where {{ elementary.edr_cast_as_timestamp('detected_at') }} >= {{ elementary_cli.get_alerts_time_limit(days_back) }} ) select diff --git a/elementary/monitor/dbt_project/macros/alerts/update_sent_alerts.sql b/elementary/monitor/dbt_project/macros/alerts/update_sent_alerts.sql index 7ff98ff73..2e5bc53c9 100644 --- a/elementary/monitor/dbt_project/macros/alerts/update_sent_alerts.sql +++ b/elementary/monitor/dbt_project/macros/alerts/update_sent_alerts.sql @@ -4,7 +4,7 @@ {% set update_sent_alerts_query %} UPDATE {{ ref(table_name) }} set suppression_status = 'sent', sent_at = {{ "'{}'".format(sent_at) }}, alert_sent = TRUE WHERE alert_id IN {{ elementary.strings_list_to_tuple(alert_ids) }} and suppression_status = 'pending' and - {{ elementary.edr_cast_as_timestamp('detected_at') }} >= {{ get_alerts_time_limit() }} + {{ elementary.edr_cast_as_timestamp('detected_at') }} >= {{ elementary_cli.get_alerts_time_limit() }} {% endset %} {% do elementary.run_query(update_sent_alerts_query) %} {% endif %} diff --git a/elementary/monitor/dbt_project/macros/alerts/update_skipped_alerts.sql b/elementary/monitor/dbt_project/macros/alerts/update_skipped_alerts.sql index 98f344648..8fbfc8375 100644 --- a/elementary/monitor/dbt_project/macros/alerts/update_skipped_alerts.sql +++ b/elementary/monitor/dbt_project/macros/alerts/update_skipped_alerts.sql @@ -4,7 +4,7 @@ {% set update_skipped_alerts_query %} UPDATE {{ ref(table_name) }} set suppression_status = 'skipped' WHERE alert_id IN {{ elementary.strings_list_to_tuple(alert_ids) }} and suppression_status = 'pending' and - {{ elementary.edr_cast_as_timestamp('detected_at') }} >= {{ get_alerts_time_limit() }} + {{ elementary.edr_cast_as_timestamp('detected_at') }} >= {{ elementary_cli.get_alerts_time_limit() }} {% endset %} {% do elementary.run_query(update_skipped_alerts_query) %} {% endif %} diff --git a/elementary/monitor/dbt_project/macros/get_test_results.sql b/elementary/monitor/dbt_project/macros/get_test_results.sql index 4982a51c0..59c469dac 100644 --- a/elementary/monitor/dbt_project/macros/get_test_results.sql +++ b/elementary/monitor/dbt_project/macros/get_test_results.sql @@ -1,7 +1,7 @@ {%- macro get_test_results(days_back = 7, invocations_per_test = 720, disable_passed_test_metrics = false) -%} {% set select_test_results %} with test_results as ( - {{ elementary_internal.current_tests_run_results_query(days_back=days_back) }} + {{ elementary_cli.current_tests_run_results_query(days_back=days_back) }} ), ordered_test_results as ( @@ -50,7 +50,7 @@ {%- endset -%} {% set test_results_agate = elementary.run_query(select_test_results) %} - {% set test_result_rows_agate = elementary_internal.get_result_rows_agate(days_back) %} + {% set test_result_rows_agate = elementary_cli.get_result_rows_agate(days_back) %} {% set tests = elementary.agate_to_dicts(test_results_agate) %} {%- for test in tests -%} {% set test_rows_sample = none %} @@ -63,7 +63,7 @@ {% do elementary_tests_allowlist_status.append('pass') %} {% endif %} {%- if (test_type == 'dbt_test' and status in ['fail', 'warn']) or (test_type != 'dbt_test' and status in elementary_tests_allowlist_status) -%} - {% set test_rows_sample = elementary_internal.get_test_rows_sample(test.result_rows, test_result_rows_agate.get(test.id)) %} + {% set test_rows_sample = elementary_cli.get_test_rows_sample(test.result_rows, test_result_rows_agate.get(test.id)) %} {# Dimension anomalies return multiple dimensions for the test rows sample, and needs to be handle differently. #} {# Currently we show only the anomalous for all of the dimensions. #} {% if test.test_sub_type == 'dimension' %} diff --git a/elementary/monitor/dbt_project/models/alerts/alerts.sql b/elementary/monitor/dbt_project/models/alerts/alerts.sql index 98dd5a531..aa8ee9d06 100644 --- a/elementary/monitor/dbt_project/models/alerts/alerts.sql +++ b/elementary/monitor/dbt_project/models/alerts/alerts.sql @@ -37,5 +37,5 @@ select from failed_tests {%- if is_incremental() %} - {{ get_new_alerts_where_clause(this) }} + {{ elementary_cli.get_new_alerts_where_clause(this) }} {%- endif %} diff --git a/elementary/monitor/dbt_project/models/alerts/alerts_models.sql b/elementary/monitor/dbt_project/models/alerts/alerts_models.sql index fa464994b..4c3f6b12c 100644 --- a/elementary/monitor/dbt_project/models/alerts/alerts_models.sql +++ b/elementary/monitor/dbt_project/models/alerts/alerts_models.sql @@ -17,8 +17,8 @@ {{ elementary.edr_cast_as_string('NULL') }} as sent_at from {{ error_models_relation }} {% if is_incremental() %} - {{ get_new_alerts_where_clause(this) }} + {{ elementary_cli.get_new_alerts_where_clause(this) }} {% endif %} {% else %} - {{ empty_alerts_models() }} + {{ elementary_cli.empty_alerts_models() }} {% endif %} diff --git a/elementary/monitor/dbt_project/models/alerts/alerts_source_freshness.sql b/elementary/monitor/dbt_project/models/alerts/alerts_source_freshness.sql index 334c37f10..9a5ccbcef 100644 --- a/elementary/monitor/dbt_project/models/alerts/alerts_source_freshness.sql +++ b/elementary/monitor/dbt_project/models/alerts/alerts_source_freshness.sql @@ -17,8 +17,8 @@ {{ elementary.edr_cast_as_string('NULL') }} as sent_at from {{ alerts_source_freshness_relation }} {% if is_incremental() %} - {{ get_new_alerts_where_clause(this) }} + {{ elementary_cli.get_new_alerts_where_clause(this) }} {% endif %} {% else %} - {{ empty_alerts_source_freshness() }} + {{ elementary_cli.empty_alerts_source_freshness() }} {% endif %} diff --git a/elementary/monitor/dbt_project/models/elementary.yml b/elementary/monitor/dbt_project/models/elementary.yml index 06aba784c..14dc3a669 100644 --- a/elementary/monitor/dbt_project/models/elementary.yml +++ b/elementary/monitor/dbt_project/models/elementary.yml @@ -8,34 +8,34 @@ sources: columns: - name: unique_id tests: - - elementary_internal.unique_if_exists + - elementary_cli.unique_if_exists - name: dbt_tests columns: - name: unique_id tests: - - elementary_internal.unique_if_exists + - elementary_cli.unique_if_exists - name: dbt_sources columns: - name: unique_id tests: - - elementary_internal.unique_if_exists + - elementary_cli.unique_if_exists - name: dbt_snapshots columns: - name: unique_id tests: - - elementary_internal.unique_if_exists + - elementary_cli.unique_if_exists - name: dbt_metrics columns: - name: unique_id tests: - - elementary_internal.unique_if_exists + - elementary_cli.unique_if_exists - name: dbt_exposures columns: - name: unique_id tests: - - elementary_internal.unique_if_exists + - elementary_cli.unique_if_exists diff --git a/elementary/monitor/dbt_project/models/update_alerts/update_sent_alerts.sql b/elementary/monitor/dbt_project/models/update_alerts/update_sent_alerts.sql index 11b07ee50..ddcd5b4e4 100644 --- a/elementary/monitor/dbt_project/models/update_alerts/update_sent_alerts.sql +++ b/elementary/monitor/dbt_project/models/update_alerts/update_sent_alerts.sql @@ -8,5 +8,5 @@ -- depends_on: {{ ref('alerts_models') }} -- depends_on: {{ ref('alerts_source_freshness') }} -{% do update_sent_alerts(var("alert_ids"), var("sent_at"), var("table_name")) %} +{% do elementary_cli.update_sent_alerts(var("alert_ids"), var("sent_at"), var("table_name")) %} {{ elementary.no_results_query() }} diff --git a/elementary/monitor/dbt_project/models/update_alerts/update_skipped_alerts.sql b/elementary/monitor/dbt_project/models/update_alerts/update_skipped_alerts.sql index 5d3f76a23..1aced5b23 100644 --- a/elementary/monitor/dbt_project/models/update_alerts/update_skipped_alerts.sql +++ b/elementary/monitor/dbt_project/models/update_alerts/update_skipped_alerts.sql @@ -8,5 +8,5 @@ -- depends_on: {{ ref('alerts_models') }} -- depends_on: {{ ref('alerts_source_freshness') }} -{% do update_skipped_alerts(var("alert_ids"), var("table_name")) %} +{% do elementary_cli.update_skipped_alerts(var("alert_ids"), var("table_name")) %} {{ elementary.no_results_query() }} diff --git a/elementary/monitor/debug.py b/elementary/monitor/debug.py index 5284b33ce..5ed46388c 100644 --- a/elementary/monitor/debug.py +++ b/elementary/monitor/debug.py @@ -18,7 +18,7 @@ def run(self) -> bool: ) try: - dbt_runner.run_operation("test_conn", quiet=True) + dbt_runner.run_operation("elementary_cli.test_conn", quiet=True) except DbtCommandError as err: logs = ( "\n".join(str(log) for log in err.logs) diff --git a/elementary/monitor/fetchers/alerts/alerts.py b/elementary/monitor/fetchers/alerts/alerts.py index 5703c88f8..62719749e 100644 --- a/elementary/monitor/fetchers/alerts/alerts.py +++ b/elementary/monitor/fetchers/alerts/alerts.py @@ -34,7 +34,7 @@ def skip_alerts(self, alerts_to_skip: Sequence[Alert], table_name: str): logger.info(f'Update skipped alerts at "{table_name}"') for alert_ids_chunk in alert_ids_chunks: self.dbt_runner.run( - select="update_alerts.update_skipped_alerts", + select="elementary_cli.update_alerts.update_skipped_alerts", vars={ "alert_ids": alert_ids_chunk, "table_name": table_name, @@ -48,7 +48,7 @@ def query_pending_test_alerts( logger.info("Querying test alerts.") return self._query_alert_type( { - "macro_name": "get_pending_test_alerts", + "macro_name": "elementary_cli.get_pending_test_alerts", "macro_args": { "days_back": days_back, "disable_samples": disable_samples, @@ -63,7 +63,7 @@ def query_pending_model_alerts( logger.info("Querying model alerts.") res = self._query_alert_type( { - "macro_name": "get_pending_model_alerts", + "macro_name": "elementary_cli.get_pending_model_alerts", "macro_args": {"days_back": days_back}, }, ModelAlert, @@ -76,7 +76,7 @@ def query_pending_source_freshness_alerts( logger.info("Querying source freshness alerts.") return self._query_alert_type( { - "macro_name": "get_pending_source_freshness_alerts", + "macro_name": "elementary_cli.get_pending_source_freshness_alerts", "macro_args": {"days_back": days_back}, }, SourceFreshnessAlert, @@ -85,7 +85,7 @@ def query_pending_source_freshness_alerts( def query_last_test_alert_times(self, days_back: int) -> Dict[str, str]: logger.info("Querying test alerts last sent times.") response = self.dbt_runner.run_operation( - macro_name="get_last_test_alert_sent_times", + macro_name="elementary_cli.get_last_test_alert_sent_times", macro_args={"days_back": days_back}, ) return json.loads(response[0]) @@ -93,7 +93,7 @@ def query_last_test_alert_times(self, days_back: int) -> Dict[str, str]: def query_last_model_alert_times(self, days_back: int) -> Dict[str, str]: logger.info("Querying model alerts last sent times.") response = self.dbt_runner.run_operation( - macro_name="get_last_model_alert_sent_times", + macro_name="elementary_cli.get_last_model_alert_sent_times", macro_args={"days_back": days_back}, ) return json.loads(response[0]) @@ -101,7 +101,7 @@ def query_last_model_alert_times(self, days_back: int) -> Dict[str, str]: def query_last_source_freshness_alert_times(self, days_back: int) -> Dict[str, str]: logger.info("Querying source freshness alerts last sent times.") response = self.dbt_runner.run_operation( - macro_name="get_last_source_freshness_alert_sent_times", + macro_name="elementary_cli.get_last_source_freshness_alert_sent_times", macro_args={"days_back": days_back}, ) return json.loads(response[0]) @@ -142,7 +142,7 @@ def update_sent_alerts(self, alert_ids: List[str], table_name: str) -> None: logger.info(f'Update sent alerts at "{table_name}"') for alert_ids_chunk in alert_ids_chunks: self.dbt_runner.run( - select="update_alerts.update_sent_alerts", + select="elementary_cli.update_alerts.update_sent_alerts", vars={ "alert_ids": alert_ids_chunk, "sent_at": get_now_utc_str(), diff --git a/elementary/monitor/fetchers/invocations/invocations.py b/elementary/monitor/fetchers/invocations/invocations.py index 8f57a8ee4..cddde925f 100644 --- a/elementary/monitor/fetchers/invocations/invocations.py +++ b/elementary/monitor/fetchers/invocations/invocations.py @@ -13,7 +13,7 @@ def get_test_last_invocation( self, macro_args: Optional[dict] = None ) -> DbtInvocationSchema: invocation_response = self.dbt_runner.run_operation( - macro_name="get_test_last_invocation", macro_args=macro_args + macro_name="elementary_cli.get_test_last_invocation", macro_args=macro_args ) invocation = json.loads(invocation_response[0]) if invocation_response else None if invocation: @@ -24,7 +24,7 @@ def get_test_last_invocation( def get_models_latest_invocations_data(self) -> List[DbtInvocationSchema]: invocations_response = self.dbt_runner.run_operation( - macro_name="get_models_latest_invocations_data" + macro_name="elementary_cli.get_models_latest_invocations_data" ) invocation_results = ( json.loads(invocations_response[0]) if invocations_response else [] @@ -37,7 +37,7 @@ def get_models_latest_invocations_data(self) -> List[DbtInvocationSchema]: def get_models_latest_invocation(self) -> Dict[str, str]: response = self.dbt_runner.run_operation( - macro_name="get_models_latest_invocation" + macro_name="elementary_cli.get_models_latest_invocation" ) models_latest_invocation_results = json.loads(response[0]) if response else [] diff --git a/elementary/monitor/fetchers/lineage/lineage.py b/elementary/monitor/fetchers/lineage/lineage.py index 407d82382..4aa1d752c 100644 --- a/elementary/monitor/fetchers/lineage/lineage.py +++ b/elementary/monitor/fetchers/lineage/lineage.py @@ -11,7 +11,7 @@ def get_nodes_depends_on_nodes( ) -> List[NodeDependsOnNodesSchema]: nodes_depends_on_nodes = [] nodes_depends_on_nodes_results = self.dbt_runner.run_operation( - macro_name="get_nodes_depends_on_nodes", + macro_name="elementary_cli.get_nodes_depends_on_nodes", macro_args={"exclude_elementary": exclude_elementary_models}, ) if nodes_depends_on_nodes_results: diff --git a/elementary/monitor/fetchers/models/models.py b/elementary/monitor/fetchers/models/models.py index bb11ae4c4..94220ffd9 100644 --- a/elementary/monitor/fetchers/models/models.py +++ b/elementary/monitor/fetchers/models/models.py @@ -22,7 +22,7 @@ def get_models_runs( self, days_back: Optional[int] = 7, exclude_elementary_models: bool = False ) -> List[ModelRunSchema]: run_operation_response = self.dbt_runner.run_operation( - macro_name="get_models_runs", + macro_name="elementary_cli.get_models_runs", macro_args={ "days_back": days_back, "exclude_elementary": exclude_elementary_models, @@ -36,7 +36,7 @@ def get_models_runs( def get_models(self, exclude_elementary_models: bool = False) -> List[ModelSchema]: run_operation_response = self.dbt_runner.run_operation( - macro_name="get_models", + macro_name="elementary_cli.get_models", macro_args={"exclude_elementary": exclude_elementary_models}, ) models = json.loads(run_operation_response[0]) if run_operation_response else [] @@ -44,7 +44,9 @@ def get_models(self, exclude_elementary_models: bool = False) -> List[ModelSchem return models def get_sources(self) -> List[SourceSchema]: - run_operation_response = self.dbt_runner.run_operation(macro_name="get_sources") + run_operation_response = self.dbt_runner.run_operation( + macro_name="elementary_cli.get_sources" + ) sources = ( json.loads(run_operation_response[0]) if run_operation_response else [] ) @@ -53,7 +55,7 @@ def get_sources(self) -> List[SourceSchema]: def get_exposures(self) -> List[ExposureSchema]: run_operation_response = self.dbt_runner.run_operation( - macro_name="get_exposures" + macro_name="elementary_cli.get_exposures" ) exposures = ( json.loads(run_operation_response[0]) if run_operation_response else [] @@ -63,7 +65,7 @@ def get_exposures(self) -> List[ExposureSchema]: def get_test_coverages(self) -> List[ModelTestCoverage]: run_operation_response = self.dbt_runner.run_operation( - macro_name="get_dbt_models_test_coverage" + macro_name="elementary_cli.get_dbt_models_test_coverage" ) coverages = ( json.loads(run_operation_response[0]) if run_operation_response else [] diff --git a/elementary/monitor/fetchers/test_management/test_management.py b/elementary/monitor/fetchers/test_management/test_management.py index 7d4d4b044..7c1048148 100644 --- a/elementary/monitor/fetchers/test_management/test_management.py +++ b/elementary/monitor/fetchers/test_management/test_management.py @@ -27,7 +27,7 @@ def get_models( columns: Optional[DefaultDict[str, List[ResourceColumnModel]]] = None, ) -> List[ResourceModel]: run_operation_response = self.dbt_runner.run_operation( - macro_name="get_model_resources", + macro_name="elementary_cli.get_model_resources", macro_args=dict(exclude_elementary=exclude_elementary), ) models_results = ( @@ -41,7 +41,7 @@ def get_sources( columns: Optional[DefaultDict[str, List[ResourceColumnModel]]] = None, ) -> List[ResourceModel]: run_operation_response = self.dbt_runner.run_operation( - macro_name="get_source_resources", + macro_name="elementary_cli.get_source_resources", macro_args=dict(exclude_elementary=exclude_elementary), ) sources_results = ( @@ -51,7 +51,7 @@ def get_sources( def get_resources_columns(self) -> DefaultDict[str, List[ResourceColumnModel]]: run_operation_response = self.dbt_runner.run_operation( - macro_name="get_resources_columns" + macro_name="elementary_cli.get_resources_columns" ) resources_columns_results = ( json.loads(run_operation_response[0]) if run_operation_response else {} @@ -107,7 +107,7 @@ def get_resources(self, exclude_elementary: bool = True) -> ResourcesModel: def get_tags(self) -> TagsModel: run_operation_response = self.dbt_runner.run_operation( - macro_name="get_project_tags" + macro_name="elementary_cli.get_project_tags" ) tags_results = ( json.loads(run_operation_response[0]) if run_operation_response else [] @@ -119,7 +119,9 @@ def get_tags(self) -> TagsModel: return TagsModel(tags=all_tags) def get_tests(self) -> List[TestModel]: - run_operation_response = self.dbt_runner.run_operation(macro_name="get_tests") + run_operation_response = self.dbt_runner.run_operation( + macro_name="elementary_cli.get_tests" + ) test_results = ( json.loads(run_operation_response[0]) if run_operation_response else [] ) @@ -161,7 +163,7 @@ def get_tests(self) -> List[TestModel]: def get_project_owners(self) -> List[str]: run_operation_response = self.dbt_runner.run_operation( - macro_name="get_project_owners" + macro_name="elementary_cli.get_project_owners" ) owners_results = ( json.loads(run_operation_response[0]) if run_operation_response else [] @@ -177,7 +179,7 @@ def get_project_owners(self) -> List[str]: def get_project_subscribers(self) -> List[str]: run_operation_response = self.dbt_runner.run_operation( - macro_name="get_resources_meta" + macro_name="elementary_cli.get_resources_meta" ) resources_meta_results = ( json.loads(run_operation_response[0]) if run_operation_response else [] diff --git a/elementary/monitor/fetchers/tests/tests.py b/elementary/monitor/fetchers/tests/tests.py index 20751fa80..13f75531b 100644 --- a/elementary/monitor/fetchers/tests/tests.py +++ b/elementary/monitor/fetchers/tests/tests.py @@ -20,7 +20,7 @@ def get_all_test_results_db_rows( disable_passed_test_metrics: bool = False, ) -> List[TestResultDBRowSchema]: run_operation_response = self.dbt_runner.run_operation( - macro_name="get_test_results", + macro_name="elementary_cli.get_test_results", macro_args=dict( days_back=days_back, invocations_per_test=invocations_per_test, diff --git a/elementary/operations/upload_source_freshness.py b/elementary/operations/upload_source_freshness.py index e007c1a21..c08fe0874 100644 --- a/elementary/operations/upload_source_freshness.py +++ b/elementary/operations/upload_source_freshness.py @@ -50,7 +50,7 @@ def upload_results(self, results: dict): for chunk in upload_with_progress_bar: results_segment = results[chunk : chunk + chunk_size] dbt_runner.run_operation( - "elementary_internal.upload_source_freshness", + "elementary_cli.upload_source_freshness", macro_args={"results": json.dumps(results_segment)}, quiet=True, ) diff --git a/pyproject.toml b/pyproject.toml index b780a5794..87cd6901a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ packages = [{include = "elementary"}] [tool.poetry.dependencies] python = ">=3.6.2" click = ">=7.0,<8.1.4" -pyfiglet = "*" +pyfiglet = "<1.0.0" dbt-core = ">=0.20,<2.0.0" requests = ">=2.28.1,<3.0.0" beautifulsoup4 = "<5.0.0" diff --git a/tests/unit/monitor/fetchers/test_alerts_fetcher.py b/tests/unit/monitor/fetchers/test_alerts_fetcher.py index 68ed95eab..458ff4465 100644 --- a/tests/unit/monitor/fetchers/test_alerts_fetcher.py +++ b/tests/unit/monitor/fetchers/test_alerts_fetcher.py @@ -37,7 +37,7 @@ def test_update_sent_alerts( # Test that update_sent_alerts has been called with alert_ids as arguments. assert call_args[0][0][1] == "run" assert call_args[0][0][2] == "-s" - assert call_args[0][0][3] == "update_alerts.update_sent_alerts" + assert call_args[0][0][3] == "elementary_cli.update_alerts.update_sent_alerts" dbt_run_params = json.loads(call_args[0][0][9]) assert "alert_ids" in dbt_run_params assert "table_name" in dbt_run_params @@ -62,7 +62,9 @@ def test_skip_alerts(mock_subprocess_run, alerts_fetcher_mock: MockAlertsFetcher # Test that update_skipped_alerts has been called with alert_ids as arguments. assert call_args[0][0][1] == "run" assert call_args[0][0][2] == "-s" - assert call_args[0][0][3] == "update_alerts.update_skipped_alerts" + assert ( + call_args[0][0][3] == "elementary_cli.update_alerts.update_skipped_alerts" + ) dbt_run_params = json.loads(call_args[0][0][9]) assert "alert_ids" in dbt_run_params assert "table_name" in dbt_run_params