filter out "airbyte_internal" in get_schemas

DalgoT4D · Apr 28, 2024 · 3b966cb · 3b966cb
1 parent 78ef9c5
commit 3b966cb
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/dbt_automation/utils/bigquery.py b/dbt_automation/utils/bigquery.py
@@ -42,7 +42,7 @@ def get_tables(self, schema: str) -> list:
     def get_schemas(self) -> list:
         """returns the list of schema names in the given connection"""
         datasets = self.bqclient.list_datasets()
-        return [x.dataset_id for x in datasets]
+        return [x.dataset_id for x in datasets if x.dataset_id != "airbyte_internal"]
 
     def get_table_columns(self, schema: str, table: str) -> list:
         """fetch the list of columns from a BigQuery table along with their data types."""
@@ -262,6 +262,6 @@ def get_column_data_types(self) -> list:
             "STRING",
             "STRUCT",
             "TIME",
-            "TIMESTAMP"
+            "TIMESTAMP",
         ]
         return bigquery_data_types
diff --git a/dbt_automation/utils/postgres.py b/dbt_automation/utils/postgres.py
@@ -121,7 +121,7 @@ def get_schemas(self) -> list:
             """
             SELECT nspname
             FROM pg_namespace
-            WHERE nspname NOT LIKE 'pg_%' AND nspname != 'information_schema';
+            WHERE nspname NOT LIKE 'pg_%' AND nspname != 'information_schema' AND nspname != 'airbyte_internal';
             """
         )
         return [x[0] for x in resultset]