Merge pull request #114 from DalgoT4D/113-dont-return-airbyte_interna…

…l-in-the-list-of-schemas

filter out "airbyte_internal" in get_schemas

dbt_automation/utils/bigquery.py

@@ -42,7 +42,7 @@ def get_tables(self, schema: str) -> list:
     def get_schemas(self) -> list:
         """returns the list of schema names in the given connection"""
         datasets = self.bqclient.list_datasets()
-        return [x.dataset_id for x in datasets]
+        return [x.dataset_id for x in datasets if x.dataset_id != "airbyte_internal"]
 
     def get_table_columns(self, schema: str, table: str) -> list:
         """fetch the list of columns from a BigQuery table along with their data types."""
@@ -262,6 +262,6 @@ def get_column_data_types(self) -> list:
             "STRING",
             "STRUCT",
             "TIME",
-            "TIMESTAMP"
+            "TIMESTAMP",
         ]
         return bigquery_data_types

dbt_automation/utils/postgres.py

@@ -121,7 +121,7 @@ def get_schemas(self) -> list:
             """
             SELECT nspname
             FROM pg_namespace
-            WHERE nspname NOT LIKE 'pg_%' AND nspname != 'information_schema';
+            WHERE nspname NOT LIKE 'pg_%' AND nspname != 'information_schema' AND nspname != 'airbyte_internal';
             """
         )
         return [x[0] for x in resultset]