Skip to content

Commit

Permalink
Remove non-current iceberg columns from catalog
Browse files Browse the repository at this point in the history
  • Loading branch information
svdimchenko committed Sep 30, 2024
1 parent 78fae87 commit fd81fb3
Show file tree
Hide file tree
Showing 2 changed files with 4 additions and 6 deletions.
8 changes: 3 additions & 5 deletions dbt/adapters/athena/impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -548,8 +548,7 @@ def _s3_path_exists(self, s3_bucket: str, s3_prefix: str) -> bool:
response = s3_client.list_objects_v2(Bucket=s3_bucket, Prefix=s3_prefix)
return True if "Contents" in response else False

@staticmethod
def _get_one_table_for_catalog(table: TableTypeDef, database: str) -> List[Dict[str, Any]]:
def _get_one_table_for_catalog(self, table: TableTypeDef, database: str) -> List[Dict[str, Any]]:
table_catalog = {
"table_database": database,
"table_schema": table["DatabaseName"],
Expand All @@ -568,6 +567,7 @@ def _get_one_table_for_catalog(table: TableTypeDef, database: str) -> List[Dict[
},
}
for idx, col in enumerate(table["StorageDescriptor"]["Columns"] + table.get("PartitionKeys", []))
if self._is_current_column(col)
]

@staticmethod
Expand Down Expand Up @@ -1098,9 +1098,7 @@ def _is_current_column(col: ColumnTypeDef) -> bool:
"""
Check if a column is explicitly set as not current. If not, it is considered as current.
"""
if col.get("Parameters", {}).get("iceberg.field.current") == "false":
return False
return True
return col.get("Parameters", {}).get("iceberg.field.current") != "false"

@available
def get_columns_in_relation(self, relation: AthenaRelation) -> List[AthenaColumn]:
Expand Down
2 changes: 1 addition & 1 deletion tests/functional/adapter/test_docs.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ class TestDocsGenerate(BaseDocsGenerate):
"""

@pytest.fixture(scope="class")
def expected_catalog(self, project):
def expected_catalog(self, project, profile_user):
return base_expected_catalog(
project,
role="test",
Expand Down

0 comments on commit fd81fb3

Please sign in to comment.