diff --git a/ingestion/setup.py b/ingestion/setup.py index df5f3a7e1fcb..965df998aaba 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -25,7 +25,7 @@ "boto3": "boto3>=1.20,<2.0", # No need to add botocore separately. It's a dep from boto3 "geoalchemy2": "GeoAlchemy2~=0.12", "google-cloud-monitoring": "google-cloud-monitoring>=2.0.0", - "google-cloud-storage": "google-cloud-storage==1.43.0", + "google-cloud-storage": "google-cloud-storage>=1.43.0", "gcsfs": "gcsfs>=2023.1.0", "great-expectations": "great-expectations>=0.18.0,<0.18.14", "grpc-tools": "grpcio-tools>=1.47.2", @@ -212,6 +212,7 @@ "datalake-s3": { # vendoring 'boto3' to keep all dependencies aligned (s3fs, boto3, botocore, aiobotocore) "s3fs[boto3]", + VERSIONS["boto3"], *COMMONS["datalake"], }, "deltalake": { @@ -240,12 +241,14 @@ "impyla~=0.18.0", }, "iceberg": { - "pyiceberg==0.5.1", + "pyiceberg[hive,glue,dynamodb,s3fs,adlfs,gcsfs]==0.7.1", # Forcing the version of a few packages so it plays nicely with other requirements. VERSIONS["pydantic"], VERSIONS["adlfs"], VERSIONS["gcsfs"], VERSIONS["pyarrow"], + VERSIONS["boto3"], + *COMMONS["hive"], }, "impala": { "presto-types-parser>=0.0.2", diff --git a/ingestion/src/metadata/ingestion/source/database/iceberg/catalog/rest.py b/ingestion/src/metadata/ingestion/source/database/iceberg/catalog/rest.py index 23ec182acb59..04af28ce2f0a 100644 --- a/ingestion/src/metadata/ingestion/source/database/iceberg/catalog/rest.py +++ b/ingestion/src/metadata/ingestion/source/database/iceberg/catalog/rest.py @@ -51,6 +51,7 @@ def get_catalog(cls, catalog: IcebergCatalog) -> Catalog: "warehouse": catalog.warehouseLocation, "uri": str(catalog.connection.uri), "credential": credential, + "scope": catalog.connection.credential.scope, "token": catalog.connection.token.get_secret_value() if catalog.connection.token else None, diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/iceberg/restCatalogConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/iceberg/restCatalogConnection.json index 45a6df742ad3..7406b22852a8 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/iceberg/restCatalogConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/iceberg/restCatalogConnection.json @@ -28,6 +28,11 @@ "description": "OAuth2 Client Secret", "type": "string", "format": "password" + }, + "scope": { + "title": "Scope", + "description": "OAuth2 Scope", + "type": "string" } }, "additionalProperties": false diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Iceberg.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Iceberg.md index cf8f75036adf..94a1f543462d 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Iceberg.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Iceberg.md @@ -97,6 +97,14 @@ To get the client secret, follow these steps: $$ $$section +### Scope $(id="scope") + +#### REST Catalog +OAuth2 Scope to use for the Authentication Flow + +$$ +$$section + ### Token $(id="token") #### REST Catalog