diff --git a/ingestion/src/metadata/ingestion/source/database/snowflake/connection.py b/ingestion/src/metadata/ingestion/source/database/snowflake/connection.py index 3ac44a8129b7..d11aa82d7bd7 100644 --- a/ingestion/src/metadata/ingestion/source/database/snowflake/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/snowflake/connection.py @@ -98,6 +98,9 @@ def get_connection(connection: SnowflakeConnection) -> Engine: """ Create connection """ + if not connection.connectionArguments: + connection.connectionArguments = init_empty_connection_arguments() + if connection.privateKey: snowflake_private_key_passphrase = ( connection.snowflakePrivatekeyPassphrase.get_secret_value() @@ -120,10 +123,12 @@ def get_connection(connection: SnowflakeConnection) -> Engine: encryption_algorithm=serialization.NoEncryption(), ) - if connection.privateKey: - if not connection.connectionArguments: - connection.connectionArguments = init_empty_connection_arguments() - connection.connectionArguments.__root__["private_key"] = pkb + connection.connectionArguments.__root__["private_key"] = pkb + + if connection.clientSessionKeepAlive: + connection.connectionArguments.__root__[ + "client_session_keep_alive" + ] = connection.clientSessionKeepAlive return create_generic_db_connection( connection=connection, diff --git a/openmetadata-docs/content/v1.1.0-snapshot/connectors/database/snowflake/airflow.md b/openmetadata-docs/content/v1.1.0-snapshot/connectors/database/snowflake/airflow.md index 3068112b8e61..bd7147fde2ac 100644 --- a/openmetadata-docs/content/v1.1.0-snapshot/connectors/database/snowflake/airflow.md +++ b/openmetadata-docs/content/v1.1.0-snapshot/connectors/database/snowflake/airflow.md @@ -171,6 +171,12 @@ This is a sample config for Snowflake: {% /codeInfo %} +{% codeInfo srNumber=43 %} + +**clientSessionKeepAlive**: Optional Configuration to keep the session active in case the ingestion job runs for longer duration. + +{% /codeInfo %} + {% codeInfo srNumber=7 %} **privateKey**: If you have configured the key pair authentication for the given user you will have to pass the private key associated with the user in this field. You can checkout [this](https://docs.snowflake.com/en/user-guide/key-pair-auth) doc to get more details about key-pair authentication. @@ -271,6 +277,9 @@ source: ```yaml {% srNumber=6 %} includeTempTables: false ``` +```yaml {% srNumber=43 %} + clientSessionKeepAlive: false +``` ```yaml {% srNumber=7 %} # privateKey: ``` diff --git a/openmetadata-docs/content/v1.1.0-snapshot/connectors/database/snowflake/cli.md b/openmetadata-docs/content/v1.1.0-snapshot/connectors/database/snowflake/cli.md index 7cf68e446409..c1cab774676e 100644 --- a/openmetadata-docs/content/v1.1.0-snapshot/connectors/database/snowflake/cli.md +++ b/openmetadata-docs/content/v1.1.0-snapshot/connectors/database/snowflake/cli.md @@ -170,6 +170,12 @@ This is a sample config for Snowflake: {% /codeInfo %} +{% codeInfo srNumber=39 %} + +**clientSessionKeepAlive**: Optional Configuration to keep the session active in case the ingestion job runs for longer duration. + +{% /codeInfo %} + {% codeInfo srNumber=7 %} **privateKey**: If you have configured the key pair authentication for the given user you will have to pass the private key associated with the user in this field. You can checkout [this](https://docs.snowflake.com/en/user-guide/key-pair-auth) doc to get more details about key-pair authentication. @@ -271,6 +277,9 @@ source: ```yaml {% srNumber=6 %} includeTempTables: false ``` +```yaml {% srNumber=39 %} + clientSessionKeepAlive: false +``` ```yaml {% srNumber=7 %} # privateKey: ``` diff --git a/openmetadata-docs/content/v1.1.0-snapshot/connectors/database/snowflake/index.md b/openmetadata-docs/content/v1.1.0-snapshot/connectors/database/snowflake/index.md index 6ccb1f3ea9f2..c491f2177709 100644 --- a/openmetadata-docs/content/v1.1.0-snapshot/connectors/database/snowflake/index.md +++ b/openmetadata-docs/content/v1.1.0-snapshot/connectors/database/snowflake/index.md @@ -245,6 +245,7 @@ desired. - **Snowflake Passphrase Key (Optional)**: If you have configured the encrypted key pair authentication for the given user you will have to pass the paraphrase associated with the private key in this field. You can checkout [this](https://docs.snowflake.com/en/user-guide/key-pair-auth) doc to get more details about key-pair authentication. - **Include Temporary and Transient Tables**: Optional configuration for ingestion of `TRANSIENT` and `TEMPORARY` tables, By default, it will skip the `TRANSIENT` and `TEMPORARY` tables. +- **Client Session Keep Alive**: Optional Configuration to keep the session active in case the ingestion job runs for longer duration. - **Connection Options (Optional)**: Enter the details for any additional connection options that can be sent to Snowflake during the connection. These details must be added as Key-Value pairs. - **Connection Arguments (Optional)**: Enter the details for any additional connection arguments such as security or protocol configs that can be sent to Snowflake during the connection. These details must be added as Key-Value pairs. - In case you are using Single-Sign-On (SSO) for authentication, add the `authenticator` details in the Connection Arguments as a Key-Value pair as follows: `"authenticator" : "sso_login_url"` @@ -310,6 +311,7 @@ caption="Configure Metadata Ingestion Page" /%} - **Include tags (toggle)**: Set the 'Include Tags' toggle to control whether to include tags as part of metadata ingestion. - **Enable Debug Log (toggle)**: Set the Enable Debug Log toggle to set the default log level to debug, these logs can be viewed later in Airflow. + - **Mark Deleted Tables (toggle)**: Set the Mark Deleted Tables toggle to flag tables as soft-deleted if they are not present anymore in the source system. - **Mark Deleted Tables from Filter Only (toggle)**: Set the Mark Deleted Tables from Filter Only toggle to flag tables as soft-deleted if they are not present anymore within the filtered schema or database only. This flag is useful when you have more than one ingestion pipelines. For example if you have a schema diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/snowflakeConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/snowflakeConnection.json index 1191ab1dabd4..f0a460b64ee4 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/snowflakeConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/snowflakeConnection.json @@ -86,6 +86,12 @@ "type": "boolean", "default": false }, + "clientSessionKeepAlive": { + "title": "Client Session Keep Alive", + "description": "Optional configuration for ingestion to keep the client session active in case the ingestion process runs for longer durations.", + "type": "boolean", + "default": false + }, "connectionOptions": { "title": "Connection Options", "$ref": "../connectionBasicType.json#/definitions/connectionOptions" diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Snowflake.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Snowflake.md index 1c4ad0cbf365..a045d3769919 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Snowflake.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/Snowflake.md @@ -152,6 +152,12 @@ In Snowflake, we also have `TRANSIENT` and `TEMPORARY` tables, which will be ign Enable this setting to ingest them during the metadata workflow. $$ +$$section +### Client Session Keep Alive $(id="clientSessionKeepAlive") + +Optional Configuration to keep the session active in case the ingestion job runs for longer duration. +$$ + $$section ### Connection Options $(id="connectionOptions")