Fix Dynamodb connection (open-metadata#11473)

--------- Co-authored-by: Onkar Ravgan <[email protected]> Co-authored-by: Nahuel Verdugo Revigliono <[email protected]>
madsemildahlgaard · May 8, 2023 · 65fc94a · 65fc94a
1 parent 890433a
commit 65fc94a
Show file tree

Hide file tree

Showing 4 changed files with 36 additions and 15 deletions.
diff --git a/ingestion/src/metadata/ingestion/source/database/dynamodb/connection.py b/ingestion/src/metadata/ingestion/source/database/dynamodb/connection.py
@@ -12,6 +12,7 @@
 """
 Source connection handler
 """
+from functools import partial
 from typing import Optional
 
 from metadata.clients.aws_client import AWSClient
@@ -32,6 +33,16 @@ def get_connection(connection: DynamoDBConnection):
     return AWSClient(connection.awsConfig).get_dynamo_client()
 
 
+def check_list_tables(client):
+    """
+    Test ListTables under dynamodb
+    client.tables.all() passes even if wrong credentials are passed
+    Having it iterate does the list call which can be used to test connection.
+    """
+    for _ in client.tables.all():
+        return True
+
+
 def test_connection(
     metadata: OpenMetadata,
     client: AWSClient,
@@ -44,7 +55,7 @@ def test_connection(
     """
 
     test_fn = {
-        "GetTables": client.tables.all,
+        "ListTables": partial(check_list_tables, client),
     }
 
     test_connection_steps(

diff --git a/ingestion/src/metadata/ingestion/source/database/dynamodb/metadata.py b/ingestion/src/metadata/ingestion/source/database/dynamodb/metadata.py
@@ -66,6 +66,7 @@ def __init__(self, config: WorkflowSource, metadata_config: OpenMetadataConnecti
         self.dynamodb = get_connection(self.service_connection)
         self.connection_obj = self.dynamodb
         self.database_source_state = set()
+        self.test_connection()
 
     @classmethod
     def create(cls, config_dict, metadata_config: OpenMetadataConnection):

diff --git a/...openmetadata/service/resources/services/connections/TestConnectionDefinitionResource.java b/...openmetadata/service/resources/services/connections/TestConnectionDefinitionResource.java
@@ -1,5 +1,7 @@
 package org.openmetadata.service.resources.services.connections;
 
+import static org.openmetadata.service.Entity.ADMIN_USER_NAME;
+
 import io.swagger.annotations.Api;
 import io.swagger.v3.oas.annotations.Hidden;
 import io.swagger.v3.oas.annotations.Operation;
@@ -35,6 +37,7 @@
 import org.openmetadata.service.resources.Collection;
 import org.openmetadata.service.resources.EntityResource;
 import org.openmetadata.service.security.Authorizer;
+import org.openmetadata.service.util.EntityUtil;
 import org.openmetadata.service.util.RestUtil;
 import org.openmetadata.service.util.ResultList;
 
@@ -66,6 +69,12 @@ public TestConnectionDefinitionResource(CollectionDAO dao, Authorizer authorizer
   public void initialize(OpenMetadataApplicationConfig config) throws IOException {
     List<TestConnectionDefinition> testConnectionDefinitions =
         dao.getEntitiesFromSeedData(".*json/data/testConnections/.*\\.json$");
+
+    for (TestConnectionDefinition testConnectionDefinition :
+        dao.listAll(EntityUtil.Fields.EMPTY_FIELDS, new ListFilter(Include.ALL))) {
+      dao.delete(ADMIN_USER_NAME, testConnectionDefinition.getId(), true, true);
+    }
+
     for (TestConnectionDefinition testConnectionDefinition : testConnectionDefinitions) {
       dao.initializeEntity(testConnectionDefinition);
     }

diff --git a/openmetadata-service/src/main/resources/json/data/testConnections/database/dynamodb.json b/openmetadata-service/src/main/resources/json/data/testConnections/database/dynamodb.json
@@ -1,15 +1,15 @@
 {
-    "name": "DynamoDB",
-    "fullyQualifiedName": "DynamoDB",
-    "displayName": "DynamoDB Test Connection",
-    "description": "This Test Connection validates the access against the database and basic metadata extraction of schemas and tables.",
-    "steps": [
-      {
-        "name": "GetTables",
-        "description": "List all the tables available to the user.",
-        "errorMessage": "Failed to fetch tables, please validate the credentials & validate if user has access to fetch tables",
-        "shortCircuit": true,
-        "mandatory": true
-      }
-    ]
-  }
+  "name": "DynamoDB",
+  "fullyQualifiedName": "DynamoDB",
+  "displayName": "DynamoDB Test Connection",
+  "description": "This Test Connection validates the access against the database and basic metadata extraction of schemas and tables.",
+  "steps": [
+    {
+      "name": "ListTables",
+      "description": "List all the tables available to the user.",
+      "errorMessage": "Failed to fetch tables, please validate the credentials & validate if user has access to list tables",
+      "shortCircuit": true,
+      "mandatory": true
+    }
+  ]
+}