Improve filtering for lineage query (open-metadata#11457)

madsemildahlgaard · May 8, 2023 · a897954 · a897954
1 parent 9d61d63
commit a897954
Show file tree

Hide file tree

Showing 8 changed files with 38 additions and 12 deletions.
diff --git a/ingestion/src/metadata/ingestion/source/database/bigquery/lineage.py b/ingestion/src/metadata/ingestion/source/database/bigquery/lineage.py
@@ -27,5 +27,8 @@ class BigqueryLineageSource(BigqueryQueryParserSource, LineageSource):
     sql_stmt = BIGQUERY_STATEMENT
 
     filters = """
-        AND statement_type IN ("INSERT", "MERGE", "CREATE_TABLE_AS_SELECT", "UPDATE")
+        AND (
+            statement_type IN ("MERGE", "CREATE_TABLE_AS_SELECT", "UPDATE") 
+            OR (statement_type = "INSERT" and UPPER(query) like '%%INSERT%%INTO%%SELECT%%')
+        )
     """
diff --git a/ingestion/src/metadata/ingestion/source/database/clickhouse/lineage.py b/ingestion/src/metadata/ingestion/source/database/clickhouse/lineage.py
@@ -29,7 +29,10 @@ class ClickhouseLineageSource(ClickhouseQueryParserSource, LineageSource):
     sql_stmt = CLICKHOUSE_SQL_STATEMENT
 
     filters = """
-        and query_kind in ('Create', 'Insert')
+        and (
+            query_kind='Create' 
+            or (query_kind='Insert' and query ilike '%%insert%%into%%select%%')
+        )
     """
 
     database_field = ""

diff --git a/ingestion/src/metadata/ingestion/source/database/mssql/lineage.py b/ingestion/src/metadata/ingestion/source/database/mssql/lineage.py
@@ -19,4 +19,11 @@
 class MssqlLineageSource(MssqlQueryParserSource, LineageSource):
     sql_stmt = MSSQL_SQL_STATEMENT
 
-    filters = ""  # No filtering in the queries
+    filters = """
+        AND (
+            lower(t.text) LIKE '%%select%%into%%'
+            OR lower(t.text) LIKE '%%insert%%into%%select%%'
+            OR lower(t.text) LIKE '%%update%%'
+            OR lower(t.text) LIKE '%%merge%%'
+        )
+    """
diff --git a/ingestion/src/metadata/ingestion/source/database/postgres/lineage.py b/ingestion/src/metadata/ingestion/source/database/postgres/lineage.py
@@ -34,7 +34,9 @@ class PostgresLineageSource(PostgresQueryParserSource, LineageSource):
     filters = """
                 AND (
                     s.query ILIKE '%%create table%%as%%select%%'
-                    OR s.query ILIKE '%%insert%%'
+                    OR s.query ILIKE '%%insert%%into%%select%%'
+                    OR s.query ILIKE '%%update%%'
+                    OR s.query ILIKE '%%merge%%'
                 )
             """
 

diff --git a/ingestion/src/metadata/ingestion/source/database/redshift/lineage.py b/ingestion/src/metadata/ingestion/source/database/redshift/lineage.py
@@ -40,8 +40,10 @@
 class RedshiftLineageSource(RedshiftQueryParserSource, LineageSource):
     filters = """
         AND (
-          querytxt ILIKE '%%create table%%as%%select%%'
-          OR querytxt ILIKE '%%insert%%'
+          querytxt ILIKE '%%create%%table%%as%%select%%'
+          OR querytxt ILIKE '%%insert%%into%%select%%'
+          OR querytxt ILIKE '%%update%%'
+          OR querytxt ILIKE '%%merge%%'
         )
     """
 

diff --git a/ingestion/src/metadata/ingestion/source/database/redshift/usage.py b/ingestion/src/metadata/ingestion/source/database/redshift/usage.py
@@ -20,10 +20,10 @@
 
 class RedshiftUsageSource(RedshiftQueryParserSource, UsageSource):
     filters = """
-        AND querytxt NOT ILIKE 'fetch %%'
-        AND querytxt NOT ILIKE 'padb_fetch_sample: %%'
-        AND querytxt NOT ILIKE 'Undoing %% transactions on table %% with current xid%%'
-        AND querytxt NOT ILIKE '%%create table%%as%%select%%'
+        AND querytxt NOT ILIKE 'fetch%%'
+        AND querytxt NOT ILIKE 'padb_fetch_sample:%%'
+        AND querytxt NOT ILIKE 'Undoing%%transactions%%on%%table%%with%%current%%xid%%'
+        AND querytxt NOT ILIKE '%%create%%table%%as%%select%%'
         AND querytxt NOT ILIKE '%%insert%%'
     """
 

diff --git a/ingestion/src/metadata/ingestion/source/database/snowflake/lineage.py b/ingestion/src/metadata/ingestion/source/database/snowflake/lineage.py
@@ -27,5 +27,8 @@ class SnowflakeLineageSource(SnowflakeQueryParserSource, LineageSource):
     sql_stmt = SNOWFLAKE_SQL_STATEMENT
 
     filters = """
-        AND QUERY_TYPE IN ('INSERT', 'MERGE', 'UPDATE','CREATE_TABLE_AS_SELECT')
+        AND (
+            QUERY_TYPE IN ('MERGE', 'UPDATE','CREATE_TABLE_AS_SELECT')
+            OR (QUERY_TYPE = 'INSERT' and query_text ILIKE '%%insert%%into%%select%%')
+        )
     """
diff --git a/ingestion/src/metadata/ingestion/source/database/vertica/lineage.py b/ingestion/src/metadata/ingestion/source/database/vertica/lineage.py
@@ -24,7 +24,13 @@
 class VerticaLineageSource(VerticaQueryParserSource, LineageSource):
     sql_stmt = VERTICA_SQL_STATEMENT
 
-    filters = "AND query_type in ('INSERT', 'UPDATE', 'QUERY', 'DDL')"
+    filters = """
+        AND (
+            query_type in ('UPDATE', 'DDL')
+            OR ( query_type IN ('INSERT','QUERY') and p.query ilike '%%INSERT%%INTO%%SELECT%%')
+            OR ( query_type = 'QUERY' and p.query not ilike '%%INSERT%%INTO%%')
+        )
+    """
 
     database_field = "DBNAME()"