Skip to content

Commit

Permalink
fix(ingest/redshift): reduce sequence limit for LISTAGG (datahub-proj…
Browse files Browse the repository at this point in the history 
…ect#11621)

Co-authored-by: treff7es <[email protected]>
Co-authored-by: Aseem Bansal <[email protected]>
  • Loading branch information
@hsheth2 @treff7es
@anshbansal @aviv-julienjehannet
3 people authored and aviv-julienjehannet committed Oct 21, 2024
1 parent cc1c7cf commit 7d0a8b0
Show file tree
Hide file tree
Showing 2 changed files with 13 additions and 0 deletions.
9 changes: 9 additions & 0 deletions metadata-ingestion/src/datahub/ingestion/source/redshift/query.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -569,7 +569,12 @@ def temp_table_ddl_query(start_time: datetime, end_time: datetime) -> str:

end_time_str: str = end_time.strftime(redshift_datetime_format)

<<<<<<< HEAD
return rf"""-- DataHub Redshift Source temp table DDL query
=======
return rf"""\
-- DataHub Redshift Source temp table DDL query
>>>>>>> e9060b8afbe (fix(ingest/redshift): reduce sequence limit for LISTAGG (#11621))
select
*
from (
Expand Down Expand Up @@ -644,7 +649,11 @@ def temp_table_ddl_query(start_time: datetime, end_time: datetime) -> str:
)
where
rn = 1
<<<<<<< HEAD
"""
=======
"""
>>>>>>> e9060b8afbe (fix(ingest/redshift): reduce sequence limit for LISTAGG (#11621))

# Add this join to the sql query for more metrics on completed queries
# LEFT JOIN svl_query_metrics_summary sqms ON ss.query = sqms.query
Expand Down
4 changes: 4 additions & 0 deletions metadata-ingestion/tests/unit/redshift/redshift_query_mocker.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,11 @@ def mock_stl_insert_table_cursor(cursor: MagicMock) -> None:

query_vs_cursor_mocker = {
(
<<<<<<< HEAD
"-- DataHub Redshift Source temp table DDL query\nselect\n *\nfrom (\n select\n session_id,\n transaction_id,\n start_time,\n userid,\n REGEXP_REPLACE(REGEXP_SUBSTR(REGEXP_REPLACE(query_text,'\\\\\\\\n','\\\\n'), '(CREATE(?:[\\\\n\\\\s\\\\t]+(?:temp|temporary))?(?:[\\\\n\\\\s\\\\t]+)table(?:[\\\\n\\\\s\\\\t]+)[^\\\\n\\\\s\\\\t()-]+)', 0, 1, 'ipe'),'[\\\\n\\\\s\\\\t]+',' ',1,'p') as create_command,\n query_text,\n row_number() over (\n partition by session_id, TRIM(query_text)\n order by start_time desc\n ) rn\n from (\n select\n pid as session_id,\n xid as transaction_id,\n starttime as start_time,\n type,\n query_text,\n userid\n from (\n select\n starttime,\n pid,\n xid,\n type,\n userid,\n LISTAGG(case\n when LEN(RTRIM(text)) = 0 then text\n else RTRIM(text)\n end,\n '') within group (\n order by sequence\n ) as query_text\n from\n SVL_STATEMENTTEXT\n where\n type in ('DDL', 'QUERY')\n AND starttime >= '2024-01-01 12:00:00'\n AND starttime < '2024-01-10 12:00:00'\n AND sequence < 290\n group by\n starttime,\n pid,\n xid,\n type,\n userid\n order by\n starttime,\n pid,\n xid,\n type,\n userid\n asc\n )\n where\n type in ('DDL', 'QUERY')\n )\n where\n (create_command ilike 'create temp table %'\n or create_command ilike 'create temporary table %'\n -- we want to get all the create table statements and not just temp tables if non temp table is created and dropped in the same transaction\n or create_command ilike 'create table %')\n -- Redshift creates temp tables with the following names: volt_tt_%. We need to filter them out.\n and query_text not ilike 'CREATE TEMP TABLE volt_tt_%'\n and create_command not like 'CREATE TEMP TABLE volt_tt_'\n -- We need to filter out our query and it was not possible earlier when we did not have any comment in the query\n and query_text not ilike '%https://stackoverflow.com/questions/72770890/redshift-result-size-exceeds-listagg-limit-on-svl-statementtext%'\n\n)\nwhere\n rn = 1\n"
=======
"\\\n-- DataHub Redshift Source temp table DDL query\nselect\n *\nfrom (\n select\n session_id,\n transaction_id,\n start_time,\n userid,\n REGEXP_REPLACE(REGEXP_SUBSTR(REGEXP_REPLACE(query_text,'\\\\\\\\n','\\\\n'), '(CREATE(?:[\\\\n\\\\s\\\\t]+(?:temp|temporary))?(?:[\\\\n\\\\s\\\\t]+)table(?:[\\\\n\\\\s\\\\t]+)[^\\\\n\\\\s\\\\t()-]+)', 0, 1, 'ipe'),'[\\\\n\\\\s\\\\t]+',' ',1,'p') as create_command,\n query_text,\n row_number() over (\n partition by session_id, TRIM(query_text)\n order by start_time desc\n ) rn\n from (\n select\n pid as session_id,\n xid as transaction_id,\n starttime as start_time,\n type,\n query_text,\n userid\n from (\n select\n starttime,\n pid,\n xid,\n type,\n userid,\n LISTAGG(case\n when LEN(RTRIM(text)) = 0 then text\n else RTRIM(text)\n end,\n '') within group (\n order by sequence\n ) as query_text\n from\n SVL_STATEMENTTEXT\n where\n type in ('DDL', 'QUERY')\n AND starttime >= '2024-01-01 12:00:00'\n AND starttime < '2024-01-10 12:00:00'\n AND sequence < 290\n group by\n starttime,\n pid,\n xid,\n type,\n userid\n order by\n starttime,\n pid,\n xid,\n type,\n userid\n asc\n )\n where\n type in ('DDL', 'QUERY')\n )\n where\n (create_command ilike 'create temp table %'\n or create_command ilike 'create temporary table %'\n -- we want to get all the create table statements and not just temp tables if non temp table is created and dropped in the same transaction\n or create_command ilike 'create table %')\n -- Redshift creates temp tables with the following names: volt_tt_%. We need to filter them out.\n and query_text not ilike 'CREATE TEMP TABLE volt_tt_%'\n and create_command not like 'CREATE TEMP TABLE volt_tt_'\n -- We need to filter out our query and it was not possible earlier when we did not have any comment in the query\n and query_text not ilike '%https://stackoverflow.com/questions/72770890/redshift-result-size-exceeds-listagg-limit-on-svl-statementtext%'\n\n)\nwhere\n rn = 1\n "
>>>>>>> e9060b8afbe (fix(ingest/redshift): reduce sequence limit for LISTAGG (#11621))
): mock_temp_table_cursor,
"select * from test_collapse_temp_lineage": mock_stl_insert_table_cursor,
}
Expand Down

0 comments on commit 7d0a8b0

Please sign in to comment.