From 8909a3f08f8857b5d3485e3ba4efc67b1bec5547 Mon Sep 17 00:00:00 2001 From: Rainer Niedermayr Date: Wed, 16 Oct 2024 11:47:04 +0200 Subject: [PATCH 1/2] postgres consistency: extend ignore entry regarding table functions --- .../ignore_filter/pg_inconsistency_ignore_filter.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/misc/python/materialize/postgres_consistency/ignore_filter/pg_inconsistency_ignore_filter.py b/misc/python/materialize/postgres_consistency/ignore_filter/pg_inconsistency_ignore_filter.py index 2c26c1d87963..ff4958d723e7 100644 --- a/misc/python/materialize/postgres_consistency/ignore_filter/pg_inconsistency_ignore_filter.py +++ b/misc/python/materialize/postgres_consistency/ignore_filter/pg_inconsistency_ignore_filter.py @@ -1089,14 +1089,17 @@ def matches_nullif(expression: Expression) -> bool: "database-issues#8293: ALL and ANY with NULL or empty array" ) - if query_template.matches_specific_select_or_filter_expression( - col_index, + # use here matches_any_expression instead of matches_specific_select_or_filter_expression on purpose because + # the concerned expression may affect other columns as well + if query_template.matches_any_expression( is_table_function, True, ) and (query_template.has_offset() or query_template.has_limit()): # When table functions are used, a row-order insensitive comparison will be conducted in the result # comparator. However, this is not sufficient when a LIMIT or OFFSET clause is present. - return YesIgnore("Different sort order") + return YesIgnore( + "Different sort order (partially a consequence of database-issues#6620 and database-issues#7812)" + ) if query_template.matches_specific_select_or_filter_expression( col_index, From 45805139f1d7a112cd48102fd4588d52f08f6dbb Mon Sep 17 00:00:00 2001 From: Rainer Niedermayr Date: Wed, 16 Oct 2024 11:55:39 +0200 Subject: [PATCH 2/2] postgres consistency: narrow entry regarding table functions --- .../operations/table_operations_provider.py | 12 ++++++++++-- .../ignore_filter/pg_inconsistency_ignore_filter.py | 7 ++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/misc/python/materialize/output_consistency/input_data/operations/table_operations_provider.py b/misc/python/materialize/output_consistency/input_data/operations/table_operations_provider.py index 56ed269a5891..24d7fe89bf3e 100644 --- a/misc/python/materialize/output_consistency/input_data/operations/table_operations_provider.py +++ b/misc/python/materialize/output_consistency/input_data/operations/table_operations_provider.py @@ -61,6 +61,10 @@ OperationRelevance, ) +TAG_TABLE_FUNCTION_WITH_NON_NUMERIC_SORT_ORDER = ( + "table_function_with_non_numeric_sort_order" +) + TABLE_OPERATION_TYPES: list[DbOperationOrFunction] = [] TABLE_OPERATION_TYPES.append( @@ -129,6 +133,7 @@ NumericReturnTypeSpec(only_integer=True), is_table_function=True, comment="Returns indices of the specified array dimension", + tags={TAG_TABLE_FUNCTION_WITH_NON_NUMERIC_SORT_ORDER}, ), ) @@ -141,7 +146,7 @@ ], StringReturnTypeSpec(), is_table_function=True, - tags={TAG_REGEX}, + tags={TAG_REGEX, TAG_TABLE_FUNCTION_WITH_NON_NUMERIC_SORT_ORDER}, ), ) @@ -153,7 +158,7 @@ is_table_function=True, # It is not possible to specify the order so that inconsistencies are inevitable. is_pg_compatible=False, - tags={TAG_REGEX}, + tags={TAG_REGEX, TAG_TABLE_FUNCTION_WITH_NON_NUMERIC_SORT_ORDER}, ), ) @@ -165,6 +170,7 @@ ], CollectionEntryReturnTypeSpec(param_index_to_take_type=0), is_table_function=True, + tags={TAG_TABLE_FUNCTION_WITH_NON_NUMERIC_SORT_ORDER}, ), ) @@ -176,6 +182,7 @@ ], CollectionEntryReturnTypeSpec(param_index_to_take_type=0), is_table_function=True, + tags={TAG_TABLE_FUNCTION_WITH_NON_NUMERIC_SORT_ORDER}, ), ) @@ -187,5 +194,6 @@ ], RecordReturnTypeSpec(), is_table_function=True, + tags={TAG_TABLE_FUNCTION_WITH_NON_NUMERIC_SORT_ORDER}, ), ) diff --git a/misc/python/materialize/postgres_consistency/ignore_filter/pg_inconsistency_ignore_filter.py b/misc/python/materialize/postgres_consistency/ignore_filter/pg_inconsistency_ignore_filter.py index ff4958d723e7..3b6d4d6afba2 100644 --- a/misc/python/materialize/postgres_consistency/ignore_filter/pg_inconsistency_ignore_filter.py +++ b/misc/python/materialize/postgres_consistency/ignore_filter/pg_inconsistency_ignore_filter.py @@ -73,6 +73,9 @@ from materialize.output_consistency.input_data.operations.string_operations_provider import ( TAG_REGEX, ) +from materialize.output_consistency.input_data.operations.table_operations_provider import ( + TAG_TABLE_FUNCTION_WITH_NON_NUMERIC_SORT_ORDER, +) from materialize.output_consistency.input_data.return_specs.number_return_spec import ( NumericReturnTypeSpec, ) @@ -1092,7 +1095,9 @@ def matches_nullif(expression: Expression) -> bool: # use here matches_any_expression instead of matches_specific_select_or_filter_expression on purpose because # the concerned expression may affect other columns as well if query_template.matches_any_expression( - is_table_function, + partial( + is_operation_tagged, tag=TAG_TABLE_FUNCTION_WITH_NON_NUMERIC_SORT_ORDER + ), True, ) and (query_template.has_offset() or query_template.has_limit()): # When table functions are used, a row-order insensitive comparison will be conducted in the result