Skip to content

Commit

Permalink
Fix e2e
Browse files Browse the repository at this point in the history
  • Loading branch information
sfc-gh-wtrefon committed Dec 6, 2024
1 parent bb50d9d commit c8916d1
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 11 deletions.
4 changes: 2 additions & 2 deletions test/test_suit/base_iceberg_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ def __init__(self, driver, name_salt: str, config_file_name: str):
"null_object": {"key": "value"},
"empty_array": [1, 2, 3],
"some_object": {
"null_key": None,
"null_key": "solnik",
"string_key": "string_key",
"another_string_key": "another_string_key",
"inner_object": {
Expand Down Expand Up @@ -308,7 +308,7 @@ def _verify_iceberg_content_for_schema_evolution_2(self, content: dict):
assert_equals([1, 2, 3], content['null_array'])
assert_equals('value', content['null_object']['key'])
assert_equals([1, 2, 3], content['empty_array'])
assert_equals(None, content['some_object']['null_key'])
assert_equals("solnik", content['some_object']['null_key'])
assert_equals('string_key', content['some_object']['string_key'])
assert_equals('another_string_key', content['some_object']['another_string_key'])
assert_equals(456, content['some_object']['inner_object']['inner_object_key'])
Expand Down
19 changes: 10 additions & 9 deletions test/test_suit/iceberg_schema_evolution_json_aws.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,22 +16,23 @@ def setup(self):

def send(self):
self._send_json_values(self.test_message_from_docs, 100)
self._send_json_values(self.test_message_for_schema_evolution_1, 100)
# TODO SNOW-1731264
# net.snowflake.ingest.utils.SFException: The given row cannot be converted to the internal format: Object of type java.util.LinkedHashMap cannot be ingested into Snowflake column NULL_OBJECT of type STRING, rowIndex:0. Allowed Java types: String, Number, boolean, char
# self._send_json_values(self.test_message_for_schema_evolution_2, 100)
# first 10 messages should be discarded due to lack of schema for null fields
self._send_json_values(self.test_message_for_schema_evolution_1, 10)
self._send_json_values(self.test_message_for_schema_evolution_2, 100)
# now with the schema coming from test_message_for_schema_evolution_2 we should be able to insert null values
self._send_json_values(self.test_message_for_schema_evolution_1, 10)


def verify(self, round):
self._assert_number_of_records_in_table(200)
self._assert_number_of_records_in_table(210)

actual_record_from_docs_dict = self._select_schematized_record_with_offset(1)
self._verify_iceberg_content_from_docs(actual_record_from_docs_dict)

actual_record_for_schema_evolution_1 = self._select_schematized_record_with_offset(100)
self._verify_iceberg_content_for_schema_evolution_1(actual_record_for_schema_evolution_1)
actual_record_for_schema_evolution_2 = self._select_schematized_record_with_offset(100)
self._verify_iceberg_content_for_schema_evolution_2(actual_record_for_schema_evolution_2)

# TODO SNOW-1731264
# actual_record_for_schema_evolution_2 = self._select_schematized_record_with_offset(200)
actual_record_for_schema_evolution_1 = self._select_schematized_record_with_offset(200)
self._verify_iceberg_content_for_schema_evolution_1(actual_record_for_schema_evolution_1)


0 comments on commit c8916d1

Please sign in to comment.