diff --git a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py index c4d01be52ae7d..977282303e37c 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py @@ -1,7 +1,8 @@ +import json import logging import os import time -from typing import Dict, Iterable, List +from typing import Any, Dict, Iterable, List from urllib.parse import urlparse from deltalake import DeltaTable @@ -35,23 +36,22 @@ read_delta_table, ) from datahub.ingestion.source.delta_lake.report import DeltaLakeSourceReport -from datahub.ingestion.source.schema_inference.csv_tsv import tableschema_type_map +from datahub.metadata._schema_classes import SchemaFieldClass from datahub.metadata.com.linkedin.pegasus2avro.common import Status from datahub.metadata.com.linkedin.pegasus2avro.metadata.snapshot import DatasetSnapshot from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent from datahub.metadata.com.linkedin.pegasus2avro.schema import ( SchemaField, - SchemaFieldDataType, SchemaMetadata, ) from datahub.metadata.schema_classes import ( DatasetPropertiesClass, - NullTypeClass, OperationClass, OperationTypeClass, OtherSchemaClass, ) from datahub.telemetry import telemetry +from datahub.utilities.hive_schema_to_avro import get_schema_fields_for_hive_column logging.getLogger("py4j").setLevel(logging.ERROR) logger: logging.Logger = logging.getLogger(__name__) @@ -126,26 +126,57 @@ def create(cls, config_dict: dict, ctx: PipelineContext) -> "Source": config = DeltaLakeSourceConfig.parse_obj(config_dict) return cls(config, ctx) + def delta_type_to_hive_type(self, field_type: Any) -> str: + if isinstance(field_type, str): + """ + return the field type + """ + return field_type + else: + if field_type.get("type") == "array": + """ + if array is of complex type, recursively parse the + fields and create the native datatype + """ + return ( + "array<" + + self.delta_type_to_hive_type(field_type.get("elementType")) + + ">" + ) + elif field_type.get("type") == "struct": + parsed_struct = "" + for field in field_type.get("fields"): + """ + if field is of complex type, recursively parse + and create the native datatype + """ + parsed_struct += ( + "{0}:{1}".format( + field.get("name"), + self.delta_type_to_hive_type(field.get("type")), + ) + + "," + ) + return "struct<" + parsed_struct.rstrip(",") + ">" + return "" + + def _parse_datatype(self, raw_field_json_str: str) -> List[SchemaFieldClass]: + raw_field_json = json.loads(raw_field_json_str) + + # get the parent field name and type + field_name = raw_field_json.get("name") + field_type = self.delta_type_to_hive_type(raw_field_json.get("type")) + + return get_schema_fields_for_hive_column(field_name, field_type) + def get_fields(self, delta_table: DeltaTable) -> List[SchemaField]: fields: List[SchemaField] = [] for raw_field in delta_table.schema().fields: - field = SchemaField( - fieldPath=raw_field.name, - type=SchemaFieldDataType( - tableschema_type_map.get(raw_field.type.type, NullTypeClass)() - ), - nativeDataType=raw_field.type.type, - recursive=False, - nullable=raw_field.nullable, - description=str(raw_field.metadata), - isPartitioningKey=True - if raw_field.name in delta_table.metadata().partition_columns - else False, - ) - fields.append(field) - fields = sorted(fields, key=lambda f: f.fieldPath) + parsed_data_list = self._parse_datatype(raw_field.to_json()) + fields = fields + parsed_data_list + fields = sorted(fields, key=lambda f: f.fieldPath) return fields def _create_operation_aspect_wu( diff --git a/metadata-ingestion/tests/integration/delta_lake/delta_lake_minio_mces_golden.json b/metadata-ingestion/tests/integration/delta_lake/delta_lake_minio_mces_golden.json index ed65d74037796..9c731fe9a1ba5 100644 --- a/metadata-ingestion/tests/integration/delta_lake/delta_lake_minio_mces_golden.json +++ b/metadata-ingestion/tests/integration/delta_lake/delta_lake_minio_mces_golden.json @@ -45,23 +45,21 @@ }, "fields": [ { - "fieldPath": "customer", + "fieldPath": "[version=2.0].[type=float].total_cost", "nullable": true, - "description": "{}", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "string", + "nativeDataType": "float", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"float\", \"_nullable\": true}" }, { - "fieldPath": "day", + "fieldPath": "[version=2.0].[type=int].day", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -70,12 +68,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "month", + "fieldPath": "[version=2.0].[type=int].month", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -84,49 +81,46 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "sale_id", + "fieldPath": "[version=2.0].[type=int].year", "nullable": true, - "description": "{}", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "string", + "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "total_cost", + "fieldPath": "[version=2.0].[type=string].customer", "nullable": true, - "description": "{}", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "float", + "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, { - "fieldPath": "year", + "fieldPath": "[version=2.0].[type=string].sale_id", "nullable": true, - "description": "{}", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "integer", + "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" } ] } @@ -380,6 +374,68 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-test-bucket/delta_tables/sales,DEV)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:34fc0473e206bb1f4307aadf4177b2fd", + "urn": "urn:li:container:34fc0473e206bb1f4307aadf4177b2fd" + }, + { + "id": "urn:li:container:acebf8bcf966274632d3d2b710ef4947", + "urn": "urn:li:container:acebf8bcf966274632d3d2b710ef4947" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1672531200000, + "runId": "delta-lake-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:34fc0473e206bb1f4307aadf4177b2fd", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1672531200000, + "runId": "delta-lake-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:acebf8bcf966274632d3d2b710ef4947", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:34fc0473e206bb1f4307aadf4177b2fd", + "urn": "urn:li:container:34fc0473e206bb1f4307aadf4177b2fd" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1672531200000, + "runId": "delta-lake-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-test-bucket/delta_tables/sales,DEV)", diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_allow_table.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_allow_table.json index 6ec6eb2809a10..e71ec17250854 100644 --- a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_allow_table.json +++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_allow_table.json @@ -45,9 +45,8 @@ }, "fields": [ { - "fieldPath": "bar", + "fieldPath": "[version=2.0].[type=int].foo", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -56,12 +55,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "foo", + "fieldPath": "[version=2.0].[type=int].bar", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -70,12 +68,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "zip", + "fieldPath": "[version=2.0].[type=string].zip", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -84,7 +81,7 @@ "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" } ] } @@ -924,23 +921,21 @@ }, "fields": [ { - "fieldPath": "customer", + "fieldPath": "[version=2.0].[type=int].year", "nullable": true, - "description": "{}", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "string", + "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "day", + "fieldPath": "[version=2.0].[type=int].month", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -949,12 +944,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "month", + "fieldPath": "[version=2.0].[type=int].day", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -963,12 +957,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "sale_id", + "fieldPath": "[version=2.0].[type=string].sale_id", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -977,35 +970,33 @@ "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, { - "fieldPath": "total_cost", + "fieldPath": "[version=2.0].[type=string].customer", "nullable": true, - "description": "{}", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "float", + "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, { - "fieldPath": "year", + "fieldPath": "[version=2.0].[type=float].total_cost", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "integer", + "nativeDataType": "float", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"float\", \"_nullable\": true}" } ] } @@ -1150,9 +1141,8 @@ }, "fields": [ { - "fieldPath": "bar", + "fieldPath": "[version=2.0].[type=int].foo", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1161,12 +1151,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "foo", + "fieldPath": "[version=2.0].[type=int].bar", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1175,12 +1164,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "zip", + "fieldPath": "[version=2.0].[type=string].zip", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1189,7 +1177,7 @@ "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" } ] } @@ -1455,9 +1443,8 @@ }, "fields": [ { - "fieldPath": "bar", + "fieldPath": "[version=2.0].[type=int].foo", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1466,12 +1453,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "foo", + "fieldPath": "[version=2.0].[type=int].bar", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1480,12 +1466,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "zip", + "fieldPath": "[version=2.0].[type=string].zip", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1494,7 +1479,7 @@ "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" } ] } @@ -1803,6 +1788,360 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:a282913be26fceff334523c2be119df1", + "urn": "urn:li:container:a282913be26fceff334523c2be119df1" + }, + { + "id": "urn:li:container:3df8f6b0f3a70d42cf70612a2fe5e5ef", + "urn": "urn:li:container:3df8f6b0f3a70d42cf70612a2fe5e5ef" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:189046201d696e7810132cfa64dad337", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:a282913be26fceff334523c2be119df1", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:a282913be26fceff334523c2be119df1", + "urn": "urn:li:container:a282913be26fceff334523c2be119df1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/sales,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:a282913be26fceff334523c2be119df1", + "urn": "urn:li:container:a282913be26fceff334523c2be119df1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:a282913be26fceff334523c2be119df1", + "urn": "urn:li:container:a282913be26fceff334523c2be119df1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:3df8f6b0f3a70d42cf70612a2fe5e5ef", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:a282913be26fceff334523c2be119df1", + "urn": "urn:li:container:a282913be26fceff334523c2be119df1" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "allow_table.json", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_inner_table.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_inner_table.json index 715beebfe22fb..11d6924296f77 100644 --- a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_inner_table.json +++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_inner_table.json @@ -45,9 +45,8 @@ }, "fields": [ { - "fieldPath": "bar", + "fieldPath": "[version=2.0].[type=int].foo", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -56,12 +55,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "foo", + "fieldPath": "[version=2.0].[type=int].bar", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -70,12 +68,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "zip", + "fieldPath": "[version=2.0].[type=string].zip", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -84,7 +81,7 @@ "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" } ] } @@ -889,23 +886,21 @@ }, "fields": [ { - "fieldPath": "customer", + "fieldPath": "[version=2.0].[type=int].year", "nullable": true, - "description": "{}", "type": { "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} + "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "string", + "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "day", + "fieldPath": "[version=2.0].[type=int].month", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -914,12 +909,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "month", + "fieldPath": "[version=2.0].[type=int].day", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -928,12 +922,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "sale_id", + "fieldPath": "[version=2.0].[type=string].sale_id", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -942,35 +935,33 @@ "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, { - "fieldPath": "total_cost", + "fieldPath": "[version=2.0].[type=string].customer", "nullable": true, - "description": "{}", "type": { "type": { - "com.linkedin.pegasus2avro.schema.NullType": {} + "com.linkedin.pegasus2avro.schema.StringType": {} } }, - "nativeDataType": "float", + "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, { - "fieldPath": "year", + "fieldPath": "[version=2.0].[type=float].total_cost", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} } }, - "nativeDataType": "integer", + "nativeDataType": "float", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"float\", \"_nullable\": true}" } ] } @@ -1111,9 +1102,8 @@ }, "fields": [ { - "fieldPath": "bar", + "fieldPath": "[version=2.0].[type=int].foo", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1122,12 +1112,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "foo", + "fieldPath": "[version=2.0].[type=int].bar", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1136,12 +1125,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "zip", + "fieldPath": "[version=2.0].[type=string].zip", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1150,7 +1138,7 @@ "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" } ] } @@ -1412,9 +1400,8 @@ }, "fields": [ { - "fieldPath": "bar", + "fieldPath": "[version=2.0].[type=int].foo", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1423,12 +1410,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "foo", + "fieldPath": "[version=2.0].[type=int].bar", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1437,12 +1423,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "zip", + "fieldPath": "[version=2.0].[type=string].zip", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1451,7 +1436,7 @@ "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" } ] } @@ -1754,6 +1739,319 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + }, + { + "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" + }, + { + "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" + }, + { + "id": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "urn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab" + }, + { + "id": "urn:li:container:6bb6dc6de93177210067d00b45b481bb", + "urn": "urn:li:container:6bb6dc6de93177210067d00b45b481bb" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + }, + { + "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + }, + { + "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" + }, + { + "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + }, + { + "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" + }, + { + "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" + }, + { + "id": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "urn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/sales,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + }, + { + "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" + }, + { + "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" + }, + { + "id": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "urn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_no_name,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + }, + { + "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" + }, + { + "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" + }, + { + "id": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "urn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:6bb6dc6de93177210067d00b45b481bb", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + }, + { + "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" + }, + { + "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" + }, + { + "id": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "urn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "inner_table.json", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/level1/my_table_inner,UAT)", diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json index 2076ec4096f68..ec1912a7e2c49 100644 --- a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json +++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_relative_path.json @@ -45,9 +45,8 @@ }, "fields": [ { - "fieldPath": "bar", + "fieldPath": "[version=2.0].[type=int].foo", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -56,12 +55,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "foo", + "fieldPath": "[version=2.0].[type=int].bar", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -70,12 +68,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "zip", + "fieldPath": "[version=2.0].[type=string].zip", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -84,7 +81,7 @@ "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" } ] } @@ -350,6 +347,43 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:85267d161e1a2ffa647cec6c1188549f", + "urn": "urn:li:container:85267d161e1a2ffa647cec6c1188549f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "relative_path.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:85267d161e1a2ffa647cec6c1188549f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "relative_path.json", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,delta_tables/my_table_basic,UAT)", diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json index 42e3b19612c2b..ffb4040d17bff 100644 --- a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json +++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_single_table.json @@ -44,9 +44,8 @@ }, "fields": [ { - "fieldPath": "bar", + "fieldPath": "[version=2.0].[type=int].foo", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -55,12 +54,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "foo", + "fieldPath": "[version=2.0].[type=int].bar", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -69,12 +67,11 @@ "nativeDataType": "integer", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": true + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" }, { - "fieldPath": "zip", + "fieldPath": "[version=2.0].[type=string].zip", "nullable": true, - "description": "{}", "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -83,7 +80,7 @@ "nativeDataType": "string", "recursive": false, "isPartOfKey": false, - "isPartitioningKey": false + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" } ] } @@ -686,6 +683,167 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + }, + { + "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" + }, + { + "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" + }, + { + "id": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "urn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + }, + { + "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ad4b596846e8e010114b1ec82b324fab", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf", + "urn": "urn:li:container:bdfaaacd66870755e65612e0b88dd4bf" + }, + { + "id": "urn:li:container:974a39dc631803eddedc699cc9bb9759", + "urn": "urn:li:container:974a39dc631803eddedc699cc9bb9759" + }, + { + "id": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6", + "urn": "urn:li:container:dae543a1ed7ecfea4079a971dc7805a6" + }, + { + "id": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4", + "urn": "urn:li:container:ee050cda8eca59687021c24cbc0bb8a4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "single_table.json", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,tests/integration/delta_lake/test_data/delta_tables/my_table_basic,UAT)", diff --git a/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_tables_with_nested_datatypes.json b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_tables_with_nested_datatypes.json new file mode 100644 index 0000000000000..c8bf54efa46c2 --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/golden_files/local/golden_mces_tables_with_nested_datatypes.json @@ -0,0 +1,2231 @@ +[ +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "number_of_files": "2", + "partition_columns": "[]", + "table_creation_time": "1709535902908", + "id": "363c76ca-3357-48c6-b14d-71262be23dbc", + "version": "0", + "location": "tests/integration/delta_lake/test_data/delta_tables_nested_datatype" + }, + "name": "table_with_nested_struct_1", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "table_with_nested_struct_1", + "platform": "urn:li:dataPlatform:delta-lake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=string].id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct<_1:string,_2:struct<_1:long,_2:string>>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct<_1:string,_2:struct<_1:long,_2:string>>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data.[type=string]._1", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data.[type=struct]._2", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct<_1:long,_2:string>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct<_1:long,_2:string>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data.[type=struct]._2.[type=null]._1", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "long", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"long\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data.[type=struct]._2.[type=string]._2", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:189046201d696e7810132cfa64dad337", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "delta-lake", + "instance": "my-platform", + "env": "UAT", + "folder_abs_path": "tests" + }, + "name": "tests" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:189046201d696e7810132cfa64dad337", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:189046201d696e7810132cfa64dad337", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:delta-lake", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:189046201d696e7810132cfa64dad337", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:189046201d696e7810132cfa64dad337", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "delta-lake", + "instance": "my-platform", + "env": "UAT", + "folder_abs_path": "tests/integration" + }, + "name": "integration" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:delta-lake", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:189046201d696e7810132cfa64dad337" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "delta-lake", + "instance": "my-platform", + "env": "UAT", + "folder_abs_path": "tests/integration/delta_lake" + }, + "name": "delta_lake" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:delta-lake", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "delta-lake", + "instance": "my-platform", + "env": "UAT", + "folder_abs_path": "tests/integration/delta_lake/test_data" + }, + "name": "test_data" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:delta-lake", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:1876d057d0ee364677b85427342e2c82" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "delta-lake", + "instance": "my-platform", + "env": "UAT", + "folder_abs_path": "tests/integration/delta_lake/test_data/delta_tables_nested_datatype" + }, + "name": "delta_tables_nested_datatype" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:delta-lake", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1,UAT)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1,UAT)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "operationType": "CUSTOM", + "customOperationType": "WRITE", + "customProperties": { + "engineInfo": "Apache-Spark/3.5.0 Delta-Lake/3.1.0", + "isBlindAppend": "False", + "isolationLevel": "Serializable", + "operationMetrics_numFiles": "3", + "operationMetrics_numOutputBytes": "3473", + "operationMetrics_numOutputRows": "2", + "operationParameters_mode": "Overwrite", + "operationParameters_partitionBy": "[]", + "txnId": "95fab5d1-7811-4326-96d2-6244adace432", + "version": "0" + }, + "lastUpdatedTimestamp": 1709535906725 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "urn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "number_of_files": "2", + "partition_columns": "[]", + "table_creation_time": "1709110539186", + "id": "73cae4a6-3988-4337-89ca-af58dd528b0b", + "version": "0", + "location": "tests/integration/delta_lake/test_data/delta_tables_nested_datatype" + }, + "name": "table_with_nested_struct", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "table_with_nested_struct", + "platform": "urn:li:dataPlatform:delta-lake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=string].id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct<_1:string,_2:struct<_1:long,_2:string>>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct<_1:string,_2:struct<_1:long,_2:string>>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data.[type=string]._1", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data.[type=struct]._2", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct<_1:long,_2:string>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct<_1:long,_2:string>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data.[type=struct]._2.[type=null]._1", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "long", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"long\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data.[type=struct]._2.[type=string]._2", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct,UAT)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct,UAT)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "operationType": "CUSTOM", + "customOperationType": "WRITE", + "customProperties": { + "engineInfo": "Apache-Spark/3.5.0 Delta-Lake/3.1.0", + "isBlindAppend": "False", + "isolationLevel": "Serializable", + "operationMetrics_numFiles": "3", + "operationMetrics_numOutputBytes": "3473", + "operationMetrics_numOutputRows": "2", + "operationParameters_mode": "Overwrite", + "operationParameters_partitionBy": "[]", + "txnId": "e78cea2a-7fd7-4aab-9b50-776e7638fb54", + "version": "0" + }, + "lastUpdatedTimestamp": 1709110542636 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "urn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "number_of_files": "2", + "partition_columns": "[]", + "table_creation_time": "1708329075098", + "id": "bccc302c-006d-4de9-b572-0c1939e64fc5", + "version": "0", + "location": "tests/integration/delta_lake/test_data/delta_tables_nested_datatype" + }, + "name": "table_with_string_and_array", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "table_with_string_and_array", + "platform": "urn:li:dataPlatform:delta-lake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=string].id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=string].data", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "string" + ] + } + } + }, + "nativeDataType": "array", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array,UAT)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array,UAT)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "operationType": "CUSTOM", + "customOperationType": "WRITE", + "customProperties": { + "engineInfo": "Apache-Spark/3.5.0 Delta-Lake/3.1.0", + "isBlindAppend": "False", + "isolationLevel": "Serializable", + "operationMetrics_numFiles": "3", + "operationMetrics_numOutputBytes": "2027", + "operationMetrics_numOutputRows": "2", + "operationParameters_mode": "Overwrite", + "operationParameters_partitionBy": "[]", + "txnId": "c1497bc8-403a-451f-b651-adeaad02a74d", + "version": "0" + }, + "lastUpdatedTimestamp": 1708329078869 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "urn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "number_of_files": "2", + "partition_columns": "[]", + "table_creation_time": "1708329893422", + "id": "6f3b01e4-ae1e-415a-979a-34861346809b", + "version": "0", + "location": "tests/integration/delta_lake/test_data/delta_tables_nested_datatype" + }, + "name": "table_with_string_and_array_of_struct", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "table_with_string_and_array_of_struct", + "platform": "urn:li:dataPlatform:delta-lake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=string].id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].data", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "record" + ] + } + } + }, + "nativeDataType": "array>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].data.[type=int].value", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"integer\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].data.[type=string].name", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct,UAT)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct,UAT)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "operationType": "CUSTOM", + "customOperationType": "WRITE", + "customProperties": { + "engineInfo": "Apache-Spark/3.5.0 Delta-Lake/3.1.0", + "isBlindAppend": "False", + "isolationLevel": "Serializable", + "operationMetrics_numFiles": "3", + "operationMetrics_numOutputBytes": "2943", + "operationMetrics_numOutputRows": "2", + "operationParameters_mode": "Overwrite", + "operationParameters_partitionBy": "[]", + "txnId": "dd7ddb85-1b3a-4f18-bd8f-1466adfe66b0", + "version": "0" + }, + "lastUpdatedTimestamp": 1708329897384 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "urn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "number_of_files": "2", + "partition_columns": "[]", + "table_creation_time": "1709536362734", + "id": "58b63865-38fa-4478-98cf-3a7fd3425e24", + "version": "0", + "location": "tests/integration/delta_lake/test_data/delta_tables_nested_datatype" + }, + "name": "table_with_nested_struct_2", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "table_with_nested_struct_2", + "platform": "urn:li:dataPlatform:delta-lake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=string].id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct<_1:struct<_1:long,_2:string>,_2:struct<_1:long,_2:string>>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct<_1:struct<_1:long,_2:string>,_2:struct<_1:long,_2:string>>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data.[type=struct]._1", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct<_1:long,_2:string>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct<_1:long,_2:string>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data.[type=struct]._1.[type=null]._1", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "long", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"long\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data.[type=struct]._1.[type=string]._2", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data.[type=struct]._2", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.RecordType": {} + } + }, + "nativeDataType": "struct<_1:long,_2:string>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"struct<_1:long,_2:string>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data.[type=struct]._2.[type=null]._1", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "long", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"long\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=struct].data.[type=struct]._2.[type=string]._2", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2,UAT)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2,UAT)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "operationType": "CUSTOM", + "customOperationType": "WRITE", + "customProperties": { + "engineInfo": "Apache-Spark/3.5.0 Delta-Lake/3.1.0", + "isBlindAppend": "False", + "isolationLevel": "Serializable", + "operationMetrics_numFiles": "3", + "operationMetrics_numOutputBytes": "4292", + "operationMetrics_numOutputRows": "2", + "operationParameters_mode": "Overwrite", + "operationParameters_partitionBy": "[]", + "txnId": "0ead04f5-e706-42f1-bae9-4829f2001edb", + "version": "0" + }, + "lastUpdatedTimestamp": 1709536366367 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "urn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers,UAT)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "number_of_files": "2", + "partition_columns": "[]", + "table_creation_time": "1708330174792", + "id": "9b656138-6370-412a-826b-4f20e659d5c2", + "version": "0", + "location": "tests/integration/delta_lake/test_data/delta_tables_nested_datatype" + }, + "name": "table_with_string_and_nested_array_of_numbers", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "table_with_string_and_nested_array_of_numbers", + "platform": "urn:li:dataPlatform:delta-lake", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=string].id", + "nullable": true, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=array].[type=array].[type=null].data", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "array" + ] + } + } + }, + "nativeDataType": "array>>", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array>>\"}" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers,UAT)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers,UAT)", + "changeType": "UPSERT", + "aspectName": "operation", + "aspect": { + "json": { + "timestampMillis": 1615443388097, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "operationType": "CUSTOM", + "customOperationType": "WRITE", + "customProperties": { + "engineInfo": "Apache-Spark/3.5.0 Delta-Lake/3.1.0", + "isBlindAppend": "False", + "isolationLevel": "Serializable", + "operationMetrics_numFiles": "3", + "operationMetrics_numOutputBytes": "2773", + "operationMetrics_numOutputRows": "2", + "operationParameters_mode": "Overwrite", + "operationParameters_partitionBy": "[]", + "txnId": "96c8e2d9-55f7-416d-84de-55d01936e58e", + "version": "0" + }, + "lastUpdatedTimestamp": 1708330178404 + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "urn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:189046201d696e7810132cfa64dad337", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "urn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "urn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "urn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "urn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "urn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:delta-lake,my-platform.tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers,UAT)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:delta-lake,my-platform)" + }, + { + "id": "urn:li:container:189046201d696e7810132cfa64dad337", + "urn": "urn:li:container:189046201d696e7810132cfa64dad337" + }, + { + "id": "urn:li:container:acf0f3806f475a7397ee745329ef2967", + "urn": "urn:li:container:acf0f3806f475a7397ee745329ef2967" + }, + { + "id": "urn:li:container:1876d057d0ee364677b85427342e2c82", + "urn": "urn:li:container:1876d057d0ee364677b85427342e2c82" + }, + { + "id": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4", + "urn": "urn:li:container:7888b6dab77b7e77709699c9a1b81aa4" + }, + { + "id": "urn:li:container:401e53437a2ce6094ab3021cb32919d9", + "urn": "urn:li:container:401e53437a2ce6094ab3021cb32919d9" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "tables_with_nested_datatypes.json", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/delta_lake/sources/local/tables_with_nested_datatypes.json b/metadata-ingestion/tests/integration/delta_lake/sources/local/tables_with_nested_datatypes.json new file mode 100644 index 0000000000000..1eb720b9579cd --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/sources/local/tables_with_nested_datatypes.json @@ -0,0 +1,11 @@ +{ + "type": "delta-lake", + "config": { + "env": "UAT", + "base_path": "tests/integration/delta_lake/test_data/delta_tables_nested_datatype", + "platform_instance": "my-platform", + "table_pattern": { + "allow": ["s*"] + } + } +} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/.part-00000-2d6d7c61-bb5a-49db-8406-cec97e7e5d47-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/.part-00000-2d6d7c61-bb5a-49db-8406-cec97e7e5d47-c000.snappy.parquet.crc new file mode 100644 index 0000000000000..6910f58465840 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/.part-00000-2d6d7c61-bb5a-49db-8406-cec97e7e5d47-c000.snappy.parquet.crc differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/.part-00003-3d49fff8-55a5-46f7-8cda-9b17f7c45c20-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/.part-00003-3d49fff8-55a5-46f7-8cda-9b17f7c45c20-c000.snappy.parquet.crc new file mode 100644 index 0000000000000..397c487f1e597 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/.part-00003-3d49fff8-55a5-46f7-8cda-9b17f7c45c20-c000.snappy.parquet.crc differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/.part-00007-0a29067b-2a26-4986-8b51-0eeb441092eb-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/.part-00007-0a29067b-2a26-4986-8b51-0eeb441092eb-c000.snappy.parquet.crc new file mode 100644 index 0000000000000..d5200fddb2fa0 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/.part-00007-0a29067b-2a26-4986-8b51-0eeb441092eb-c000.snappy.parquet.crc differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/_delta_log/.00000000000000000000.json.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/_delta_log/.00000000000000000000.json.crc new file mode 100644 index 0000000000000..092d22f5ce7bc Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/_delta_log/.00000000000000000000.json.crc differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/_delta_log/00000000000000000000.json b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/_delta_log/00000000000000000000.json new file mode 100644 index 0000000000000..ad5a3581b381a --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/_delta_log/00000000000000000000.json @@ -0,0 +1,5 @@ +{"commitInfo":{"timestamp":1709110542636,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"3","numOutputRows":"2","numOutputBytes":"3473"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0","txnId":"e78cea2a-7fd7-4aab-9b50-776e7638fb54"}} +{"metaData":{"id":"73cae4a6-3988-4337-89ca-af58dd528b0b","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"data\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"_1\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"_2\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"_1\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"_2\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1709110539186}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"add":{"path":"part-00003-3d49fff8-55a5-46f7-8cda-9b17f7c45c20-c000.snappy.parquet","partitionValues":{},"size":1380,"modificationTime":1709110541918,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":\"row1\",\"data\":{\"_1\":\"value1\",\"_2\":{\"_1\":20,\"_2\":\"test\"}}},\"maxValues\":{\"id\":\"row1\",\"data\":{\"_1\":\"value1\",\"_2\":{\"_1\":20,\"_2\":\"test\"}}},\"nullCount\":{\"id\":0,\"data\":{\"_1\":0,\"_2\":{\"_1\":0,\"_2\":0}}}}"}} +{"add":{"path":"part-00007-0a29067b-2a26-4986-8b51-0eeb441092eb-c000.snappy.parquet","partitionValues":{},"size":1380,"modificationTime":1709110541918,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":\"row2\",\"data\":{\"_1\":\"value2\",\"_2\":{\"_1\":30,\"_2\":\"test\"}}},\"maxValues\":{\"id\":\"row2\",\"data\":{\"_1\":\"value2\",\"_2\":{\"_1\":30,\"_2\":\"test\"}}},\"nullCount\":{\"id\":0,\"data\":{\"_1\":0,\"_2\":{\"_1\":0,\"_2\":0}}}}"}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/part-00000-2d6d7c61-bb5a-49db-8406-cec97e7e5d47-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/part-00000-2d6d7c61-bb5a-49db-8406-cec97e7e5d47-c000.snappy.parquet new file mode 100644 index 0000000000000..ba7c91adbe124 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/part-00000-2d6d7c61-bb5a-49db-8406-cec97e7e5d47-c000.snappy.parquet differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/part-00003-3d49fff8-55a5-46f7-8cda-9b17f7c45c20-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/part-00003-3d49fff8-55a5-46f7-8cda-9b17f7c45c20-c000.snappy.parquet new file mode 100644 index 0000000000000..ffbe7fa6aef61 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/part-00003-3d49fff8-55a5-46f7-8cda-9b17f7c45c20-c000.snappy.parquet differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/part-00007-0a29067b-2a26-4986-8b51-0eeb441092eb-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/part-00007-0a29067b-2a26-4986-8b51-0eeb441092eb-c000.snappy.parquet new file mode 100644 index 0000000000000..21c725ed0402f Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct/part-00007-0a29067b-2a26-4986-8b51-0eeb441092eb-c000.snappy.parquet differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/.part-00000-4dfde8c6-b3c0-45bf-a021-4198a3fdd2a9-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/.part-00000-4dfde8c6-b3c0-45bf-a021-4198a3fdd2a9-c000.snappy.parquet.crc new file mode 100644 index 0000000000000..f43bfa6cc5468 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/.part-00000-4dfde8c6-b3c0-45bf-a021-4198a3fdd2a9-c000.snappy.parquet.crc differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/.part-00003-32acad0a-77b4-4fdd-93c4-917f41926d19-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/.part-00003-32acad0a-77b4-4fdd-93c4-917f41926d19-c000.snappy.parquet.crc new file mode 100644 index 0000000000000..d34bf9efcad34 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/.part-00003-32acad0a-77b4-4fdd-93c4-917f41926d19-c000.snappy.parquet.crc differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/.part-00007-4c1cd124-4b9b-49ed-9b42-1df4d0e934c6-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/.part-00007-4c1cd124-4b9b-49ed-9b42-1df4d0e934c6-c000.snappy.parquet.crc new file mode 100644 index 0000000000000..78386f43b8fe1 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/.part-00007-4c1cd124-4b9b-49ed-9b42-1df4d0e934c6-c000.snappy.parquet.crc differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/_delta_log/.00000000000000000000.json.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/_delta_log/.00000000000000000000.json.crc new file mode 100644 index 0000000000000..a3313c598afc9 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/_delta_log/.00000000000000000000.json.crc differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/_delta_log/00000000000000000000.json b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/_delta_log/00000000000000000000.json new file mode 100644 index 0000000000000..85138b9f9efed --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/_delta_log/00000000000000000000.json @@ -0,0 +1,5 @@ +{"commitInfo":{"timestamp":1709535906725,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"3","numOutputRows":"2","numOutputBytes":"3473"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0","txnId":"95fab5d1-7811-4326-96d2-6244adace432"}} +{"metaData":{"id":"363c76ca-3357-48c6-b14d-71262be23dbc","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"data\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"_1\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"_2\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"_1\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"_2\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1709535902908}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"add":{"path":"part-00003-32acad0a-77b4-4fdd-93c4-917f41926d19-c000.snappy.parquet","partitionValues":{},"size":1380,"modificationTime":1709535905678,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"data\":{\"_1\":\"value1\",\"_2\":{\"_1\":20,\"_2\":\"test\"}},\"id\":\"row1\"},\"maxValues\":{\"data\":{\"_1\":\"value1\",\"_2\":{\"_1\":20,\"_2\":\"test\"}},\"id\":\"row1\"},\"nullCount\":{\"data\":{\"_1\":0,\"_2\":{\"_1\":0,\"_2\":0}},\"id\":0}}"}} +{"add":{"path":"part-00007-4c1cd124-4b9b-49ed-9b42-1df4d0e934c6-c000.snappy.parquet","partitionValues":{},"size":1380,"modificationTime":1709535905678,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"data\":{\"_1\":\"value2\",\"_2\":{\"_1\":30,\"_2\":\"test\"}},\"id\":\"row2\"},\"maxValues\":{\"data\":{\"_1\":\"value2\",\"_2\":{\"_1\":30,\"_2\":\"test\"}},\"id\":\"row2\"},\"nullCount\":{\"data\":{\"_1\":0,\"_2\":{\"_1\":0,\"_2\":0}},\"id\":0}}"}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/part-00000-4dfde8c6-b3c0-45bf-a021-4198a3fdd2a9-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/part-00000-4dfde8c6-b3c0-45bf-a021-4198a3fdd2a9-c000.snappy.parquet new file mode 100644 index 0000000000000..3db87037e0232 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/part-00000-4dfde8c6-b3c0-45bf-a021-4198a3fdd2a9-c000.snappy.parquet differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/part-00003-32acad0a-77b4-4fdd-93c4-917f41926d19-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/part-00003-32acad0a-77b4-4fdd-93c4-917f41926d19-c000.snappy.parquet new file mode 100644 index 0000000000000..84638657d66e2 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/part-00003-32acad0a-77b4-4fdd-93c4-917f41926d19-c000.snappy.parquet differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/part-00007-4c1cd124-4b9b-49ed-9b42-1df4d0e934c6-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/part-00007-4c1cd124-4b9b-49ed-9b42-1df4d0e934c6-c000.snappy.parquet new file mode 100644 index 0000000000000..783419e5969d7 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_1/part-00007-4c1cd124-4b9b-49ed-9b42-1df4d0e934c6-c000.snappy.parquet differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/.part-00000-8711f6dd-96f0-4db6-b331-a5ceb2b48436-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/.part-00000-8711f6dd-96f0-4db6-b331-a5ceb2b48436-c000.snappy.parquet.crc new file mode 100644 index 0000000000000..247ed10c40e25 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/.part-00000-8711f6dd-96f0-4db6-b331-a5ceb2b48436-c000.snappy.parquet.crc differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/.part-00003-c8d92698-1f00-40fb-943e-254432f372a2-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/.part-00003-c8d92698-1f00-40fb-943e-254432f372a2-c000.snappy.parquet.crc new file mode 100644 index 0000000000000..c84f84ae75851 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/.part-00003-c8d92698-1f00-40fb-943e-254432f372a2-c000.snappy.parquet.crc differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/.part-00007-b7aee5c1-0e76-4bfb-b154-65bc9d495f79-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/.part-00007-b7aee5c1-0e76-4bfb-b154-65bc9d495f79-c000.snappy.parquet.crc new file mode 100644 index 0000000000000..da7aa656d8095 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/.part-00007-b7aee5c1-0e76-4bfb-b154-65bc9d495f79-c000.snappy.parquet.crc differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/_delta_log/.00000000000000000000.json.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/_delta_log/.00000000000000000000.json.crc new file mode 100644 index 0000000000000..f1ebfa5144737 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/_delta_log/.00000000000000000000.json.crc differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/_delta_log/00000000000000000000.json b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/_delta_log/00000000000000000000.json new file mode 100644 index 0000000000000..d31c179dbb9ac --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/_delta_log/00000000000000000000.json @@ -0,0 +1,5 @@ +{"commitInfo":{"timestamp":1709536366367,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"3","numOutputRows":"2","numOutputBytes":"4292"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0","txnId":"0ead04f5-e706-42f1-bae9-4829f2001edb"}} +{"metaData":{"id":"58b63865-38fa-4478-98cf-3a7fd3425e24","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"data\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"_1\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"_1\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"_2\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"_2\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"_1\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"_2\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1709536362734}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"add":{"path":"part-00003-c8d92698-1f00-40fb-943e-254432f372a2-c000.snappy.parquet","partitionValues":{},"size":1710,"modificationTime":1709536365577,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"data\":{\"_1\":{\"_1\":20,\"_2\":\"test\"},\"_2\":{\"_1\":20,\"_2\":\"test\"}},\"id\":\"row1\"},\"maxValues\":{\"data\":{\"_1\":{\"_1\":20,\"_2\":\"test\"},\"_2\":{\"_1\":20,\"_2\":\"test\"}},\"id\":\"row1\"},\"nullCount\":{\"data\":{\"_1\":{\"_1\":0,\"_2\":0},\"_2\":{\"_1\":0,\"_2\":0}},\"id\":0}}"}} +{"add":{"path":"part-00007-b7aee5c1-0e76-4bfb-b154-65bc9d495f79-c000.snappy.parquet","partitionValues":{},"size":1710,"modificationTime":1709536365576,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"data\":{\"_1\":{\"_1\":20,\"_2\":\"test\"},\"_2\":{\"_1\":30,\"_2\":\"test\"}},\"id\":\"row2\"},\"maxValues\":{\"data\":{\"_1\":{\"_1\":20,\"_2\":\"test\"},\"_2\":{\"_1\":30,\"_2\":\"test\"}},\"id\":\"row2\"},\"nullCount\":{\"data\":{\"_1\":{\"_1\":0,\"_2\":0},\"_2\":{\"_1\":0,\"_2\":0}},\"id\":0}}"}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/part-00000-8711f6dd-96f0-4db6-b331-a5ceb2b48436-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/part-00000-8711f6dd-96f0-4db6-b331-a5ceb2b48436-c000.snappy.parquet new file mode 100644 index 0000000000000..7e32df7323c8b Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/part-00000-8711f6dd-96f0-4db6-b331-a5ceb2b48436-c000.snappy.parquet differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/part-00003-c8d92698-1f00-40fb-943e-254432f372a2-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/part-00003-c8d92698-1f00-40fb-943e-254432f372a2-c000.snappy.parquet new file mode 100644 index 0000000000000..cbf52efddf12d Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/part-00003-c8d92698-1f00-40fb-943e-254432f372a2-c000.snappy.parquet differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/part-00007-b7aee5c1-0e76-4bfb-b154-65bc9d495f79-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/part-00007-b7aee5c1-0e76-4bfb-b154-65bc9d495f79-c000.snappy.parquet new file mode 100644 index 0000000000000..67dba6c735232 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_nested_struct_2/part-00007-b7aee5c1-0e76-4bfb-b154-65bc9d495f79-c000.snappy.parquet differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/.part-00000-d49ffc34-b6da-4c6e-8d1a-3b0e4fb3db80-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/.part-00000-d49ffc34-b6da-4c6e-8d1a-3b0e4fb3db80-c000.snappy.parquet.crc new file mode 100644 index 0000000000000..6f6b2373dfeba Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/.part-00000-d49ffc34-b6da-4c6e-8d1a-3b0e4fb3db80-c000.snappy.parquet.crc differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/.part-00003-7a130f72-44b3-4072-b50c-b353c7380ad0-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/.part-00003-7a130f72-44b3-4072-b50c-b353c7380ad0-c000.snappy.parquet.crc new file mode 100644 index 0000000000000..e8c4254da796f Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/.part-00003-7a130f72-44b3-4072-b50c-b353c7380ad0-c000.snappy.parquet.crc differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/.part-00007-7f5b112b-1800-4b3e-804e-322d552a034e-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/.part-00007-7f5b112b-1800-4b3e-804e-322d552a034e-c000.snappy.parquet.crc new file mode 100644 index 0000000000000..df6ca78dd2571 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/.part-00007-7f5b112b-1800-4b3e-804e-322d552a034e-c000.snappy.parquet.crc differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/_delta_log/.00000000000000000000.json.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/_delta_log/.00000000000000000000.json.crc new file mode 100644 index 0000000000000..9ec7421e7f791 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/_delta_log/.00000000000000000000.json.crc differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/_delta_log/00000000000000000000.json b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/_delta_log/00000000000000000000.json new file mode 100644 index 0000000000000..0236fbb37d2e8 --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/_delta_log/00000000000000000000.json @@ -0,0 +1,5 @@ +{"commitInfo":{"timestamp":1708329078869,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"3","numOutputRows":"2","numOutputBytes":"2027"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0","txnId":"c1497bc8-403a-451f-b651-adeaad02a74d"}} +{"metaData":{"id":"bccc302c-006d-4de9-b572-0c1939e64fc5","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"data\",\"type\":{\"type\":\"array\",\"elementType\":\"string\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1708329075098}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"add":{"path":"part-00003-7a130f72-44b3-4072-b50c-b353c7380ad0-c000.snappy.parquet","partitionValues":{},"size":779,"modificationTime":1708329077928,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":\"row1\"},\"maxValues\":{\"id\":\"row1\"},\"nullCount\":{\"id\":0,\"data\":0}}"}} +{"add":{"path":"part-00007-7f5b112b-1800-4b3e-804e-322d552a034e-c000.snappy.parquet","partitionValues":{},"size":779,"modificationTime":1708329077928,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":\"row2\"},\"maxValues\":{\"id\":\"row2\"},\"nullCount\":{\"id\":0,\"data\":0}}"}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/part-00000-d49ffc34-b6da-4c6e-8d1a-3b0e4fb3db80-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/part-00000-d49ffc34-b6da-4c6e-8d1a-3b0e4fb3db80-c000.snappy.parquet new file mode 100644 index 0000000000000..5ecd72a84a87e Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/part-00000-d49ffc34-b6da-4c6e-8d1a-3b0e4fb3db80-c000.snappy.parquet differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/part-00003-7a130f72-44b3-4072-b50c-b353c7380ad0-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/part-00003-7a130f72-44b3-4072-b50c-b353c7380ad0-c000.snappy.parquet new file mode 100644 index 0000000000000..d1b251a2be55f Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/part-00003-7a130f72-44b3-4072-b50c-b353c7380ad0-c000.snappy.parquet differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/part-00007-7f5b112b-1800-4b3e-804e-322d552a034e-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/part-00007-7f5b112b-1800-4b3e-804e-322d552a034e-c000.snappy.parquet new file mode 100644 index 0000000000000..2013c3c531f68 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array/part-00007-7f5b112b-1800-4b3e-804e-322d552a034e-c000.snappy.parquet differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/.part-00000-1b381498-a903-486a-baf1-57414786faa2-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/.part-00000-1b381498-a903-486a-baf1-57414786faa2-c000.snappy.parquet.crc new file mode 100644 index 0000000000000..a4eff05371f7c Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/.part-00000-1b381498-a903-486a-baf1-57414786faa2-c000.snappy.parquet.crc differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/.part-00003-a52fed7e-93c6-4402-a13b-9f7cf9560ad2-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/.part-00003-a52fed7e-93c6-4402-a13b-9f7cf9560ad2-c000.snappy.parquet.crc new file mode 100644 index 0000000000000..b33edc31f6e7f Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/.part-00003-a52fed7e-93c6-4402-a13b-9f7cf9560ad2-c000.snappy.parquet.crc differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/.part-00007-97e055ec-b063-4bf0-8007-bcb36dc0a8f6-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/.part-00007-97e055ec-b063-4bf0-8007-bcb36dc0a8f6-c000.snappy.parquet.crc new file mode 100644 index 0000000000000..3914d912a54c3 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/.part-00007-97e055ec-b063-4bf0-8007-bcb36dc0a8f6-c000.snappy.parquet.crc differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/_delta_log/.00000000000000000000.json.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/_delta_log/.00000000000000000000.json.crc new file mode 100644 index 0000000000000..b7ff32e8f6ce9 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/_delta_log/.00000000000000000000.json.crc differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/_delta_log/00000000000000000000.json b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/_delta_log/00000000000000000000.json new file mode 100644 index 0000000000000..00e2ad37347fb --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/_delta_log/00000000000000000000.json @@ -0,0 +1,5 @@ +{"commitInfo":{"timestamp":1708329897384,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"3","numOutputRows":"2","numOutputBytes":"2943"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0","txnId":"dd7ddb85-1b3a-4f18-bd8f-1466adfe66b0"}} +{"metaData":{"id":"6f3b01e4-ae1e-415a-979a-34861346809b","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"data\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"value\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1708329893422}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"add":{"path":"part-00003-a52fed7e-93c6-4402-a13b-9f7cf9560ad2-c000.snappy.parquet","partitionValues":{},"size":1149,"modificationTime":1708329896608,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":\"row1\"},\"maxValues\":{\"id\":\"row1\"},\"nullCount\":{\"id\":0,\"data\":0}}"}} +{"add":{"path":"part-00007-97e055ec-b063-4bf0-8007-bcb36dc0a8f6-c000.snappy.parquet","partitionValues":{},"size":1153,"modificationTime":1708329896608,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":\"row2\"},\"maxValues\":{\"id\":\"row2\"},\"nullCount\":{\"id\":0,\"data\":0}}"}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/part-00000-1b381498-a903-486a-baf1-57414786faa2-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/part-00000-1b381498-a903-486a-baf1-57414786faa2-c000.snappy.parquet new file mode 100644 index 0000000000000..5d518b0151a55 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/part-00000-1b381498-a903-486a-baf1-57414786faa2-c000.snappy.parquet differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/part-00003-a52fed7e-93c6-4402-a13b-9f7cf9560ad2-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/part-00003-a52fed7e-93c6-4402-a13b-9f7cf9560ad2-c000.snappy.parquet new file mode 100644 index 0000000000000..cc97be744b01b Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/part-00003-a52fed7e-93c6-4402-a13b-9f7cf9560ad2-c000.snappy.parquet differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/part-00007-97e055ec-b063-4bf0-8007-bcb36dc0a8f6-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/part-00007-97e055ec-b063-4bf0-8007-bcb36dc0a8f6-c000.snappy.parquet new file mode 100644 index 0000000000000..231ba1ba975fb Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_array_of_struct/part-00007-97e055ec-b063-4bf0-8007-bcb36dc0a8f6-c000.snappy.parquet differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/.part-00000-56bf8d99-9483-49cb-bc4a-f4daa12b85cb-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/.part-00000-56bf8d99-9483-49cb-bc4a-f4daa12b85cb-c000.snappy.parquet.crc new file mode 100644 index 0000000000000..3c5bb71746af1 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/.part-00000-56bf8d99-9483-49cb-bc4a-f4daa12b85cb-c000.snappy.parquet.crc differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/.part-00003-5890b536-c464-4b14-89dd-0a2e03f8d95b-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/.part-00003-5890b536-c464-4b14-89dd-0a2e03f8d95b-c000.snappy.parquet.crc new file mode 100644 index 0000000000000..6b34fbafe876b Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/.part-00003-5890b536-c464-4b14-89dd-0a2e03f8d95b-c000.snappy.parquet.crc differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/.part-00007-5ae5a06d-3e76-4d2b-b22e-60c196d352c1-c000.snappy.parquet.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/.part-00007-5ae5a06d-3e76-4d2b-b22e-60c196d352c1-c000.snappy.parquet.crc new file mode 100644 index 0000000000000..5316ea328f0d9 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/.part-00007-5ae5a06d-3e76-4d2b-b22e-60c196d352c1-c000.snappy.parquet.crc differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/_delta_log/.00000000000000000000.json.crc b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/_delta_log/.00000000000000000000.json.crc new file mode 100644 index 0000000000000..a895d840fa06c Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/_delta_log/.00000000000000000000.json.crc differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/_delta_log/00000000000000000000.json b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/_delta_log/00000000000000000000.json new file mode 100644 index 0000000000000..81732bf30841d --- /dev/null +++ b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/_delta_log/00000000000000000000.json @@ -0,0 +1,5 @@ +{"commitInfo":{"timestamp":1708330178404,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"3","numOutputRows":"2","numOutputBytes":"2773"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0","txnId":"96c8e2d9-55f7-416d-84de-55d01936e58e"}} +{"metaData":{"id":"9b656138-6370-412a-826b-4f20e659d5c2","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"data\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"array\",\"elementType\":{\"type\":\"array\",\"elementType\":\"long\",\"containsNull\":true},\"containsNull\":true},\"containsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1708330174792}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"add":{"path":"part-00003-5890b536-c464-4b14-89dd-0a2e03f8d95b-c000.snappy.parquet","partitionValues":{},"size":1074,"modificationTime":1708330177598,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":\"row1\"},\"maxValues\":{\"id\":\"row1\"},\"nullCount\":{\"id\":0,\"data\":0}}"}} +{"add":{"path":"part-00007-5ae5a06d-3e76-4d2b-b22e-60c196d352c1-c000.snappy.parquet","partitionValues":{},"size":1074,"modificationTime":1708330177598,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":\"row2\"},\"maxValues\":{\"id\":\"row2\"},\"nullCount\":{\"id\":0,\"data\":0}}"}} diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/part-00000-56bf8d99-9483-49cb-bc4a-f4daa12b85cb-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/part-00000-56bf8d99-9483-49cb-bc4a-f4daa12b85cb-c000.snappy.parquet new file mode 100644 index 0000000000000..bd636bb9c2d88 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/part-00000-56bf8d99-9483-49cb-bc4a-f4daa12b85cb-c000.snappy.parquet differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/part-00003-5890b536-c464-4b14-89dd-0a2e03f8d95b-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/part-00003-5890b536-c464-4b14-89dd-0a2e03f8d95b-c000.snappy.parquet new file mode 100644 index 0000000000000..a248d84055b9d Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/part-00003-5890b536-c464-4b14-89dd-0a2e03f8d95b-c000.snappy.parquet differ diff --git a/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/part-00007-5ae5a06d-3e76-4d2b-b22e-60c196d352c1-c000.snappy.parquet b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/part-00007-5ae5a06d-3e76-4d2b-b22e-60c196d352c1-c000.snappy.parquet new file mode 100644 index 0000000000000..4809a08baac66 Binary files /dev/null and b/metadata-ingestion/tests/integration/delta_lake/test_data/delta_tables_nested_datatype/table_with_string_and_nested_array_of_numbers/part-00007-5ae5a06d-3e76-4d2b-b22e-60c196d352c1-c000.snappy.parquet differ