From 1b3bbf8234adf6c04b90fb6d0f62ce0254b7ec09 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Wed, 6 Mar 2024 09:21:42 -0600 Subject: [PATCH 1/4] fix(test): nested domains (#9993) --- .../tests/cypress/cypress/e2e/domains/nested_domains.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/smoke-test/tests/cypress/cypress/e2e/domains/nested_domains.js b/smoke-test/tests/cypress/cypress/e2e/domains/nested_domains.js index 3910a280a303c..3152174e17072 100644 --- a/smoke-test/tests/cypress/cypress/e2e/domains/nested_domains.js +++ b/smoke-test/tests/cypress/cypress/e2e/domains/nested_domains.js @@ -153,13 +153,13 @@ describe("Verify nested domains test functionalities", () => { cy.waitTextVisible("Test Label") cy.clickOptionWithTestId("add-owners-button") cy.waitTextVisible("Find a user or group") - cy.clickTextOptionWithClass(".rc-virtual-list-holder-inner", "DataHub") + cy.clickTextOptionWithClass(".rc-virtual-list-holder-inner", Cypress.env('ADMIN_DISPLAYNAME')) cy.clickOptionWithText("Find a user or group") cy.clickOptionWithId('#addOwnerButton') - cy.waitTextVisible("DataHub") + cy.waitTextVisible(Cypress.env('ADMIN_DISPLAYNAME')) cy.goToDomainList(); cy.waitTextVisible("Test documentation") - cy.waitTextVisible("DataHub") + cy.waitTextVisible(Cypress.env('ADMIN_DISPLAYNAME')) cy.clickOptionWithText(domainName) cy.clickOptionWithText("Documentation") clearAndDelete() From 7de517c36329c7738e200139d50d209d7d106b34 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 6 Mar 2024 11:43:05 -0800 Subject: [PATCH 2/4] fix(ci): refactor build-and-test command (#9999) --- .github/workflows/build-and-test.yml | 12 +++++++++++- .github/workflows/metadata-ingestion.yml | 2 ++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index cba752e92e046..df223e3603e1e 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -76,7 +76,17 @@ jobs: - name: Gradle build (and test) for NOT metadata ingestion if: ${{ matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }} run: | - ./gradlew build -x :metadata-ingestion:build -x :metadata-ingestion:check -x docs-website:build -x :metadata-integration:java:spark-lineage:test -x :metadata-io:test -x :metadata-ingestion-modules:airflow-plugin:build -x :metadata-ingestion-modules:airflow-plugin:check -x :datahub-frontend:build -x :datahub-web-react:build --parallel + ./gradlew build \ + -x :metadata-ingestion:build \ + -x :metadata-ingestion:check \ + -x docs-website:build \ + -x :metadata-integration:java:spark-lineage:test \ + -x :metadata-io:test \ + -x :metadata-ingestion-modules:airflow-plugin:build \ + -x :metadata-ingestion-modules:airflow-plugin:check \ + -x :datahub-frontend:build \ + -x :datahub-web-react:build \ + --parallel - name: Gradle build (and test) for frontend if: ${{ matrix.command == 'frontend' && needs.setup.outputs.frontend_change == 'true' }} run: | diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml index 82b78cef4c23d..c978a8ff09b5e 100644 --- a/.github/workflows/metadata-ingestion.yml +++ b/.github/workflows/metadata-ingestion.yml @@ -6,6 +6,7 @@ on: paths: - ".github/workflows/metadata-ingestion.yml" - "metadata-ingestion/**" + - "metadata-ingestion-modules/**" - "metadata-models/**" pull_request: branches: @@ -13,6 +14,7 @@ on: paths: - ".github/workflows/metadata-ingestion.yml" - "metadata-ingestion/**" + - "metadata-ingestion-modules/**" - "metadata-models/**" release: types: [published] From 8047d0b514fd416a71fc4eb071683a77d2055287 Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Thu, 7 Mar 2024 04:41:37 +0530 Subject: [PATCH 3/4] feat(ingest/snowflake): generate query nodes for snowflake (#9966) Co-authored-by: Harshal Sheth --- .../source/snowflake/snowflake_config.py | 2 +- .../source/snowflake/snowflake_lineage_v2.py | 447 ++--- .../source/snowflake/snowflake_query.py | 177 +- .../source/snowflake/snowflake_report.py | 22 +- .../source/snowflake/snowflake_v2.py | 98 +- .../sql_parsing/sql_parsing_aggregator.py | 2 +- .../tests/integration/snowflake/common.py | 66 +- .../snowflake/snowflake_golden.json | 1632 +++++++++++++---- .../snowflake_privatelink_golden.json | 252 ++- .../integration/snowflake/test_snowflake.py | 1 + .../snowflake/test_snowflake_failures.py | 33 - .../tests/unit/config/test_datetime_parser.py | 4 + 12 files changed, 1964 insertions(+), 772 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py index aad4a6ed27cb8..c1fbb2cdc1f3f 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py @@ -220,7 +220,7 @@ def validate_unsupported_configs(cls, values: Dict) -> Dict: and values["stateful_ingestion"].remove_stale_metadata ) - # TODO: Allow lineage extraction and profiling irrespective of basic schema extraction, + # TODO: Allow profiling irrespective of basic schema extraction, # as it seems possible with some refactor if not include_technical_schema and any( [include_profiles, delete_detection_enabled] diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py index 47434f8b9a963..4aeed983ff701 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage_v2.py @@ -1,26 +1,25 @@ import json import logging -from collections import defaultdict from dataclasses import dataclass from datetime import datetime from typing import ( + Any, Callable, Collection, - Dict, Iterable, List, - MutableMapping, Optional, Sequence, Set, Tuple, + Type, ) +from pydantic import BaseModel, validator from snowflake.connector import SnowflakeConnection -import datahub.emitter.mce_builder as builder +from datahub.configuration.datetimes import parse_absolute_time from datahub.emitter.mcp import MetadataChangeProposalWrapper -from datahub.emitter.sql_parsing_builder import SqlParsingBuilder from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.aws.s3_util import make_s3_urn_for_lineage from datahub.ingestion.source.snowflake.constants import ( @@ -41,13 +40,17 @@ ) from datahub.metadata.com.linkedin.pegasus2avro.dataset import ( FineGrainedLineage, - FineGrainedLineageDownstreamType, - FineGrainedLineageUpstreamType, UpstreamLineage, ) from datahub.metadata.schema_classes import DatasetLineageTypeClass, UpstreamClass -from datahub.sql_parsing.schema_resolver import SchemaResolver -from datahub.sql_parsing.sqlglot_lineage import SqlParsingResult, sqlglot_lineage +from datahub.sql_parsing.sql_parsing_aggregator import ( + ColumnLineageInfo, + ColumnRef, + KnownQueryLineageInfo, + SqlParsingAggregator, + UrnStr, +) +from datahub.sql_parsing.sqlglot_lineage import DownstreamColumnRef from datahub.utilities.perf_timer import PerfTimer from datahub.utilities.time import ts_millis_to_datetime @@ -58,11 +61,60 @@ VIEW_LINEAGE = "view_lineage" +def pydantic_parse_json(field: str) -> classmethod: + def _parse_from_json(cls: Type, v: Any) -> dict: + if isinstance(v, str): + return json.loads(v) + return v + + return validator(field, pre=True, allow_reuse=True)(_parse_from_json) + + +class UpstreamColumnNode(BaseModel): + object_name: str + object_domain: str + column_name: str + + +class ColumnUpstreamJob(BaseModel): + column_upstreams: List[UpstreamColumnNode] + query_id: str + + +class ColumnUpstreamLineage(BaseModel): + column_name: str + upstreams: List[ColumnUpstreamJob] + + +class UpstreamTableNode(BaseModel): + upstream_object_domain: str + upstream_object_name: str + query_id: str + + +class Query(BaseModel): + query_id: str + query_text: str + start_time: str + + +class UpstreamLineageEdge(BaseModel): + DOWNSTREAM_TABLE_NAME: str + DOWNSTREAM_TABLE_DOMAIN: str + UPSTREAM_TABLES: Optional[List[UpstreamTableNode]] + UPSTREAM_COLUMNS: Optional[List[ColumnUpstreamLineage]] + QUERIES: Optional[List[Query]] + + _json_upstream_tables = pydantic_parse_json("UPSTREAM_TABLES") + _json_upstream_columns = pydantic_parse_json("UPSTREAM_COLUMNS") + _json_queries = pydantic_parse_json("QUERIES") + + @dataclass(frozen=True) class SnowflakeColumnId: - columnName: str - objectName: str - objectDomain: Optional[str] = None + column_name: str + object_name: str + object_domain: Optional[str] = None class SnowflakeLineageExtractor( @@ -72,13 +124,12 @@ class SnowflakeLineageExtractor( Extracts Lineage from Snowflake. Following lineage edges are considered. - 1. "Table to View" lineage via `snowflake.account_usage.object_dependencies` view - 2. "S3 to Table" lineage via `show external tables` query. - 3. "View to Table" lineage via `snowflake.account_usage.access_history` view (requires Snowflake Enterprise Edition or above) - 4. "Table to Table" lineage via `snowflake.account_usage.access_history` view (requires Snowflake Enterprise Edition or above) - 5. "S3 to Table" lineage via `snowflake.account_usage.access_history` view (requires Snowflake Enterprise Edition or above) + 1. "Table to View" lineage via `snowflake.account_usage.object_dependencies` view + View definition SQL parsing. + 2. "S3 to Table" lineage via `show external tables` query and `snowflake.account_usage.copy_history view. + 3. "View to Table" and "Table to Table" lineage via `snowflake.account_usage.access_history` view (requires Snowflake Enterprise Edition or above) - Edition Note - Snowflake Standard Edition does not have Access History Feature. So it does not support lineage extraction for edges 3, 4, 5 mentioned above. + Edition Note - Snowflake Standard Edition does not have Access History Feature. + So it does not support lineage extraction for point 3 edges mentioned above. """ def __init__( @@ -87,13 +138,14 @@ def __init__( report: SnowflakeV2Report, dataset_urn_builder: Callable[[str], str], redundant_run_skip_handler: Optional[RedundantLineageRunSkipHandler], + sql_aggregator: SqlParsingAggregator, ) -> None: - self._external_lineage_map: Dict[str, Set[str]] = defaultdict(set) self.config = config self.report = report self.logger = logger self.dataset_urn_builder = dataset_urn_builder self.connection: Optional[SnowflakeConnection] = None + self.sql_aggregator = sql_aggregator self.redundant_run_skip_handler = redundant_run_skip_handler self.start_time, self.end_time = ( @@ -125,8 +177,6 @@ def get_workunits( self, discovered_tables: List[str], discovered_views: List[str], - schema_resolver: SchemaResolver, - view_definitions: MutableMapping[str, str], ) -> Iterable[MetadataWorkUnit]: if not self._should_ingest_lineage(): return @@ -135,21 +185,14 @@ def get_workunits( if self.connection is None: return - self._populate_external_lineage_map(discovered_tables) - if self.config.include_view_lineage: - if len(discovered_views) > 0: - yield from self.get_view_upstream_workunits( - discovered_views=discovered_views, - schema_resolver=schema_resolver, - view_definitions=view_definitions, - ) - else: - logger.info("No views found. Skipping View Lineage Extraction.") + # s3 dataset -> snowflake table + self._populate_external_upstreams(discovered_tables) - yield from self.get_table_upstream_workunits(discovered_tables) + # snowflake view/table -> snowflake table + self.populate_table_upstreams(discovered_tables) - if self._external_lineage_map: # Some external lineage is yet to be emitted - yield from self.get_table_external_upstream_workunits() + for mcp in self.sql_aggregator.gen_metadata(): + yield mcp.as_workunit() if self.redundant_run_skip_handler: # Update the checkpoint state for this run. @@ -162,23 +205,10 @@ def get_workunits( self.config.end_time, ) - def get_table_external_upstream_workunits(self) -> Iterable[MetadataWorkUnit]: - for ( - dataset_name, - external_lineage, - ) in self._external_lineage_map.items(): - upstreams = self.get_external_upstreams(external_lineage) - if upstreams: - self.report.num_tables_with_external_upstreams_only += 1 - yield self._create_upstream_lineage_workunit(dataset_name, upstreams) - logger.info( - f"Only upstream external lineage detected for {self.report.num_tables_with_external_upstreams_only} tables.", - ) - - def get_table_upstream_workunits( - self, discovered_tables: List[str] - ) -> Iterable[MetadataWorkUnit]: + def populate_table_upstreams(self, discovered_tables: List[str]) -> None: if self.report.edition == SnowflakeEdition.STANDARD: + # TODO: use sql_aggregator.add_observed_query to report queries from + # snowflake.account_usage.query_history and let Datahub generate lineage, usage and operations logger.info( "Snowflake Account is Standard Edition. Table to Table and View to Table Lineage Feature is not supported." ) # See Edition Note above for why @@ -189,120 +219,39 @@ def get_table_upstream_workunits( if not results: return - yield from self._gen_workunits_from_query_result( - discovered_tables, results - ) + self.populate_known_query_lineage(discovered_tables, results) self.report.table_lineage_query_secs = timer.elapsed_seconds() logger.info( - f"Upstream lineage detected for {self.report.num_tables_with_upstreams} tables.", + f"Upstream lineage detected for {self.report.num_tables_with_known_upstreams} tables.", ) - def _gen_workunits_from_query_result( + def populate_known_query_lineage( self, discovered_assets: Collection[str], - results: Iterable[dict], - upstream_for_view: bool = False, - ) -> Iterable[MetadataWorkUnit]: + results: Iterable[UpstreamLineageEdge], + ) -> None: for db_row in results: dataset_name = self.get_dataset_identifier_from_qualified_name( - db_row["DOWNSTREAM_TABLE_NAME"] + db_row.DOWNSTREAM_TABLE_NAME ) - if dataset_name not in discovered_assets: + if dataset_name not in discovered_assets or not db_row.QUERIES: continue - ( - upstreams, - fine_upstreams, - ) = self.get_upstreams_from_query_result_row(dataset_name, db_row) - if upstreams: - if upstream_for_view: - self.report.num_views_with_upstreams += 1 - else: - self.report.num_tables_with_upstreams += 1 - yield self._create_upstream_lineage_workunit( - dataset_name, upstreams, fine_upstreams - ) - else: - logger.debug(f"No lineage found for {dataset_name}") - def get_view_upstream_workunits( - self, - discovered_views: List[str], - schema_resolver: SchemaResolver, - view_definitions: MutableMapping[str, str], - ) -> Iterable[MetadataWorkUnit]: - views_failed_parsing = set() - if self.config.include_view_column_lineage: - with PerfTimer() as timer: - builder = SqlParsingBuilder( - generate_lineage=True, - generate_usage_statistics=False, - generate_operations=False, - ) - for view_identifier, view_definition in view_definitions.items(): - result = self._run_sql_parser( - view_identifier, view_definition, schema_resolver - ) - if result and result.out_tables: - self.report.num_views_with_upstreams += 1 - # This does not yield any workunits but we use - # yield here to execute this method - yield from builder.process_sql_parsing_result( - result=result, - query=view_definition, - is_view_ddl=True, - ) - else: - views_failed_parsing.add(view_identifier) - - yield from builder.gen_workunits() - self.report.view_lineage_parse_secs = timer.elapsed_seconds() - - with PerfTimer() as timer: - results = self._fetch_upstream_lineages_for_views() - - if results: - yield from self._gen_workunits_from_query_result( - views_failed_parsing, - results, - upstream_for_view=True, + for query in db_row.QUERIES: + known_lineage = self.get_known_query_lineage( + query, dataset_name, db_row ) - self.report.view_upstream_lineage_query_secs = timer.elapsed_seconds() - logger.info( - f"Upstream lineage detected for {self.report.num_views_with_upstreams} views.", - ) - - def _run_sql_parser( - self, dataset_identifier: str, query: str, schema_resolver: SchemaResolver - ) -> Optional[SqlParsingResult]: - try: - database, schema, _view = dataset_identifier.split(".") - except ValueError: - logger.warning(f"Invalid view identifier: {dataset_identifier}") - return None - raw_lineage = sqlglot_lineage( - query, - schema_resolver=schema_resolver, - default_db=database, - default_schema=schema, - ) - if raw_lineage.debug_info.table_error: - logger.debug( - f"Failed to parse lineage for view {dataset_identifier}: " - f"{raw_lineage.debug_info.table_error}" - ) - self.report.num_view_definitions_failed_parsing += 1 - return None - elif raw_lineage.debug_info.column_error: - self.report.num_view_definitions_failed_column_parsing += 1 - else: - self.report.num_view_definitions_parsed += 1 - return raw_lineage + if known_lineage and known_lineage.upstreams: + self.report.num_tables_with_known_upstreams += 1 + self.sql_aggregator.add_known_query_lineage(known_lineage, True) + else: + logger.debug(f"No lineage found for {dataset_name}") def _create_upstream_lineage_workunit( self, dataset_name: str, upstreams: Sequence[UpstreamClass], - fine_upstreams: Sequence[FineGrainedLineage] = (), + fine_upstreams: Sequence[FineGrainedLineage], ) -> MetadataWorkUnit: logger.debug( f"Upstream lineage of '{dataset_name}': {[u.dataset for u in upstreams]}" @@ -322,35 +271,36 @@ def _create_upstream_lineage_workunit( entityUrn=self.dataset_urn_builder(dataset_name), aspect=upstream_lineage ).as_workunit() - def get_upstreams_from_query_result_row( - self, dataset_name: str, db_row: dict - ) -> Tuple[List[UpstreamClass], List[FineGrainedLineage]]: - upstreams: List[UpstreamClass] = [] - fine_upstreams: List[FineGrainedLineage] = [] - - if "UPSTREAM_TABLES" in db_row and db_row["UPSTREAM_TABLES"] is not None: - upstreams = self.map_query_result_upstreams( - json.loads(db_row["UPSTREAM_TABLES"]) - ) + def get_known_query_lineage( + self, query: Query, dataset_name: str, db_row: UpstreamLineageEdge + ) -> Optional[KnownQueryLineageInfo]: - if ( - self.config.include_column_lineage - and "UPSTREAM_COLUMNS" in db_row - and db_row["UPSTREAM_COLUMNS"] is not None - ): - fine_upstreams = self.map_query_result_fine_upstreams( - self.dataset_urn_builder(dataset_name), - json.loads(db_row["UPSTREAM_COLUMNS"]), - ) + if not db_row.UPSTREAM_TABLES: + return None - # Populate the external-table-lineage(s3->snowflake), if present - if dataset_name in self._external_lineage_map: - external_lineage = self._external_lineage_map.pop(dataset_name) - upstreams += self.get_external_upstreams(external_lineage) + downstream_table_urn = self.dataset_urn_builder(dataset_name) + + known_lineage = KnownQueryLineageInfo( + query_text=query.query_text, + downstream=downstream_table_urn, + upstreams=self.map_query_result_upstreams( + db_row.UPSTREAM_TABLES, query.query_id + ), + column_lineage=( + self.map_query_result_fine_upstreams( + downstream_table_urn, + db_row.UPSTREAM_COLUMNS, + query.query_id, + ) + if (self.config.include_column_lineage and db_row.UPSTREAM_COLUMNS) + else None + ), + timestamp=parse_absolute_time(query.start_time), + ) - return upstreams, fine_upstreams + return known_lineage - def _populate_external_lineage_map(self, discovered_tables: List[str]) -> None: + def _populate_external_upstreams(self, discovered_tables: List[str]) -> None: with PerfTimer() as timer: self.report.num_external_table_edges_scanned = 0 @@ -367,8 +317,6 @@ def _populate_external_lineage_map(self, discovered_tables: List[str]) -> None: ) self.report.external_lineage_queries_secs = timer.elapsed_seconds() - if len(self._external_lineage_map.keys()) == 0: - logger.debug("No external lineage found.") # Handles the case for explicitly created external tables. # NOTE: Snowflake does not log this information to the access_history table. @@ -384,10 +332,15 @@ def _populate_external_lineage_from_show_query( if key not in discovered_tables: continue - self._external_lineage_map[key].add(db_row["location"]) - logger.debug( - f"ExternalLineage[Table(Down)={key}]:External(Up)={self._external_lineage_map[key]} via show external tables" - ) + if db_row["location"].startswith("s3://"): + self.sql_aggregator.add_known_lineage_mapping( + downstream_urn=self.dataset_urn_builder(key), + upstream_urn=make_s3_urn_for_lineage( + db_row["location"], self.config.env + ), + ) + self.report.num_external_table_edges_scanned += 1 + self.report.num_external_table_edges_scanned += 1 except Exception as e: logger.debug(e, exc_info=e) @@ -439,15 +392,14 @@ def _process_external_lineage_result_row( external_locations = json.loads(db_row["UPSTREAM_LOCATIONS"]) for loc in external_locations: - if loc not in self._external_lineage_map[key]: - self._external_lineage_map[key].add(loc) + if loc.startswith("s3://"): + self.sql_aggregator.add_known_lineage_mapping( + downstream_urn=self.dataset_urn_builder(key), + upstream_urn=make_s3_urn_for_lineage(loc, self.config.env), + ) self.report.num_external_table_edges_scanned += 1 - logger.debug( - f"ExternalLineage[Table(Down)={key}]:External(Up)={self._external_lineage_map[key]} via access_history" - ) - - def _fetch_upstream_lineages_for_tables(self) -> Iterable[Dict]: + def _fetch_upstream_lineages_for_tables(self) -> Iterable[UpstreamLineageEdge]: query: str = SnowflakeQuery.table_to_table_lineage_history_v2( start_time_millis=int(self.start_time.timestamp() * 1000), end_time_millis=int(self.end_time.timestamp() * 1000), @@ -457,7 +409,7 @@ def _fetch_upstream_lineages_for_tables(self) -> Iterable[Dict]: ) try: for db_row in self.query(query): - yield db_row + yield UpstreamLineageEdge.parse_obj(db_row) except Exception as e: if isinstance(e, SnowflakePermissionError): error_msg = "Failed to get table/view to table lineage. Please grant imported privileges on SNOWFLAKE database. " @@ -471,46 +423,41 @@ def _fetch_upstream_lineages_for_tables(self) -> Iterable[Dict]: self.report_status(TABLE_LINEAGE, False) def map_query_result_upstreams( - self, upstream_tables: Optional[List[dict]] - ) -> List[UpstreamClass]: + self, upstream_tables: Optional[List[UpstreamTableNode]], query_id: str + ) -> List[UrnStr]: if not upstream_tables: return [] - upstreams: List[UpstreamClass] = [] + upstreams: List[UrnStr] = [] for upstream_table in upstream_tables: - if upstream_table: + if upstream_table and upstream_table.query_id == query_id: try: - self._process_add_single_upstream(upstreams, upstream_table) + upstream_name = self.get_dataset_identifier_from_qualified_name( + upstream_table.upstream_object_name + ) + if upstream_name and self._is_dataset_pattern_allowed( + upstream_name, + upstream_table.upstream_object_domain, + is_upstream=True, + ): + upstreams.append(self.dataset_urn_builder(upstream_name)) except Exception as e: logger.debug(e, exc_info=e) return upstreams - def _process_add_single_upstream( - self, upstreams: List[UpstreamClass], upstream_table: dict - ) -> None: - upstream_name = self.get_dataset_identifier_from_qualified_name( - upstream_table["upstream_object_name"] - ) - if upstream_name and self._is_dataset_pattern_allowed( - upstream_name, upstream_table["upstream_object_domain"], is_upstream=True - ): - upstreams.append( - UpstreamClass( - dataset=self.dataset_urn_builder(upstream_name), - type=DatasetLineageTypeClass.TRANSFORMED, - ) - ) - def map_query_result_fine_upstreams( - self, dataset_urn: str, column_wise_upstreams: Optional[List[dict]] - ) -> List[FineGrainedLineage]: + self, + dataset_urn: str, + column_wise_upstreams: Optional[List[ColumnUpstreamLineage]], + query_id: str, + ) -> List[ColumnLineageInfo]: if not column_wise_upstreams: return [] - fine_upstreams: List[FineGrainedLineage] = [] + fine_upstreams: List[ColumnLineageInfo] = [] for column_with_upstreams in column_wise_upstreams: if column_with_upstreams: try: self._process_add_single_column_upstream( - dataset_urn, fine_upstreams, column_with_upstreams + dataset_urn, fine_upstreams, column_with_upstreams, query_id ) except Exception as e: logger.debug(e, exc_info=e) @@ -519,97 +466,71 @@ def map_query_result_fine_upstreams( def _process_add_single_column_upstream( self, dataset_urn: str, - fine_upstreams: List[FineGrainedLineage], - column_with_upstreams: Dict, + fine_upstreams: List[ColumnLineageInfo], + column_with_upstreams: ColumnUpstreamLineage, + query_id: str, ) -> None: - column_name = column_with_upstreams["column_name"] - upstream_jobs = column_with_upstreams["upstreams"] + column_name = column_with_upstreams.column_name + upstream_jobs = column_with_upstreams.upstreams if column_name and upstream_jobs: - for upstream_columns in upstream_jobs: - if not upstream_columns: + for upstream_job in upstream_jobs: + if not upstream_job or upstream_job.query_id != query_id: continue fine_upstream = self.build_finegrained_lineage( dataset_urn=dataset_urn, col=column_name, upstream_columns={ SnowflakeColumnId( - columnName=col["column_name"], - objectName=col["object_name"], - objectDomain=col["object_domain"], + column_name=col.column_name, + object_name=col.object_name, + object_domain=col.object_domain, ) - for col in upstream_columns + for col in upstream_job.column_upstreams }, ) if not fine_upstream: continue fine_upstreams.append(fine_upstream) - def _fetch_upstream_lineages_for_views(self): - # NOTE: This query captures only the upstream lineage of a view (with no column lineage). - # For more details see: https://docs.snowflake.com/en/user-guide/object-dependencies.html#object-dependencies - # and also https://docs.snowflake.com/en/sql-reference/account-usage/access_history.html#usage-notes for current limitations on capturing the lineage for views. - view_upstream_lineage_query: str = SnowflakeQuery.view_dependencies_v2() - - try: - for db_row in self.query(view_upstream_lineage_query): - yield db_row - except Exception as e: - if isinstance(e, SnowflakePermissionError): - error_msg = "Failed to get table to view lineage. Please grant imported privileges on SNOWFLAKE database." - self.warn_if_stateful_else_error(LINEAGE_PERMISSION_ERROR, error_msg) - else: - logger.debug(e, exc_info=e) - self.report_warning( - "view-upstream-lineage", - f"Extracting the upstream view lineage from Snowflake failed due to error {e}.", - ) - self.report_status(VIEW_LINEAGE, False) - def build_finegrained_lineage( self, dataset_urn: str, col: str, upstream_columns: Set[SnowflakeColumnId], - ) -> Optional[FineGrainedLineage]: + ) -> Optional[ColumnLineageInfo]: column_upstreams = self.build_finegrained_lineage_upstreams(upstream_columns) if not column_upstreams: return None - finegrained_lineage_entry = FineGrainedLineage( - upstreamType=FineGrainedLineageUpstreamType.FIELD_SET, - # Sorting the list of upstream lineage events in order to avoid creating multiple aspects in backend - # even if the lineage is same but the order is different. + column_lineage = ColumnLineageInfo( + downstream=DownstreamColumnRef( + table=dataset_urn, column=self.snowflake_identifier(col) + ), upstreams=sorted(column_upstreams), - downstreamType=FineGrainedLineageDownstreamType.FIELD, - downstreams=[ - builder.make_schema_field_urn( - dataset_urn, self.snowflake_identifier(col) - ) - ], ) - return finegrained_lineage_entry + return column_lineage def build_finegrained_lineage_upstreams( self, upstream_columms: Set[SnowflakeColumnId] - ) -> List[str]: + ) -> List[ColumnRef]: column_upstreams = [] for upstream_col in upstream_columms: if ( - upstream_col.objectName - and upstream_col.columnName + upstream_col.object_name + and upstream_col.column_name and self._is_dataset_pattern_allowed( - upstream_col.objectName, - upstream_col.objectDomain, + upstream_col.object_name, + upstream_col.object_domain, is_upstream=True, ) ): upstream_dataset_name = self.get_dataset_identifier_from_qualified_name( - upstream_col.objectName + upstream_col.object_name ) column_upstreams.append( - builder.make_schema_field_urn( - self.dataset_urn_builder(upstream_dataset_name), - self.snowflake_identifier(upstream_col.columnName), + ColumnRef( + table=self.dataset_urn_builder(upstream_dataset_name), + column=self.snowflake_identifier(upstream_col.column_name), ) ) return column_upstreams diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py index 93e29f3fb99bb..5c04af3290ab1 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py @@ -515,7 +515,7 @@ def show_external_tables() -> str: def copy_lineage_history( start_time_millis: int, end_time_millis: int, - downstreams_deny_pattern: List[str], + downstreams_deny_pattern: List[str] = DEFAULT_TABLES_DENY_LIST, ) -> str: temp_table_filter = create_deny_regex_sql_filter( downstreams_deny_pattern, @@ -802,7 +802,7 @@ def table_upstreams_with_column_lineage( SELECT downstream_table_name, downstream_column_name, - ANY_VALUE(query_start_time), + ANY_VALUE(query_start_time) as query_start_time, query_id, ARRAY_UNIQUE_AGG( OBJECT_CONSTRUCT( @@ -820,25 +820,73 @@ def table_upstreams_with_column_lineage( downstream_table_name, downstream_column_name, query_id - ), - column_upstreams AS ( + ),-- one row per column's upstream job (repetitive query possible) + column_upstream_jobs_unique AS ( SELECT downstream_table_name, downstream_column_name, - ARRAY_UNIQUE_AGG(upstream_columns_for_job) as upstreams + upstream_columns_for_job, + MAX_BY(query_id, query_start_time) as query_id, + MAX(query_start_time) as query_start_time FROM column_upstream_jobs + GROUP BY + downstream_table_name, + downstream_column_name, + upstream_columns_for_job + ),-- one row per column's unique upstream job (keep only latest query) + column_upstreams AS ( + SELECT + downstream_table_name, + downstream_column_name, + ARRAY_UNIQUE_AGG( + OBJECT_CONSTRUCT ( + 'column_upstreams', upstream_columns_for_job, + 'query_id', query_id + ) + ) as upstreams + FROM + column_upstream_jobs_unique GROUP BY downstream_table_name, downstream_column_name - ) + ), -- one row per downstream column + table_upstream_jobs_unique AS ( + SELECT + downstream_table_name, + ANY_VALUE(downstream_table_domain) as downstream_table_domain, + upstream_table_name, + ANY_VALUE(upstream_table_domain) as upstream_table_domain, + MAX_BY(query_id, query_start_time) as query_id + FROM + column_lineage_history + GROUP BY + downstream_table_name, + upstream_table_name + ), -- one row per downstream table + query_ids AS ( + SELECT distinct downstream_table_name, query_id from table_upstream_jobs_unique + UNION + select distinct downstream_table_name, query_id from column_upstream_jobs_unique + ), + queries AS ( + select qid.downstream_table_name, qid.query_id, query_history.query_text, query_history.start_time + from query_ids qid + LEFT JOIN ( + SELECT * FROM snowflake.account_usage.query_history + WHERE query_history.start_time >= to_timestamp_ltz({start_time_millis}, 3) + AND query_history.start_time < to_timestamp_ltz({end_time_millis}, 3) + ) query_history + on qid.query_id = query_history.query_id + ) SELECT h.downstream_table_name AS "DOWNSTREAM_TABLE_NAME", ANY_VALUE(h.downstream_table_domain) AS "DOWNSTREAM_TABLE_DOMAIN", ARRAY_UNIQUE_AGG( OBJECT_CONSTRUCT( 'upstream_object_name', h.upstream_table_name, - 'upstream_object_domain', h.upstream_table_domain + 'upstream_object_domain', h.upstream_table_domain, + 'query_id', h.query_id ) ) AS "UPSTREAM_TABLES", ARRAY_UNIQUE_AGG( @@ -846,11 +894,20 @@ def table_upstreams_with_column_lineage( 'column_name', column_upstreams.downstream_column_name, 'upstreams', column_upstreams.upstreams ) - ) AS "UPSTREAM_COLUMNS" + ) AS "UPSTREAM_COLUMNS", + ARRAY_UNIQUE_AGG( + OBJECT_CONSTRUCT( + 'query_id', q.query_id, + 'query_text', q.query_text, + 'start_time', q.start_time + ) + ) as "QUERIES" FROM - column_lineage_history h + table_upstream_jobs_unique h LEFT JOIN column_upstreams column_upstreams on h.downstream_table_name = column_upstreams.downstream_table_name + LEFT JOIN queries q + on h.downstream_table_name = q.downstream_table_name GROUP BY h.downstream_table_name ORDER BY @@ -876,42 +933,78 @@ def table_upstreams_only( ["upstream_table_name"], ) return f""" - WITH table_lineage_history AS ( - SELECT - r.value:"objectName"::varchar AS upstream_table_name, - r.value:"objectDomain"::varchar AS upstream_table_domain, - r.value:"columns" AS upstream_table_columns, - w.value:"objectName"::varchar AS downstream_table_name, - w.value:"objectDomain"::varchar AS downstream_table_domain, - w.value:"columns" AS downstream_table_columns, - t.query_start_time AS query_start_time - FROM - (SELECT * from snowflake.account_usage.access_history) t, - lateral flatten(input => t.DIRECT_OBJECTS_ACCESSED) r, - lateral flatten(input => t.OBJECTS_MODIFIED) w - WHERE r.value:"objectId" IS NOT NULL - AND w.value:"objectId" IS NOT NULL - AND w.value:"objectName" NOT LIKE '%.GE_TMP_%' - AND w.value:"objectName" NOT LIKE '%.GE_TEMP_%' + WITH table_lineage_history AS ( + SELECT + r.value : "objectName" :: varchar AS upstream_table_name, + r.value : "objectDomain" :: varchar AS upstream_table_domain, + w.value : "objectName" :: varchar AS downstream_table_name, + w.value : "objectDomain" :: varchar AS downstream_table_domain, + t.query_start_time AS query_start_time, + t.query_id AS query_id + FROM + (SELECT * from snowflake.account_usage.access_history) t, + lateral flatten(input => t.DIRECT_OBJECTS_ACCESSED) r, + lateral flatten(input => t.OBJECTS_MODIFIED) w + WHERE + r.value : "objectId" IS NOT NULL + AND w.value : "objectId" IS NOT NULL + AND w.value : "objectName" NOT LIKE '%.GE_TMP_%' + AND w.value : "objectName" NOT LIKE '%.GE_TEMP_%' AND t.query_start_time >= to_timestamp_ltz({start_time_millis}, 3) AND t.query_start_time < to_timestamp_ltz({end_time_millis}, 3) AND upstream_table_domain in {allowed_upstream_table_domains} AND downstream_table_domain = '{SnowflakeObjectDomain.TABLE.capitalize()}' {("AND " + upstream_sql_filter) if upstream_sql_filter else ""} - ) + ), + table_upstream_jobs_unique AS ( SELECT - downstream_table_name AS "DOWNSTREAM_TABLE_NAME", - ANY_VALUE(downstream_table_domain) as "DOWNSTREAM_TABLE_DOMAIN", - ARRAY_UNIQUE_AGG( - OBJECT_CONSTRUCT( - 'upstream_object_name', upstream_table_name, - 'upstream_object_domain', upstream_table_domain - ) - ) as "UPSTREAM_TABLES" - FROM - table_lineage_history - GROUP BY - downstream_table_name - ORDER BY - downstream_table_name - """ + downstream_table_name, + ANY_VALUE(downstream_table_domain) as downstream_table_domain, + upstream_table_name, + ANY_VALUE(upstream_table_domain) as upstream_table_domain, + MAX_BY(query_id, query_start_time) as query_id + FROM + table_lineage_history + GROUP BY + downstream_table_name, + upstream_table_name + ), -- one row per downstream table + query_ids AS ( + SELECT distinct downstream_table_name, query_id from table_upstream_jobs_unique + ), + queries AS ( + select qid.downstream_table_name, qid.query_id, query_history.query_text, query_history.start_time + from query_ids qid + LEFT JOIN ( + SELECT * FROM snowflake.account_usage.query_history + WHERE query_history.start_time >= to_timestamp_ltz({start_time_millis}, 3) + AND query_history.start_time < to_timestamp_ltz({end_time_millis}, 3) + ) query_history + on qid.query_id = query_history.query_id + ) + SELECT + h.downstream_table_name AS "DOWNSTREAM_TABLE_NAME", + ANY_VALUE(h.downstream_table_domain) AS "DOWNSTREAM_TABLE_DOMAIN", + ARRAY_UNIQUE_AGG( + OBJECT_CONSTRUCT( + 'upstream_object_name', h.upstream_table_name, + 'upstream_object_domain', h.upstream_table_domain, + 'query_id', h.query_id + ) + ) AS "UPSTREAM_TABLES", + ARRAY_UNIQUE_AGG( + OBJECT_CONSTRUCT( + 'query_id', q.query_id, + 'query_text', q.query_text, + 'start_time', q.start_time + ) + ) as "QUERIES" + FROM + table_upstream_jobs_unique h + LEFT JOIN queries q + on h.downstream_table_name = q.downstream_table_name + GROUP BY + h.downstream_table_name + ORDER BY + h.downstream_table_name + """ diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py index f67b359dedb11..40112eed5a463 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py @@ -10,6 +10,7 @@ ) from datahub.ingestion.source_report.ingestion_stage import IngestionStageReport from datahub.ingestion.source_report.time_window import BaseTimeWindowReport +from datahub.sql_parsing.sql_parsing_aggregator import SqlAggregatorReport @dataclass @@ -58,6 +59,9 @@ class SnowflakeReport(ProfilingSqlReport, BaseTimeWindowReport): profile_if_updated_since: Optional[datetime] = None profile_candidates: Dict[str, List[str]] = field(default_factory=dict) + # lineage/usage v2 + sql_aggregator: Optional[SqlAggregatorReport] = None + @dataclass class SnowflakeV2Report( @@ -81,9 +85,9 @@ class SnowflakeV2Report( usage_aggregation_query_secs: float = -1 table_lineage_query_secs: float = -1 - view_lineage_parse_secs: float = -1 - view_upstream_lineage_query_secs: float = -1 - view_downstream_lineage_query_secs: float = -1 + # view_lineage_parse_secs: float = -1 + # view_upstream_lineage_query_secs: float = -1 + # view_downstream_lineage_query_secs: float = -1 external_lineage_queries_secs: float = -1 # Reports how many times we reset in-memory `functools.lru_cache` caches of data, @@ -111,13 +115,13 @@ class SnowflakeV2Report( edition: Optional[SnowflakeEdition] = None - num_tables_with_external_upstreams_only: int = 0 - num_tables_with_upstreams: int = 0 - num_views_with_upstreams: int = 0 + # num_tables_with_external_upstreams_only: int = 0 + num_tables_with_known_upstreams: int = 0 + # num_views_with_upstreams: int = 0 - num_view_definitions_parsed: int = 0 - num_view_definitions_failed_parsing: int = 0 - num_view_definitions_failed_column_parsing: int = 0 + # num_view_definitions_parsed: int = 0 + # num_view_definitions_failed_parsing: int = 0 + # num_view_definitions_failed_column_parsing: int = 0 def report_entity_scanned(self, name: str, ent_type: str = "table") -> None: """ diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py index 87f8a30bbe05f..591bdffed5819 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py @@ -133,8 +133,7 @@ TimeType, ) from datahub.metadata.com.linkedin.pegasus2avro.tag import TagProperties -from datahub.sql_parsing.schema_resolver import SchemaResolver -from datahub.utilities.file_backed_collections import FileBackedDict +from datahub.sql_parsing.sql_parsing_aggregator import SqlParsingAggregator from datahub.utilities.perf_timer import PerfTimer from datahub.utilities.registries.domain_registry import DomainRegistry @@ -236,9 +235,20 @@ def __init__(self, ctx: PipelineContext, config: SnowflakeV2Config): # For database, schema, tables, views, etc self.data_dictionary = SnowflakeDataDictionary() - self.lineage_extractor: Optional[SnowflakeLineageExtractor] = None + self.aggregator: Optional[SqlParsingAggregator] = None + if self.config.include_table_lineage: + self.aggregator = SqlParsingAggregator( + platform=self.platform, + platform_instance=self.config.platform_instance, + env=self.config.env, + graph=self.ctx.graph, + generate_usage_statistics=False, + generate_operations=False, + ) + self.report.sql_aggregator = self.aggregator.report + redundant_lineage_run_skip_handler: Optional[ RedundantLineageRunSkipHandler ] = None @@ -254,6 +264,7 @@ def __init__(self, ctx: PipelineContext, config: SnowflakeV2Config): self.report, dataset_urn_builder=self.gen_dataset_urn, redundant_run_skip_handler=redundant_lineage_run_skip_handler, + sql_aggregator=self.aggregator, ) self.usage_extractor: Optional[SnowflakeUsageExtractor] = None @@ -301,12 +312,8 @@ def __init__(self, ctx: PipelineContext, config: SnowflakeV2Config): # Caches tables for a single database. Consider moving to disk or S3 when possible. self.db_tables: Dict[str, List[SnowflakeTable]] = {} - - self.view_definitions: FileBackedDict[str] = FileBackedDict() self.add_config_to_report() - self.sql_parser_schema_resolver = self._init_schema_resolver() - @classmethod def create(cls, config_dict: dict, ctx: PipelineContext) -> "Source": config = SnowflakeV2Config.parse_obj(config_dict) @@ -491,24 +498,6 @@ def query(query): return _report - def _init_schema_resolver(self) -> SchemaResolver: - if not self.config.include_technical_schema and self.config.parse_view_ddl: - if self.ctx.graph: - return self.ctx.graph.initialize_schema_resolver_from_datahub( - platform=self.platform, - platform_instance=self.config.platform_instance, - env=self.config.env, - ) - else: - logger.warning( - "Failed to load schema info from DataHub as DataHubGraph is missing.", - ) - return SchemaResolver( - platform=self.platform, - platform_instance=self.config.platform_instance, - env=self.config.env, - ) - def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: return [ *super().get_workunit_processors(), @@ -600,8 +589,6 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: yield from self.lineage_extractor.get_workunits( discovered_tables=discovered_tables, discovered_views=discovered_views, - schema_resolver=self.sql_parser_schema_resolver, - view_definitions=self.view_definitions, ) if ( @@ -609,7 +596,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: ) and self.usage_extractor: yield from self.usage_extractor.get_usage_workunits(discovered_datasets) - def report_cache_info(self): + def report_cache_info(self) -> None: lru_cache_functions: List[Callable] = [ self.data_dictionary.get_tables_for_database, self.data_dictionary.get_views_for_database, @@ -620,7 +607,7 @@ def report_cache_info(self): for func in lru_cache_functions: self.report.lru_cache_info[func.__name__] = func.cache_info()._asdict() # type: ignore - def report_warehouse_failure(self): + def report_warehouse_failure(self) -> None: if self.config.warehouse is not None: self.report_error( GENERIC_PERMISSION_ERROR_KEY, @@ -656,7 +643,9 @@ def get_databases(self) -> Optional[List[SnowflakeDatabase]]: ) return ischema_databases - def get_databases_from_ischema(self, databases): + def get_databases_from_ischema( + self, databases: List[SnowflakeDatabase] + ) -> List[SnowflakeDatabase]: ischema_databases: List[SnowflakeDatabase] = [] for database in databases: try: @@ -791,11 +780,22 @@ def _process_schema( if self.config.include_views: views = self.fetch_views_for_schema(snowflake_schema, db_name, schema_name) - if self.config.parse_view_ddl: + if ( + self.aggregator + and self.config.include_view_lineage + and self.config.parse_view_ddl + ): for view in views: - key = self.get_dataset_identifier(view.name, schema_name, db_name) + view_identifier = self.get_dataset_identifier( + view.name, schema_name, db_name + ) if view.view_definition: - self.view_definitions[key] = view.view_definition + self.aggregator.add_view_definition( + view_urn=self.gen_dataset_urn(view_identifier), + view_definition=view.view_definition, + default_db=db_name, + default_schema=schema_name, + ) if self.config.include_technical_schema: for view in views: @@ -945,8 +945,12 @@ def fetch_sample_data_for_classification( ) def fetch_foreign_keys_for_table( - self, table, schema_name, db_name, table_identifier - ): + self, + table: SnowflakeTable, + schema_name: str, + db_name: str, + table_identifier: str, + ) -> None: try: table.foreign_keys = self.get_fk_constraints_for_table( table.name, schema_name, db_name @@ -958,7 +962,13 @@ def fetch_foreign_keys_for_table( ) self.report_warning("Failed to get foreign key for table", table_identifier) - def fetch_pk_for_table(self, table, schema_name, db_name, table_identifier): + def fetch_pk_for_table( + self, + table: SnowflakeTable, + schema_name: str, + db_name: str, + table_identifier: str, + ) -> None: try: table.pk = self.get_pk_constraints_for_table( table.name, schema_name, db_name @@ -970,7 +980,13 @@ def fetch_pk_for_table(self, table, schema_name, db_name, table_identifier): ) self.report_warning("Failed to get primary key for table", table_identifier) - def fetch_columns_for_table(self, table, schema_name, db_name, table_identifier): + def fetch_columns_for_table( + self, + table: SnowflakeTable, + schema_name: str, + db_name: str, + table_identifier: str, + ) -> None: try: table.columns = self.get_columns_for_table(table.name, schema_name, db_name) table.column_count = len(table.columns) @@ -1240,10 +1256,8 @@ def gen_schema_metadata( foreignKeys=foreign_keys, ) - if self.config.parse_view_ddl: - self.sql_parser_schema_resolver.add_schema_metadata( - urn=dataset_urn, schema_metadata=schema_metadata - ) + if self.aggregator and self.config.parse_view_ddl: + self.aggregator.register_schema(urn=dataset_urn, schema=schema_metadata) return schema_metadata @@ -1688,8 +1702,6 @@ def _snowflake_clear_ocsp_cache(self) -> None: def close(self) -> None: super().close() StatefulIngestionSourceBase.close(self) - self.view_definitions.close() - self.sql_parser_schema_resolver.close() if self.lineage_extractor: self.lineage_extractor.close() if self.usage_extractor: diff --git a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py index 49b58ddd22b83..d553fa8c07c32 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py +++ b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py @@ -425,7 +425,7 @@ def add_known_lineage_mapping( def add_view_definition( self, - view_urn: DatasetUrn, + view_urn: Union[DatasetUrn, UrnStr], view_definition: str, default_db: Optional[str] = None, default_schema: Optional[str] = None, diff --git a/metadata-ingestion/tests/integration/snowflake/common.py b/metadata-ingestion/tests/integration/snowflake/common.py index 53b87636068bf..74c89bf73bdef 100644 --- a/metadata-ingestion/tests/integration/snowflake/common.py +++ b/metadata-ingestion/tests/integration/snowflake/common.py @@ -341,6 +341,7 @@ def default_query_results( # noqa: C901 include_column_lineage=True, ), ): + return [ { "DOWNSTREAM_TABLE_NAME": "TEST_DB.TEST_SCHEMA.TABLE_{}".format(op_idx), @@ -350,6 +351,7 @@ def default_query_results( # noqa: C901 { "upstream_object_name": "TEST_DB.TEST_SCHEMA.TABLE_2", "upstream_object_domain": "TABLE", + "query_id": f"01b2576e-0804-4957-0034-7d83066cd0ee{op_idx}", } ] + ( # This additional upstream is only for TABLE_1 @@ -357,10 +359,12 @@ def default_query_results( # noqa: C901 { "upstream_object_name": "TEST_DB.TEST_SCHEMA.VIEW_1", "upstream_object_domain": "VIEW", + "query_id": f"01b2576e-0804-4957-0034-7d83066cd0ee{op_idx}", }, { "upstream_object_name": "OTHER_DB.OTHER_SCHEMA.TABLE_1", "upstream_object_domain": "TABLE", + "query_id": f"01b2576e-0804-4957-0034-7d83066cd0ee{op_idx}", }, ] if op_idx == 1 @@ -370,15 +374,18 @@ def default_query_results( # noqa: C901 "UPSTREAM_COLUMNS": json.dumps( [ { - "column_name": "COL_{}".format(col_idx), + "column_name": f"COL_{col_idx}", "upstreams": [ - [ - { - "object_name": "TEST_DB.TEST_SCHEMA.TABLE_2", - "object_domain": "Table", - "column_name": "COL_{}".format(col_idx), - } - ] + { + "query_id": f"01b2576e-0804-4957-0034-7d83066cd0ee{op_idx}", + "column_upstreams": [ + { + "object_name": "TEST_DB.TEST_SCHEMA.TABLE_2", + "object_domain": "Table", + "column_name": f"COL_{col_idx}", + } + ], + } ], } for col_idx in range(1, num_cols + 1) @@ -388,13 +395,16 @@ def default_query_results( # noqa: C901 { "column_name": "COL_1", "upstreams": [ - [ - { - "object_name": "OTHER_DB.OTHER_SCHEMA.TABLE_1", - "object_domain": "Table", - "column_name": "COL_1", - } - ] + { + "query_id": f"01b2576e-0804-4957-0034-7d83066cd0ee{op_idx}", + "column_upstreams": [ + { + "object_name": "OTHER_DB.OTHER_SCHEMA.TABLE_1", + "object_domain": "Table", + "column_name": "COL_1", + } + ], + } ], } ] @@ -402,6 +412,15 @@ def default_query_results( # noqa: C901 else [] ) ), + "QUERIES": json.dumps( + [ + { + "query_text": f"INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_{op_idx} SELECT * FROM TEST_DB.TEST_SCHEMA.TABLE_2", + "query_id": f"01b2576e-0804-4957-0034-7d83066cd0ee{op_idx}", + "start_time": "06-06-2022", + } + ] + ), } for op_idx in range(1, num_ops + 1) ] @@ -422,6 +441,7 @@ def default_query_results( # noqa: C901 { "upstream_object_name": "TEST_DB.TEST_SCHEMA.TABLE_2", "upstream_object_domain": "TABLE", + "query_id": f"01b2576e-0804-4957-0034-7d83066cd0ee{op_idx}", }, ] + ( # This additional upstream is only for TABLE_1 @@ -429,12 +449,24 @@ def default_query_results( # noqa: C901 { "upstream_object_name": "OTHER_DB.OTHER_SCHEMA.TABLE_1", "upstream_object_domain": "TABLE", + "query_id": f"01b2576e-0804-4957-0034-7d83066cd0ee{op_idx}", }, ] if op_idx == 1 else [] ) ), + "QUERIES": json.dumps( + [ + { + "query_text": f"INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_{op_idx} SELECT * FROM TEST_DB.TEST_SCHEMA.TABLE_2", + "query_id": f"01b2576e-0804-4957-0034-7d83066cd0ee{op_idx}", + "start_time": datetime(2022, 6, 6, 0, 0, 0, 0).replace( + tzinfo=timezone.utc + ), + } + ] + ), } for op_idx in range(1, num_ops + 1) ] @@ -511,6 +543,10 @@ def default_query_results( # noqa: C901 snowflake_query.SnowflakeQuery.view_dependencies_v2(), snowflake_query.SnowflakeQuery.view_dependencies(), snowflake_query.SnowflakeQuery.show_external_tables(), + snowflake_query.SnowflakeQuery.copy_lineage_history( + 1654473600000, + 1654586220000, + ), ]: return [] diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json index ece54f00eeaa0..a064e4d1d58dc 100644 --- a/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json +++ b/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json @@ -4167,11 +4167,16 @@ "upstreams": [ { "auditStamp": { + "time": 1615443388097, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { "time": 0, - "actor": "urn:li:corpuser:unknown" + "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", - "type": "VIEW" + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" } ], "fineGrainedLineages": [ @@ -4184,18 +4189,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_1)" ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_10)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_10)" - ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -4206,7 +4201,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_2)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -4217,7 +4213,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_3)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -4228,7 +4225,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_4)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -4239,7 +4237,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_5)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -4250,7 +4249,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_6)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -4261,7 +4261,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_7)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -4272,7 +4273,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_8)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -4283,14 +4285,55 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_9)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD),col_10)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2023_10_06-17_59_03", + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_1 SELECT * FROM TEST_DB.TEST_SCHEMA.TABLE_2", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", "lastRunId": "no-run-id-provided" } }, @@ -4304,11 +4347,16 @@ "upstreams": [ { "auditStamp": { + "time": 1615443388097, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { "time": 0, - "actor": "urn:li:corpuser:unknown" + "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "type": "VIEW" + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" } ], "fineGrainedLineages": [ @@ -4321,18 +4369,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_1)" ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_10)" - ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -4343,7 +4381,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_2)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -4354,7 +4393,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_3)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -4365,7 +4405,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_4)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -4376,7 +4417,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_5)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -4387,7 +4429,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_6)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -4398,7 +4441,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_7)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -4409,7 +4453,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_8)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -4420,14 +4465,56 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_9)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD),col_10)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29" } ] } }, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "snowflake-2023_10_06-17_59_03", + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,other_db.other_schema.table_1,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", "lastRunId": "no-run-id-provided" } }, @@ -4441,62 +4528,57 @@ "upstreams": [ { "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" + "time": 1615443388097, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "type": "TRANSFORMED" + "type": "TRANSFORMED", + "query": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b" }, { "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" + "time": 1615443388097, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)", - "type": "TRANSFORMED" + "type": "TRANSFORMED", + "query": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b" }, { "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" + "time": 1615443388097, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,other_db.other_schema.table_1,PROD)", - "type": "TRANSFORMED" + "type": "TRANSFORMED", + "query": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b" } ], "fineGrainedLineages": [ { "upstreamType": "FIELD_SET", "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)", "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,other_db.other_schema.table_1,PROD),col_1)" ], "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_1)" ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_1)" - ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_10)" - ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b" }, { "upstreamType": "FIELD_SET", @@ -4507,7 +4589,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_2)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b" }, { "upstreamType": "FIELD_SET", @@ -4518,7 +4601,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_3)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b" }, { "upstreamType": "FIELD_SET", @@ -4529,7 +4613,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_4)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b" }, { "upstreamType": "FIELD_SET", @@ -4540,7 +4625,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_5)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b" }, { "upstreamType": "FIELD_SET", @@ -4551,7 +4637,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_6)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b" }, { "upstreamType": "FIELD_SET", @@ -4562,7 +4649,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_7)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b" }, { "upstreamType": "FIELD_SET", @@ -4573,7 +4661,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_8)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b" }, { "upstreamType": "FIELD_SET", @@ -4584,14 +4673,27 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_9)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD),col_10)" + ], + "confidenceScore": 1.0, + "query": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b" } ] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", "lastRunId": "no-run-id-provided" } }, @@ -4605,11 +4707,16 @@ "upstreams": [ { "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" + "time": 1615443388097, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "type": "TRANSFORMED" + "type": "TRANSFORMED", + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" } ], "fineGrainedLineages": [ @@ -4622,18 +4729,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_1)" ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" - ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" }, { "upstreamType": "FIELD_SET", @@ -4644,7 +4741,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_2)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" }, { "upstreamType": "FIELD_SET", @@ -4655,7 +4753,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_3)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" }, { "upstreamType": "FIELD_SET", @@ -4666,7 +4765,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_4)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" }, { "upstreamType": "FIELD_SET", @@ -4677,7 +4777,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_5)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" }, { "upstreamType": "FIELD_SET", @@ -4688,7 +4789,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_6)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" }, { "upstreamType": "FIELD_SET", @@ -4699,7 +4801,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_7)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" }, { "upstreamType": "FIELD_SET", @@ -4710,7 +4813,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_8)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" }, { "upstreamType": "FIELD_SET", @@ -4721,14 +4825,83 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_9)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "confidenceScore": 1.0, + "query": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9" } ] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_10 SELECT * FROM TEST_DB.TEST_SCHEMA.TABLE_2", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_2 SELECT * FROM TEST_DB.TEST_SCHEMA.TABLE_2", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", "lastRunId": "no-run-id-provided" } }, @@ -4742,11 +4915,16 @@ "upstreams": [ { "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" + "time": 1615443388097, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "type": "TRANSFORMED" + "type": "TRANSFORMED", + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" } ], "fineGrainedLineages": [ @@ -4759,18 +4937,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_1)" ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_10)" - ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" }, { "upstreamType": "FIELD_SET", @@ -4781,7 +4949,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_2)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" }, { "upstreamType": "FIELD_SET", @@ -4792,7 +4961,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_3)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" }, { "upstreamType": "FIELD_SET", @@ -4803,7 +4973,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_4)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" }, { "upstreamType": "FIELD_SET", @@ -4814,7 +4985,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_5)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" }, { "upstreamType": "FIELD_SET", @@ -4825,7 +4997,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_6)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" }, { "upstreamType": "FIELD_SET", @@ -4836,7 +5009,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_7)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" }, { "upstreamType": "FIELD_SET", @@ -4847,7 +5021,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_8)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" }, { "upstreamType": "FIELD_SET", @@ -4858,14 +5033,73 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_9)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD),col_10)" + ], + "confidenceScore": 1.0, + "query": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8" } ] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", "lastRunId": "no-run-id-provided" } }, @@ -4879,11 +5113,16 @@ "upstreams": [ { "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" + "time": 1615443388097, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "type": "TRANSFORMED" + "type": "TRANSFORMED", + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" } ], "fineGrainedLineages": [ @@ -4896,18 +5135,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_1)" ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_10)" - ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" }, { "upstreamType": "FIELD_SET", @@ -4918,7 +5147,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_2)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" }, { "upstreamType": "FIELD_SET", @@ -4929,7 +5159,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_3)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" }, { "upstreamType": "FIELD_SET", @@ -4940,7 +5171,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_4)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" }, { "upstreamType": "FIELD_SET", @@ -4951,7 +5183,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_5)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" }, { "upstreamType": "FIELD_SET", @@ -4962,7 +5195,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_6)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" }, { "upstreamType": "FIELD_SET", @@ -4973,7 +5207,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_7)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" }, { "upstreamType": "FIELD_SET", @@ -4984,7 +5219,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_8)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" }, { "upstreamType": "FIELD_SET", @@ -4995,14 +5231,27 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_9)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD),col_10)" + ], + "confidenceScore": 1.0, + "query": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632" } ] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", "lastRunId": "no-run-id-provided" } }, @@ -5016,11 +5265,16 @@ "upstreams": [ { "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" + "time": 1615443388097, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "type": "TRANSFORMED" + "type": "TRANSFORMED", + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" } ], "fineGrainedLineages": [ @@ -5033,18 +5287,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_1)" ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_10)" - ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" }, { "upstreamType": "FIELD_SET", @@ -5055,7 +5299,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_2)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" }, { "upstreamType": "FIELD_SET", @@ -5066,7 +5311,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_3)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" }, { "upstreamType": "FIELD_SET", @@ -5077,7 +5323,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_4)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" }, { "upstreamType": "FIELD_SET", @@ -5088,7 +5335,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_5)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" }, { "upstreamType": "FIELD_SET", @@ -5099,7 +5347,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_6)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" }, { "upstreamType": "FIELD_SET", @@ -5110,7 +5359,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_7)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" }, { "upstreamType": "FIELD_SET", @@ -5121,7 +5371,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_8)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" }, { "upstreamType": "FIELD_SET", @@ -5132,14 +5383,55 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_9)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD),col_10)" + ], + "confidenceScore": 1.0, + "query": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2" } ] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_4 SELECT * FROM TEST_DB.TEST_SCHEMA.TABLE_2", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", "lastRunId": "no-run-id-provided" } }, @@ -5153,11 +5445,16 @@ "upstreams": [ { "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" + "time": 1615443388097, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "type": "TRANSFORMED" + "type": "TRANSFORMED", + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" } ], "fineGrainedLineages": [ @@ -5170,18 +5467,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_1)" ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_10)" - ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" }, { "upstreamType": "FIELD_SET", @@ -5192,7 +5479,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_2)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" }, { "upstreamType": "FIELD_SET", @@ -5203,7 +5491,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_3)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" }, { "upstreamType": "FIELD_SET", @@ -5214,7 +5503,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_4)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" }, { "upstreamType": "FIELD_SET", @@ -5225,7 +5515,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_5)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" }, { "upstreamType": "FIELD_SET", @@ -5236,7 +5527,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_6)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" }, { "upstreamType": "FIELD_SET", @@ -5247,7 +5539,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_7)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" }, { "upstreamType": "FIELD_SET", @@ -5258,7 +5551,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_8)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" }, { "upstreamType": "FIELD_SET", @@ -5269,14 +5563,78 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_9)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD),col_10)" + ], + "confidenceScore": 1.0, + "query": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12" } ] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_5 SELECT * FROM TEST_DB.TEST_SCHEMA.TABLE_2", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", "lastRunId": "no-run-id-provided" } }, @@ -5290,11 +5648,16 @@ "upstreams": [ { "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" + "time": 1615443388097, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "type": "TRANSFORMED" + "type": "TRANSFORMED", + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" } ], "fineGrainedLineages": [ @@ -5307,18 +5670,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_1)" ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_10)" - ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" }, { "upstreamType": "FIELD_SET", @@ -5329,7 +5682,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_2)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" }, { "upstreamType": "FIELD_SET", @@ -5340,7 +5694,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_3)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" }, { "upstreamType": "FIELD_SET", @@ -5351,7 +5706,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_4)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" }, { "upstreamType": "FIELD_SET", @@ -5362,7 +5718,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_5)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" }, { "upstreamType": "FIELD_SET", @@ -5373,7 +5730,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_6)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" }, { "upstreamType": "FIELD_SET", @@ -5384,7 +5742,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_7)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" }, { "upstreamType": "FIELD_SET", @@ -5395,7 +5754,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_8)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" }, { "upstreamType": "FIELD_SET", @@ -5406,14 +5766,50 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_9)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD),col_10)" + ], + "confidenceScore": 1.0, + "query": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01" } ] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", "lastRunId": "no-run-id-provided" } }, @@ -5427,11 +5823,16 @@ "upstreams": [ { "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" + "time": 1615443388097, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "type": "TRANSFORMED" + "type": "TRANSFORMED", + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" } ], "fineGrainedLineages": [ @@ -5444,18 +5845,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_1)" ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_10)" - ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" }, { "upstreamType": "FIELD_SET", @@ -5466,7 +5857,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_2)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" }, { "upstreamType": "FIELD_SET", @@ -5477,7 +5869,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_3)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" }, { "upstreamType": "FIELD_SET", @@ -5488,7 +5881,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_4)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" }, { "upstreamType": "FIELD_SET", @@ -5499,7 +5893,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_5)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" }, { "upstreamType": "FIELD_SET", @@ -5510,7 +5905,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_6)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" }, { "upstreamType": "FIELD_SET", @@ -5521,7 +5917,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_7)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" }, { "upstreamType": "FIELD_SET", @@ -5532,7 +5929,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_8)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" }, { "upstreamType": "FIELD_SET", @@ -5543,14 +5941,55 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_9)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD),col_10)" + ], + "confidenceScore": 1.0, + "query": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200" } ] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_3 SELECT * FROM TEST_DB.TEST_SCHEMA.TABLE_2", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", "lastRunId": "no-run-id-provided" } }, @@ -5564,11 +6003,16 @@ "upstreams": [ { "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" + "time": 1615443388097, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "type": "TRANSFORMED" + "type": "TRANSFORMED", + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" } ], "fineGrainedLineages": [ @@ -5581,18 +6025,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_1)" ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_10)" - ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" }, { "upstreamType": "FIELD_SET", @@ -5603,7 +6037,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_2)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" }, { "upstreamType": "FIELD_SET", @@ -5614,7 +6049,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_3)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" }, { "upstreamType": "FIELD_SET", @@ -5625,7 +6061,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_4)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" }, { "upstreamType": "FIELD_SET", @@ -5636,7 +6073,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_5)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" }, { "upstreamType": "FIELD_SET", @@ -5647,7 +6085,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_6)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" }, { "upstreamType": "FIELD_SET", @@ -5658,7 +6097,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_7)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" }, { "upstreamType": "FIELD_SET", @@ -5669,7 +6109,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_8)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" }, { "upstreamType": "FIELD_SET", @@ -5680,14 +6121,50 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_9)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD),col_10)" + ], + "confidenceScore": 1.0, + "query": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4" } ] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", "lastRunId": "no-run-id-provided" } }, @@ -5701,11 +6178,16 @@ "upstreams": [ { "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" + "time": 1615443388097, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", - "type": "TRANSFORMED" + "type": "TRANSFORMED", + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" } ], "fineGrainedLineages": [ @@ -5718,18 +6200,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_1)" ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_10)" - ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" }, { "upstreamType": "FIELD_SET", @@ -5740,7 +6212,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_2)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" }, { "upstreamType": "FIELD_SET", @@ -5751,7 +6224,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_3)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" }, { "upstreamType": "FIELD_SET", @@ -5762,7 +6236,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_4)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" }, { "upstreamType": "FIELD_SET", @@ -5773,7 +6248,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_5)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" }, { "upstreamType": "FIELD_SET", @@ -5784,7 +6260,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_6)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" }, { "upstreamType": "FIELD_SET", @@ -5795,7 +6272,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_7)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" }, { "upstreamType": "FIELD_SET", @@ -5806,7 +6284,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_8)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" }, { "upstreamType": "FIELD_SET", @@ -5817,14 +6296,27 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_9)" ], - "confidenceScore": 1.0 + "confidenceScore": 1.0, + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD),col_10)" + ], + "confidenceScore": 1.0, + "query": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37" } ] } }, "systemMetadata": { - "lastObserved": 1654621200000, - "runId": "snowflake-2022_06_07-17_00_00", + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", "lastRunId": "no-run-id-provided" } }, @@ -5974,18 +6466,46 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "entityType": "query", + "entityUrn": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12", "changeType": "UPSERT", - "aspectName": "datasetUsageStatistics", + "aspectName": "queryProperties", "aspect": { "json": { - "timestampMillis": 1654473600000, - "eventGranularity": { - "unit": "DAY", - "multiple": 1 + "statement": { + "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_6 SELECT * FROM TEST_DB.TEST_SCHEMA.TABLE_2", + "language": "SQL" }, - "partitionSpec": { + "source": "SYSTEM", + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetUsageStatistics", + "aspect": { + "json": { + "timestampMillis": 1654473600000, + "eventGranularity": { + "unit": "DAY", + "multiple": 1 + }, + "partitionSpec": { "type": "FULL_TABLE", "partition": "FULL_TABLE_SNAPSHOT" }, @@ -6002,6 +6522,29 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", @@ -6031,6 +6574,34 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_8 SELECT * FROM TEST_DB.TEST_SCHEMA.TABLE_2", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", @@ -6060,6 +6631,108 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "create view view_1 as select * from table_1", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1709290787484, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_1,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_9 SELECT * FROM TEST_DB.TEST_SCHEMA.TABLE_2", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", @@ -6089,6 +6762,29 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", @@ -6118,6 +6814,34 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "INSERT INTO TEST_DB.TEST_SCHEMA.TABLE_7 SELECT * FROM TEST_DB.TEST_SCHEMA.TABLE_2", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1654473600000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)", @@ -6147,6 +6871,80 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "create view view_2 as select * from table_2", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1709290787501, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.view_2,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", @@ -6485,5 +7283,197 @@ "runId": "snowflake-2022_06_07-17_00_00", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_01-16_29_41", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json index 5e55860483d24..7c39a53e243c1 100644 --- a/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json +++ b/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json @@ -1113,11 +1113,16 @@ "upstreams": [ { "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { "time": 0, - "actor": "urn:li:corpuser:unknown" + "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD)", - "type": "VIEW" + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_1%2CPROD%29" } ], "fineGrainedLineages": [ @@ -1130,18 +1135,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_1)" ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_10)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_10)" - ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_1%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -1152,7 +1147,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_2)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_1%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -1163,7 +1159,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_3)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_1%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -1174,7 +1171,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_4)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_1%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -1185,7 +1183,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_5)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_1%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -1196,7 +1195,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_6)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_1%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -1207,7 +1207,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_7)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_1%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -1218,7 +1219,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_8)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_1%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -1229,7 +1231,20 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_9)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_1%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD),col_10)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_1%2CPROD%29" } ] } @@ -3841,6 +3856,57 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "create view view_1 as select * from table_1", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1654621200000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_1,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_1,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD)", @@ -3851,11 +3917,16 @@ "upstreams": [ { "auditStamp": { + "time": 1654621200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { "time": 0, - "actor": "urn:li:corpuser:unknown" + "actor": "urn:li:corpuser:_ingestion" }, "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD)", - "type": "VIEW" + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_2%2CPROD%29" } ], "fineGrainedLineages": [ @@ -3868,18 +3939,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_1)" ], - "confidenceScore": 1.0 - }, - { - "upstreamType": "FIELD_SET", - "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_10)" - ], - "downstreamType": "FIELD", - "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_10)" - ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_2%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -3890,7 +3951,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_2)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_2%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -3901,7 +3963,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_3)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_2%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -3912,7 +3975,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_4)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_2%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -3923,7 +3987,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_5)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_2%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -3934,7 +3999,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_6)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_2%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -3945,7 +4011,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_7)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_2%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -3956,7 +4023,8 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_8)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_2%2CPROD%29" }, { "upstreamType": "FIELD_SET", @@ -3967,7 +4035,71 @@ "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_9)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_2%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD),col_10)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD),col_10)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_2%2CPROD%29" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_2%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "create view view_2 as select * from table_2", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1654621200000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_2%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.table_2,PROD)" } ] } @@ -3977,5 +4109,37 @@ "runId": "snowflake-2022_06_07-17_00_00", "lastRunId": "no-run-id-provided" } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_2%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00", + "lastRunId": "no-run-id-provided" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/snowflake/test_snowflake.py b/metadata-ingestion/tests/integration/snowflake/test_snowflake.py index 39a62056a7e4a..88354ba74c417 100644 --- a/metadata-ingestion/tests/integration/snowflake/test_snowflake.py +++ b/metadata-ingestion/tests/integration/snowflake/test_snowflake.py @@ -162,6 +162,7 @@ def test_snowflake_basic(pytestconfig, tmp_path, mock_time, mock_datahub_graph): pipeline.run() pipeline.pretty_print_summary() pipeline.raise_from_status() + assert not pipeline.source.get_report().warnings # Verify the output. diff --git a/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py b/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py index 4b0dd2b1045a3..65c259e8acdc3 100644 --- a/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py +++ b/metadata-ingestion/tests/integration/snowflake/test_snowflake_failures.py @@ -1,5 +1,4 @@ from datetime import datetime, timezone -from typing import cast from unittest import mock from freezegun import freeze_time @@ -262,35 +261,3 @@ def test_snowflake_missing_snowflake_operations_permission_causes_pipeline_failu pipeline = Pipeline(snowflake_pipeline_config) pipeline.run() assert "usage-permission-error" in pipeline.source.get_report().failures.keys() - - -@freeze_time(FROZEN_TIME) -def test_snowflake_unexpected_snowflake_view_lineage_error_causes_pipeline_warning( - pytestconfig, - snowflake_pipeline_config, -): - with mock.patch("snowflake.connector.connect") as mock_connect: - sf_connection = mock.MagicMock() - sf_cursor = mock.MagicMock() - mock_connect.return_value = sf_connection - sf_connection.cursor.return_value = sf_cursor - - # Error in getting view lineage - sf_cursor.execute.side_effect = query_permission_error_override( - default_query_results, - [snowflake_query.SnowflakeQuery.view_dependencies_v2()], - "Unexpected Error", - ) - - snowflake_pipeline_config1 = snowflake_pipeline_config.copy() - config = cast( - SnowflakeV2Config, - cast(PipelineConfig, snowflake_pipeline_config1).source.config, - ) - config.include_table_lineage = True - config.include_view_lineage = True - - pipeline = Pipeline(snowflake_pipeline_config1) - pipeline.run() - pipeline.raise_from_status() # pipeline should not fail - assert "view-upstream-lineage" in pipeline.source.get_report().warnings.keys() diff --git a/metadata-ingestion/tests/unit/config/test_datetime_parser.py b/metadata-ingestion/tests/unit/config/test_datetime_parser.py index ad88677fe8844..7b1a1f83f8373 100644 --- a/metadata-ingestion/tests/unit/config/test_datetime_parser.py +++ b/metadata-ingestion/tests/unit/config/test_datetime_parser.py @@ -19,6 +19,10 @@ def test_user_time_parser(): 2022, 1, 1, 3, 2, 3, tzinfo=timezone.utc ) + assert parse_user_datetime("2024-03-01 00:46:33.000 -0800") == datetime( + 2024, 3, 1, 8, 46, 33, tzinfo=timezone.utc + ) + # Times with no timestamp are assumed to be in UTC. assert parse_user_datetime("2022-01-01 01:02:03") == datetime( 2022, 1, 1, 1, 2, 3, tzinfo=timezone.utc From e963047dd08810a31e7e5117639ad4f5b4e0d247 Mon Sep 17 00:00:00 2001 From: dushayntAW <158567391+dushayntAW@users.noreply.github.com> Date: Thu, 7 Mar 2024 14:44:06 +0530 Subject: [PATCH 4/4] fix(ingest/unity): creating group urn in case of group (#9951) --- .../src/datahub/ingestion/source/unity/proxy.py | 10 ++++++++++ .../src/datahub/ingestion/source/unity/source.py | 11 +++++++++++ 2 files changed, 21 insertions(+) diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py index b414f3f188c23..20aa10305fa8f 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py +++ b/metadata-ingestion/src/datahub/ingestion/source/unity/proxy.py @@ -187,6 +187,16 @@ def service_principals(self) -> Iterable[ServicePrincipal]: if optional_sp: yield optional_sp + def groups(self): + """ + fetch the list of the groups belongs to the workspace, using the workspace client + create the list of group's display name, iterating through the list of groups fetched by the workspace client + """ + group_list: List[Optional[str]] = [] + for group in self._workspace_client.groups.list(): + group_list.append(group.display_name) + return group_list + def workspace_notebooks(self) -> Iterable[Notebook]: for obj in self._workspace_client.workspace.list("/", recursive=True): if obj.object_type == ObjectType.NOTEBOOK and obj.object_id and obj.path: diff --git a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py index ac77cd7e5e2be..ad5a75c4d73d4 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/unity/source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/unity/source.py @@ -9,6 +9,7 @@ make_dataplatform_instance_urn, make_dataset_urn_with_platform_instance, make_domain_urn, + make_group_urn, make_schema_field_urn, make_user_urn, ) @@ -184,6 +185,7 @@ def __init__(self, ctx: PipelineContext, config: UnityCatalogSourceConfig): # Global map of service principal application id -> ServicePrincipal self.service_principals: Dict[str, ServicePrincipal] = {} + self.groups: List[str] = [] # Global set of table refs self.table_refs: Set[TableReference] = set() self.view_refs: Set[TableReference] = set() @@ -256,6 +258,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: if self.config.include_ownership: self.report.report_ingestion_stage_start("Ingest service principals") self.build_service_principal_map() + self.build_groups_map() if self.config.include_notebooks: self.report.report_ingestion_stage_start("Ingest notebooks") yield from self.process_notebooks() @@ -315,6 +318,12 @@ def build_service_principal_map(self) -> None: "service-principals", f"Unable to fetch service principals: {e}" ) + def build_groups_map(self) -> None: + try: + self.groups += self.unity_catalog_api_proxy.groups() + except Exception as e: + self.report.report_warning("groups", f"Unable to fetch groups: {e}") + def process_notebooks(self) -> Iterable[MetadataWorkUnit]: for notebook in self.unity_catalog_api_proxy.workspace_notebooks(): if not self.config.notebook_pattern.allowed(notebook.path): @@ -600,6 +609,8 @@ def _get_domain_aspect(self, dataset_name: str) -> Optional[DomainsClass]: def get_owner_urn(self, user: Optional[str]) -> Optional[str]: if self.config.include_ownership and user is not None: + if user in self.groups: + return make_group_urn(user) return self.gen_user_urn(user) return None