Skip to content

Commit

Permalink
fix(ingest/powerbi): change default for use_powerbi_email (#11742)
Browse files Browse the repository at this point in the history 
Co-authored-by: sid-acryl <[email protected]>
  • Loading branch information
@hsheth2 @sid-acryl
hsheth2 and sid-acryl authored Nov 6, 2024
1 parent fa97f66 commit d5d4810
Show file tree
Hide file tree
Showing 15 changed files with 621 additions and 618 deletions.
1 change: 1 addition & 0 deletions docs/how/updating-datahub.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
`datahub delete --platform powerbi --soft` and then re-ingest with the latest CLI version, ensuring the `include_workspace_name_in_dataset_urn` configuration is set to true.

- #11701: The Fivetran `sources_to_database` field is deprecated in favor of setting directly within `sources_to_platform_instance.<key>.database`.
- #11742: For PowerBi ingestion, `use_powerbi_email` is now enabled by default when extracting ownership information.

### Breaking Changes

Expand Down
4 changes: 3 additions & 1 deletion metadata-ingestion/src/datahub/ingestion/source/powerbi/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,9 @@ class OwnershipMapping(ConfigModel):
default=True, description="Whether ingest PowerBI user as Datahub Corpuser"
)
use_powerbi_email: bool = pydantic.Field(
default=False,
# TODO: Deprecate and remove this config, since the non-email format
# doesn't make much sense.
default=True,
description="Use PowerBI User email to ingest as corpuser, default is powerbi user identifier",
)
remove_email_suffix: bool = pydantic.Field(
Expand Down
12 changes: 6 additions & 6 deletions metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
[
{
"entityType": "corpuser",
"entityUrn": "urn:li:corpuser:users.[email protected]",
"entityUrn": "urn:li:corpuser:[email protected]",
"changeType": "UPSERT",
"aspectName": "corpUserKey",
"aspect": {
Expand All @@ -17,7 +17,7 @@
},
{
"entityType": "corpuser",
"entityUrn": "urn:li:corpuser:users.[email protected]",
"entityUrn": "urn:li:corpuser:[email protected]",
"changeType": "UPSERT",
"aspectName": "corpUserKey",
"aspect": {
Expand Down Expand Up @@ -402,11 +402,11 @@
"json": {
"owners": [
{
"owner": "urn:li:corpuser:users.[email protected]",
"owner": "urn:li:corpuser:[email protected]",
"type": "NONE"
},
{
"owner": "urn:li:corpuser:users.[email protected]",
"owner": "urn:li:corpuser:[email protected]",
"type": "NONE"
}
],
Expand Down Expand Up @@ -445,7 +445,7 @@
},
{
"entityType": "corpuser",
"entityUrn": "urn:li:corpuser:users.[email protected]",
"entityUrn": "urn:li:corpuser:[email protected]",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
Expand All @@ -461,7 +461,7 @@
},
{
"entityType": "corpuser",
"entityUrn": "urn:li:corpuser:users.[email protected]",
"entityUrn": "urn:li:corpuser:[email protected]",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
Expand Down
20 changes: 10 additions & 10 deletions metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -875,7 +875,7 @@
},
{
"entityType": "corpuser",
"entityUrn": "urn:li:corpuser:users.[email protected]",
"entityUrn": "urn:li:corpuser:[email protected]",
"changeType": "UPSERT",
"aspectName": "corpUserKey",
"aspect": {
Expand All @@ -891,7 +891,7 @@
},
{
"entityType": "corpuser",
"entityUrn": "urn:li:corpuser:users.[email protected]",
"entityUrn": "urn:li:corpuser:[email protected]",
"changeType": "UPSERT",
"aspectName": "corpUserKey",
"aspect": {
Expand Down Expand Up @@ -1302,11 +1302,11 @@
"json": {
"owners": [
{
"owner": "urn:li:corpuser:users.[email protected]",
"owner": "urn:li:corpuser:[email protected]",
"type": "NONE"
},
{
"owner": "urn:li:corpuser:users.[email protected]",
"owner": "urn:li:corpuser:[email protected]",
"type": "NONE"
}
],
Expand Down Expand Up @@ -1922,7 +1922,7 @@
},
{
"entityType": "corpuser",
"entityUrn": "urn:li:corpuser:users.[email protected]",
"entityUrn": "urn:li:corpuser:[email protected]",
"changeType": "UPSERT",
"aspectName": "corpUserKey",
"aspect": {
Expand All @@ -1938,7 +1938,7 @@
},
{
"entityType": "corpuser",
"entityUrn": "urn:li:corpuser:users.[email protected]",
"entityUrn": "urn:li:corpuser:[email protected]",
"changeType": "UPSERT",
"aspectName": "corpUserKey",
"aspect": {
Expand Down Expand Up @@ -2074,11 +2074,11 @@
"json": {
"owners": [
{
"owner": "urn:li:corpuser:users.[email protected]",
"owner": "urn:li:corpuser:[email protected]",
"type": "NONE"
},
{
"owner": "urn:li:corpuser:users.[email protected]",
"owner": "urn:li:corpuser:[email protected]",
"type": "NONE"
}
],
Expand Down Expand Up @@ -2117,7 +2117,7 @@
},
{
"entityType": "corpuser",
"entityUrn": "urn:li:corpuser:users.[email protected]",
"entityUrn": "urn:li:corpuser:[email protected]",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
Expand All @@ -2133,7 +2133,7 @@
},
{
"entityType": "corpuser",
"entityUrn": "urn:li:corpuser:users.[email protected]",
"entityUrn": "urn:li:corpuser:[email protected]",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
Expand Down
132 changes: 66 additions & 66 deletions metadata-ingestion/tests/integration/powerbi/golden_test_cll.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,22 +17,6 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "corpuser",
"entityUrn": "urn:li:corpuser:[email protected]",
"changeType": "UPSERT",
"aspectName": "corpUserKey",
"aspect": {
"json": {
"username": "[email protected]"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "powerbi-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
Expand Down Expand Up @@ -71,22 +55,6 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "corpuser",
"entityUrn": "urn:li:corpuser:[email protected]",
"changeType": "UPSERT",
"aspectName": "corpUserKey",
"aspect": {
"json": {
"username": "[email protected]"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "powerbi-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.public_issue_history,DEV)",
Expand Down Expand Up @@ -992,6 +960,38 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "corpuser",
"entityUrn": "urn:li:corpuser:[email protected]",
"changeType": "UPSERT",
"aspectName": "corpUserKey",
"aspect": {
"json": {
"username": "[email protected]"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "powerbi-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "corpuser",
"entityUrn": "urn:li:corpuser:[email protected]",
"changeType": "UPSERT",
"aspectName": "corpUserKey",
"aspect": {
"json": {
"username": "[email protected]"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "powerbi-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(powerbi,dashboards.7D668CAD-7FFC-4505-9215-655BCA5BEBAE)",
Expand Down Expand Up @@ -1159,11 +1159,11 @@
"json": {
"owners": [
{
"owner": "urn:li:corpuser:users.[email protected]",
"owner": "urn:li:corpuser:[email protected]",
"type": "NONE"
},
{
"owner": "urn:li:corpuser:users.[email protected]",
"owner": "urn:li:corpuser:[email protected]",
"type": "NONE"
}
],
Expand Down Expand Up @@ -1200,38 +1200,6 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "corpuser",
"entityUrn": "urn:li:corpuser:[email protected]",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "powerbi-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "corpuser",
"entityUrn": "urn:li:corpuser:[email protected]",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "powerbi-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:powerbi,library-dataset.job-history,DEV)",
Expand Down Expand Up @@ -1503,5 +1471,37 @@
"runId": "powerbi-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "corpuser",
"entityUrn": "urn:li:corpuser:[email protected]",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "powerbi-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "corpuser",
"entityUrn": "urn:li:corpuser:[email protected]",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "powerbi-test",
"lastRunId": "no-run-id-provided"
}
}
]
Loading

0 comments on commit d5d4810

Please sign in to comment.