From 2324b665b205de52b58e9251a9bcb052130cb4be Mon Sep 17 00:00:00 2001 From: "Ching Yi, Chan" Date: Mon, 25 Sep 2023 15:27:03 +0800 Subject: [PATCH 01/11] Add example for show disclaimer Signed-off-by: Ching Yi, Chan --- static_report/src/components/Common/Main.tsx | 1 + static_report/src/components/Overview/Overview.tsx | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/static_report/src/components/Common/Main.tsx b/static_report/src/components/Common/Main.tsx index 22afd0404..4fdcb237c 100644 --- a/static_report/src/components/Common/Main.tsx +++ b/static_report/src/components/Common/Main.tsx @@ -34,6 +34,7 @@ export function Main({ children, isSingleReport, ...props }: Props) { color={color[colorMode]} minHeight="100vh" > + diff --git a/static_report/src/components/Overview/Overview.tsx b/static_report/src/components/Overview/Overview.tsx index 3b9ceebb8..b219c1926 100644 --- a/static_report/src/components/Overview/Overview.tsx +++ b/static_report/src/components/Overview/Overview.tsx @@ -364,6 +364,14 @@ export function Overview({ singleOnly }: Props) { return ( <> + + + + Schemas and reports generated from manifest files are limited to + information in those manifests. For a more detailed report, follow + these instructions + + {singleOnly ? 'Overview' : 'Impact Summary'} From 9fcdfe137f05e7b0d4af70265fea9b593914e365 Mon Sep 17 00:00:00 2001 From: "Wei-Chun, Chang" Date: Mon, 25 Sep 2023 19:06:35 +0800 Subject: [PATCH 02/11] Add skip datasource alert Signed-off-by: Wei-Chun, Chang --- .../src/components/Common/SkipDatasource.tsx | 14 ++++++++++++++ static_report/src/components/Overview/Overview.tsx | 10 ++-------- static_report/src/pages/CRTableDetailPage.tsx | 3 ++- static_report/src/pages/SRTableDetailPage.tsx | 2 ++ 4 files changed, 20 insertions(+), 9 deletions(-) create mode 100644 static_report/src/components/Common/SkipDatasource.tsx diff --git a/static_report/src/components/Common/SkipDatasource.tsx b/static_report/src/components/Common/SkipDatasource.tsx new file mode 100644 index 000000000..1236c58e2 --- /dev/null +++ b/static_report/src/components/Common/SkipDatasource.tsx @@ -0,0 +1,14 @@ +import { Alert, AlertIcon, AlertDescription } from '@chakra-ui/react'; + +export function SkipDatasource() { + return ( + + + + Schemas and reports generated from manifest files are limited to + information in those manifests. For a more detailed report, follow these + instructions + + + ); +} diff --git a/static_report/src/components/Overview/Overview.tsx b/static_report/src/components/Overview/Overview.tsx index b219c1926..0582fbf6a 100644 --- a/static_report/src/components/Overview/Overview.tsx +++ b/static_report/src/components/Overview/Overview.tsx @@ -30,6 +30,7 @@ import { LineageGraphData } from '../../utils/dbt'; import { topologySort } from '../../utils/graph'; import { CompDbtNodeEntryItem, useReportStore } from '../../utils/store'; import { SearchTextInput } from '../Common/SearchTextInput'; +import { SkipDatasource } from '../Common/SkipDatasource'; import { ModelList } from './ModelList'; import { MetricList } from './MetricList'; import { NodeList } from './NodeList'; @@ -364,14 +365,7 @@ export function Overview({ singleOnly }: Props) { return ( <> - - - - Schemas and reports generated from manifest files are limited to - information in those manifests. For a more detailed report, follow - these instructions - - + {singleOnly ? 'Overview' : 'Impact Summary'} diff --git a/static_report/src/pages/CRTableDetailPage.tsx b/static_report/src/pages/CRTableDetailPage.tsx index d5ee500b3..8b97aedb8 100644 --- a/static_report/src/pages/CRTableDetailPage.tsx +++ b/static_report/src/pages/CRTableDetailPage.tsx @@ -25,6 +25,7 @@ import { useTableRoute } from '../utils/routes'; import { NO_VALUE } from '../components/Columns/constants'; import { TableGeneralStats } from '../components/Tables/TableMetrics/TableGeneralStats'; import { DupedTableRowsWidget } from '../components/Widgets/DupedTableRowsWidget'; +import { SkipDatasource } from '../components/Common/SkipDatasource'; export default function CRTableDetailPage() { let { tableName, uniqueId } = useTableRoute(); @@ -178,6 +179,7 @@ export default function CRTableDetailPage() { return ( + */} - diff --git a/static_report/src/pages/SRTableDetailPage.tsx b/static_report/src/pages/SRTableDetailPage.tsx index 7730af1ad..f9992e15f 100644 --- a/static_report/src/pages/SRTableDetailPage.tsx +++ b/static_report/src/pages/SRTableDetailPage.tsx @@ -12,6 +12,7 @@ import { import { TableColumnHeader } from '../components/Tables/TableColumnHeader'; import { useReportStore } from '../utils/store'; import { useTableRoute } from '../utils/routes'; +import { SkipDatasource } from '../components/Common/SkipDatasource'; export default function SRTableDetailPage() { let { tableName, uniqueId } = useTableRoute(); @@ -59,6 +60,7 @@ export default function SRTableDetailPage() { return ( <> + Date: Tue, 26 Sep 2023 11:55:45 +0800 Subject: [PATCH 03/11] Read report datasource from rawData Signed-off-by: Wei-Chun, Chang --- .../src/components/Common/SkipDatasource.tsx | 10 +++++++++- static_report/src/components/Overview/Overview.tsx | 6 +++++- static_report/src/pages/CRTableDetailPage.tsx | 2 +- static_report/src/pages/SRTableDetailPage.tsx | 5 +++-- static_report/src/sdlc/single-report-schema.ts | 1 + static_report/src/utils/store.ts | 12 ++++++++++++ 6 files changed, 31 insertions(+), 5 deletions(-) diff --git a/static_report/src/components/Common/SkipDatasource.tsx b/static_report/src/components/Common/SkipDatasource.tsx index 1236c58e2..4058c6706 100644 --- a/static_report/src/components/Common/SkipDatasource.tsx +++ b/static_report/src/components/Common/SkipDatasource.tsx @@ -1,6 +1,14 @@ import { Alert, AlertIcon, AlertDescription } from '@chakra-ui/react'; -export function SkipDatasource() { +export function SkipDatasource({ + skipDataSource, +}: { + skipDataSource: boolean | undefined; +}) { + if (!skipDataSource) { + return <>; + } + return ( diff --git a/static_report/src/components/Overview/Overview.tsx b/static_report/src/components/Overview/Overview.tsx index 0582fbf6a..5fb82bf04 100644 --- a/static_report/src/components/Overview/Overview.tsx +++ b/static_report/src/components/Overview/Overview.tsx @@ -214,6 +214,7 @@ export function Overview({ singleOnly }: Props) { tableColumnsOnly = [], lineageGraph, rawData, + reportDataSource, } = useReportStore.getState(); const isBrokenByMetrics = rawData.broken_by_metrics || false; const [sortMethod, setSortMethod] = useState('topology'); @@ -232,6 +233,9 @@ export function Overview({ singleOnly }: Props) { } }; + const skipDataSource = + reportDataSource?.base?.skip_datasource || + reportDataSource?.target?.skip_datasource; const tabItems = getTabItems(tableColumnsOnly); const resourceType = tabItems[resourceIndex].resourceType; const isNoProfiled = singleOnly @@ -365,7 +369,7 @@ export function Overview({ singleOnly }: Props) { return ( <> - + {singleOnly ? 'Overview' : 'Impact Summary'} diff --git a/static_report/src/pages/CRTableDetailPage.tsx b/static_report/src/pages/CRTableDetailPage.tsx index 8b97aedb8..61f370503 100644 --- a/static_report/src/pages/CRTableDetailPage.tsx +++ b/static_report/src/pages/CRTableDetailPage.tsx @@ -179,7 +179,7 @@ export default function CRTableDetailPage() { return ( - + ; } + const skipDataSource = reportDataSource?.base?.skip_datasource || undefined; const [, { base: data }, { columns }] = currentTableEntry; const name = data?.name; const description = data?.description || undefined; @@ -60,7 +61,7 @@ export default function SRTableDetailPage() { return ( <> - + >; tableColumnsOnly?: CompDbtNodeEntryItem[]; assertionsOnly?: ComparableData; @@ -167,6 +172,12 @@ const getReportTitle = (rawData: ComparableReport) => { return title; }; +const getReportDataSource = (rawData: ComparableReport) => { + const baseDatasource = rawData.base?.datasource ?? undefined; + const targetDatasource = rawData.input?.datasource ?? undefined; + + return { base: baseDatasource, target: targetDatasource }; +}; /** * returns an aligned, compared (base/target), and normalized entries for profiler's tables and columns, making it easier to iterate and render over them. Each entry is equipped with a 3-element entry item that contains [key, {base, target}, metadata]. @@ -379,6 +390,7 @@ export const useReportStore = create()( reportTime: getReportTime(rawData), reportDisplayTime: getReportDisplayTime(rawData), reportTitle: getReportTitle(rawData), + reportDataSource: getReportDataSource(rawData), /*Report */ reportOnly: getReportOnly(rawData), tableColumnsOnly, From e5c4cf89539bb4d306461e6e09826da682315b88 Mon Sep 17 00:00:00 2001 From: "Ching Yi, Chan" Date: Tue, 26 Sep 2023 10:37:21 +0800 Subject: [PATCH 04/11] Add skip_datasource Signed-off-by: Ching Yi, Chan --- piperider_cli/runner.py | 4 +++- tests/test_runner.py | 11 ++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/piperider_cli/runner.py b/piperider_cli/runner.py index 5d9e15df8..6aeeeeb16 100644 --- a/piperider_cli/runner.py +++ b/piperider_cli/runner.py @@ -844,7 +844,9 @@ def _slim_dbt_manifest(manifest): run_result['id'] = run_id run_result['created_at'] = datetime_to_str(created_at) git_branch, git_sha = get_git_branch() - run_result['datasource'] = dict(name=ds.name, type=ds.type_name, git_branch=git_branch, git_sha=git_sha) + run_result['datasource'] = dict(name=ds.name, type=ds.type_name, + git_branch=git_branch, git_sha=git_sha, + skip_datasource=skip_datasource_connection) decorate_with_metadata(run_result) diff --git a/tests/test_runner.py b/tests/test_runner.py index eeeb69817..f7438650e 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -1,3 +1,4 @@ +import json import os import shutil from unittest import TestCase @@ -154,8 +155,15 @@ def setUp(self) -> None: Configuration.cleanup() pass + def read_skip_datasource_value(self): + with open(os.path.join(self.dbt_project_dir, "run.json"), "r") as fh: + ds = json.loads(fh.read()).get('datasource') + result = ds['skip_datasource'] + return result + def test_skip_datasource(self): - rc = Runner.exec(skip_datasource_connection=True) + rc = Runner.exec(skip_datasource_connection=True, output=self.dbt_project_dir) + self.assertTrue(self.read_skip_datasource_value()) assert rc == 0 def test_non_skip_datasource(self): @@ -171,6 +179,7 @@ def tearDown(self) -> None: piperider_output_dir = os.path.join(self.dbt_project_dir, '.piperider', 'outputs') try: shutil.rmtree(piperider_output_dir) + os.unlink(os.path.join(self.dbt_project_dir, 'run.json')) except FileNotFoundError: pass Configuration.cleanup() From 7978c3203835da57f7f93a24fc5a6d4314a9d31e Mon Sep 17 00:00:00 2001 From: "Ching Yi, Chan" Date: Tue, 26 Sep 2023 10:40:42 +0800 Subject: [PATCH 05/11] Update json-schema Signed-off-by: Ching Yi, Chan --- piperider_cli/profiler/schema.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/piperider_cli/profiler/schema.json b/piperider_cli/profiler/schema.json index f27efbba8..3eb5f7871 100644 --- a/piperider_cli/profiler/schema.json +++ b/piperider_cli/profiler/schema.json @@ -388,6 +388,9 @@ }, "git_sha": { "type": "string" + }, + "skip_datasource": { + "type": "boolean" } } }, From a9cfb9d85cd50ca7384c104495e149eedb89209f Mon Sep 17 00:00:00 2001 From: "Wei-Chun, Chang" Date: Tue, 26 Sep 2023 18:59:02 +0800 Subject: [PATCH 06/11] Add source to report context bar Signed-off-by: Wei-Chun, Chang Co-authored-by: Ching Yi, Chan --- ...{SkipDatasource.tsx => SkipDataSource.tsx} | 10 +-- .../src/components/Overview/Overview.tsx | 4 +- .../components/Reports/ReportContextBar.tsx | 73 ++++++++++++++++++- static_report/src/pages/CRTableDetailPage.tsx | 9 ++- static_report/src/pages/SRTableDetailPage.tsx | 5 +- 5 files changed, 83 insertions(+), 18 deletions(-) rename static_report/src/components/Common/{SkipDatasource.tsx => SkipDataSource.tsx} (72%) diff --git a/static_report/src/components/Common/SkipDatasource.tsx b/static_report/src/components/Common/SkipDataSource.tsx similarity index 72% rename from static_report/src/components/Common/SkipDatasource.tsx rename to static_report/src/components/Common/SkipDataSource.tsx index 4058c6706..54664597b 100644 --- a/static_report/src/components/Common/SkipDatasource.tsx +++ b/static_report/src/components/Common/SkipDataSource.tsx @@ -1,14 +1,6 @@ import { Alert, AlertIcon, AlertDescription } from '@chakra-ui/react'; -export function SkipDatasource({ - skipDataSource, -}: { - skipDataSource: boolean | undefined; -}) { - if (!skipDataSource) { - return <>; - } - +export function SkipDataSource() { return ( diff --git a/static_report/src/components/Overview/Overview.tsx b/static_report/src/components/Overview/Overview.tsx index 5fb82bf04..221a9843e 100644 --- a/static_report/src/components/Overview/Overview.tsx +++ b/static_report/src/components/Overview/Overview.tsx @@ -30,7 +30,7 @@ import { LineageGraphData } from '../../utils/dbt'; import { topologySort } from '../../utils/graph'; import { CompDbtNodeEntryItem, useReportStore } from '../../utils/store'; import { SearchTextInput } from '../Common/SearchTextInput'; -import { SkipDatasource } from '../Common/SkipDatasource'; +import { SkipDataSource } from '../Common/SkipDataSource'; import { ModelList } from './ModelList'; import { MetricList } from './MetricList'; import { NodeList } from './NodeList'; @@ -369,7 +369,7 @@ export function Overview({ singleOnly }: Props) { return ( <> - + {skipDataSource && } {singleOnly ? 'Overview' : 'Impact Summary'} diff --git a/static_report/src/components/Reports/ReportContextBar.tsx b/static_report/src/components/Reports/ReportContextBar.tsx index 2f1f8ab12..af5b2ea30 100644 --- a/static_report/src/components/Reports/ReportContextBar.tsx +++ b/static_report/src/components/Reports/ReportContextBar.tsx @@ -1,9 +1,20 @@ -import { Box, Flex, FlexProps, Link, Text } from '@chakra-ui/react'; +import { + Box, + Flex, + FlexProps, + Icon, + Link, + Text, + Tooltip, +} from '@chakra-ui/react'; import { ReactNode } from 'react'; import { BiPlug } from 'react-icons/bi'; import { BsGearWideConnected } from 'react-icons/bs'; +import { FiInfo } from 'react-icons/fi'; import { GoGitBranch } from 'react-icons/go'; +import { TbBuildingWarehouse } from 'react-icons/tb'; import { VscGitPullRequest } from 'react-icons/vsc'; + import { Comparable, ComparisonReportSchema, @@ -31,6 +42,8 @@ export function ReportContextBar({ let gitBranch: string | undefined = undefined; let githubPr: string | undefined = undefined; let githubPrUrl: string | undefined = undefined; + let reportFrom: string | undefined = undefined; + let skipDataSource: boolean | undefined = false; if (data) { if (singleOnly) { @@ -38,6 +51,10 @@ export function ReportContextBar({ datasource = report.datasource?.name; version = report.version; gitBranch = report.datasource?.git_branch; + skipDataSource = report.datasource?.skip_datasource; + reportFrom = report.datasource?.skip_datasource + ? 'Manifest File' + : report.datasource.type; } else { const report = data as ComparisonReportSchema; const fallback = report.input ?? report.base; @@ -60,9 +77,60 @@ export function ReportContextBar({ githubPrUrl = report.metadata?.github_pr_url; } + + skipDataSource = + report.base?.datasource.skip_datasource || + report.input?.datasource.skip_datasource; + + const baseFrom = report.base?.datasource.skip_datasource + ? 'Manifest File' + : report.base?.datasource.type; + const targetFrom = report.input?.datasource.skip_datasource + ? 'Manifest File' + : report.input?.datasource.type; + + if (baseFrom !== targetFrom) { + reportFrom = `${baseFrom} ↔ ${targetFrom}`; + } else { + reportFrom = baseFrom; + } } } + const showIcon = (skipDataSource: boolean | undefined) => { + return skipDataSource ? ( + + + + ) : ( + + ); + }; + + const showReportSource = ( + reportFrom: string | undefined, + skipDataSource: boolean | undefined, + ) => { + return ( + + {showIcon(skipDataSource)} + + source: {reportFrom} + + + ); + }; + return ( )} {gitBranch && ( - + )} + {showReportSource(reportFrom, skipDataSource)} {actionArea && {actionArea}} diff --git a/static_report/src/pages/CRTableDetailPage.tsx b/static_report/src/pages/CRTableDetailPage.tsx index 61f370503..b778a4817 100644 --- a/static_report/src/pages/CRTableDetailPage.tsx +++ b/static_report/src/pages/CRTableDetailPage.tsx @@ -25,7 +25,7 @@ import { useTableRoute } from '../utils/routes'; import { NO_VALUE } from '../components/Columns/constants'; import { TableGeneralStats } from '../components/Tables/TableMetrics/TableGeneralStats'; import { DupedTableRowsWidget } from '../components/Widgets/DupedTableRowsWidget'; -import { SkipDatasource } from '../components/Common/SkipDatasource'; +import { SkipDataSource } from '../components/Common/SkipDataSource'; export default function CRTableDetailPage() { let { tableName, uniqueId } = useTableRoute(); @@ -39,7 +39,7 @@ export default function CRTableDetailPage() { }, }); - const { tableColumnsOnly = [] } = useReportStore.getState(); + const { tableColumnsOnly = [], reportDataSource } = useReportStore.getState(); const nodeKey = uniqueId ? uniqueId : `table.${tableName}`; const currentTableEntry = tableColumnsOnly.find(([key]) => key === nodeKey); @@ -47,6 +47,9 @@ export default function CRTableDetailPage() { return ; } + const skipDataSource = + reportDataSource?.base?.skip_datasource || + reportDataSource?.target?.skip_datasource; const [, { base, target }] = currentTableEntry; const fallback = target || base; @@ -179,7 +182,7 @@ export default function CRTableDetailPage() { return ( - + {skipDataSource && } - + {skipDataSource && } Date: Wed, 27 Sep 2023 00:31:16 +0800 Subject: [PATCH 07/11] Remove blank line Signed-off-by: Wei-Chun, Chang --- static_report/src/components/Common/Main.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/static_report/src/components/Common/Main.tsx b/static_report/src/components/Common/Main.tsx index 4fdcb237c..22afd0404 100644 --- a/static_report/src/components/Common/Main.tsx +++ b/static_report/src/components/Common/Main.tsx @@ -34,7 +34,6 @@ export function Main({ children, isSingleReport, ...props }: Props) { color={color[colorMode]} minHeight="100vh" > - From d64f0d3126706dae6727424364ca34d2f0f3f367 Mon Sep 17 00:00:00 2001 From: "Wei-Chun, Chang" Date: Wed, 27 Sep 2023 11:51:02 +0800 Subject: [PATCH 08/11] Adjust skip datasource data type to fit convention Signed-off-by: Wei-Chun, Chang --- .../components/Reports/ReportContextBar.tsx | 59 +++++++++---------- .../TableList/TableColumnSchemaList.tsx | 10 +++- static_report/src/pages/CRTableDetailPage.tsx | 16 +++-- static_report/src/pages/SRTableDetailPage.tsx | 7 ++- 4 files changed, 53 insertions(+), 39 deletions(-) diff --git a/static_report/src/components/Reports/ReportContextBar.tsx b/static_report/src/components/Reports/ReportContextBar.tsx index af5b2ea30..0da172059 100644 --- a/static_report/src/components/Reports/ReportContextBar.tsx +++ b/static_report/src/components/Reports/ReportContextBar.tsx @@ -1,12 +1,4 @@ -import { - Box, - Flex, - FlexProps, - Icon, - Link, - Text, - Tooltip, -} from '@chakra-ui/react'; +import { Box, Flex, FlexProps, Link, Text, Tooltip } from '@chakra-ui/react'; import { ReactNode } from 'react'; import { BiPlug } from 'react-icons/bi'; import { BsGearWideConnected } from 'react-icons/bs'; @@ -42,7 +34,8 @@ export function ReportContextBar({ let gitBranch: string | undefined = undefined; let githubPr: string | undefined = undefined; let githubPrUrl: string | undefined = undefined; - let reportFrom: string | undefined = undefined; + let baseFrom: string | undefined = undefined; + let targetFrom: string | undefined = undefined; let skipDataSource: boolean | undefined = false; if (data) { @@ -52,9 +45,9 @@ export function ReportContextBar({ version = report.version; gitBranch = report.datasource?.git_branch; skipDataSource = report.datasource?.skip_datasource; - reportFrom = report.datasource?.skip_datasource - ? 'Manifest File' - : report.datasource.type; + baseFrom = report.datasource?.skip_datasource + ? 'Manifest file' + : `${report.datasource.type} connection`; } else { const report = data as ComparisonReportSchema; const fallback = report.input ?? report.base; @@ -82,18 +75,12 @@ export function ReportContextBar({ report.base?.datasource.skip_datasource || report.input?.datasource.skip_datasource; - const baseFrom = report.base?.datasource.skip_datasource - ? 'Manifest File' - : report.base?.datasource.type; - const targetFrom = report.input?.datasource.skip_datasource - ? 'Manifest File' - : report.input?.datasource.type; - - if (baseFrom !== targetFrom) { - reportFrom = `${baseFrom} ↔ ${targetFrom}`; - } else { - reportFrom = baseFrom; - } + baseFrom = report.base?.datasource.skip_datasource + ? 'Manifest file' + : `${report.base?.datasource.type} connection`; + targetFrom = report.input?.datasource.skip_datasource + ? 'Manifest file' + : `${report.input?.datasource.type} connection`; } } @@ -104,17 +91,29 @@ export function ReportContextBar({ label="Connect PipeRider to your datasource for full schema info" placement="top-start" > - + ) : ( - + ); }; const showReportSource = ( - reportFrom: string | undefined, + singleOnly: boolean | undefined, + baseFrom: string | undefined, + targetFrom: string | undefined, skipDataSource: boolean | undefined, ) => { + let source: string | undefined = undefined; + if (singleOnly) { + source = baseFrom; + } else { + if (baseFrom !== targetFrom) { + source = `${baseFrom} ↔ ${targetFrom}`; + } else { + source = baseFrom; + } + } return ( {showIcon(skipDataSource)} @@ -125,7 +124,7 @@ export function ReportContextBar({ textOverflow="ellipsis" overflow="hidden" > - source: {reportFrom} + source: {source} ); @@ -206,7 +205,7 @@ export function ReportContextBar({ )} - {showReportSource(reportFrom, skipDataSource)} + {showReportSource(singleOnly, baseFrom, targetFrom, skipDataSource)} {actionArea && {actionArea}} diff --git a/static_report/src/components/Tables/TableList/TableColumnSchemaList.tsx b/static_report/src/components/Tables/TableList/TableColumnSchemaList.tsx index 15bd60689..6422e6b41 100644 --- a/static_report/src/components/Tables/TableList/TableColumnSchemaList.tsx +++ b/static_report/src/components/Tables/TableList/TableColumnSchemaList.tsx @@ -24,11 +24,13 @@ import { CompColEntryItem } from '../../../lib'; interface Props extends Comparable { columns?: CompColEntryItem[]; visibleDetail?: boolean; //for reuse in other pages + skipDataSource?: boolean; } export function TableColumnSchemaList({ columns, singleOnly, visibleDetail = false, + skipDataSource, }: Props) { const isNotSingle = !singleOnly; @@ -101,7 +103,9 @@ export function TableColumnSchemaList({ borderRight={isNotSingle ? '1px solid lightgray' : ''} > - {baseColumn?.schema_type ?? NO_VALUE} + {skipDataSource + ? '-' + : baseColumn?.schema_type ?? NO_VALUE} {isNotSingle ? ( @@ -128,7 +132,9 @@ export function TableColumnSchemaList({ } > - {targetColumn?.schema_type ?? NO_VALUE} + {skipDataSource + ? '-' + : targetColumn?.schema_type ?? NO_VALUE} diff --git a/static_report/src/pages/CRTableDetailPage.tsx b/static_report/src/pages/CRTableDetailPage.tsx index b778a4817..3011ad107 100644 --- a/static_report/src/pages/CRTableDetailPage.tsx +++ b/static_report/src/pages/CRTableDetailPage.tsx @@ -210,7 +210,11 @@ export default function CRTableDetailPage() { - + @@ -247,7 +251,7 @@ function ComparableGridHeader() { ); } -function TableColumnSchemaCompList({ tableEntry }) { +function TableColumnSchemaCompList({ tableEntry, skipBase, skipTarget }) { const [ , { base: baseTableColEntry, target: targetTableColEntry }, @@ -300,7 +304,7 @@ function TableColumnSchemaCompList({ tableEntry }) { borderRightColor="gray.200" > - {baseColumn?.schema_type ?? NO_VALUE} + {skipBase ? '-' : baseColumn?.schema_type ?? NO_VALUE} @@ -320,7 +324,9 @@ function TableColumnSchemaCompList({ tableEntry }) { - {targetColumn?.schema_type ?? NO_VALUE} + {skipTarget + ? '-' + : targetColumn?.schema_type ?? NO_VALUE} @@ -369,7 +375,7 @@ function TableColumnSchemaCompList({ tableEntry }) { - {column?.schema_type ?? NO_VALUE} + {skipBase ? '-' : column?.schema_type ?? NO_VALUE} diff --git a/static_report/src/pages/SRTableDetailPage.tsx b/static_report/src/pages/SRTableDetailPage.tsx index 690e5bf65..12a4f23f8 100644 --- a/static_report/src/pages/SRTableDetailPage.tsx +++ b/static_report/src/pages/SRTableDetailPage.tsx @@ -13,7 +13,6 @@ import { TableColumnHeader } from '../components/Tables/TableColumnHeader'; import { useReportStore } from '../utils/store'; import { useTableRoute } from '../utils/routes'; import { SkipDataSource } from '../components/Common/SkipDataSource'; -import { sk } from 'date-fns/locale'; export default function SRTableDetailPage() { let { tableName, uniqueId } = useTableRoute(); @@ -88,7 +87,11 @@ export default function SRTableDetailPage() { - + ); From caeb939a033c3a57e3f8764484c8811a6e11389d Mon Sep 17 00:00:00 2001 From: "Wei-Chun, Chang" Date: Wed, 27 Sep 2023 16:15:52 +0800 Subject: [PATCH 09/11] Leave empty string to schema type as initial value Signed-off-by: Wei-Chun, Chang --- piperider_cli/profiler/profiler.py | 2 +- .../TableList/TableColumnSchemaList.tsx | 10 ++++------ static_report/src/pages/CRTableDetailPage.tsx | 20 +++++++++---------- static_report/src/pages/SRTableDetailPage.tsx | 6 +----- 4 files changed, 16 insertions(+), 22 deletions(-) diff --git a/piperider_cli/profiler/profiler.py b/piperider_cli/profiler/profiler.py index a5f5e1ba9..6771eb51a 100644 --- a/piperider_cli/profiler/profiler.py +++ b/piperider_cli/profiler/profiler.py @@ -304,7 +304,7 @@ async def _collect_metadata_from_dbt_manifest(self, dbt_manifest, metadata_subje columns[key] = dict( name=name, type='other', - schema_type='other', + schema_type='', description=description) profiled_tables[ref_id] = dict(name=table_name, columns=columns, ref_id=ref_id) completed += 1 diff --git a/static_report/src/components/Tables/TableList/TableColumnSchemaList.tsx b/static_report/src/components/Tables/TableList/TableColumnSchemaList.tsx index 6422e6b41..c0727df10 100644 --- a/static_report/src/components/Tables/TableList/TableColumnSchemaList.tsx +++ b/static_report/src/components/Tables/TableList/TableColumnSchemaList.tsx @@ -24,13 +24,11 @@ import { CompColEntryItem } from '../../../lib'; interface Props extends Comparable { columns?: CompColEntryItem[]; visibleDetail?: boolean; //for reuse in other pages - skipDataSource?: boolean; } export function TableColumnSchemaList({ columns, singleOnly, visibleDetail = false, - skipDataSource, }: Props) { const isNotSingle = !singleOnly; @@ -103,8 +101,8 @@ export function TableColumnSchemaList({ borderRight={isNotSingle ? '1px solid lightgray' : ''} > - {skipDataSource - ? '-' + {baseColumn?.schema_type === '' + ? NO_VALUE : baseColumn?.schema_type ?? NO_VALUE} @@ -132,8 +130,8 @@ export function TableColumnSchemaList({ } > - {skipDataSource - ? '-' + {targetColumn?.schema_type === '' + ? NO_VALUE : targetColumn?.schema_type ?? NO_VALUE} diff --git a/static_report/src/pages/CRTableDetailPage.tsx b/static_report/src/pages/CRTableDetailPage.tsx index 3011ad107..dd1a40f9f 100644 --- a/static_report/src/pages/CRTableDetailPage.tsx +++ b/static_report/src/pages/CRTableDetailPage.tsx @@ -210,11 +210,7 @@ export default function CRTableDetailPage() { - + @@ -251,7 +247,7 @@ function ComparableGridHeader() { ); } -function TableColumnSchemaCompList({ tableEntry, skipBase, skipTarget }) { +function TableColumnSchemaCompList({ tableEntry }) { const [ , { base: baseTableColEntry, target: targetTableColEntry }, @@ -304,7 +300,9 @@ function TableColumnSchemaCompList({ tableEntry, skipBase, skipTarget }) { borderRightColor="gray.200" > - {skipBase ? '-' : baseColumn?.schema_type ?? NO_VALUE} + {baseColumn?.schema_type === '' + ? NO_VALUE + : baseColumn?.schema_type ?? NO_VALUE} @@ -324,8 +322,8 @@ function TableColumnSchemaCompList({ tableEntry, skipBase, skipTarget }) { - {skipTarget - ? '-' + {targetColumn?.schema_type === '' + ? NO_VALUE : targetColumn?.schema_type ?? NO_VALUE} @@ -375,7 +373,9 @@ function TableColumnSchemaCompList({ tableEntry, skipBase, skipTarget }) { - {skipBase ? '-' : column?.schema_type ?? NO_VALUE} + {column?.schema_type === '' + ? NO_VALUE + : column?.schema_type ?? NO_VALUE} diff --git a/static_report/src/pages/SRTableDetailPage.tsx b/static_report/src/pages/SRTableDetailPage.tsx index 12a4f23f8..2028e0b94 100644 --- a/static_report/src/pages/SRTableDetailPage.tsx +++ b/static_report/src/pages/SRTableDetailPage.tsx @@ -87,11 +87,7 @@ export default function SRTableDetailPage() { - + ); From 5975660c569ce8ea864b7fa19e73b3d2466f980c Mon Sep 17 00:00:00 2001 From: "Wei-Chun, Chang" Date: Wed, 27 Sep 2023 16:27:03 +0800 Subject: [PATCH 10/11] Keep report store simple if no transformation Signed-off-by: Wei-Chun, Chang --- static_report/src/components/Overview/Overview.tsx | 5 ++--- static_report/src/pages/CRTableDetailPage.tsx | 6 +++--- static_report/src/pages/SRTableDetailPage.tsx | 4 ++-- static_report/src/utils/store.ts | 12 ------------ 4 files changed, 7 insertions(+), 20 deletions(-) diff --git a/static_report/src/components/Overview/Overview.tsx b/static_report/src/components/Overview/Overview.tsx index 221a9843e..737f46be5 100644 --- a/static_report/src/components/Overview/Overview.tsx +++ b/static_report/src/components/Overview/Overview.tsx @@ -214,7 +214,6 @@ export function Overview({ singleOnly }: Props) { tableColumnsOnly = [], lineageGraph, rawData, - reportDataSource, } = useReportStore.getState(); const isBrokenByMetrics = rawData.broken_by_metrics || false; const [sortMethod, setSortMethod] = useState('topology'); @@ -234,8 +233,8 @@ export function Overview({ singleOnly }: Props) { }; const skipDataSource = - reportDataSource?.base?.skip_datasource || - reportDataSource?.target?.skip_datasource; + rawData.base?.datasource.skip_datasource || + rawData.input?.datasource.skip_datasource; const tabItems = getTabItems(tableColumnsOnly); const resourceType = tabItems[resourceIndex].resourceType; const isNoProfiled = singleOnly diff --git a/static_report/src/pages/CRTableDetailPage.tsx b/static_report/src/pages/CRTableDetailPage.tsx index dd1a40f9f..9560b0e28 100644 --- a/static_report/src/pages/CRTableDetailPage.tsx +++ b/static_report/src/pages/CRTableDetailPage.tsx @@ -39,7 +39,7 @@ export default function CRTableDetailPage() { }, }); - const { tableColumnsOnly = [], reportDataSource } = useReportStore.getState(); + const { tableColumnsOnly = [], rawData } = useReportStore.getState(); const nodeKey = uniqueId ? uniqueId : `table.${tableName}`; const currentTableEntry = tableColumnsOnly.find(([key]) => key === nodeKey); @@ -48,8 +48,8 @@ export default function CRTableDetailPage() { } const skipDataSource = - reportDataSource?.base?.skip_datasource || - reportDataSource?.target?.skip_datasource; + rawData.base?.datasource.skip_datasource || + rawData.input?.datasource.skip_datasource; const [, { base, target }] = currentTableEntry; const fallback = target || base; diff --git a/static_report/src/pages/SRTableDetailPage.tsx b/static_report/src/pages/SRTableDetailPage.tsx index 2028e0b94..875cdea2a 100644 --- a/static_report/src/pages/SRTableDetailPage.tsx +++ b/static_report/src/pages/SRTableDetailPage.tsx @@ -24,7 +24,7 @@ export default function SRTableDetailPage() { }, }); - const { tableColumnsOnly = [], reportDataSource } = useReportStore.getState(); + const { tableColumnsOnly = [], rawData } = useReportStore.getState(); const tableKey = tableName || uniqueId; if (tableKey === undefined) { @@ -37,7 +37,7 @@ export default function SRTableDetailPage() { return ; } - const skipDataSource = reportDataSource?.base?.skip_datasource || undefined; + const skipDataSource = rawData.base?.datasource.skip_datasource || undefined; const [, { base: data }, { columns }] = currentTableEntry; const name = data?.name; const description = data?.description || undefined; diff --git a/static_report/src/utils/store.ts b/static_report/src/utils/store.ts index f324c62c7..f73c61cd9 100644 --- a/static_report/src/utils/store.ts +++ b/static_report/src/utils/store.ts @@ -5,7 +5,6 @@ import { ColumnSchema, ComparableData, ComparisonReportSchema, - DataSource, DbtNode, SaferSRSchema, } from '../types/index'; @@ -101,10 +100,6 @@ export interface ReportState { reportTitle?: string; reportTime?: string; reportDisplayTime?: string; - reportDataSource?: { - base: DataSource | undefined; - target: DataSource | undefined; - }; reportOnly?: ComparableData>; tableColumnsOnly?: CompDbtNodeEntryItem[]; assertionsOnly?: ComparableData; @@ -172,12 +167,6 @@ const getReportTitle = (rawData: ComparableReport) => { return title; }; -const getReportDataSource = (rawData: ComparableReport) => { - const baseDatasource = rawData.base?.datasource ?? undefined; - const targetDatasource = rawData.input?.datasource ?? undefined; - - return { base: baseDatasource, target: targetDatasource }; -}; /** * returns an aligned, compared (base/target), and normalized entries for profiler's tables and columns, making it easier to iterate and render over them. Each entry is equipped with a 3-element entry item that contains [key, {base, target}, metadata]. @@ -390,7 +379,6 @@ export const useReportStore = create()( reportTime: getReportTime(rawData), reportDisplayTime: getReportDisplayTime(rawData), reportTitle: getReportTitle(rawData), - reportDataSource: getReportDataSource(rawData), /*Report */ reportOnly: getReportOnly(rawData), tableColumnsOnly, From 85c61b655e59a857e6ccb10b5bf82ebf74d93ba5 Mon Sep 17 00:00:00 2001 From: "Wei-Chun, Chang" Date: Wed, 27 Sep 2023 16:40:35 +0800 Subject: [PATCH 11/11] Adjust context bar UI and add doc url Signed-off-by: Wei-Chun, Chang --- .../src/components/Common/SkipDataSource.tsx | 11 +++++++-- .../components/Reports/ReportContextBar.tsx | 23 ++++++++----------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/static_report/src/components/Common/SkipDataSource.tsx b/static_report/src/components/Common/SkipDataSource.tsx index 54664597b..915f0a52d 100644 --- a/static_report/src/components/Common/SkipDataSource.tsx +++ b/static_report/src/components/Common/SkipDataSource.tsx @@ -6,8 +6,15 @@ export function SkipDataSource() { Schemas and reports generated from manifest files are limited to - information in those manifests. For a more detailed report, follow these - instructions + information in those manifests. For a more detailed report, follow these{' '} + + instructions + ); diff --git a/static_report/src/components/Reports/ReportContextBar.tsx b/static_report/src/components/Reports/ReportContextBar.tsx index 0da172059..a3141ab29 100644 --- a/static_report/src/components/Reports/ReportContextBar.tsx +++ b/static_report/src/components/Reports/ReportContextBar.tsx @@ -2,7 +2,6 @@ import { Box, Flex, FlexProps, Link, Text, Tooltip } from '@chakra-ui/react'; import { ReactNode } from 'react'; import { BiPlug } from 'react-icons/bi'; import { BsGearWideConnected } from 'react-icons/bs'; -import { FiInfo } from 'react-icons/fi'; import { GoGitBranch } from 'react-icons/go'; import { TbBuildingWarehouse } from 'react-icons/tb'; import { VscGitPullRequest } from 'react-icons/vsc'; @@ -46,8 +45,8 @@ export function ReportContextBar({ gitBranch = report.datasource?.git_branch; skipDataSource = report.datasource?.skip_datasource; baseFrom = report.datasource?.skip_datasource - ? 'Manifest file' - : `${report.datasource.type} connection`; + ? 'Manifest File' + : `${report.datasource.type} Connection`; } else { const report = data as ComparisonReportSchema; const fallback = report.input ?? report.base; @@ -76,11 +75,11 @@ export function ReportContextBar({ report.input?.datasource.skip_datasource; baseFrom = report.base?.datasource.skip_datasource - ? 'Manifest file' - : `${report.base?.datasource.type} connection`; + ? 'Manifest File' + : `${report.base?.datasource.type} Connection`; targetFrom = report.input?.datasource.skip_datasource - ? 'Manifest file' - : `${report.input?.datasource.type} connection`; + ? 'Manifest File' + : `${report.input?.datasource.type} Connection`; } } @@ -91,7 +90,7 @@ export function ReportContextBar({ label="Connect PipeRider to your datasource for full schema info" placement="top-start" > - + ) : ( @@ -108,11 +107,7 @@ export function ReportContextBar({ if (singleOnly) { source = baseFrom; } else { - if (baseFrom !== targetFrom) { - source = `${baseFrom} ↔ ${targetFrom}`; - } else { - source = baseFrom; - } + source = targetFrom ?? baseFrom; } return ( @@ -124,7 +119,7 @@ export function ReportContextBar({ textOverflow="ellipsis" overflow="hidden" > - source: {source} + {source} );