From 6436f06d0844af403dcda2b891c1ab5401cdeaf4 Mon Sep 17 00:00:00 2001 From: Gabe Lyons Date: Mon, 29 Nov 2021 21:07:06 -0800 Subject: [PATCH 01/14] feat(autorender): Auto render aspects that don't have frontend components in the UI (#3597) --- .../datahub/graphql/GmsGraphQLEngine.java | 11 ++- .../graphql/WeaklyTypedAspectsResolver.java | 84 ++++++++++++++++ .../src/main/resources/entity.graphql | 58 +++++++++++ datahub-web-react/src/Mocks.tsx | 11 +++ .../containers/profile/EntityProfile.tsx | 29 +++++- .../profile/__tests__/EntityProfile.test.tsx | 96 +++++++++++++++++++ .../DynamicPropertiesTab.tsx | 49 ++++++++++ .../Entity/weaklyTypedAspects/DynamicTab.tsx | 50 ++++++++++ .../weaklyTypedAspects/DynamicTabularTab.tsx | 29 ++++++ .../weaklyTypedAspects/TableValueElement.tsx | 29 ++++++ .../src/app/entity/shared/types.ts | 2 + datahub-web-react/src/graphql/dataset.graphql | 9 ++ docs/modeling/extending-the-metadata-model.md | 6 ++ .../linkedin/metadata/models/AspectSpec.java | 11 ++- .../models/annotation/AspectAnnotation.java | 9 +- .../PluginEntityRegistryLoaderTest.java | 2 +- .../factory/graphql/GraphQLEngineFactory.java | 24 ++++- .../linkedin/entity/client/EntityClient.java | 7 ++ .../entity/client/JavaEntityClient.java | 16 ++++ .../entity/client/RestliEntityClient.java | 8 ++ 20 files changed, 527 insertions(+), 13 deletions(-) create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/WeaklyTypedAspectsResolver.java create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Entity/weaklyTypedAspects/DynamicPropertiesTab.tsx create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Entity/weaklyTypedAspects/DynamicTab.tsx create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Entity/weaklyTypedAspects/DynamicTabularTab.tsx create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Entity/weaklyTypedAspects/TableValueElement.tsx diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index 46704647bf64b..67857eb5ab06d 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -110,6 +110,7 @@ import com.linkedin.datahub.graphql.types.usage.UsageType; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.recommendation.RecommendationsService; import com.linkedin.metadata.graph.GraphClient; import com.linkedin.usage.UsageClient; @@ -149,6 +150,7 @@ public class GmsGraphQLEngine { private final EntityService entityService; private final AnalyticsService analyticsService; private final RecommendationsService recommendationsService; + private final EntityRegistry entityRegistry; private final TokenService tokenService; private final DatasetType datasetType; @@ -203,6 +205,7 @@ public GmsGraphQLEngine() { null, null, null, + null, null); } @@ -213,7 +216,9 @@ public GmsGraphQLEngine( final AnalyticsService analyticsService, final EntityService entityService, final RecommendationsService recommendationsService, - final TokenService tokenService) { + final TokenService tokenService, + final EntityRegistry entityRegistry + ) { this.entityClient = entityClient; this.graphClient = graphClient; @@ -223,6 +228,7 @@ public GmsGraphQLEngine( this.entityService = entityService; this.recommendationsService = recommendationsService; this.tokenService = tokenService; + this.entityRegistry = entityRegistry; this.datasetType = new DatasetType(entityClient); this.corpUserType = new CorpUserType(entityClient); @@ -559,6 +565,9 @@ private void configureDatasetResolvers(final RuntimeWiring.Builder builder) { .dataFetcher("schemaMetadata", new AuthenticatedResolver<>( new AspectResolver()) ) + .dataFetcher("aspects", new AuthenticatedResolver<>( + new WeaklyTypedAspectsResolver(entityClient, entityRegistry)) + ) .dataFetcher("subTypes", new AuthenticatedResolver(new SubTypesResolver( this.entityClient, "dataset", diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/WeaklyTypedAspectsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/WeaklyTypedAspectsResolver.java new file mode 100644 index 0000000000000..6372e46b11a44 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/WeaklyTypedAspectsResolver.java @@ -0,0 +1,84 @@ +package com.linkedin.datahub.graphql; + +import com.linkedin.data.DataMap; + +import com.linkedin.data.codec.JacksonDataCodec; +import com.linkedin.datahub.graphql.generated.AspectParams; +import com.linkedin.datahub.graphql.generated.AspectRenderSpec; +import com.linkedin.datahub.graphql.generated.Entity; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.RawAspect; +import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.models.AspectSpec; +import com.linkedin.metadata.models.EntitySpec; +import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.r2.RemoteInvocationException; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; + + +@Slf4j +@AllArgsConstructor +public class WeaklyTypedAspectsResolver implements DataFetcher>> { + + private final EntityClient _entityClient; + private final EntityRegistry _entityRegistry; + private static final JacksonDataCodec CODEC = new JacksonDataCodec(); + + private boolean shouldReturnAspect(AspectSpec aspectSpec, AspectParams params) { + return !params.getAutoRenderOnly() || aspectSpec.isAutoRender(); + } + + @Override + public CompletableFuture> get(DataFetchingEnvironment environment) throws Exception { + return CompletableFuture.supplyAsync(() -> { + List results = new ArrayList<>(); + + final QueryContext context = environment.getContext(); + final String urn = ((Entity) environment.getSource()).getUrn(); + final EntityType entityType = ((Entity) environment.getSource()).getType(); + final String entityTypeName = EntityTypeMapper.getName(entityType); + final AspectParams input = bindArgument(environment.getArgument("input"), AspectParams.class); + + EntitySpec entitySpec = _entityRegistry.getEntitySpec(entityTypeName); + entitySpec.getAspectSpecs().stream().filter(aspectSpec -> shouldReturnAspect(aspectSpec, input)).forEach(aspectSpec -> { + try { + RawAspect result = new RawAspect(); + DataMap resolvedAspect = + _entityClient.getRawAspect(urn, aspectSpec.getName(), 0L, context.getAuthentication()); + if (resolvedAspect == null || resolvedAspect.keySet().size() != 1) { + return; + } + + DataMap aspectPayload = resolvedAspect.getDataMap(resolvedAspect.keySet().iterator().next()); + + result.setPayload(CODEC.mapToString(aspectPayload)); + result.setAspectName(aspectSpec.getName()); + + DataMap renderSpec = aspectSpec.getRenderSpec(); + + AspectRenderSpec resultRenderSpec = new AspectRenderSpec(); + + resultRenderSpec.setDisplayType(renderSpec.getString("displayType")); + resultRenderSpec.setDisplayName(renderSpec.getString("displayName")); + resultRenderSpec.setKey(renderSpec.getString("key")); + result.setRenderSpec(resultRenderSpec); + + results.add(result); + } catch (IOException | RemoteInvocationException e) { + throw new RuntimeException("Failed to fetch aspect " + aspectSpec.getName() + " for urn " + urn + " ", e); + } + }); + return results; + }); + } +} diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index ee93f5dc1c8c9..37012bda3c7d5 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -611,6 +611,64 @@ type Dataset implements EntityWithRelationships & Entity { View related properties. Only relevant if subtypes field contains VIEW. """ viewProperties: ViewProperties + + """ + Experimental API. + For fetching extra entities that do not have custom UI code yet + """ + aspects(input: AspectParams): [RawAspect!] +} + + +""" +Params to configure what list of aspects should be fetched by the aspects property +""" +input AspectParams { + """ + Only fetch auto render aspects + """ + autoRenderOnly: Boolean +} + + +""" +Payload representing data about a single aspect +""" +type RawAspect { + """ + The name of the aspect + """ + aspectName: String! + + """ + JSON string containing the aspect's payload + """ + payload: String + + """ + Details for the frontend on how the raw aspect should be rendered + """ + renderSpec: AspectRenderSpec +} + +""" +Details for the frontend on how the raw aspect should be rendered +""" +type AspectRenderSpec { + """ + Format the aspect should be displayed in for the UI. Powered by the renderSpec annotation on the aspect model + """ + displayType: String + + """ + Name to refer to the aspect type by for the UI. Powered by the renderSpec annotation on the aspect model + """ + displayName: String + + """ + Field in the aspect payload to index into for rendering. + """ + key: String } """ diff --git a/datahub-web-react/src/Mocks.tsx b/datahub-web-react/src/Mocks.tsx index abd5e590d27b8..dbf4b919653d8 100644 --- a/datahub-web-react/src/Mocks.tsx +++ b/datahub-web-react/src/Mocks.tsx @@ -420,6 +420,17 @@ export const dataset3 = { ], subTypes: null, viewProperties: null, + autoRenderAspects: [ + { + aspectName: 'autoRenderAspect', + payload: '{ "values": [{ "autoField1": "autoValue1", "autoField2": "autoValue2" }] }', + renderSpec: { + displayType: 'tabular', + displayName: 'Auto Render Aspect Custom Tab Name', + key: 'values', + }, + }, + ], } as Dataset; export const dataset4 = { diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx index aabcf8644a001..318679c56ec4d 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/EntityProfile.tsx @@ -17,6 +17,7 @@ import useIsLineageMode from '../../../../lineage/utils/useIsLineageMode'; import { useEntityRegistry } from '../../../../useEntityRegistry'; import LineageExplorer from '../../../../lineage/LineageExplorer'; import CompactContext from '../../../../shared/CompactContext'; +import DynamicTab from '../../tabs/Entity/weaklyTypedAspects/DynamicTab'; type Props = { urn: string; @@ -105,7 +106,6 @@ export const EntityProfile = ({ tabs, sidebarSections, }: Props): JSX.Element => { - const routedTab = useRoutedTab(tabs); const isLineageMode = useIsLineageMode(); const entityRegistry = useEntityRegistry(); const history = useHistory(); @@ -131,7 +131,9 @@ export const EntityProfile = ({ [history, entityType, urn, entityRegistry], ); - const { loading, error, data, refetch } = useEntityQuery({ variables: { urn } }); + const { loading, error, data, refetch } = useEntityQuery({ + variables: { urn }, + }); const [updateEntity] = useUpdateQuery({ onCompleted: () => refetch(), @@ -142,6 +144,24 @@ export const EntityProfile = ({ const lineage = entityData ? entityRegistry.getLineageVizConfig(entityType, entityData) : undefined; + const autoRenderTabs: EntityTab[] = + entityData?.autoRenderAspects?.map((aspect) => ({ + name: aspect.renderSpec?.displayName || aspect.aspectName, + component: () => ( + + ), + display: { + visible: () => true, + enabled: () => true, + }, + })) || []; + + const routedTab = useRoutedTab([...tabsWithDefaults, ...autoRenderTabs]); + if (isCompact) { return ( ({
- +
{routedTab && } diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/__tests__/EntityProfile.test.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/__tests__/EntityProfile.test.tsx index fb37e7558c2c8..8c0d0b12db211 100644 --- a/datahub-web-react/src/app/entity/shared/containers/profile/__tests__/EntityProfile.test.tsx +++ b/datahub-web-react/src/app/entity/shared/containers/profile/__tests__/EntityProfile.test.tsx @@ -371,4 +371,100 @@ describe('EntityProfile', () => { await waitFor(() => expect(getByText('Tags')).toBeInTheDocument()); await waitFor(() => expect(getByText('abc-sample-tag')).toBeInTheDocument()); }); + + it('renders autorender aspects', async () => { + const { getByText } = render( + + + ({})} + tabs={[ + { + name: 'Schema', + component: SchemaTab, + }, + { + name: 'Documentation', + component: DocumentationTab, + }, + { + name: 'Properties', + component: PropertiesTab, + }, + { + name: 'Lineage', + component: LineageTab, + display: { + visible: (_, _1) => true, + enabled: (_, dataset: GetDatasetQuery) => + (dataset?.dataset?.upstreamLineage?.entities?.length || 0) > 0 || + (dataset?.dataset?.downstreamLineage?.entities?.length || 0) > 0, + }, + }, + { + name: 'Queries', + component: QueriesTab, + display: { + visible: (_, _1) => true, + enabled: (_, dataset: GetDatasetQuery) => + (dataset?.dataset?.usageStats?.buckets?.length && true) || false, + }, + }, + { + name: 'Stats', + component: StatsTab, + display: { + enabled: (_, _1) => true, + visible: (_, dataset: GetDatasetQuery) => + (dataset?.dataset?.datasetProfiles?.length && true) || + (dataset?.dataset?.usageStats?.buckets?.length && true) || + false, + }, + }, + ]} + sidebarSections={[ + { + component: SidebarAboutSection, + }, + { + component: SidebarStatsSection, + display: { + visible: (_, dataset: GetDatasetQuery) => + (dataset?.dataset?.datasetProfiles?.length && true) || + (dataset?.dataset?.usageStats?.buckets?.length && true) || + false, + }, + }, + { + component: SidebarTagsSection, + }, + { + component: SidebarOwnerSection, + }, + ]} + /> + + , + ); + + // find the tab name + await waitFor(() => expect(getByText('Auto Render Aspect Custom Tab Name')).toBeInTheDocument()); + + // open the custom tab + fireEvent( + getByText('Auto Render Aspect Custom Tab Name'), + new MouseEvent('click', { + bubbles: true, + cancelable: true, + }), + ); + + // find the tab contents + await waitFor(() => expect(getByText('autoField1')).toBeInTheDocument()); + await waitFor(() => expect(getByText('autoValue1')).toBeInTheDocument()); + }); }); diff --git a/datahub-web-react/src/app/entity/shared/tabs/Entity/weaklyTypedAspects/DynamicPropertiesTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Entity/weaklyTypedAspects/DynamicPropertiesTab.tsx new file mode 100644 index 0000000000000..81a9398afa418 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Entity/weaklyTypedAspects/DynamicPropertiesTab.tsx @@ -0,0 +1,49 @@ +import React from 'react'; +import { Typography } from 'antd'; +import styled from 'styled-components'; + +import { StyledTable } from '../../../components/styled/StyledTable'; +import { ANTD_GRAY } from '../../../constants'; +import TableValueElement from './TableValueElement'; + +type Props = { + payload: string | undefined | null; +}; + +const NameText = styled(Typography.Text)` + font-weight: 600; + font-size: 12px; + color: ${ANTD_GRAY[9]}; +`; + +export default function DynamicTabularTab({ payload: rawPayload }: Props) { + const aspectData = JSON.parse(rawPayload || '{}'); + const transformedRowData = Object.keys(aspectData).map((key) => ({ key, value: aspectData[key] })); + + const propertyTableColumns = [ + { + width: 210, + title: 'Name', + dataIndex: 'key', + sorter: (a, b) => a?.key.localeCompare(b?.key || '') || 0, + defaultSortOrder: 'ascend', + render: (name: string) => {name}, + }, + { + title: 'Value', + dataIndex: 'value', + render: (value: string) => , + }, + ]; + + return ( + + ); +} diff --git a/datahub-web-react/src/app/entity/shared/tabs/Entity/weaklyTypedAspects/DynamicTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Entity/weaklyTypedAspects/DynamicTab.tsx new file mode 100644 index 0000000000000..419a2ae9f37ea --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Entity/weaklyTypedAspects/DynamicTab.tsx @@ -0,0 +1,50 @@ +import React from 'react'; +import { Typography } from 'antd'; +import styled from 'styled-components'; +import { Prism as SyntaxHighlighter } from 'react-syntax-highlighter'; + +import { ANTD_GRAY } from '../../../constants'; +import DynamicTabularTab from './DynamicTabularTab'; +import DynamicPropertiesTab from './DynamicPropertiesTab'; +import { AspectRenderSpec } from '../../../../../../types.generated'; + +type Props = { + payload: string | undefined | null; + type: string | undefined | null; + renderSpec: AspectRenderSpec | undefined | null; +}; + +const QueryText = styled(Typography.Paragraph)` + margin: 20px; + &&& pre { + background-color: ${ANTD_GRAY[2]}; + border: none; + } +`; + +// NOTE: Yes, using `!important` is a shame. However, the SyntaxHighlighter is applying styles directly +// to the component, so there's no way around this +const NestedSyntax = styled(SyntaxHighlighter)` + background-color: transparent !important; + border: none !important; +`; + +export default function DynamicTab({ renderSpec, payload, type }: Props) { + if (type === 'tabular') { + return ; + } + if (type === 'properties') { + return ; + } + + // Default fallback behavior + return ( + <> + + 
+                    {JSON.stringify(JSON.parse(payload || '{}'), null, 2)}
+
+ + + ); +} diff --git a/datahub-web-react/src/app/entity/shared/tabs/Entity/weaklyTypedAspects/DynamicTabularTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Entity/weaklyTypedAspects/DynamicTabularTab.tsx new file mode 100644 index 0000000000000..ea30dcc0b4605 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Entity/weaklyTypedAspects/DynamicTabularTab.tsx @@ -0,0 +1,29 @@ +import React from 'react'; +import { StyledTable } from '../../../components/styled/StyledTable'; +import TableValueElement from './TableValueElement'; + +type Props = { + payload: string | undefined | null; + tableKey: string | undefined | null; +}; + +export default function DynamicTabularTab({ payload: rawPayload, tableKey }: Props) { + const aspectData = JSON.parse(rawPayload || '{}'); + const rowData = tableKey ? aspectData[tableKey] : aspectData[Object.keys(aspectData)[0]]; + const columns = Object.keys(rowData[0]).map((columnName) => ({ + title: columnName, + dataIndex: columnName, + render: (value: any) => , + })); + + return ( + + ); +} diff --git a/datahub-web-react/src/app/entity/shared/tabs/Entity/weaklyTypedAspects/TableValueElement.tsx b/datahub-web-react/src/app/entity/shared/tabs/Entity/weaklyTypedAspects/TableValueElement.tsx new file mode 100644 index 0000000000000..13532d10dbe59 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Entity/weaklyTypedAspects/TableValueElement.tsx @@ -0,0 +1,29 @@ +import React from 'react'; + +function isValidHttpUrl(string) { + let url; + + try { + url = new URL(string); + } catch (_) { + return false; + } + + return url.protocol === 'http:' || url.protocol === 'https:'; +} + +export default function TableValueElement({ value }: { value: any }) { + if (typeof value === 'boolean') { + return {String(value)}; + } + if (typeof value === 'string') { + if (isValidHttpUrl(value)) { + return {value}; + } + return {value}; + } + if (typeof value === 'number') { + return {value}; + } + return null; +} diff --git a/datahub-web-react/src/app/entity/shared/types.ts b/datahub-web-react/src/app/entity/shared/types.ts index 9f93faf80a845..af6f560d3f053 100644 --- a/datahub-web-react/src/app/entity/shared/types.ts +++ b/datahub-web-react/src/app/entity/shared/types.ts @@ -4,6 +4,7 @@ import { DataPlatform, DatasetEditableProperties, DatasetEditablePropertiesUpdate, + RawAspect, EditableSchemaMetadata, EditableSchemaMetadataUpdate, EntityType, @@ -56,6 +57,7 @@ export type GenericEntityProperties = { /** Dataset specific- TODO, migrate these out */ editableSchemaMetadata?: Maybe; editableProperties?: Maybe; + autoRenderAspects?: Maybe>; }; export type GenericEntityUpdate = { diff --git a/datahub-web-react/src/graphql/dataset.graphql b/datahub-web-react/src/graphql/dataset.graphql index 955c2367a4829..dd26a55b8846f 100644 --- a/datahub-web-react/src/graphql/dataset.graphql +++ b/datahub-web-react/src/graphql/dataset.graphql @@ -115,6 +115,15 @@ query getDataset($urn: String!) { ...fullRelationshipResults } ...viewProperties + autoRenderAspects: aspects(input: { autoRenderOnly: true }) { + aspectName + payload + renderSpec { + displayType + displayName + key + } + } } } diff --git a/docs/modeling/extending-the-metadata-model.md b/docs/modeling/extending-the-metadata-model.md index 2f9795b109318..4b3dd0f6feaa0 100644 --- a/docs/modeling/extending-the-metadata-model.md +++ b/docs/modeling/extending-the-metadata-model.md @@ -337,6 +337,12 @@ It takes the following parameters: - **name**: string - A common name used to identify the Aspect. Must be unique among all aspects DataHub is aware of. - **type**: string (optional) - set to "timeseries" to mark this aspect as timeseries. Check out this [doc](metadata-model.md#timeseries-aspects) for details. +- **autoRender**: boolean (optional) - defaults to false. When set to true, the aspect will automatically be displayed + on entity pages in a tab using a default renderer. +- **renderSpec**: RenderSpec (optional) - config for autoRender aspects that controls how they are displayed. Contains three fields: + - **displayType**: One of `tabular`, `properties`. Tabular should be uesd for a list of data elements- proprties for a single data bag. + - **displayName**: How the aspect should be referred to in the UI. Determines the name of the tab on the entity page. + - **key**: For `tabular` aspects only. Specifies the key in which the array to render may be found. ##### Example diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/AspectSpec.java b/entity-registry/src/main/java/com/linkedin/metadata/models/AspectSpec.java index 8a2977291702a..334651a4f38f4 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/AspectSpec.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/AspectSpec.java @@ -1,5 +1,6 @@ package com.linkedin.metadata.models; +import com.linkedin.data.DataMap; import com.linkedin.data.schema.RecordDataSchema; import com.linkedin.data.template.RecordTemplate; import com.linkedin.metadata.models.annotation.AspectAnnotation; @@ -59,6 +60,14 @@ public boolean isTimeseries() { return _aspectAnnotation.isTimeseries(); } + public Boolean isAutoRender() { + return _aspectAnnotation.isAutoRender(); + } + + public DataMap getRenderSpec() { + return _aspectAnnotation.getRenderSpec(); + } + public Map getSearchableFieldSpecMap() { return _searchableFieldSpecs; } @@ -99,5 +108,3 @@ public Class getDataTemplateClass() { return _aspectClass; } } - - diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/annotation/AspectAnnotation.java b/entity-registry/src/main/java/com/linkedin/metadata/models/annotation/AspectAnnotation.java index 5623b4c9efae5..d116170e10d22 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/annotation/AspectAnnotation.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/annotation/AspectAnnotation.java @@ -1,5 +1,6 @@ package com.linkedin.metadata.models.annotation; +import com.linkedin.data.DataMap; import com.linkedin.metadata.models.ModelValidationException; import java.util.Map; import java.util.Optional; @@ -16,11 +17,15 @@ public class AspectAnnotation { public static final String ANNOTATION_NAME = "Aspect"; public static final String NAME_FIELD = "name"; private static final String TYPE_FIELD = "type"; + private static final String RENDER_SPEC_FIELD = "renderSpec"; + private static final String AUTO_RENDER_FIELD = "autoRender"; private static final String IS_KEY_FIELD = "isKey"; private static final String TIMESERIES_TYPE = "timeseries"; String name; boolean isTimeseries; + boolean autoRender; + DataMap renderSpec; @Nonnull public static AspectAnnotation fromSchemaProperty( @@ -48,7 +53,9 @@ public static AspectAnnotation fromSchemaProperty( final Optional type = AnnotationUtils.getField(map, TYPE_FIELD, String.class); boolean isTimeseries = type.isPresent() && type.get().equals(TIMESERIES_TYPE); + Optional autoRender = AnnotationUtils.getField(map, AUTO_RENDER_FIELD, Boolean.class); + Optional renderSpec = AnnotationUtils.getField(map, RENDER_SPEC_FIELD, DataMap.class); - return new AspectAnnotation(name.get(), isTimeseries); + return new AspectAnnotation(name.get(), isTimeseries, autoRender.orElseGet(() -> false), renderSpec.orElseGet(() -> null)); } } diff --git a/entity-registry/src/test/java/com/linkedin/metadata/models/registry/PluginEntityRegistryLoaderTest.java b/entity-registry/src/test/java/com/linkedin/metadata/models/registry/PluginEntityRegistryLoaderTest.java index 8fba3c0d581f5..bb4162c1d07db 100644 --- a/entity-registry/src/test/java/com/linkedin/metadata/models/registry/PluginEntityRegistryLoaderTest.java +++ b/entity-registry/src/test/java/com/linkedin/metadata/models/registry/PluginEntityRegistryLoaderTest.java @@ -57,7 +57,7 @@ public Map getEntitySpecs() { private EntityRegistry getBaseEntityRegistry() { final AspectSpec keyAspectSpec = - new AspectSpec(new AspectAnnotation("datasetKey", false), Collections.emptyList(), Collections.emptyList(), + new AspectSpec(new AspectAnnotation("datasetKey", false, false, null), Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), (RecordDataSchema) DataSchemaFactory.getInstance().getAspectSchema("datasetKey").get(), DataSchemaFactory.getInstance().getAspectClass("datasetKey").get()); diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java index 57599ba3275d8..3fabbbc11f456 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java @@ -1,6 +1,7 @@ package com.linkedin.gms.factory.graphql; import com.datahub.authentication.token.TokenService; + import com.linkedin.datahub.graphql.GmsGraphQLEngine; import com.linkedin.datahub.graphql.GraphQLEngine; import com.linkedin.datahub.graphql.analytics.service.AnalyticsService; @@ -8,9 +9,11 @@ import com.linkedin.gms.factory.auth.DataHubTokenServiceFactory; import com.linkedin.gms.factory.common.IndexConventionFactory; import com.linkedin.gms.factory.common.RestHighLevelClientFactory; +import com.linkedin.gms.factory.entityregistry.EntityRegistryFactory; import com.linkedin.gms.factory.entity.RestliEntityClientFactory; import com.linkedin.gms.factory.recommendation.RecommendationServiceFactory; import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.recommendation.RecommendationsService; import com.linkedin.metadata.graph.GraphClient; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; @@ -26,8 +29,14 @@ @Configuration -@Import({RestHighLevelClientFactory.class, IndexConventionFactory.class, RestliEntityClientFactory.class, RecommendationServiceFactory.class, - DataHubTokenServiceFactory.class}) +@Import({ + RestHighLevelClientFactory.class, + IndexConventionFactory.class, + RestliEntityClientFactory.class, + RecommendationServiceFactory.class, + EntityRegistryFactory.class, + DataHubTokenServiceFactory.class +}) public class GraphQLEngineFactory { @Autowired @Qualifier("elasticSearchRestHighLevelClient") @@ -60,6 +69,10 @@ public class GraphQLEngineFactory { @Qualifier("dataHubTokenService") private TokenService _tokenService; + @Autowired + @Qualifier("entityRegistry") + private EntityRegistry _entityRegistry; + @Value("${platformAnalytics.enabled}") // TODO: Migrate to DATAHUB_ANALYTICS_ENABLED private Boolean isAnalyticsEnabled; @@ -74,8 +87,8 @@ protected GraphQLEngine getInstance() { new AnalyticsService(elasticClient, indexConvention.getPrefix()), _entityService, _recommendationsService, - _tokenService - ).builder().build(); + _tokenService, + _entityRegistry).builder().build(); } return new GmsGraphQLEngine( _entityClient, @@ -84,6 +97,7 @@ protected GraphQLEngine getInstance() { null, _entityService, _recommendationsService, - _tokenService).builder().build(); + _tokenService, + _entityRegistry).builder().build(); } } diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClient.java index 8d58b4ec611b5..c6dfe859dbc69 100644 --- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClient.java +++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/EntityClient.java @@ -2,6 +2,7 @@ import com.datahub.authentication.Authentication; import com.linkedin.common.urn.Urn; +import com.linkedin.data.DataMap; import com.linkedin.data.template.RecordTemplate; import com.linkedin.data.template.StringArray; import com.linkedin.entity.Entity; @@ -262,4 +263,10 @@ public Optional getVersionedAspect( @Nonnull Long version, @Nonnull Class aspectClass, @Nonnull Authentication authentication) throws RemoteInvocationException; + + public DataMap getRawAspect( + @Nonnull String urn, + @Nonnull String aspect, + @Nonnull Long version, + @Nonnull Authentication authentication) throws RemoteInvocationException; } diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/JavaEntityClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/JavaEntityClient.java index f8d78a2ff9911..d4457c6c06ad0 100644 --- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/JavaEntityClient.java +++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/JavaEntityClient.java @@ -372,4 +372,20 @@ public Optional getVersionedAspect(@Nonnull String } return Optional.empty(); } + + @SneakyThrows + public DataMap getRawAspect(@Nonnull String urn, @Nonnull String aspect, + @Nonnull Long version, @Nonnull Authentication authentication) throws RemoteInvocationException { + VersionedAspect entity = _entityService.getVersionedAspect(Urn.createFromString(urn), aspect, version); + if (entity == null) { + return null; + } + + if (entity.hasAspect()) { + DataMap rawAspect = ((DataMap) entity.data().get("aspect")); + return rawAspect; + } + + return null; + } } diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java index b74c4e5c7d54c..8401dcc74efb3 100644 --- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java +++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java @@ -55,6 +55,8 @@ import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import javax.mail.MethodNotSupportedException; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import static com.linkedin.metadata.search.utils.QueryUtils.newFilter; @@ -556,4 +558,10 @@ public Optional getVersionedAspect( return Optional.empty(); } + @SneakyThrows + @Override + public DataMap getRawAspect(@Nonnull String urn, @Nonnull String aspect, @Nonnull Long version, + @Nonnull Authentication authentication) throws RemoteInvocationException { + throw new MethodNotSupportedException(); + } } From 996b0491f69de8adfb5eca3b5073129e5b6ffa0e Mon Sep 17 00:00:00 2001 From: Gabe Lyons Date: Mon, 29 Nov 2021 23:53:08 -0800 Subject: [PATCH 02/14] docs(business glossary): document the business glossary file format (#3639) --- .../source_docs/business_glossary.md | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/source_docs/business_glossary.md b/metadata-ingestion/source_docs/business_glossary.md index cf0ed1dcb3bf6..b2ebce6bc8ba0 100644 --- a/metadata-ingestion/source_docs/business_glossary.md +++ b/metadata-ingestion/source_docs/business_glossary.md @@ -37,7 +37,41 @@ Note that a `.` is used to denote nested fields in the YAML recipe. ### Business Glossary File Format -The business glossary file format should be pretty easy to understand using the sample business glossary checked in [here](../examples/bootstrap_data/business_glossary.yml) +The business glossary source file should be a `.yml` file with the following top-level keys: + +**Glossary**: the top level keys of the business glossary file +- **version**: the version of business glossary file config the config conforms to. Currently the only version released is `1`. +- **source**: the source format of the terms. Currently only supports `DataHub` +- **owners**: (optional) owners contains two nested fields + - **users**: (optional) a list of user ids + - **groups**: (optional) a list of group ids +- **url**: (optional) external url pointing to where the glossary is defined externally, if applicable. +- **nodes**: list of **GlossaryNode** objects, as defined below. + + +**GlossaryNode**: a container of **GlossaryNode** and **GlossaryTerm** objects +- **name**: name of the node +- **description**: (optional) description of the node +- **owners**: (optional) owners contains two nested fields + - **users**: (optional) a list of user ids + - **groups**: (optional) a list of group ids +- **terms**: (optional) list of child **GlossaryTerm** objects +- **nodes**: (optional) list of child **GlossaryNode** objects + +**GlossaryTerm**: a term in your business glossary +- **name**: name of the term +- **description**: (optional) description of the term +- **owners**: (optional) owners contains two nested fields + - **users**: (optional) a list of user ids + - **groups**: (optional) a list of group ids +- **term_source**: One of `EXTERNAL` or `INTERNAL`. Whether the term is coming from an external glossary or one defined in your organization. +- **source_ref**: (optional) If external, what is the name of the source the glossary term is coming from? +- **source_url**: (optional) If external, what is the url of the source definition? +- **inherits**: (optional) List of **GlossaryTerm** that this term inherits from +- **contains**: (optional) List of **GlossaryTerm** that this term contains +- **custom_properties**: A map of key/value pairs of arbitrary custom properties + +You can also view an example business glossary file checked in [here](../examples/bootstrap_data/business_glossary.yml) ## Compatibility From 314f877aeba9b02ef8e096196e221554450ac137 Mon Sep 17 00:00:00 2001 From: Ravindra Lanka Date: Mon, 29 Nov 2021 23:54:35 -0800 Subject: [PATCH 03/14] fix(ingest): enhance supported and unsupported base_objects_accessed for snowflake (#3608) --- .../ingestion/source/usage/snowflake_usage.py | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/usage/snowflake_usage.py b/metadata-ingestion/src/datahub/ingestion/source/usage/snowflake_usage.py index 05e9a3b8619e0..6c8fd051c2c35 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/usage/snowflake_usage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/usage/snowflake_usage.py @@ -3,7 +3,7 @@ import json import logging from datetime import datetime, timezone -from typing import Dict, Iterable, List, Optional +from typing import Any, Dict, Iterable, List, Optional import pydantic import pydantic.dataclasses @@ -70,10 +70,11 @@ class Config: class SnowflakeObjectAccessEntry(PermissiveModel): - columns: List[SnowflakeColumnReference] + columns: Optional[List[SnowflakeColumnReference]] objectDomain: str objectId: int objectName: str + stageKind: Optional[str] class SnowflakeJoinedAccessEvent(PermissiveModel): @@ -160,9 +161,15 @@ def _get_snowflake_history(self) -> Iterable[SnowflakeJoinedAccessEvent]: if event_dict["query_text"] is None: continue - event_dict["base_objects_accessed"] = json.loads( - event_dict["base_objects_accessed"] - ) + def is_unsupported_base_object_accessed(obj: Dict[str, Any]) -> bool: + unsupported_keys = ["locations"] + return any([obj.get(key) is not None for key in unsupported_keys]) + + event_dict["base_objects_accessed"] = [ + obj + for obj in json.loads(event_dict["base_objects_accessed"]) + if not is_unsupported_base_object_accessed(obj) + ] event_dict["query_start_time"] = ( event_dict["query_start_time"] ).astimezone(tz=timezone.utc) @@ -170,7 +177,8 @@ def _get_snowflake_history(self) -> Iterable[SnowflakeJoinedAccessEvent]: try: # big hammer try block to ensure we don't fail on parsing events event = SnowflakeJoinedAccessEvent(**event_dict) yield event - except Exception: + except Exception as e: + logger.warning(f"Failed to parse usage line {event_dict}", e) self.report.report_warning( "usage", f"Failed to parse usage line {event_dict}" ) @@ -197,7 +205,9 @@ def _aggregate_access_events( agg_bucket.add_read_entry( event.email, event.query_text, - [colRef.columnName.lower() for colRef in object.columns], + [colRef.columnName.lower() for colRef in object.columns] + if object.columns is not None + else [], ) return datasets From 66328ee23f279b6356f68759b4628f2f3e28fce8 Mon Sep 17 00:00:00 2001 From: Enrico Minack Date: Tue, 30 Nov 2021 08:57:47 +0100 Subject: [PATCH 04/14] feat(quickstart): simplify docker generate and compare script (#3434) --- docker/quickstart.sh | 2 +- ... docker-compose.monitoring.quickstart.yml} | 0 docker/quickstart/generate_and_compare.sh | 37 +++++++------------ .../quickstart/generate_docker_quickstart.sh | 2 +- 4 files changed, 15 insertions(+), 26 deletions(-) rename docker/quickstart/{docker-compose.quickstart.monitoring.yml => docker-compose.monitoring.quickstart.yml} (100%) diff --git a/docker/quickstart.sh b/docker/quickstart.sh index 9ae5be5ee2c4a..6bb42da6fb71e 100755 --- a/docker/quickstart.sh +++ b/docker/quickstart.sh @@ -2,7 +2,7 @@ MONITORING_COMPOSE="" if [[ $MONITORING == true ]]; then - MONITORING_COMPOSE="-f quickstart/docker-compose.quickstart.monitoring.yml" + MONITORING_COMPOSE="-f quickstart/docker-compose.monitoring.quickstart.yml" fi CONSUMERS_COMPOSE="" diff --git a/docker/quickstart/docker-compose.quickstart.monitoring.yml b/docker/quickstart/docker-compose.monitoring.quickstart.yml similarity index 100% rename from docker/quickstart/docker-compose.quickstart.monitoring.yml rename to docker/quickstart/docker-compose.monitoring.quickstart.yml diff --git a/docker/quickstart/generate_and_compare.sh b/docker/quickstart/generate_and_compare.sh index 28ce77583672a..58124ebe1037b 100755 --- a/docker/quickstart/generate_and_compare.sh +++ b/docker/quickstart/generate_and_compare.sh @@ -1,5 +1,8 @@ #!/bin/bash +# this scripts checks if docker-compose$flavour.quickstart.yml is up to date for these 'flavours': +FLAVOURS=("" "-without-neo4j" ".monitoring") + DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" cd "$DIR" @@ -10,30 +13,16 @@ source venv/bin/activate pip install -r requirements.txt python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.override.yml temp.quickstart.yml - python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../docker-compose-without-neo4j.override.yml temp-without-neo4j.quickstart.yml - -python generate_docker_quickstart.py ../monitoring/docker-compose.monitoring.yml temp.quickstart.monitoring.yml - -if cmp docker-compose.quickstart.yml temp.quickstart.yml; then - printf 'docker-compose.quickstart.yml is up to date.' -else - printf 'docker-compose.quickstart.yml is out of date.' +python generate_docker_quickstart.py ../monitoring/docker-compose.monitoring.yml temp.monitoring.quickstart.yml + +for flavour in "${flavours[@]}" +do + if cmp docker-compose$flavour.quickstart.yml temp$flavour.quickstart.yml; then + echo "docker-compose$flavour.quickstart.yml is up to date." + else + echo "docker-compose$flavour.quickstart.yml is out of date." exit 1 -fi + fi +done -if cmp docker-compose-without-neo4j.quickstart.yml temp-without-neo4j.quickstart.yml; then - printf 'docker-compose-without-neo4j.quickstart.yml is up to date.' - exit 0 -else - printf 'docker-compose-without-neo4j.quickstart.yml is out of date.' - exit 1 -fi - -if cmp docker-compose.quickstart.monitoring.yml temp.quickstart.monitoring.yml; then - printf 'docker-compose.quickstart.monitoring.yml is up to date.' - exit 0 -else - printf 'docker-compose.quickstart.monitoring.yml is out of date.' - exit 1 -fi diff --git a/docker/quickstart/generate_docker_quickstart.sh b/docker/quickstart/generate_docker_quickstart.sh index 39c98becc49ed..d8427a67b593a 100755 --- a/docker/quickstart/generate_docker_quickstart.sh +++ b/docker/quickstart/generate_docker_quickstart.sh @@ -11,4 +11,4 @@ source venv/bin/activate pip install -r requirements.txt python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.override.yml docker-compose.quickstart.yml python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../docker-compose-without-neo4j.override.yml docker-compose-without-neo4j.quickstart.yml -python generate_docker_quickstart.py ../monitoring/docker-compose.monitoring.yml docker-compose.quickstart.monitoring.yml +python generate_docker_quickstart.py ../monitoring/docker-compose.monitoring.yml docker-compose.monitoring.quickstart.yml From 50122cbd9819cc621a48f2bdbfd886de97ab0866 Mon Sep 17 00:00:00 2001 From: Swaroop Jagadish <67564030+swaroopjagadish@users.noreply.github.com> Date: Tue, 30 Nov 2021 00:12:17 -0800 Subject: [PATCH 05/14] =?UTF-8?q?fix(docs):=20small=20fixes=20to=20docs=20?= =?UTF-8?q?and=20docker=20images=20for=20custom=20metadata=20=E2=80=A6=20(?= =?UTF-8?q?#3640)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../docker-compose-without-neo4j.override.yml | 2 ++ docker/docker-compose.override.yml | 2 ++ ...cker-compose-without-neo4j-m1.quickstart.yml | 2 ++ .../docker-compose-without-neo4j.quickstart.yml | 2 ++ docker/quickstart/docker-compose.quickstart.yml | 2 ++ docker/quickstart/generate_and_compare.sh | 2 +- docs/modeling/extending-the-metadata-model.md | 13 ++++++------- metadata-models-custom/README.md | 8 +++++--- .../scripts/data/dq_rule.json | 17 +++++++++++++++++ metadata-models-custom/scripts/insert_one.sh | 1 + 10 files changed, 40 insertions(+), 11 deletions(-) create mode 100644 metadata-models-custom/scripts/data/dq_rule.json create mode 100755 metadata-models-custom/scripts/insert_one.sh diff --git a/docker/docker-compose-without-neo4j.override.yml b/docker/docker-compose-without-neo4j.override.yml index 7a934a8aa25d6..5b5df8c1fb9c4 100644 --- a/docker/docker-compose-without-neo4j.override.yml +++ b/docker/docker-compose-without-neo4j.override.yml @@ -28,6 +28,8 @@ services: env_file: datahub-gms/env/docker-without-neo4j.env depends_on: - mysql + volumes: + - ${HOME}/.datahub/plugins:/etc/datahub/plugins volumes: mysqldata: diff --git a/docker/docker-compose.override.yml b/docker/docker-compose.override.yml index e08d1c29cd22c..1e8231e20532d 100644 --- a/docker/docker-compose.override.yml +++ b/docker/docker-compose.override.yml @@ -29,6 +29,8 @@ services: env_file: datahub-gms/env/docker.env depends_on: - mysql + volumes: + - ${HOME}/.datahub/plugins/:/etc/datahub/plugins volumes: mysqldata: diff --git a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml index a9f5db27370fe..cff1e717852b4 100644 --- a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml @@ -64,6 +64,8 @@ services: image: linkedin/datahub-gms:${DATAHUB_VERSION:-head} ports: - 8080:8080 + volumes: + - ${HOME}/.datahub/plugins:/etc/datahub/plugins elasticsearch: container_name: elasticsearch environment: diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml index cddd1e0f88817..7a70b77bc6f6e 100644 --- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml @@ -64,6 +64,8 @@ services: image: linkedin/datahub-gms:${DATAHUB_VERSION:-head} ports: - 8080:8080 + volumes: + - ${HOME}/.datahub/plugins:/etc/datahub/plugins elasticsearch: container_name: elasticsearch environment: diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml index 5aaf3fd3e8c68..1232e63a69add 100644 --- a/docker/quickstart/docker-compose.quickstart.yml +++ b/docker/quickstart/docker-compose.quickstart.yml @@ -68,6 +68,8 @@ services: image: linkedin/datahub-gms:${DATAHUB_VERSION:-head} ports: - 8080:8080 + volumes: + - ${HOME}/.datahub/plugins/:/etc/datahub/plugins elasticsearch: container_name: elasticsearch environment: diff --git a/docker/quickstart/generate_and_compare.sh b/docker/quickstart/generate_and_compare.sh index 58124ebe1037b..6939eae7c2d1a 100755 --- a/docker/quickstart/generate_and_compare.sh +++ b/docker/quickstart/generate_and_compare.sh @@ -16,7 +16,7 @@ python generate_docker_quickstart.py ../docker-compose.yml ../docker-compose.ove python generate_docker_quickstart.py ../docker-compose-without-neo4j.yml ../docker-compose-without-neo4j.override.yml temp-without-neo4j.quickstart.yml python generate_docker_quickstart.py ../monitoring/docker-compose.monitoring.yml temp.monitoring.quickstart.yml -for flavour in "${flavours[@]}" +for flavour in "${FLAVOURS[@]}" do if cmp docker-compose$flavour.quickstart.yml temp$flavour.quickstart.yml; then echo "docker-compose$flavour.quickstart.yml is up to date." diff --git a/docs/modeling/extending-the-metadata-model.md b/docs/modeling/extending-the-metadata-model.md index 4b3dd0f6feaa0..54d7eb9ab2e1c 100644 --- a/docs/modeling/extending-the-metadata-model.md +++ b/docs/modeling/extending-the-metadata-model.md @@ -258,12 +258,12 @@ After you create your Aspect, you need to attach to all the entities that it app At the beginning of this document, we walked you through a flow-chart that should help you decide whether you need to maintain a fork of the open source DataHub repo for your model extensions, or whether you can just use a model extension repository that can stay independent of the DataHub repo. Depending on what path you took, the place you store your aspect model files (the .pdl files) and the entity-registry files (the yaml file called `entity-registry.yaml` or `entity-registry.yml`) will vary. -- Open source Fork: Aspect files go under [`metadata-models`](../../metadata-models) module in the main repo, entity registry goes into `metadata-models/src/resources/entity-registry.yml`. Read on for more details in Step 6. -- Custom repository: Read the [metadata-models-custom](../../metadata-models-custom/README.md) documentation to figure out how to store and version your aspect models and registry. +- Open source Fork: Aspect files go under [`metadata-models`](../../metadata-models) module in the main repo, entity registry goes into [`metadata-models/src/main/resources/entity-registry.yml`](../../metadata-models/src/main/resources/entity-registry.yml). Read on for more details in [Step 6](#step_6). +- Custom repository: Read the [metadata-models-custom](../../metadata-models-custom/README.md) documentation to learn how to store and version your aspect models and registry. ### Step 6: Attaching your non-key Aspect(s) to the Entity -Attaching non-key aspects to an entity can be done simply by editing the `entity-registry.yml` file located under the metadata-models module [here](../../metadata-models/src/main/resources). +Attaching non-key aspects to an entity can be done simply by adding them to the entity registry yaml file. The location of this file differs based on whether you are following the oss-fork path or the custom-repository path. Here is an minimal example of adding our new `DashboardInfo` aspect to the `Dashboard` entity. @@ -271,12 +271,11 @@ Here is an minimal example of adding our new `DashboardInfo` aspect to the `Dash entities: - name: dashboard aspects: - - dashboardInfo # the name of the aspect must be the same as that on the @Aspect annotation on the class + # the name of the aspect must be the same as that on the @Aspect annotation on the class + - dashboardInfo ``` -Store this entity registry in the appropriate place depending on whether you are following the open-source fork approach or the custom repository approach. - -### Step 7 (Oss-Fork only): Re-build DataHub to have access to your new or updated entity +### Step 7 (Oss-Fork approach): Re-build DataHub to have access to your new or updated entity If you opted for the open-source fork approach, where you are editing models in the `metadata-models` repository of DataHub, you will need to re-build the DataHub metadata service using the steps below. If you are following the custom model repository approach, you just need to build your custom model repository and deploy it to a running metadata service instance to read and write metadata using your new model extensions. diff --git a/metadata-models-custom/README.md b/metadata-models-custom/README.md index 31bf45b698cbc..c4a089c0c9f2f 100644 --- a/metadata-models-custom/README.md +++ b/metadata-models-custom/README.md @@ -1,6 +1,6 @@ # A Custom Metadata Model -This module hosts a gradle project where you can store your custom metadata model. It contains an example extension for you to follow. +[This module](./) hosts a gradle project where you can store your custom metadata model. It contains an example extension for you to follow. ### Caveats @@ -12,8 +12,10 @@ Before proceeding further, make sure you understand the DataHub Metadata Model c ## Create your new aspect(s) -Follow the regular process in creating a new aspect by adding it to the [`src/main/pegasus`](metadata-models-custom/src/main/pegasus) folder. e.g. This repository has an Aspect called `customDataQualityRules` hosted in the [`DataQualityRules.pdl`](metadata-models-custom/src/main/pegasus/com/mycompany/dq/DataQualityRules.pdl) file that you can follow. -Feel free to delete the sample aspects that are stored in this repo using a simple `rm -rf src/main/pegasus/*` +Follow the regular process in creating a new aspect by adding it to the [`src/main/pegasus`](./src/main/pegasus) folder. e.g. This repository has an Aspect called `customDataQualityRules` hosted in the [`DataQualityRules.pdl`](./src/main/pegasus/com/mycompany/dq/DataQualityRules.pdl) file that you can follow. +Once you've gone through this exercise, feel free to delete the sample aspects that are stored in this module. + +**_Tip_**: PDL requires that the name of the file must match the name of the class that is defined in it, so keep that in mind when you create your aspect pdl file. ## Add your aspect(s) to the entity registry diff --git a/metadata-models-custom/scripts/data/dq_rule.json b/metadata-models-custom/scripts/data/dq_rule.json new file mode 100644 index 0000000000000..698308a3b3507 --- /dev/null +++ b/metadata-models-custom/scripts/data/dq_rule.json @@ -0,0 +1,17 @@ +{"rules": [ + { + "field": "my_event_data", + "isFieldLevel": false, + "type": "isNull", + "checkDefinition": "n/a", + "url": "https://github.com/linkedin/datahub/blob/master/checks/nonNull.sql" + }, + { + "field": "timestamp", + "isFieldLevel": true, + "type": "increasing", + "checkDefinition": "n/a", + "url": "https://github.com/linkedin/datahub/blob/master/checks/increasing.sql" + } + ] +} diff --git a/metadata-models-custom/scripts/insert_one.sh b/metadata-models-custom/scripts/insert_one.sh new file mode 100755 index 0000000000000..c82bc2ce648cb --- /dev/null +++ b/metadata-models-custom/scripts/insert_one.sh @@ -0,0 +1 @@ +datahub put --urn "urn:li:dataset:(urn:li:dataPlatform:hive,logging_events,PROD)" --aspect customDataQualityRules --aspect-data data/dq_rule.json From a5ec05e2b2ff312a3b6e0b4133039082a58b33ea Mon Sep 17 00:00:00 2001 From: varunbharill Date: Tue, 30 Nov 2021 11:33:33 -0800 Subject: [PATCH 06/14] fix(ingest): mongodb enable version check for document size filter. (#3644) --- .../src/datahub/ingestion/source/mongodb.py | 34 +++++++++++++++---- .../integration/mongodb/setup/mongo_init.js | 2 ++ .../tests/integration/mongodb/test_mongodb.py | 1 + 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/mongodb.py b/metadata-ingestion/src/datahub/ingestion/source/mongodb.py index 3a65f53f77e14..3ee30246050e6 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/mongodb.py +++ b/metadata-ingestion/src/datahub/ingestion/source/mongodb.py @@ -8,6 +8,7 @@ import bson import pymongo from mypy_extensions import TypedDict +from packaging import version from pydantic import PositiveInt, validator from pymongo.mongo_client import MongoClient @@ -295,6 +296,7 @@ def construct_schema_pymongo( delimiter: str, use_random_sampling: bool, max_document_size: int, + is_version_gte_4_4: bool, sample_size: Optional[int] = None, ) -> Dict[Tuple[str, ...], SchemaDescription]: """ @@ -317,12 +319,14 @@ def construct_schema_pymongo( """ doc_size_field = "temporary_doc_size_field" - # create a temporary field to store the size of the document. filter on it and then remove it. - aggregations = [ - {"$addFields": {doc_size_field: {"$bsonSize": "$$ROOT"}}}, - {"$match": {doc_size_field: {"$lt": max_document_size}}}, - {"$project": {doc_size_field: 0}}, - ] + aggregations: List[Dict] = [] + if is_version_gte_4_4: + # create a temporary field to store the size of the document. filter on it and then remove it. + aggregations = [ + {"$addFields": {doc_size_field: {"$bsonSize": "$$ROOT"}}}, + {"$match": {doc_size_field: {"$lt": max_document_size}}}, + {"$project": {doc_size_field: 0}}, + ] if use_random_sampling: # get sample documents in collection aggregations.append({"$sample": {"size": sample_size}}) @@ -462,6 +466,7 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: delimiter=".", use_random_sampling=self.config.useRandomSampling, max_document_size=self.config.maxDocumentSize, + is_version_gte_4_4=self.is_server_version_gte_4_4(), sample_size=self.config.schemaSamplingSize, ) @@ -533,6 +538,23 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: self.report.report_workunit(wu) yield wu + def is_server_version_gte_4_4(self) -> bool: + try: + server_version = self.mongo_client.server_info().get("versionArray") + if server_version: + logger.info( + f"Mongodb version for current connection - {server_version}" + ) + server_version_str_list = [str(i) for i in server_version] + required_version = "4.4" + return version.parse( + ".".join(server_version_str_list) + ) >= version.parse(required_version) + except Exception as e: + logger.error("Error while getting version of the mongodb server %s", e) + + return False + def get_report(self) -> MongoDBSourceReport: return self.report diff --git a/metadata-ingestion/tests/integration/mongodb/setup/mongo_init.js b/metadata-ingestion/tests/integration/mongodb/setup/mongo_init.js index 3ef7decc0cd94..104982232406a 100644 --- a/metadata-ingestion/tests/integration/mongodb/setup/mongo_init.js +++ b/metadata-ingestion/tests/integration/mongodb/setup/mongo_init.js @@ -102,6 +102,7 @@ db.secondCollection.insertMany([ }, ]); +var large_field_value = (new Array(250001)).join('x'); db.emptyCollection.createIndex({ stringField: 1 }, { unique: true }); db.largeCollection.insertMany([ {'field_9': 'uchjbphlsw', 'field_24': 'ttothndxpu', 'field_26': 'abgudwfllh', 'field_31': 'wgkuntylkf', 'field_46': 'hiinxfysod', 'field_49': 'ljvnavbole', 'field_61': 'waakkniukq', 'field_72': 'lmytgyfcdv', 'field_75': 'vjawoijfxa', 'field_81': 'ccxtkxxlom', 'field_86': 'ufckjmjgjg', 'field_103': 'oqhoqaktqx', 'field_110': 'fszysiwxtk', 'field_118': 'mlwkyhrhqf', 'field_129': 'hotbyiwqov', 'field_172': 'exeuyhyosu', 'field_176': 'jodeptaoyx', 'field_182': 'aawdiurayf', 'field_185': 'odshxvjrzm', 'field_188': 'akpetbvcpb', 'field_194': 'abklngtjnx', 'field_200': 'aydboztnyy', 'field_201': 'vgsyykxaxw', 'field_202': 'jgnymcwzwd', 'field_203': 'sxktncbbrz', 'field_205': 'wtblurrqgb', 'field_206': 'qpgmhchtgr', 'field_208': 'jevmagdint', 'field_209': 'hvtwxgxsdn', 'field_210': 'zxujywoaub', 'field_211': 'ocztwjayor', 'field_213': 'zoeoyumbhq', 'field_214': 'oylzthhncg', 'field_215': 'hrzjsnqrjk', 'field_216': 'zyyoxqztnx', 'field_217': 'nfcectwcyd', 'field_218': 'nksoxqtdwe', 'field_219': 'tjzynamwmf', 'field_220': 'msvnvbpsiy', 'field_221': 'mtbavxhgkv', 'field_222': 'lvigdxllpu', 'field_223': 'nzgrldnpnx', 'field_224': 'yjfegrgspa', 'field_225': 'mlwstgfyrc', 'field_226': 'zyvydtfmkv', 'field_227': 'hvnlrnqdzr', 'field_228': 'bdcssikloe', 'field_229': 'mihfibange', 'field_230': 'zavunfnoya', 'field_231': 'rubhenacsa', 'field_232': 'ejniazigqr', 'field_233': 'nbjaocoikq', 'field_234': 'pzabevszfk', 'field_235': 'yxculntznc', 'field_236': 'vxwlcfoisn', 'field_237': 'oncszefarh', 'field_238': 'tbvlunlxbf', 'field_239': 'nftyjgruly', 'field_240': 'fleahvitnv', 'field_241': 'xyefrzqkuz', 'field_242': 'ifabkveqmu', 'field_243': 'zxhivszgzn', 'field_244': 'maswvrqsaf', 'field_245': 'mxmvteniuv', 'field_246': 'svncrwxcrr', 'field_247': 'bwkaufywme', 'field_248': 'ymvwcoisgc', 'field_249': 'dnieugtiol', 'field_250': 'mtafidfuhr', 'field_251': 'onkhpbdaza', 'field_252': 'onsqcrtbig', 'field_253': 'csrobxsysb', 'field_255': 'mthbqlkahd', 'field_256': 'nevnwazgzj', 'field_257': 'gczfkvrqly', 'field_258': 'vraeiocfvg', 'field_259': 'jvuzudyyfg', 'field_260': 'frhzrwuuts', 'field_261': 'pfnxyqrdcl', 'field_262': 'jramdncqjy', 'field_263': 'frilycytnu', 'field_264': 'adjhjrfpyd', 'field_265': 'xvmhxlmxih', 'field_266': 'umjhiwlcax', 'field_267': 'ubhpaqaezd', 'field_268': 'rfiaepwuvh', 'field_269': 'yxtzbcxrni', 'field_270': 'ruhdsiaxuz', 'field_271': 'hmvjanuegx', 'field_272': 'etdczyzfmw', 'field_273': 'xiaadukvxy', 'field_274': 'qhqiyhzzsj', 'field_275': 'twrpcnhdaw', 'field_276': 'lmuskzxgtl', 'field_277': 'vyewwhduwb', 'field_278': 'gsjrayuscg', 'field_279': 'wmudokwuzo', 'field_280': 'cplwfwqbik', 'field_281': 'equtopxxzm', 'field_282': 'qnmhvqqpvp', 'field_283': 'zpfnutlqnw', 'field_284': 'illrdbsetg', 'field_285': 'dqzthrqjkt', 'field_286': 'bcitdneegh', 'field_287': 'dvzmxnakis', 'field_288': 'smjyqreefq', 'field_289': 'zxbhpnhtpi', 'field_290': 'wpurnfhyjo', 'field_291': 'admdxadnes', 'field_292': 'bqtctfxbdk', 'field_293': 'kjqrdvjfos', 'field_294': 'dotihaxtfb', 'field_295': 'oiylkornih', 'field_296': 'qthvoeffvr', 'field_297': 'xfnqljtuhn', 'field_298': 'obfxibrvyj', 'field_299': 'mkgtlqgnab', 'field_300': 'xunqvmmalh', 'field_301': 'mywnpuoouc', 'field_302': 'zpnmtuekut', 'field_303': 'jsjopxomgv', 'field_304': 'bcoglyekeb', 'field_305': 'pvemopdict', 'field_306': 'ezskzyuzql', 'field_307': 'zpvuoavdln', 'field_308': 'xgwikvkcqg', 'field_309': 'zwiwvqnuey', 'field_310': 'nrlzpdvxlq', 'field_311': 'lwaoxmtxpt', 'field_312': 'ztdssfsihl', 'field_313': 'jtmjqwxvrt', 'field_314': 'sncesddwok', 'field_315': 'okiymzkvam', 'field_316': 'zpsqlbpxxm', 'field_317': 'hrpkfqiowj', 'field_318': 'mtjjdbpmbd', 'field_319': 'npeklvvlil', 'field_320': 'qxlgxhskym', 'field_321': 'ojukbvxeww', 'field_322': 'pqzoypjedt', 'field_323': 'qemukgexzq', 'field_324': 'mhzbodljjh', 'field_325': 'yjmkwfkctj', 'field_326': 'ibwshurhxw', 'field_327': 'cbkzyxfuai', 'field_328': 'mtbvxiatoi', 'field_329': 'fnkidxpvqh', 'field_330': 'loomukgcdp', 'field_331': 'xqhvtwecry', 'field_332': 'ipfjpshiak', 'field_333': 'ntwrroliys', 'field_334': 'twjwswjwcf', 'field_335': 'bxebvxlpnd', 'field_336': 'qqzrgfsglw', 'field_337': 'tkhrzmtiks', 'field_338': 'grbcxkbtlc', 'field_339': 'omallaspkj', 'field_340': 'tvlspvltqp', 'field_341': 'rnnqwiqzuz', 'field_342': 'ijufyptbfp', 'field_343': 'mvnnsfehvt', 'field_344': 'yijcdfdtfi', 'field_345': 'acwgitwvum', 'field_346': 'swzijoeqeo', 'field_347': 'ggrjwbugiv', 'field_348': 'pynzegfhqt', 'field_349': 'rzyyfoexji', 'field_350': 'zrqzuwnvru', 'field_351': 'lwdzgxdmsa', 'field_352': 'hdiaiaxuhw', 'field_354': 'hmghrvfwhs', 'field_355': 'xhqxygxyps', 'field_356': 'zjxkklvbxi', 'field_357': 'gstidigitq', 'field_358': 'knnihgtvyd', 'field_359': 'wqthcdkgjg', 'field_360': 'eecuamblle', 'field_361': 'ralaubwzgb', 'field_362': 'gqcibaacbm', 'field_363': 'xleauoiymf', 'field_364': 'spuxphxlxz', 'field_365': 'lwgptqbkeo', 'field_366': 'dxliwjdiaq', 'field_367': 'javrtefbyp', 'field_368': 'foskvyvuoq', 'field_370': 'srltzglrco', 'field_371': 'lgthgpbxox', 'field_372': 'xcjxrsbibu', 'field_373': 'btdffywhym', 'field_374': 'bhgmgnplfh', 'field_375': 'hduqtpwswl', 'field_376': 'jdkyaaseqb', 'field_377': 'ygglvcowsy', 'field_378': 'rwnblemmjf', 'field_379': 'gxqkalssnr', 'field_380': 'ovxljjajyt', 'field_381': 'iypxyhmpro', 'field_382': 'psiiwpdygz', 'field_383': 'hiroklftxu', 'field_384': 'ywjwombgsj', 'field_385': 'xofindyokx', 'field_386': 'tegognwxfn', 'field_387': 'lofdgnwgmi', 'field_388': 'kmtkzfbqnt', 'field_389': 'sykbdogolp', 'field_390': 'ghqlzsxhid', 'field_391': 'xxwrmywkhi', 'field_392': 'okrkvjaymg', 'field_393': 'auvwppmlpp', 'field_394': 'nbdnuyxebe', 'field_395': 'vkzdsabgxv', 'field_396': 'yqzmnmjfdb', 'field_397': 'weaywxgfqe', 'field_398': 'halulprgwa', 'field_399': 'mqqkxochkx', 'field_400': 'mrtfqhukzv', 'field_401': 'hdibtuywgg', 'field_402': 'sjudlzcufv', 'field_403': 'yxjgakteae', 'field_404': 'zqtjpegnac', 'field_405': 'nbxlyhljnf', 'field_406': 'ntudvxeuea', 'field_407': 'tzpigfroeg', 'field_408': 'lsnmeouvgd', 'field_409': 'txronkagbe', 'field_410': 'pmhcgpvola', 'field_411': 'jeiwpzypzu', 'field_412': 'nhjmavlpix', 'field_413': 'edwpqccnwk', 'field_414': 'pxzspbwlxl', 'field_415': 'nutfwctsvo', 'field_416': 'apsbqlzspg', 'field_417': 'iubckrdhbi', 'field_418': 'jedjzgvxzi', 'field_419': 'vzvbvqrapz', 'field_420': 'xlguqpcamk', 'field_421': 'gmxxrlwhvy', 'field_422': 'gjpmymgtoo', 'field_423': 'lfhcywuwbi', 'field_424': 'kzzofhmhob', 'field_425': 'uwpmanpcyg', 'field_426': 'kpxnlkleae', 'field_427': 'hxmgostrlr', 'field_428': 'jlldncwjgk', 'field_429': 'ejgzjegnvo', 'field_430': 'mjdvjopqdl', 'field_431': 'kvappclwbw', 'field_432': 'mobwhdxtto', 'field_433': 'wjtypavbum', 'field_434': 'jtveoegntm', 'field_435': 'dsvihdnpsf', 'field_436': 'chcqopfatn', 'field_437': 'sxbbkyetso', 'field_438': 'lqcmmotbgc', 'field_439': 'gcfomhvqye', 'field_440': 'ryxbrqgiqv', 'field_441': 'tzycnlqwex', 'field_442': 'qgfyukgtvt', 'field_443': 'hlkrixwojy', 'field_444': 'ynwmifbloi', 'field_445': 'ioctihqhqe', 'field_446': 'grjcefiwwz', 'field_447': 'nvtsrwuxww', 'field_448': 'cgdxsscxws', 'field_449': 'ewwvpfkeln', 'field_450': 'priipukgeg', 'field_451': 'xddrojapap', 'field_452': 'mahozavkdl', 'field_453': 'mwybdmeqmz', 'field_454': 'oiszsfjero', 'field_455': 'keowjycukj', 'field_456': 'grhhnfegfp', 'field_457': 'gofjnjipzx', 'field_458': 'wndeptfmoo', 'field_459': 'fctxgjijfi', 'field_460': 'oerknkmsst', 'field_461': 'wvwrmnjpng', 'field_462': 'swfywdhwve', 'field_463': 'xdkifkqjix', 'field_464': 'wwdvqymkod', 'field_465': 'ndjoxiicsj', 'field_466': 'qchusvmqpr', 'field_467': 'vfhqttjtqr', 'field_468': 'alxkofcpeo', 'field_469': 'kxryigaqaq', 'field_470': 'cedctxzrqx', 'field_471': 'vjegbkiejh', 'field_472': 'ufzfsxhdny', 'field_473': 'ysszuygwlr', 'field_474': 'ittwiizhmt', 'field_475': 'jrrwmowdis', 'field_476': 'usvxcnjbgg', 'field_477': 'iglreiowmt', 'field_478': 'pnybnwenhd', 'field_479': 'retyenvact', 'field_480': 'kqssjtidvh', 'field_481': 'mjrokgklom', 'field_482': 'wrncbivdzy', 'field_483': 'jhuzkkqarn', 'field_484': 'blephauzyk', 'field_485': 'obsfbvxyvk', 'field_486': 'tdquugrzof', 'field_487': 'spspzgfnmz', 'field_488': 'ljwzofkuft', 'field_489': 'hjqblfokjp', 'field_491': 'kpwieofgeh', 'field_492': 'bfavrlbqlw', 'field_493': 'tqtlsmlbcx', 'field_494': 'wclrsiahnj', 'field_495': 'lriagtlpsx', 'field_496': 'vqqhdqfegc', 'field_497': 'drploxhnkf', 'field_498': 'ckjvkrbpmi', 'field_499': 'jrqovdbohe'}, @@ -204,4 +205,5 @@ db.largeCollection.insertMany([ {'field_8': 'aofvxkdwka', 'field_22': 'prtqghbzcw', 'field_24': 'mlzeflelcz', 'field_31': 'zfgdkjunmb', 'field_43': 'edcuvrclie', 'field_45': 'rmxmplobgn', 'field_55': 'xelgudghdl', 'field_67': 'znvqjvgiez', 'field_70': 'ufriqfytrt', 'field_80': 'wdxrlwldif', 'field_93': 'zpiwilanmx', 'field_114': 'vmuiiqberd', 'field_121': 'gcskefxywo', 'field_129': 'zrmkqcxxyq', 'field_136': 'zrahsmasbo', 'field_164': 'nrrrscbznk', 'field_165': 'hegveluvrp', 'field_177': 'kwdcisczwo', 'field_185': 'bjtedmxolz', 'field_188': 'wsfmulfbbk', 'field_200': 'xbqjpavfrm', 'field_201': 'bwvhnsklge', 'field_202': 'bkxxhpzqtm', 'field_203': 'ypacfomldy', 'field_204': 'gfwoabhytg', 'field_205': 'issjaibshz', 'field_206': 'xszyjxpwre', 'field_207': 'dsbuxpvzeb', 'field_208': 'utmmssjrtl', 'field_210': 'ybfszkfouy', 'field_211': 'vzerabdlbl', 'field_212': 'dfkwswgktt', 'field_213': 'pxyovwtcvg', 'field_214': 'bhjlvoxxms', 'field_215': 'awzjgsjcik', 'field_216': 'ymlbfqjoss', 'field_217': 'yhwxlggqux', 'field_218': 'jgnfagzslx', 'field_219': 'cnjetybguh', 'field_220': 'gaucxehwwd', 'field_221': 'tkuweckqau', 'field_222': 'mhnhuvvnbs', 'field_223': 'ipubkuxqqu', 'field_224': 'jjnacloozl', 'field_225': 'shzngbfqcs', 'field_226': 'nmtjluqgck', 'field_227': 'adplerzbzl', 'field_228': 'qvdyfegtfi', 'field_229': 'eluomdygxf', 'field_230': 'mzdhbzubiy', 'field_231': 'nmzddgfieg', 'field_232': 'ehyyowjbjx', 'field_233': 'wxgjvidaqw', 'field_234': 'ssafwrnprr', 'field_235': 'grwpazlbpy', 'field_236': 'lokskxppio', 'field_237': 'jjllrmhmdn', 'field_238': 'fwhmfzxool', 'field_239': 'btetixtxps', 'field_240': 'hpbtmuqcmc', 'field_241': 'mzjnhiudlw', 'field_242': 'sarcuahdvi', 'field_243': 'rtdpysipdr', 'field_244': 'vxczjgpqdd', 'field_245': 'khcxmxvdyk', 'field_246': 'vjoqflylnu', 'field_247': 'lfeoaiesmk', 'field_248': 'iguhhufamn', 'field_249': 'iaovlxjowy', 'field_250': 'nkkrvmlaco', 'field_251': 'hzseonkzws', 'field_252': 'yrirkhesas', 'field_253': 'jqmkcxtmxf', 'field_254': 'qyezaxkhht', 'field_255': 'fctupamztc', 'field_256': 'biwuxshrpu', 'field_257': 'doxygoyuly', 'field_258': 'dkcyyxpxso', 'field_259': 'dtiwbamapm', 'field_260': 'zbteydnyvz', 'field_261': 'ggprgieoue', 'field_262': 'oxawnzqqgk', 'field_263': 'vgoaqhogtx', 'field_264': 'yqsqgsgkku', 'field_265': 'xxrcoicpqg', 'field_266': 'mgkpulwoag', 'field_267': 'avftuoxvmr', 'field_268': 'asvodytgmq', 'field_269': 'aspvzlwxvm', 'field_270': 'ooxvkbdpkm', 'field_271': 'jcvkxbvxhd', 'field_272': 'ydcnvilpvo', 'field_273': 'qrumimcbyu', 'field_274': 'suxbzjsdoi', 'field_275': 'ywamepswec', 'field_276': 'aoapcklzah', 'field_277': 'vlmtzsifmj', 'field_278': 'qpoubgitva', 'field_279': 'dflohmlasz', 'field_280': 'hbxuohemlj', 'field_281': 'hypccdkiie', 'field_282': 'aydeuqgmpf', 'field_283': 'tepzkeeiyr', 'field_284': 'jjvwvjkyjz', 'field_285': 'zfgkkgupij', 'field_286': 'uoqwrfzcoy', 'field_287': 'ppokeiocad', 'field_288': 'gofqkfsvmv', 'field_289': 'yaxjnyptoy', 'field_290': 'yubqrraybv', 'field_291': 'yroorgxzcl', 'field_292': 'itpwbwznjj', 'field_293': 'scmasyemof', 'field_294': 'lfgjqjbbku', 'field_295': 'ujhleqlafs', 'field_296': 'xdsrjeidet', 'field_297': 'oavmtrfuyd', 'field_298': 'ibvbfgkdib', 'field_299': 'nosqjgxdqo', 'field_300': 'elwlwtywwv', 'field_301': 'wdzacnfupn', 'field_302': 'pavpzhjzvj', 'field_303': 'umnpwzsvfo', 'field_304': 'eysifkgjln', 'field_305': 'vmjtcmlseg', 'field_306': 'tojlhpgomt', 'field_307': 'mrnonezctu', 'field_308': 'alsvldxjqi', 'field_309': 'rcycgjtxcq', 'field_310': 'udyuxhvbag', 'field_311': 'zhdmnowclw', 'field_312': 'xssiquuobp', 'field_313': 'pbmifrtmrb', 'field_314': 'uuvjgjzfxs', 'field_315': 'yckjuhvpwi', 'field_316': 'bbmqzejuod', 'field_317': 'ssbicjksjl', 'field_318': 'rvgpdywjxg', 'field_319': 'sityqcvwca', 'field_320': 'hkdbtylgcj', 'field_321': 'wmmelglxyv', 'field_322': 'oqhcqdjhcd', 'field_323': 'nkfxasthpy', 'field_324': 'bmgkeimjkq', 'field_325': 'fefhdntjhn', 'field_326': 'fiqtjdwsuz', 'field_327': 'uqhohvrtbe', 'field_328': 'scqgvmqbjn', 'field_329': 'sschpsjgct', 'field_330': 'vocelahvdd', 'field_331': 'enluzegvih', 'field_332': 'dxfrvfuhtm', 'field_333': 'ftzneetjpa', 'field_334': 'ameqisklox', 'field_335': 'mztjnvcije', 'field_336': 'ccmkfwksxl', 'field_337': 'osuvbsjubz', 'field_338': 'mmkmyxhccr', 'field_339': 'qzecqchbfg', 'field_340': 'sgeyfagauj', 'field_341': 'toobdaboou', 'field_342': 'ftqjmswmdu', 'field_343': 'ipttteafei', 'field_344': 'qesokwnscp', 'field_345': 'vogxjmcpvo', 'field_346': 'asihmgwjah', 'field_347': 'zzwbwrtwci', 'field_348': 'fsilxrlxmu', 'field_349': 'foiovucpde', 'field_350': 'cnwvyswfsb', 'field_351': 'bncaxxemsj', 'field_352': 'mszrxqrkaz', 'field_353': 'jtuyzxsmvx', 'field_354': 'tvpsrpktul', 'field_355': 'ubrsftryao', 'field_356': 'lbtkuvnldy', 'field_357': 'viofbbnncd', 'field_358': 'rxmtrgpaqy', 'field_359': 'hnsuaallex', 'field_360': 'wkyfuakxtq', 'field_361': 'xfsfbamnfw', 'field_362': 'wugbssvzrx', 'field_363': 'ijioljzoda', 'field_364': 'erlltaiqdw', 'field_365': 'zecsfjomyx', 'field_366': 'eoeusgtjfq', 'field_367': 'wuveplqegp', 'field_368': 'yzillxhfgq', 'field_369': 'khcaqbyygb', 'field_370': 'xvjaagwhlp', 'field_371': 'jxxoowhjrv', 'field_372': 'ngqbvuzltr', 'field_373': 'fhzitipzut', 'field_374': 'xslpdrcxnk', 'field_375': 'jpcdbqewaq', 'field_376': 'uoiziiblsz', 'field_377': 'mabjztykni', 'field_378': 'yuwgkmzmox', 'field_379': 'qgbemjbqgv', 'field_380': 'ogiondskyw', 'field_381': 'uwotxytmik', 'field_382': 'xshssyctoj', 'field_383': 'glmxpzsqqo', 'field_384': 'aowvrvqljo', 'field_385': 'rymsghnpih', 'field_386': 'rgtxadbuhw', 'field_387': 'hgxiufmjfs', 'field_388': 'plgaxjdpbp', 'field_389': 'ampkhnpoou', 'field_391': 'bwuudhgwen', 'field_392': 'ezqymiaiah', 'field_394': 'wyvqwoipxa', 'field_395': 'pgvknbexes', 'field_396': 'lffhhqbwxk', 'field_397': 'biyibehuvg', 'field_398': 'hvxxkefpij', 'field_399': 'wuozxnwcxc', 'field_400': 'mmlupuzxlv', 'field_401': 'htiysdfbfr', 'field_402': 'bngspwspmu', 'field_403': 'oanuhquzxn', 'field_404': 'fcifyxgpiz', 'field_405': 'jhbdjvlige', 'field_406': 'qvadrejige', 'field_407': 'wktuxhcmdj', 'field_409': 'ptfusfthpr', 'field_410': 'iqpczuiobo', 'field_411': 'mjocrpfeqd', 'field_412': 'rbqtkqplqm', 'field_414': 'csmxlocvfh', 'field_415': 'nhxhusdzao', 'field_416': 'wbdaapijol', 'field_417': 'glmbczlbtz', 'field_418': 'hsqrzjoxbb', 'field_419': 'bxdhzlgleb', 'field_420': 'yufgztzyne', 'field_421': 'lsrtclkfzt', 'field_422': 'slmmyycwld', 'field_423': 'yoqcuwabzz', 'field_424': 'loasbggnpd', 'field_425': 'mhxjntcpct', 'field_426': 'bdpnjggqsc', 'field_427': 'opztmezbvl', 'field_428': 'zdytzypnzy', 'field_429': 'zkaonbkvlv', 'field_430': 'aaqzttztku', 'field_431': 'lxmcpevspd', 'field_432': 'cnuiofbiiq', 'field_434': 'ywtvnnirmj', 'field_435': 'imsdmqrnwq', 'field_436': 'srzsrhqqbc', 'field_437': 'nbqtozaqen', 'field_438': 'obqaouwybx', 'field_439': 'wogxrybxfo', 'field_440': 'mmaadvvqss', 'field_441': 'renyoniavg', 'field_442': 'lwkwmljodg', 'field_443': 'mrskvnwbde', 'field_444': 'irsebgoaxi', 'field_445': 'ilfozxkjww', 'field_446': 'cmqvepgrkx', 'field_447': 'rosaegesdq', 'field_448': 'gxqzretruo', 'field_449': 'rejrmffrpq', 'field_450': 'kldyvunfkb', 'field_451': 'xurvffccdk', 'field_452': 'ucjqqmbklz', 'field_453': 'xqdyalqgig', 'field_454': 'fuxrkvknij', 'field_455': 'hcnkbimdgc', 'field_456': 'gvvwytbasn', 'field_457': 'tdhhruxqnw', 'field_458': 'srotqxtdqq', 'field_459': 'flfgunuijd', 'field_460': 'jrgpvebtsj', 'field_461': 'crixvdwrfh', 'field_462': 'iddhjtqrvi', 'field_463': 'hhhjsqsxal', 'field_464': 'xtbfgncdhu', 'field_465': 'iulgjgrktz', 'field_466': 'ahpcqcgsfo', 'field_467': 'wcmipdrhph', 'field_468': 'vbncbkkeox', 'field_469': 'tzyixfepcv', 'field_470': 'shxfzeaxnr', 'field_471': 'jdwkqlmerp', 'field_472': 'gtdaffbefd', 'field_473': 'yjczawzeyw', 'field_474': 'tcsuymqthv', 'field_475': 'afvkrfjxqs', 'field_476': 'rirdsxvfux', 'field_477': 'rgbfnddnol', 'field_478': 'qyuclrlpjp', 'field_479': 'nnviromait', 'field_480': 'apeawrlmix', 'field_481': 'uedmfiybjx', 'field_482': 'siwcsosmoh', 'field_483': 'jcjydprlgv', 'field_484': 'dpyuyjyubt', 'field_485': 'nmwsawjacc', 'field_486': 'jadcokhfjd', 'field_487': 'rqwurchuvv', 'field_488': 'wagofgruhs', 'field_489': 'qrxqgefpgc', 'field_490': 'pquhhvezty', 'field_491': 'fpcajgfiey', 'field_492': 'wtjxpstnwc', 'field_493': 'enimbzqltq', 'field_494': 'cqmsafwuso', 'field_495': 'lazyjtslsv', 'field_496': 'jtfrzfxotz', 'field_497': 'iajbovpghz', 'field_498': 'eegyrpmhtd', 'field_499': 'hwmliugmle'}, {'field_11': 'eshdtmlgqz', 'field_45': 'vjkfkdfcev', 'field_60': 'gwmnmslgcc', 'field_62': 'mcltcbcvnv', 'field_79': 'hwvpzkbzrp', 'field_109': 'lhboijxkac', 'field_114': 'onuquboypv', 'field_121': 'swdkpjtgzj', 'field_132': 'diasefmvbi', 'field_152': 'gswhdlfhrz', 'field_170': 'pkxvcpeoee', 'field_182': 'lcfddrchqp', 'field_198': 'jenhccrdom', 'field_200': 'jutsrxxjlc', 'field_201': 'dbmhanjlgs', 'field_202': 'vmfhfrlnky', 'field_203': 'gmhpscjldc', 'field_204': 'qssczmqmip', 'field_205': 'hobjsraoph', 'field_206': 'vzatwoktjf', 'field_207': 'ywirwofjtv', 'field_208': 'htbbotyhzq', 'field_209': 'itcqdngmxs', 'field_210': 'hoqrdscnit', 'field_211': 'eiawmiwrug', 'field_212': 'eiudhynzbm', 'field_213': 'udotablaux', 'field_214': 'ysebforsfg', 'field_215': 'nmkvpqgona', 'field_216': 'brlnmlfjtm', 'field_217': 'ltstictlsp', 'field_218': 'zgselceyno', 'field_219': 'hgvcjkvizx', 'field_220': 'ohqckdjgvr', 'field_221': 'dredaneach', 'field_222': 'hlmzsoeoaj', 'field_223': 'qfvcbyqxso', 'field_224': 'vwbyxfrbrg', 'field_225': 'iaeayydrhc', 'field_226': 'bqlsrutzuu', 'field_227': 'pwsaisuakj', 'field_228': 'akzcrmrvjp', 'field_229': 'jstejbnnoj', 'field_230': 'qthjwmymid', 'field_232': 'spwrndcytt', 'field_233': 'lhcuwvjzcn', 'field_234': 'brtxekyiqj', 'field_235': 'oxsqsvqrlc', 'field_236': 'ixmizhmsri', 'field_237': 'fjrgvnyvsd', 'field_238': 'pglojhppab', 'field_239': 'veyqkzjfgx', 'field_240': 'muprrbepnc', 'field_241': 'ujjtsynotq', 'field_242': 'zwllheuutk', 'field_243': 'fddejslken', 'field_244': 'txhbisytbp', 'field_245': 'hyqwxfhzri', 'field_246': 'yzmkfrpxix', 'field_247': 'zfldaihgwa', 'field_248': 'pdixcilkbw', 'field_249': 'gkxjikulzq', 'field_250': 'ndvwebwknb', 'field_251': 'mnpnsxkldt', 'field_252': 'bfvwduopqk', 'field_253': 'dvclodxxwl', 'field_254': 'scspwqjfnh', 'field_255': 'jxmmunuqhi', 'field_256': 'yammynayse', 'field_257': 'otpyscmhwp', 'field_258': 'yrevyfksvs', 'field_259': 'uznizsjjhj', 'field_260': 'pyqzrbajwh', 'field_261': 'jjljdcawlc', 'field_262': 'bcozzgrvjy', 'field_263': 'rbxtttacjm', 'field_264': 'fpuzkilcux', 'field_265': 'twrkwgrgyf', 'field_266': 'aogssdzlev', 'field_267': 'eucepnwlwu', 'field_268': 'jtvfjgrczd', 'field_269': 'prdazfrxul', 'field_270': 'sorglomftv', 'field_271': 'uwfqoccqyy', 'field_272': 'lhkuyemcyd', 'field_273': 'yosyvoxtdf', 'field_274': 'avxfksvcwo', 'field_275': 'sopmyiimyj', 'field_276': 'hvwtchbzvv', 'field_277': 'uuoldpdgsd', 'field_278': 'sxqzqiuonb', 'field_279': 'dhffggayfa', 'field_280': 'brbldhtzaa', 'field_281': 'lmlqvxyetq', 'field_282': 'olcnvophey', 'field_283': 'dktnjxlwcw', 'field_284': 'cabvczjsxm', 'field_285': 'ychsrayupc', 'field_286': 'uywktciqck', 'field_287': 'tkwbuvepfk', 'field_288': 'dngxpreeuc', 'field_289': 'tffmwmmdju', 'field_290': 'qctkmogihq', 'field_291': 'kfcbxxwhnw', 'field_292': 'cokeuuanuo', 'field_293': 'yzgwcqkzcy', 'field_294': 'zhzovswwtn', 'field_295': 'jgnpspsgcj', 'field_296': 'lbovdhwjcr', 'field_297': 'rmrcsbjpzh', 'field_298': 'bywnddnnog', 'field_299': 'npxcjkyfin', 'field_300': 'gliinvuwcc', 'field_301': 'iiznhafyuz', 'field_302': 'urwalgadzy', 'field_303': 'tnijxxjsvr', 'field_304': 'iphgyljgwa', 'field_305': 'nzultslven', 'field_306': 'aafvttknpq', 'field_307': 'hakoinozgj', 'field_308': 'wxiajxezhe', 'field_309': 'aojtsphytw', 'field_310': 'gzlcvkxrrq', 'field_311': 'uzcaxlomxp', 'field_312': 'vktznioehm', 'field_313': 'unmyvduyit', 'field_314': 'qddbvfdvpr', 'field_315': 'gbebfchjyb', 'field_316': 'lnndnzatjk', 'field_317': 'uouhsawhuq', 'field_318': 'xtdgmsfllh', 'field_319': 'zkprbxlfiq', 'field_320': 'cimtiiooqi', 'field_321': 'fdahdpgxdl', 'field_322': 'jawrcwdmfa', 'field_323': 'muxokseirt', 'field_324': 'xvpsvqtcld', 'field_325': 'htvvtipocx', 'field_326': 'erhhrtecgf', 'field_327': 'iivflhscdx', 'field_328': 'gbgyvgawet', 'field_329': 'wonrogyjqf', 'field_330': 'eseoyxsoaz', 'field_331': 'uqtichssre', 'field_332': 'gbwllictvb', 'field_333': 'nkdzwqwmrj', 'field_334': 'pmmbpbkbxt', 'field_335': 'trgrzyganp', 'field_336': 'uzkraufegg', 'field_337': 'hnxwibmhgu', 'field_338': 'gpewmaocik', 'field_339': 'osctvzfkxf', 'field_340': 'xiaqqkajlp', 'field_341': 'yjdaxxeegi', 'field_342': 'dpeydnjcql', 'field_343': 'exikhenraa', 'field_344': 'fdcaugolfa', 'field_345': 'mwqtvhzjfm', 'field_346': 'vbdpbuyhbo', 'field_347': 'zfityqzmti', 'field_348': 'vllzlvecgq', 'field_349': 'alxhljwqoi', 'field_350': 'dldhdpvayv', 'field_351': 'muqgaykgmz', 'field_352': 'dpmavetsus', 'field_353': 'qetxuiyngt', 'field_354': 'ihwycaihdj', 'field_355': 'otziiwaxqm', 'field_357': 'trvmburhka', 'field_358': 'mauglzltsg', 'field_359': 'ntjpxmrhwp', 'field_360': 'mlswtnjlkl', 'field_361': 'ismtnuzout', 'field_362': 'xoguzcptmb', 'field_363': 'dxxyrivlke', 'field_364': 'bakonemqcf', 'field_365': 'ckaropwmcd', 'field_366': 'zjtludqudd', 'field_367': 'qhfndcyefi', 'field_368': 'rmnbbyybsq', 'field_369': 'atzqvkxmlg', 'field_370': 'imxrzscagt', 'field_371': 'fvhnbqnrnp', 'field_372': 'teuqxgqexa', 'field_373': 'uzapvizemx', 'field_374': 'nueeicqpgi', 'field_375': 'sygpxbkmlu', 'field_376': 'nwbiwvtkav', 'field_377': 'ftgvaioaii', 'field_378': 'gohqteetxk', 'field_379': 'bxfdladfsq', 'field_380': 'hgpblaoeyn', 'field_381': 'jtebdnucuv', 'field_382': 'rfsoxffovv', 'field_383': 'aplxvkmuuy', 'field_384': 'igknvlysns', 'field_385': 'svbsclxsvu', 'field_386': 'xskjteqaiy', 'field_387': 'uraxzheenk', 'field_388': 'dwrtvmzdng', 'field_389': 'iavbhenbch', 'field_390': 'djjkkwbypa', 'field_391': 'kfnhenhalu', 'field_392': 'lglsnqwupk', 'field_393': 'ggyetexkah', 'field_394': 'antyhvzven', 'field_395': 'tuoopoqlbk', 'field_396': 'asbceuftle', 'field_397': 'iejpcszhxo', 'field_398': 'qfxguagwlw', 'field_399': 'zpywfojrrd', 'field_400': 'vwerhhwcek', 'field_401': 'tplrtpehyv', 'field_402': 'vrpqcxdmci', 'field_403': 'xhvylffaxn', 'field_404': 'mvmpsfbdwt', 'field_405': 'ljctedmnxo', 'field_406': 'awcmrrmyyh', 'field_407': 'tikxulhckd', 'field_408': 'dqojoiuomo', 'field_409': 'ulokhbouix', 'field_410': 'xsfbsybkqj', 'field_411': 'gkafhunvmc', 'field_412': 'gefgfqvmuw', 'field_413': 'fwehrmswxn', 'field_414': 'zspqvefmcr', 'field_415': 'gbirpkojij', 'field_416': 'tlcsabqlvq', 'field_417': 'vovwzlwvnj', 'field_418': 'wwitojprst', 'field_419': 'xvtnsvcxqn', 'field_420': 'fljkhxmdro', 'field_421': 'bwgpdmeaph', 'field_422': 'srbdtgpdss', 'field_423': 'umrspwifod', 'field_424': 'yhjgwgrsks', 'field_425': 'aacpowoesj', 'field_426': 'xaqpuopzmq', 'field_427': 'mpjwxnrljr', 'field_428': 'frjrnqyqor', 'field_429': 'mkivwanehv', 'field_430': 'ixhprfvsls', 'field_431': 'codvxwrtvk', 'field_432': 'hppbeequwz', 'field_433': 'gsznhvfrbx', 'field_434': 'pefghygwzu', 'field_435': 'eecvksnruf', 'field_436': 'xijimxblmo', 'field_437': 'urnfefayuv', 'field_438': 'lcbaxruwjg', 'field_439': 'wncmsboysy', 'field_440': 'zchjbryuxi', 'field_441': 'mkxqobxkhr', 'field_442': 'guxbutqgmo', 'field_443': 'grfvifcqfy', 'field_444': 'vgvpczvcri', 'field_445': 'bducaejqjt', 'field_446': 'qestdgqxhx', 'field_447': 'alasdokkiu', 'field_448': 'mdgigmhvob', 'field_449': 'jyrlycbanr', 'field_450': 'fjucjcpiol', 'field_451': 'lyxchemlmp', 'field_452': 'yiqsqgjykj', 'field_453': 'bdmjnjvxge', 'field_454': 'esynvxtwjw', 'field_455': 'bwyrocjrzx', 'field_456': 'yownvflony', 'field_457': 'lvayfzfoec', 'field_459': 'pmgldzbkio', 'field_460': 'kslpatmeoq', 'field_461': 'qgycwyvtve', 'field_462': 'ldundjmwmx', 'field_463': 'ozetpwopxb', 'field_464': 'qneiyarqyk', 'field_465': 'njocawzjnt', 'field_466': 'jakxxjyygh', 'field_467': 'kggdlqbxoy', 'field_468': 'ecccrrrrpg', 'field_469': 'levtmnsbek', 'field_470': 'lrvlalexzz', 'field_471': 'awxabslfuy', 'field_472': 'slykmvbzus', 'field_473': 'zncntkxkar', 'field_474': 'ulwpbsuxoo', 'field_475': 'koeoslzzza', 'field_476': 'ztzjocrodm', 'field_477': 'vmltnyiooa', 'field_478': 'tdemlvocxh', 'field_479': 'fxwruenggr', 'field_480': 'ovofgzkvin', 'field_481': 'tzsotvgzpu', 'field_482': 'ntpeqivxkt', 'field_483': 'aqlnacywze', 'field_484': 'dkhpqclhyg', 'field_485': 'kephfjdgtl', 'field_486': 'qkzkfaevlc', 'field_487': 'pnpgckqxvd', 'field_488': 'lhbmwkolig', 'field_489': 'qgrasmlmov', 'field_490': 'hghehylogt', 'field_491': 'kdhmrzhnsh', 'field_492': 'irayqzvvkj', 'field_493': 'zmgutaboxm', 'field_494': 'tjhzudfakh', 'field_495': 'bkenrqzwyy', 'field_496': 'jjviyyhfkn', 'field_497': 'jajarerhkr', 'field_498': 'neiicxoidd', 'field_499': 'hhlxwteins'}, {'field_7': 'fvzzxxwhin', 'field_22': 'yzhtqbidst', 'field_53': 'kakdzcffjb', 'field_55': 'wkskaaalpn', 'field_61': 'edjpsmntjr', 'field_90': 'gdvaqyouhv', 'field_92': 'cmcfcoswlc', 'field_98': 'fnnipofsgw', 'field_99': 'ysoxmwjmne', 'field_131': 'wqmtlnbrub', 'field_142': 'ucpowbhble', 'field_146': 'fbygklrjuc', 'field_147': 'gwrwlzqeeq', 'field_162': 'pnawxgjjie', 'field_172': 'plfcgbgluq', 'field_174': 'ygbhfhzleo', 'field_177': 'nsujtbdwzh', 'field_186': 'ddczzccrkp', 'field_192': 'mxlmoihcvp', 'field_196': 'tmstrumxoz', 'field_200': 'govhnrcaps', 'field_201': 'zyhifdxwsk', 'field_203': 'jydvfprpeh', 'field_204': 'bkjetaxwvc', 'field_205': 'sdqnksdjou', 'field_206': 'kmvlxtiius', 'field_207': 'sypqehvlvr', 'field_208': 'udyafbehlr', 'field_209': 'xcgeuyoktf', 'field_210': 'mjfrpeiaoq', 'field_211': 'oynbtordbu', 'field_212': 'jmrxspgvhg', 'field_213': 'lzwottouei', 'field_214': 'glfoexllvu', 'field_215': 'htrectvmxo', 'field_216': 'wxqcxaadhx', 'field_217': 'xyzrqumoex', 'field_218': 'zzztbarplc', 'field_219': 'yalpdheizm', 'field_220': 'egfyvkzkif', 'field_221': 'dwmuljoncy', 'field_222': 'odwulayqlj', 'field_223': 'uygkknyvte', 'field_224': 'ghdnfdaopm', 'field_225': 'vgfecmboik', 'field_226': 'raswtflitl', 'field_227': 'rimyirqbrf', 'field_228': 'zklusrrqyc', 'field_229': 'xgdgcqmbjl', 'field_231': 'xcbspxxfqi', 'field_232': 'bqcgztpvzp', 'field_233': 'hkxjwiibyz', 'field_234': 'ehrnglptii', 'field_236': 'qwzppqitdd', 'field_237': 'ixdhaiajki', 'field_238': 'gmudsyhxrl', 'field_239': 'itovfhrhlc', 'field_240': 'itcoyxpwcy', 'field_241': 'bjwvhscrcz', 'field_242': 'vidltsfyaq', 'field_243': 'lkhxbjkclp', 'field_244': 'jmuwcbhrkx', 'field_245': 'sjedvvprcb', 'field_246': 'riihoqqkoc', 'field_247': 'rdvxyyykvd', 'field_248': 'mhhrqnrtkr', 'field_249': 'nkxtgneonf', 'field_250': 'pzijfubccb', 'field_251': 'gzqknqwhji', 'field_252': 'uhypjhwwjx', 'field_253': 'xfbltdjwwb', 'field_254': 'svuhkfklwz', 'field_255': 'ndkswqlwrm', 'field_256': 'umakmpunbr', 'field_257': 'nzgdhuhjct', 'field_258': 'zrwgsqhimc', 'field_259': 'lptrabytkv', 'field_260': 'yxhhchjctq', 'field_261': 'ttrroowjia', 'field_262': 'nfiorszjdx', 'field_263': 'siarxbigid', 'field_264': 'dulizlmvxo', 'field_265': 'igiulforbg', 'field_266': 'kpubekxqxa', 'field_267': 'vzpbtpklqm', 'field_268': 'ehuhjtcllg', 'field_269': 'fudeljojha', 'field_270': 'dgtmzbxtfg', 'field_271': 'dwljxpuhky', 'field_272': 'kgivhsuhnm', 'field_273': 'lzakdzhzfl', 'field_274': 'exozlincnv', 'field_275': 'ttgkcqqsib', 'field_276': 'hyrmkzwpmp', 'field_277': 'eshgbkmohy', 'field_278': 'vqsjsiqmjb', 'field_279': 'zbndsvfjsc', 'field_280': 'pregmribnd', 'field_281': 'hbvgsexzfb', 'field_282': 'dmlnkyswhx', 'field_283': 'ipmwpihyir', 'field_284': 'luclzvyezg', 'field_285': 'ymcvdkbsdg', 'field_286': 'mnyjuwenal', 'field_287': 'ppmsapvoar', 'field_288': 'jeuvejjckv', 'field_289': 'cjlunjgwpk', 'field_290': 'fwitcwuffo', 'field_291': 'bniujuozny', 'field_292': 'rsnpydqhfe', 'field_293': 'miqnlzxyku', 'field_294': 'aydzcwvdyv', 'field_295': 'ldqpirtgqb', 'field_296': 'kmukimzhzw', 'field_297': 'ltuwoskuns', 'field_298': 'rntoluemcm', 'field_299': 'asduxvwzut', 'field_300': 'dyyspmfphd', 'field_301': 'bjzekqyjhf', 'field_302': 'mlpckcsbqa', 'field_303': 'gmiwxlvcnm', 'field_304': 'qxdqgjqayu', 'field_305': 'qejfllddbo', 'field_306': 'gpjskumbls', 'field_307': 'kymxpneajd', 'field_308': 'prtzmjzgvf', 'field_309': 'rsildgymqx', 'field_311': 'omettutsya', 'field_312': 'uivtssinoq', 'field_313': 'zqgfbjecmz', 'field_314': 'dyvempqbpk', 'field_315': 'juemgvkskj', 'field_316': 'luhjscfppd', 'field_317': 'elujsjbxzy', 'field_318': 'meypjraubd', 'field_319': 'ahkncffrdu', 'field_320': 'yyejkozwut', 'field_322': 'atfzjmpfie', 'field_323': 'abkisshwoe', 'field_324': 'gwifhtbkax', 'field_325': 'ylvbeuhcxh', 'field_326': 'ukdikpigpr', 'field_327': 'jgcqdhymjv', 'field_328': 'pqnpbwsdsl', 'field_329': 'vxqldxhhhw', 'field_330': 'mwgxldqzsa', 'field_331': 'jdajocghbl', 'field_332': 'hvtsuhjezv', 'field_333': 'ujtxetcfcs', 'field_334': 'qakhthgdpe', 'field_335': 'qtvdtieidv', 'field_336': 'xvmuhxztqx', 'field_337': 'rwmuhzqalq', 'field_338': 'gncajpkkhe', 'field_339': 'koqamaohxk', 'field_340': 'pueybmttxs', 'field_341': 'anzmlkbtcf', 'field_342': 'chnovziwgt', 'field_343': 'ocsoyolusv', 'field_344': 'dqnxiakhge', 'field_345': 'zsplymvbzs', 'field_346': 'pfkqbakfel', 'field_347': 'fcttmggaki', 'field_348': 'hmjfztrvmg', 'field_349': 'ssxzstuydb', 'field_350': 'vcmkwgvvnl', 'field_351': 'jppkjvrgzz', 'field_352': 'vnvmdcrabm', 'field_354': 'ifntwgzjbp', 'field_355': 'ngwluuvlag', 'field_356': 'ihifzimehu', 'field_357': 'uvmtpbeabe', 'field_358': 'uqshgbnyar', 'field_359': 'lifzphdyrq', 'field_360': 'iybkolbjtg', 'field_361': 'tnjoitglzz', 'field_362': 'kgitnoagff', 'field_363': 'jtheezggsa', 'field_364': 'pansdsngel', 'field_365': 'clfxswrfsn', 'field_366': 'kvqrwqwuhn', 'field_367': 'xjajpwzzuu', 'field_368': 'xjzpzuoiiu', 'field_369': 'gwwsqyadpw', 'field_370': 'ljkvoywhgh', 'field_371': 'fefihrdger', 'field_373': 'izstnzveiz', 'field_374': 'iarhisdmrl', 'field_375': 'vrrvlbraqx', 'field_376': 'vxjjmolcsr', 'field_377': 'lccdxdbaxl', 'field_378': 'tjkmfcjzzv', 'field_379': 'bdompywlfp', 'field_380': 'bxelcnxyzx', 'field_381': 'ocqqlfphyu', 'field_382': 'htrbxkfsdb', 'field_383': 'nrdqtevvyq', 'field_384': 'zmteuhhvyi', 'field_385': 'cbshwleibh', 'field_386': 'pklxbudvka', 'field_387': 'ycmsewnpxg', 'field_388': 'gtoznhgnih', 'field_389': 'kbapbfdduv', 'field_390': 'ihwbzhucts', 'field_391': 'zjdzfenuoq', 'field_392': 'lxwvinojtc', 'field_393': 'ttqjomupnm', 'field_394': 'hxreyjddlf', 'field_395': 'lrkdfkzkbx', 'field_396': 'rfwnsjlssj', 'field_397': 'tiddjdaajs', 'field_398': 'pebcmahcks', 'field_399': 'lwqinuiarx', 'field_400': 'pqiudcshfn', 'field_401': 'jscskldikq', 'field_402': 'gqfywadipj', 'field_403': 'txtugilcmi', 'field_404': 'nxnmifvlqs', 'field_405': 'ssipohatsx', 'field_406': 'fzjwdibxpx', 'field_407': 'tiqpagjmmt', 'field_408': 'apjsyazxeg', 'field_409': 'qqqilcstng', 'field_410': 'yxmnqfdqqu', 'field_411': 'xbfcxsfqqo', 'field_412': 'zuslhtqrdo', 'field_413': 'ubujkdawxc', 'field_414': 'jqqeskaqgl', 'field_415': 'xdjdvetoew', 'field_416': 'uqkdqxbehz', 'field_417': 'veynevytbi', 'field_418': 'dmhmfrtvoy', 'field_419': 'cbkldzrfkl', 'field_420': 'xwrcvuwsgh', 'field_421': 'rvgdlkvtjq', 'field_422': 'drnqurerkn', 'field_423': 'mbkllysngu', 'field_424': 'urlhyqzdub', 'field_425': 'uxecokvcwe', 'field_426': 'mxfailhxwb', 'field_427': 'tlmgyiqvrh', 'field_428': 'roekawbvdk', 'field_429': 'lvaynzscyn', 'field_430': 'vkjceztozx', 'field_431': 'zktxjvszll', 'field_432': 'plgzsjvntl', 'field_433': 'hkubhrdiyo', 'field_434': 'scbiqjlqjg', 'field_435': 'renswyqyrz', 'field_436': 'ulylqkrxma', 'field_437': 'kxqgsdlxcw', 'field_438': 'mwjhzrooil', 'field_439': 'gmyjmuotce', 'field_440': 'eniyygdhtk', 'field_441': 'ukyzfdrysj', 'field_442': 'wndjiqcdol', 'field_443': 'sosohvdmms', 'field_444': 'cwkfhvogtt', 'field_445': 'tvhlmaocup', 'field_446': 'zixyhxdhjt', 'field_447': 'aayagslnay', 'field_448': 'rfhyyjxsmf', 'field_449': 'eudbwbrajf', 'field_450': 'gycgtaamkg', 'field_451': 'ghdfotjhbu', 'field_452': 'ykjafyadcn', 'field_453': 'mytyoiluyi', 'field_454': 'fsohnlyozt', 'field_455': 'zbmlbrnvme', 'field_456': 'kkydidmwiu', 'field_457': 'jyeruwedol', 'field_458': 'lwwfwuuqul', 'field_459': 'ffbxrqtxll', 'field_460': 'yckvxdxwzl', 'field_461': 'eulokyekjn', 'field_463': 'wjatohffhx', 'field_464': 'ufvsdzzcsv', 'field_465': 'ilumovnarr', 'field_466': 'bgzvtlzqdd', 'field_467': 'fksxkfmqeb', 'field_468': 'gxahvvbfji', 'field_470': 'xelmawtzzw', 'field_471': 'ybnygqkkkq', 'field_472': 'kcituviada', 'field_473': 'huynizeldm', 'field_474': 'xroswqilhm', 'field_475': 'yftmouwblb', 'field_476': 'mbnvlqrkgk', 'field_477': 'zcicsqqpuf', 'field_478': 'acvktajajk', 'field_479': 'phdylzgphw', 'field_480': 'rgjhccaqpb', 'field_481': 'nhgrqurrwg', 'field_482': 'loogyrytdd', 'field_483': 'tzblqdqbpr', 'field_484': 'vqdscfvsqh', 'field_485': 'qhqkjgnqia', 'field_486': 'xodfimrytg', 'field_487': 'iifvmknvrn', 'field_488': 'rsbvgejguu', 'field_489': 'giqeqxczbm', 'field_490': 'chqdmkvuxe', 'field_491': 'tfxbatjxqs', 'field_492': 'mictdhilcv', 'field_493': 'emfbfwpbqg', 'field_494': 'rmlozkuwfm', 'field_495': 'kqxrxopcci', 'field_496': 'dvbxbexmbp', 'field_497': 'onxmyqwjla', 'field_498': 'arwivelpku', 'field_499': 'beqwzeftma'}, +{'large_field': large_field_value}, ]); diff --git a/metadata-ingestion/tests/integration/mongodb/test_mongodb.py b/metadata-ingestion/tests/integration/mongodb/test_mongodb.py index 54539fcc5d77f..5228c21223e24 100644 --- a/metadata-ingestion/tests/integration/mongodb/test_mongodb.py +++ b/metadata-ingestion/tests/integration/mongodb/test_mongodb.py @@ -24,6 +24,7 @@ def test_mongodb_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time "connect_uri": "mongodb://localhost:57017", "username": "mongoadmin", "password": "examplepass", + "maxDocumentSize": 25000, }, }, "sink": { From 1781691c3a88d6875e9d41fb395fdad92505053a Mon Sep 17 00:00:00 2001 From: Maggie Hays Date: Tue, 30 Nov 2021 18:26:59 -0600 Subject: [PATCH 07/14] docs: update to DataHub Adopter logos & Townhall details (#3648) --- README.md | 7 ++++--- docs-website/src/components/Logos.js | 5 +++++ .../static/img/logos/companies/hipages.png | Bin 0 -> 22147 bytes docs/townhall-history.md | 11 +++++++++++ 4 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 docs-website/static/img/logos/companies/hipages.png diff --git a/README.md b/README.md index 9938c497ced6c..b8dce8609e303 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ HOSTED_DOCS_ONLY--> [Town Hall](https://datahubproject.io/docs/townhalls) --- -> πŸ“£β€‚Next DataHub town hall meeting on Nov 19th, 9am-10am PDT ([convert to your local time](https://greenwichmeantime.com/time/to/pacific-local/)) +> πŸ“£β€‚Next DataHub town hall meeting on Dec 17th, 9am-10am PDT ([convert to your local time](https://greenwichmeantime.com/time/to/pacific-local/)) > > - Topic Proposals: [submit here](https://docs.google.com/forms/d/1v2ynbAXjJlqY97xE_X1DAntNrXDznOFiNfryUkMPtkI/) > - Signup to get a calendar invite: [here](https://docs.google.com/forms/d/1r9bObXKS3tgKpISqqO3rw4yQog5zwuaFxg8IrJGUbvQ/) @@ -58,7 +58,7 @@ HOSTED_DOCS_ONLY--> > βœ¨β€‚Latest Update: > -> - Monthly project update: [August 2021 Edition](https://medium.com/datahub-project/datahub-project-updates-7a0b75cae2b7?source=friends_link&sk=307c9c9983a2d0c778d0455fef12e1e9). +> - Monthly project update: [October 2021 Edition](https://blog.datahubproject.io/datahub-project-update-62adced87ad0). > - Bringing The Power Of The DataHub Real-Time Metadata Graph To Everyone At Acryl Data: [Data Engineering Podcast](https://www.dataengineeringpodcast.com/acryl-data-datahub-metadata-graph-episode-230/) > - Unleashing Excellent DataOps with LinkedIn DataHub: [DataOps Unleashed Talk](https://www.youtube.com/watch?v=ccsIKK9nVxk). > - Latest blog post [DataHub: Popular Metadata Architectures Explained](https://engineering.linkedin.com/blog/2020/datahub-popular-metadata-architectures-explained) @ LinkedIn Engineering Blog. @@ -115,6 +115,7 @@ Here are the companies that have officially adopted DataHub. Please feel free to - [Experius](https://www.experius.nl) - [Geotab](https://www.geotab.com) - [Grofers](https://grofers.com) +- [hipages](https://hipages.com.au/) - [Klarna](https://www.klarna.com) - [LinkedIn](http://linkedin.com) - [Peloton](https://www.onepeloton.com) @@ -128,7 +129,7 @@ Here are the companies that have officially adopted DataHub. Please feel free to ## Select Articles & Talks -- [DataHub Project Newsletter on Medium](https://medium.com/datahub-project) +- [DataHub Blog](https://blog.datahubproject.io/) - [DataHub YouTube Channel](https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w) - [Saxo Bank: Enabling Data Discovery in Data Mesh](https://medium.com/datahub-project/enabling-data-discovery-in-a-data-mesh-the-saxo-journey-451b06969c8f) - Bringing The Power Of The DataHub Real-Time Metadata Graph To Everyone At Acryl Data: [Data Engineering Podcast](https://www.dataengineeringpodcast.com/acryl-data-datahub-metadata-graph-episode-230/) diff --git a/docs-website/src/components/Logos.js b/docs-website/src/components/Logos.js index f95d014c944f9..8f027cd1bea80 100644 --- a/docs-website/src/components/Logos.js +++ b/docs-website/src/components/Logos.js @@ -81,6 +81,11 @@ const companyLogos = [ imageUrl: "/img/logos/companies/uphold.png", size: "large", }, + { + name: "hipages", + imageUrl: "/img/logos/companies/hipages.png", + size: "default", + }, ]; const platformLogos = [ diff --git a/docs-website/static/img/logos/companies/hipages.png b/docs-website/static/img/logos/companies/hipages.png new file mode 100644 index 0000000000000000000000000000000000000000..fa941d7cef94999f3cd03744c66abf20bbc3648d GIT binary patch literal 22147 zcmX_HcRbbK|G&evxkfhErbsrCO`%BH*)z&5E3)F=vSpUCN2Fw3GP7#@xE8OGnK~4FZAa^!0R%K_Dm}2n0q@k^@hCa=o*F zKV)v2hMFKyWg^Y7BMkVT&sopd5CjSm0fA8AAkZQ32=xmD@{?)pNGA_?AfdHL>?=oMWXW<&Mx#s~<|!{60tXz2{TX zA84mFxa+eqZELoCGT#!XBqwX|*y6jA--n9LWfKG0{E{`Lm4+O4KKZw6Zz4J+xzGpL z+gPg8H7y5vDV84kqjyRUTo{BUl?vrFoeFI!j2t|9$u3`um64G#S(?U_OrAw$6%9p? zp#AZe9S}n@k_8@tdxLxBPDS5L(MlY^$27@eiLd(kww+NIHAh@wh!u| z5^Y02O}=i-0Bpy|2us>xRw;rkgFL}EKL;uQ9HN`1KW49|#+CE^w@D3`iU_!ZY-Zzg zfpk9aJy1QANKeqsyP>Ib_H$0-zG#RXNRmt~`sRmnhaqY@#yh4{1x>-q?$cj*dQ$(laB2V`dTtp^}|Jn)iDdyo3oHc>aHf!Wi@~ zg~)*`x+BGSEa!OxsT8W$6|s3*z{2QgNOT9A3TO$+OEE)cz%I%g#zttxk?rnM^ZygM z@c|(jJV4e0%339<=bS@jV(fCQ(YO8y3#`+}s{#^%Y_|E!<;o7kxk?bs@Wy%8{C`CO zyya4(hpEvtak}5?1xbO9dB?bR$lRZP`gib4PPJ9EZ>3pYf@$&k@Dn>V3#iinDCT%V z0HsldP;&Xp-x1!whxi;Hj%Po9_e-(s1W_U?1|W-JA2MJzQI2MXaKs$p*Y$FGYD zIUH4h9t1Cg{KV2@ImtrT7EjpS^y`cqLRc>n*Z^eWChUV ziw+~7p*OhIE94|f;K8e-4fAIHq6U^UNyE#5mqWfeSaJ7EEL=plIx&xK>I6rgZG0Xz zdW)C_YNA^^NPA~oELMV+1gnFCShW7Z`5U(ih`wtnsEI!Fi2wULI5C|fnI(_^&Ob@5 z(D3?UCvZt6&Zc5!l4KiO7$O_r|0I~)42=;3FP_p{IUX#W{Oj8Xu|05Ss$ zp-g*g;$#y*O%(IGFedxve}b}X>z>29p?%Th+4S>$ATK2aMVIvdkgg}(uB%EAgY!nM z$+3@b>~G`6{xj+~Ama^!G!0P{pN%7kCqKZ}p*$UqqTc^AX>j2Jo)PrUiTUdwL`M0T zDVgGwHiUCZ-aAP4=vk&q>B2(X&g zOS!hL`=0xoh7LjaG-|i@2l}5TihqNC&K1xFTp>6*l+(Wza*!vm;B_?;|B6~K9+uRT zRHBU;TON1tRo6-R20w;OHF+uhGX*_EG6p#BF7``hvueWF+zT-uIACK&8g_rpNa*cf z!gz?F+SYCt9_3;!Yjhl%L|!W%eg>byi>Lr1C5jnXYe(zihbxR-M5zOeO~e0jZM=K7 ze3-TgOeDkHin?L!k{-461^v(eCX3PWB@-&u3Eud4jX3($!dr?2D82-1^3TwcKTz`} zH>)aRVwT?JHLD8!pvIN~zAzK@^ej*3#Ai=K~0?2^1q~bm^dg9JD3xS|IkOxw*^>) z5>_3&2@+ZL>bYFSLsY2s?790_$+tnpNPyvGXYJbq9U?h#&lLmlk^ohW8roHV5md-D zR{#s3sqNSZ;ejxkLrB!e+`mlzDjAma2iW-;#t=40kzsl)V6+Afr4HfvU&MhW8Mo>3 z48{EZ?cEe>>yoI2_r$(KwL${_JD9^3z#e%MJfJm9LSXO#Gl3WT7|rufR<}VM_lTG2 z*LDLq#%Kft1ZBx=&=N)`(f>(G(G;aRI-u+7`ogme_AYRA}oq@S@tk7L?f??PL=?|JqaXD8(7%w^5RiIMYW0W^PmjD*XO`kT&Mf zPzkAqut9h=waaxrOqhEL5$XSnN5#~92sIIBX4_dBxeq-$AMAL&qzS@#|3Zo}Be2i_ z+Ctq#_5Q}PE^R);WQC>!uhNSMgS!i}Z*HNq=-Y-))eDLxjd#f_V^(sn9z~o{S_2TX z^wb_YbInRLlZ|=^XW+jiAma-cmURwu&(t5nUB5N1p?CQygqm{uS@@5s^T?2=Z6A($ zE9}X&PZ(Stk?YHIL0ahrVf$QY6>1$VY7%zH)q6=f-Jxw%hAc7( z6U=qc?de3Qn0<%YlioCAEVg=v%Sw`jYX8k5jLptO4r-!UJ8{?dAih`Mt&z(*aeP|4 z!V(zQRwT-P{N1fbH8fL8Po7enSAcB;A*R^n8^veh(fx>x0LB{+F*O6YFct<2RSkB> zp>iR<#s@Lfu&OAP+#T4h~!}#>SxX2Q{ySK)mdKM*N$dtd~GZso}h<+GpuAGx?D>&F6%9^?73%2dBqWj zY%L8x=b(JD!P0bYEvctp)L;kv*@jf%iT$I2quB)N)r4PsX_ZNN_aH7@U1QG}SsmWz zw$;AtQTO4|er$8iapaE`6+Z>B=7fZK8*~;an35yhXqyii2zRelY6qEJ2zOcLz1bc& zdt+#b`-?UCi@U~_L2)%{9_P)U&j|`O5)b6ENuHzG#QRS#Y*TVt@tG*@%`=cCm5HWF z#jn3%?nQ)%&F#HAkF-mlyx|1rcMyGg3QOtbYX5lfH>nC>^?q-4^>=C9b@TP{o7TJs*Q8hiLPRu@88&e*PE1Hd44Y^y8mfGez~2SiOp)}d-b*7E_V@@2Ph?1 zH9E;7zRw-$Zw((-ASfUthK9W}?thkDIjYbFEf2|@LLx~kJ^1|R9oVh&qu2Fw=nRIa&hkHfvZMmEpvTXV_Y`lIV7qi=CQF^97KsR_T@HBoGE=tY}rh4Nl7m>jaZ@*=2@awc=0EUJ3 zKt3|^RlVrvQ2m_Z-LEy~DY|RnXVMgc^a!v#>(LvXW^a9lCg-sLwv4%g>bH=NettuC zNaxYn5qu~El0r2$f1*=ANaAXhhC6xMqFV_FA zY(%O3Z>7R}cAc)bWg2y{nrEt@=My)h6%f4byCz=>A%A28=tpP+^d~P?(^xR%P<`tQ z-I&!qwGYhKfe)up^6lq&<5S2rHPX2FxYv#Ni03GSYsZc6?VW?_O?8StdU&RzQt$#; z4zBY}KvAdu$pgsd6+K5?Lt7CeaN_^-d>yaG5>coX8r^pIwzBm>y@%l3*ms7}ewCy~ z)UI;=naE;5K(4^?n}Y9NFW;9|7iep%wo-` zTi7>1K3w?C8QC@GHEKWzk` zQ;l1^^eaa@w8=e9EVdCy#{I5h7_{%{AA>680F;RWkMFtKYGAF zU7OIB0Bl2%ZXgknD2lp)Y6?<66Y4PNoBtN|r%P6pUXlb`{PubduYhqUzYxaaVRYNx zF6MW!%#%R_iLVDB>x0!PY`gNkvsL;=QM8_nbm&d!_y>2PR7ar~5+^L9E_*H1M8^dy zwlf+PPFnJ6PiapxGlLsjY!a$NZvpkz5qq!`1XYV^q4j4_3iul81cYTuDs%~Y-08;7#;wX2)3;^VPV5?b?e<_4 zaHMJerza;7POu`NIbr0atsENv`xQ`TnJ~r*iFKHlH?mH-04cEE=d<#|m`mf0W+CuF zDApcjjVh?L9kHk62KSqL$9AwPk2!u>n~C1Nr3RtZ;i4;4nY^M5hP2w(LJTzo2RAek z1Pk6F_&HE{;~poAQ(%JT)@2t|P~KhZ`^-WN6t#DzylrYjHqLi=M|e|A?? z((ZSle|r0ApT>sr9MbTOZq2T3#o9R=COcQx-Ii5ykn+jib=D?o+qzXLezqj#Z)PEg z`UlGRVAuMXm+bF2?}bj)^&7wKZow%#QY{L~8eJyL-gRnFj~S@7AtB_j~Y=;34`ww?7I@X zbsUtXM#3aoiVmeS7rN_u#p}Lyiq`RV8zY?Zd8a}aj&|6%T#wf@Nu*08(!;P_qG}D% zua1;$d%F5(wj>72)UcFs5|lyJ4_!L+c*R@S8){=E(Cy(FKlUv7&!rh?9VD}|tI995 zxomfK6GHuIU@!1FXVY=!@%N<)R)?p&uNBmntxxvjQrlQb&C+HbCmB5l`x{&Hle|hT zKSG~8(k*}2GHfF%JZ_~L-9$#TU5EsVzXS(Hh-3BVBAh~UB~K$2WtG1=Wjo~DpS}IK zR<(IOiqnr5-jAXlSZ>b$phmcs91l<=KyE@3!1P-TGLCRf=jtGpq>JE((1&e%t`nN^ zb#2%B>SuVPJE;ee73=-`eMZ&aCYQPkhk3E-)Thy%sHzchU8BRwKy2bTd4)~<(GdFz z6%x7!r6;F{((j_w#@n5wU0h7|7-`lwE^z#rZ}}oV2#1F4dOW^iH4xRa`+Aq5m6EH{ zrdW7E;=e&O8P z3Bej}I znK4N@S;ScVl~2|?^2rZnIx7(-IXP{S8$-Y7bQ?6=@v;P=Wf2BJ;+y z9Kbf;SKac}E!_^DJ%Thw;Y|zF;6nRg#emLjDhCG@vy|idvB3Jv;LCKEIW7;0 zSJTb^+LJ~-_;$>K&$!=wH%h+rBH7K|qiHI;%50t8YMng?)ujC(Y$I$<0`b0ch<8Og zi~1sY!uGsRy<&$!{>}{cKII=Sv^r#VC$FJ9QUR7!2yni!@#DP*pL@2!-ARbdN4S8j zn{;}59A#0zDDZ4Zg|TXvmHpc5`+2uepY7UmexLJyn1`98qFzH3V3r)IGc6eQHmX|- zqyTKSP|WJ!`C2~@@XkXe6Khn|P&I1)um5QL$u0XFRz#0?4c*So!GBdFGY7XqLZiQ~ zP??NYaS~O-%6T^nD#yNeo(7Mc43304UvQA9e(a!nGJZ%JciA*sP|l%l0$m;##~i)b zky=~DaMUvpUFFSE1XHIDuyLr@(09a|w8)wFDhdW`{uM@co8uhv-!99xCwAXJEgz}+ z3e>1xbmbs`;evz83DkVvYlEUePL{GpSf7T+$3{&HM@+u0Yow>{*o<`wdDaCfzuEI=M3wBpRW@>cm<;)Hm! zk9F?PKRg|rIj4gaAI{s_OAQAS1jj zOb9M~8eBcgMWSE(O?iJ^*%}hUOT1H~FtENDWJP%lC6JSpWfLk=Nn~pMmPluvZrjUdt0Y_ zUd_<%UZ^w7=v61y4YfUWMY-*J8sgg}mT=uNE@OHn_{98P`Z9Ope|$+Di& zQCzZIS$H zjN@B-^X8lT(_x>@e{&g0xJ;opL8PlJyOBEO?_k(>m3DAC+vzemSB#h6PFnH3I^^!0 zic6Kt63?~v3EsQ|+t=TShZRYxZv<`UpzFRDFTX0Gm8fP| z_RjptGR^BmcWJ(iUJ*t-??qKhCZ})jk-V?YN|YujL?pW1>$)bbQu2pSm|Diqg$-de zADH|0AX`0ePM9qv-S@nU$;^u%xa&W*FXgR#WqYAYe7Ra2?zUE}zFp?RmThw|x!WY@ z9xy?H6t0pQ?L9@xVgkY5N>m;-U!9oqTIr5CJ-joFon5CWo8VlWBQNpK{J}%TCQouR zP+=WG0ou&9Gs>cSqPuC~bRexBDxvQ6o@6T2W-=4LahtHGab5Z+{iEu={QWk*K5J(>SBCQ#D-13m_cl%7ME1Z{?U1Pxlnp* z&5OMRu)?F|IBMP%h6&v67o1P)pJ}^abFFZ-;SPJPMchoZ^o{#K`8w%eZcHg<>WkB1#?B{Md z?XK$4^BfZWT#0g2brVt*ck`*1-6B6zZYQpV+wXhRY`99Ia-(?Xyd=$Mt{D&Sdm)et-9rfGOBxW9}AgX&~Bf%~g68+NbN4}&`aL;y)@=Lv@ zyG}<+qa|!YJayNR8w1bT#VP|}d%U;BA53ynH1QZBFLesxvrX^v=3K*XV)8LE%2&>5 z6gF{My;nbwSCh7B<`a-srkE1C^=-})PM*2}OD8*)-GU#ae0%DwI(Afw(9lUX}Ve)Xy;jU z+9(WG5VkU+zMi<7-hQX;GBCjPp1{z&WnTUAF+4}P)Z%LU{54M=jEfOW%(!a(CmTbx zq2P3laK5Y0)vmYN$N9vM)Gfg1^`KPybhT~lrDRL3<9ys>2Z;w)(A)}zt{E0YIE-H8 z!0(97LA%o2JdcS+LG8rSmbJJk!T!!$b?aZ_CNuWK`Sj?hcctT^CB-a0^tE^JPk!@c z54s2P7Oc4cIj}row9E$yuq3BfMz1%u_x^4ILw3WXI|>n*G(LBL*n5iuQDxM)G3gP@ zGS!U^MN-~=yNE`>S*%g=$>UZI&7{!RtwDk;1sI|)yyT@Dx)+zjUeOtO{_FK$0rF1H zJz*9_*Kd!i;3eFD2HvNgG7Bb&5tFGyf0sdC1-;$6(7NP*Ag?HO5moJYCCR8gQdD9` z!%u3g;KhPo=0~*Wk(=0et?JwXOO@4bKK0|!cOZeq;UR;&!&C%ZyckU}OEXraf>^#A zMY!4OIncH@e;JawlUZnjhN4_q4o~-q`U%iV(Iu~>P7~9;nSV0qJwp+CGJ4>B!A_Cx)T1EG z_PA!+(OaVS-PeNPLv0wth0OJ*&Oj@dGU{+hN?$*pzqDpk4ew9#Yp}=)iZXj8W4EHj z>fxt0=((eh8`SQpqOSfh@9INUTd-#-;um>}nf|rdKo-!^-X-SnkrWtO&F?JNq$^4> zO$H=`s`!4ggmHmV*8o3`)@2=}XIf$I%CpcAPXeZncCBW0PyNfBcEFxrmf(?-i@>;z zZQ+J7_aXD48|iMws$4Sl(Uy>6wHv6O(<8s-Yel=X?X%=C6VNB1%fU;A=!lH+hwwsf zCN%g+F-|cr9Lh{{!42O#edOV4dlYYPop~J8r(5xE`4(G4@B>vGOSboyG7|1pBHW_j*k{X?2xq zH8cO-Q;BERvoGB{)dBYo3=6`oEj$p>g;xi}KP!U6$T$9&XQH2|MAKjsfTWNMb>&#( zsKQblxjw;rxt({%)i(B$4DA{Pj_~a9(X;#ZuAhFiceP_r5SMeFhd0E+;u%A8T!Ai> zJtL{=oow^P8}xk1*ViwcUSHQ=O#MpgvKm$dIbP_-ksesm7M-pQ+4VWN#cpcy2-X*g zIm!}TJ#Bu8?aOm;P&d%5`86>U1-#U@=$Tc%eyJ-7_qZ6hE7Sczo}h`p?jThje0QPc z8{V-1cSVUMP3GeJb5VHL(#v@gXsksjZ)TlGpe(plme8VP3WsErw6!ycw#2 z9e(0d*Gc2TfKti?dlWRBY0`UrdoEti_~fkI+pkLdlo(}8{$PZvs2?p+G71*>{!n~O z>nOOi#-Amay+FB*`a!G3kT$-R>4QNQ$}nZxH2-?ABHS5EFlx zlG`)}By-W-HiizC_A~XJGLqIuve6>{O$MY}En&j5^6P9-H*xQdy4_+GZ|#D+aVXrz zQf2?c3C<$D(@rzNM19QKdW%r`8ozzd6Klym5ctDlJNlu$@e8~SImeT_KxcgFZm1@vkwWleh4wj>`PXf)zv+N)P3Bt{n-lM9aJhHkvQ1y?|!pw9;mwvsS(b~{-Qx?i6ds%x86~SdV}+Vp}EJL zTRS$cDyZZ4;cE{dTSgjT(W>hRyBk0OTtU72!rqU!A(*5Q+G}{Ow(ObK3`CdVWlOrf z$PLH&7Yi2ChCjTwB*hU18E@&R-dfQ*Y|DnO!+*rLi+xM=QYTm-O)Beo*5Ff5=})46 zVgW!xlG{>Ij^&xb_IL?+ETZuip1 zu9PM2Fbu19h0R_&&I~U9nNE{?9@WE6x0e$3lRk2zDeLlJjg`o_xn#}TTEdX8TO zlXO7$(kUl`df+Q*H@rgAD}>)K1-ocL_`|Hr?5(N! zQeW{3v>soIe0VAij5PVz*FG5hVlTq@$uCgwPcf`@({YM2xO@yh(3E4gb$SKefVITc zV1)gkVMDW5Hl*8+93|GM`2^w59VFtPJ%120SD9f|w0|Y#&wwaLF+*lR%Wb!BX%c)h zsi}W($8|Ge^&5W>8Z%ldTr2}vg&-SgKqPuibk`lJ9QDC2(=^)DTnM*cOS~@{N%z|} z9#oPgzME|U`VK>RWhpFfpnLAMgY05qdZmhMZ?XH8F?cQ}YQ$9!v>z2&(#mO)?FSPl zQw67zfX={=$j?@0bqR~S@J!NhhDN$-gv7PHnZ(MtVCTlKrn;ND!|k{v%suI*aWZ7i zD4p~1ewmPL|7{?{zr(dUac#r1)#=x(+}pUSzWYBb2jVD8-w7u>-n!1iRPq&Tgt1F> zeObS+k-pv*$==c2V9D$yMPSQjEvRC#<~YCMY_?j}@D<1XRA@RzvO(OLJ~67pruYeD z?RN?58YXq5sBqmA7AS|;KaTP ze$dm4j}6kiE*(dF?FC!&Zc6^PT=Cq^ zkIFRqH6~a+>OSU0KosFO+{M_(lQufcPY#w;P2E^0OiFs3wmIGt29eAxnA|6V~8{ag_Zo z_fj-I19erp)vbo^D>r1Un;7bQzHD52V~!^ZGGX=5;2T3}Rd}KVFiWEHfuDBzpEnBO zm_%Wc1pDdSHC~RsQ6YdZC9&cf9x-7~vRIpn>H8d?hRwo%Zh(7V@t;SSMKZ{;enE@+ zO}=yy+Ztf}0xc;ae41jNKl=J-&Glll2JEN(%rR|i*Jj1{I1Qj1LdmB()mzat8?5@F zurGJ8m_q0Nof@~bU4%x;dz`_y73tN!JJa1ClBI7V_F0Fq;-dNiQcJPU5*as}w2_B| zcHTeeQhFDg{x_Fl^Ya=J_=RB7_ z@~7PSQ}vRfODARO(t36~!_llj~)_)_z^U@NW{j?^jtBfna#ZuV#ny13n!1wD;IU{-< z2eR@qdtK?jJ6etHw$!|3ye@cgc!8xpnm>(RLID$>rOz21ktn*3x>DCNN~WN}E~e9y zr2qO8Gq`!z~aalso6ON*12yj}riJUzqVllztn0*{P7H3YtoLEaAoG#p7iSlnpNsc^@wQ zJ<7D}Rrt(fr^+`uMX^pomZQM|?tcbl=(x-kc+z6xb8}Z()q6<;LddK~*`y=oFthDH z;fBI0b<4u+8ihNcH*c$$n-mj%JZPIDfKFgh_@IGZW*KmzG$wI@lh`dQQ9B1F%SZn^Abz?UAV zrj5HCaG`l7wd0saBHL!N3zG;<1WW4mx_xzMY#jtES6;pS3RDMCtOnExLV_Yws0FEk zyTGZHG}cc$q7G8Ma9%1dj?&J@U@!Js^o)~3GDXq_9O1kKjk*lr3B%uU&Gz5C zUjEoH`ao3_ED80mef-G$Mxq1avNGC3l--$q3Qgw`q)9#4@RRJ=)eyra;^l` z7xwk>8{pNxtGog+t$c5R1h9?W$lUWKFofBn=)MPmxIXxq0O~0?J^tX%KKx8Wr%>h- z29Z=R&*1C%R6UCe^$fU5-HF|@gh%DbF3P97L>BBO_0K5O6iK5!1o&p+;U3?gVN<=WJ1#$&f&aI7H{S%(BJ$cNSRSdi6&Dw zxF|vs)hRO7t08MF4Nc7j+lam)@i*pT0BP)GhfD#phu5d40CR|&D_{xWA#mZXu`ATP zl^IF5V6~Uu=?j%z=R93X{jEZn#m3LSpRU4cVvo?XP<4t^`ZVsqOJ^2j8pGXN&0d$< z_!PLv0m?K8r2V~B6!|ow69t%(~FKU?$~YtDph4_*L;4<3#=SIkASO^NXhsu z&cdn6uJ^f4+t*gFF>C#WhI6z_q(f?1+Zz1EJ?v}X>gkQO>s2BmJ1w^DVh#fLOkRCu z7aYg3U?Z>@-Zn)NmWe&Cz#UUx-AS%9XCg32&HGRtnbRGCn}+M#88h40KwcHvD!w(; z;6-R@;eGIBIHu9CvZv)hFb_{zM0^bD>O@iUVH)!3r+>Pc+(R{vD;$t_yV82Tfw4X@;pSQ$nwN;n9fY?|9au zfcFU9JeI$Z1GOZ7fgXec&8%Q*{xkDr%HH3)i~k*@2Hd-wFz)r-&@2Q^gPUO1N(7Ov z%~PcZST~gCdm;}y&qR-ElF?@;xknVCmtbTEfh^m0Tcu_xafTEXIxkO zprwX3SVyO@_K|e#)?qShu`kNvJ#j$^@=r}maSyiwClt7kozy3HA^%!CS&15-q zD%Or)gmzj4nO07If$VO*|ziT zP5zQzZT+n^8q~Ilj~xqB(5(Ka!Y^;eq17F3f`An}aJN@nW4}fY7}h2Q0@O0UKWhPT zbK=5|Ih`E4;1{uQ;LYE|3zJ=f&IRq?=HM;w6`$2NRDq)nvDw&otSZ(A*`vkFU|#hN z&%cg&NtVQbOT`};1nY`^zZJvGk?)6nhxz{g!*v(M-LC+Lt2;og(p~4PA1~86s1~kOUW>Pf?n5H((+`|}Tm;$}xu6{EQrt!I0f-%hJMjTS zrs2Vs5(n2q|95~r=(&!*=uewLCz~=w(Y0ozYef7~#Im8IN}7FcMX=z2p>Q@t?@sw( zz4ih%QoH`a=&4OkC^Buyzf|$E!AP_4pO5rs5r#~KEQI$mC0Efd6S85a;R2oD2uk*^ zmA?b?)-rv}xoyg1jb`gS-lhZ3V#gw9rPuStk6w*3!|JkEh#!jP2aS)o)R2;AGgp^^dl^YukZ10{ zY$V82xG_b|1-^7%FomX-^T_BCj?-P!51zOm7x4^duCXsImaaW^%3UnFyTc~=1V%`^ z8K1I#9SAKD6O37e%X%b+YR;SY%C6mae|9Rg1b~Ne=y>q6&-n-)JF+ZL|N9iFZ`9o7 zXKB;52{n}STB)Quo)m~-{}iu7+YrjsWMzpmicgNZ}1mGO!P}zR?9T(5k18E z8l%SDZsJGPr-MexDUU==6a1(mR;@}I9>yG$acOcHCGPtU+FwC=U|iFIac=5Or_dpT zGx_}SX!oy%2fx%0hpn#;< zt2Z?Jn~zR(TxlwH?>-OJvpITs4rQ1emHU!aqgG^Rh6c~Eo#lcYW8S$RKwb@L|wIP?1RH$FPQX`PRpPgX!?Te$VAg|aR`dwnu zExOzns_YI6C8Dq5&kQn<$FS+h;~X92hC|2pMUhV8N0{mC!=_?prSg)Puzm&0iiuEV zCgC-aQ@c`a)AcGg*iRlNU>YD`d}*eN9klG!%;!DT_iJw=+Sj>CS?f(SnT!j(mTrZAeI;o=9kGs?>=Rj z;KwiBHb;>=pK|OMP%ohYr^1>zp$Xh*t7E2(cP_&&p`w@$8R@}$BFiOGxT0*9_)A+5sz&TOHKq*^sriXiT&Ip2ENPxE1Y1c1{?<)c` z%}T6}S2{#}N&%-Ti}l9tqc9^Sztk;IJWKR{mg{hLa#07*_Q(LuRn z;xdKbJ)QtY{Q{|8#l-MLK~x1;UqN&URl;$S^x^lZ=g%Pe{_Q_j$yo0UcoR!zfr&V? zcP<1WoIZC;lI*5Sc>C%7lTazEaob?St?4G{)P&lA)MI+!;-(@Hk=k$Xq?xDo)MRgI zd|#RiV8cN5e!Jbk4pU9*`U6VRn{+|F2CV_~z6M#NLb&0XJ=&g-+R5+Tm5UNDcF4S* zR65C0XI7fy>bt@f!N0L)spUkSgc0DL3f>kgJv!FZtyY0;xrpA+iL5D{odQPYFHElv zCi?P>RpoZglty84zFgk_L{KEt6-q9?ye;0sA2*{Ncvh&*2R0C!X* zJ z`1sxIVCWFQpILEi5XVNt*VAnTB{w-YoI8$gC966L-_SD{XM(0d%L636Jh+33N=S5Q zXM7jTKwo(x7j$y^Y1rNfD4olpW5Ml%3UD#o8A`e6T^BZ|hkBVSxpSLLmpj&g{6VN} z661ZW>v*GhkN7o|WlWsl0!Zh2SdVA>r^UufmI5D353^9o*(+=Cm#oN{Hh%0q+;t1x zTdGIbkmQ%#knME-MCcZZJ#0oBMT3$w%QH^e9rU^6r8YJ{d~1Fj+m4l~c-dJ-MSN-T z>C(N;U+WTX!V1RV2Rqci1d#80wocb(#PaT?P?T?b2y7hUXz}9MJJ@~H6abOOLX0{B zs$7P4>GpWXdCJnL0}Uj{T=FIM1 z|63XlDRJ_Y2cfk7hosFvts~2t!iQqSeM-UQZ#j_yk#>mA)N&DLai$#FCdk^=*rRD0 zv>6b~jm52+jjMWiKXhjLv|7|Q9@qTvB>Xs~R%trd?NsOVpB?D7&-c9h$KYiF6s^Ko(y-KYUU;YroXTmrMo`?ZS@afph zaD#6lIskY1G^nE95DHX$NcztT`!)-$;~9UjTh)%umg1F;I^G^T*co)$o5yrsaqtH} ztdMyZbon^^OSw)yWzRXz^Qq6oosis)Dlhr9mMI3L04eWSHkGJN;)O=HrQQH%Gh*8!NM^Kf_wQKu=g7+f!~;;5z=k&hooBd0u%C+7$xlkgd#FkEAw>v_6@v z?;7m%$=+Nxp0Im{&9%KFIa0)VWw{Z>a+ZUwJc4{VNPywL;EyFw0`VZKSAiRW@HF@A`JcSo}+Ub=ISmOszc`huiK(9aU}(=387B z{n!{0qM+d7z*M9D&iSH0sI@cb*$3AEV?4`15Kj(c!w-AO;Dykg7k}zNqQQ-6jgS#{KB%g7k)rq-!0I zhvl!^w_nJf$*VQzfNwimnY}HAhK6s3Tr4ifo^lc+ zt_ywdCg|cs3XBn*Puzr*l<>ru6ZF+bfS4Al&_e~_k{|Jk+ckI=2+fTrPA|6+ThwzchH3e%Ha82 zb&kE?0^F-neD8Cr0N(w(!xE_*YZ;fwi%lU^zR;nM@~3}jx5I5eE#J5*_Qsyvj>hKd zfg;C6;qd8omYk^jd07+~B}UIbO#KxG zy@ZX_|DfnH`zZz)YH{BXv}Tyx{Kc`vwi)KpGwI=M1?V6o!OQJ<`I_T4H9Kc%dDZ!a z-^;N_59-PzZD7Iqf3(jjbdnm{X0pSl#(s?nELK4?*H7*vpPx<^2B}_S(3&AT=2$D% zcP7*`a*gd-Arqf(sJ`6oV;5h_#58am)F;%n8hzHrt)VvaD%bDP@hc$%XU4%K+Kc)t zH5{+)!C?~fs@^cG+a8_@4yunlyMtfnt|2U}xju&TAMD8+EP~sm z%FK*4LOu~kz#J9@%=iM|(aWG#e8oz=Fs`EWB$TphcMDcx_~io9mIgicASne2F5e?~ z$5|wmhy>7v;el)kGQG*akc+AVo0!}hu*H^HmXPF$OBRE-2cBSsu{7ex2SJsK=RI$T znF1d!@91(R{a+nd9u0N$_P@rKWwMSv#)M+X5Mz(Bi&ECI%-BL^Y+1(ASSm{;Mzo12 zk}PAFh_OY4D2!w&`x2AMzP(?a^Pcm2-}AnI%*^?o``qU~&z&=O?!C|F^J$CHf#Lff zRdb&Mw=~x#TqoEkp!-+v3%$v1^EE~+{i?p8VxUwja9D8m~hA*+JF zvO(Aj#if2!$6-G<6Bf1{hEg#^yI^P|8yYATB$o_k9$XwyYJM(GPvkm6eYb6r-rPV zNu}UnUxQ}IH9`h@7%u?dKL*eUygpM~XIj>?PJ%1z$5(dq^6ngih2gS;4r6uX9XymZ zh4T*H=hNs0>jGw#DAyowE2JDWM3%)l+uVeGg~NYTJC5=BbYmd6xhnK1mxTeZPvY|vpaBDCSH2h`!{gdKJ z#Ks;p_5+{V_Fjne^wSL`M=ntK0zT0Zu!$QbF3^uEBc3gS!nEm-msvn**L~bu~5^E|W6(XCok-)3ZS9=Ro?7FkK_g zcSrV?9Wk@CF@M#ACyptpTP4WZWN+TDV2-=Oze{dg@jz-zTFuU?l6 zW;>~dRj>#^U*j+JxL!dK%TAw~t+?aN99K5?{P8QVecxO5ZVjw=B?gFmDfn4Dzk=f&vZx zT*n9e>7uPN{C>=~F2z5Rw-ya>OJUD9a>URD2eShqj(db_rCIf_4!P^U9{|_FoD?iv z@L$OJzSvi4GexT(;MV{|$WBr?6_$+D%?Lr+! z7cMo9Kl=9Mas&etT%Ozf>3RM3x&p0bvDiV`X)TFn$j=ARspWm>Ew%kGwnlGfTr^x_ zvhkWw#qblsC-#mt-S(d5IC4k(=Xw?A=zbLE=qg`x!}4aVy%0!ZxfHrI3}PtPKy37d zFI2ErUn3pLaq&mbMAd;lae8k+0M#A=d~zu~?Ua-hcIJpWzt4OI>t_}jhy^tzHBh2n z64yf!z$V4#?34|>e6+PP>mwVqG%5Cmafb1P%Z-x{7?1vqzP!+)?jFi9dtGr-tIYZC ziM^j4gp-R*%^gyWuP?M-{gzBi(P$$V2kLIJi&M8h1XeIF8agIf>^jY3$Q!!Jz9j?( znj3SwTEAKk#r%57DpuCAKRMI1IRo{~70t<-ctyrmZCzW2O)^QQ9hU8ANKp8CX5d|O@Hcd`ul!NjJ!|n4^C*ioU7xiG8}#ZTjnxe!<+b!TV?`$F$eHOTHxSpz zWrHeuZ6>D?ql=qmxVKx?$eK1!!ctY1wyfmozaF$@P)9p!e;{9o?)I{?izSK;nG%ov zUODToTf|yUz{8NG-4O+?>&9}LZI-n}7TMyx9aUj;x8x zPTp6GRW)|D5I2x9fSBHka*_x;^YaU z`;V?V9VBd-GJO5i?mF=-1_QtvJkW@^#)>dAP!yCDV*w1r^w6CV*0HJd- zuS{oUxogkPAk_L#tkX)njhx#i`uJ;`_-apXBhvGpkKacU^-E$VxsBuml8xlZZI2pW zyD&(J+0L|edXIHEO*`qd2gf%)I$DS|OvW?TMHn~hSYL=oGzyRlVjtWV64`UdA zsLA_Z10PCnz?3&vId>Gj8k7&ya;0TLIU);$nmshU3B5)QsJLM_6MrVE+pq?J!sExZfx;If6 zY=TQ4avk0%w7y;vYa{ltFF&=Qz-N8lNbuN*0hz3G(@AvT#F|BH#?lb} zugHereO$IQ0G67-LmvX+*8j1>u7dBd_6L^5?s{4BfHFK_O)bIN+P$_vQ4UJ_BdoV7 zX9l*KMne)h)S>V?P!PtI{#OQ%KukIcjC5=H<}BzSKCz~psl1eA;uu~w>SJi|u`V1N z0$Isu7r_`xH39w1OV%Ls^rgd4M#2$5fl|aYrk2YRnlE7L01PwWtiU$cig@t2^2b*O zr4X|*&~O}K!>#u|rb_Mt$-W3+vF1b|1?d?4i*&;~dQObq3&0|ch*-@jmTP#)o48Ae63xeq&*7b`oS45V*E zLGiA$Lxvm-;>^`b>nX8eq z>6rzoXZ5m}Yo{g_19Av9)|5;TnL96MOS)UjW)aCx8ioC_oj6U|iJozZ@=&6=;+p@W zu^MQ{?J*Ph@Li}h`&VwW-P2&h1O^h)aeGjy5~6qR}Zw@V*$5o15}v<3bhuOh5{`iQqXzlDsCPyQVp}R@T=lD+9~3WqgW_5r4PaQb z`@p*ah-hb(@CdzdRrmx@cxOXHt0>ieW{m&^rbI;EoBJ^+43=tF!mH2^e^WSE&ujAszwO0x_{6u9xH>@Z(DCc z3u{vweM?@JW1!lvtt@ld!Ec&(!ffKhq4%w0=oo7QrlQBcI&MeBaiZB8-ti9X^6c07 zQ-^Me2SQR;MuiebO0(cRvOi}op~sXnSc#lC%RhJ1+T@ zks7IoskFX$sOr!z7wI}1oM!`gTu3nN{589|Jc~dqC!|?ZeY$lh-Pv0b2pgDQ{T(!G zq@ZZcDTLd`5gOyS9Z7_?&e^)7R7@UEoh0sdI~sP!B%L|M1(xp@Cd5PP69pOy;EITr zO5n6LXIUL=Q{l~qey$v&Np9Y{@n)wELixHKwuM}|nx0^w>KZ{Xi40*7bUT0ao{1_; z3W$dkEHkY{CI za}!OoQCZQK8U@3DS;tFIq-l`}iT6F3CmvHGq6b<^(kr#!aImEfXJ0rb-2Q&|id;Hd zCorjg2w@CJ+2>y*3=Qek^NLSh5eehVQ>eh*g)6`vV(jWI*x^rh(aEZ6+mBWci#>rMa<96=a zb>e>SQpyPc0ZALzYiy1;fwF*k&qEqm>Y3lMLO-Q`x1+3o3_DJPW~)u}^{DY3^iW2= z__iXF85(Vs(uxE}h;Yz246taHL_PU+Bee{p&e74wV^N}1Zz(yW+E~< zvKQUGt;v&UYH6Ws4z9NMFc(;y8S0Oz3w!nFr1CE?FEcg72mcNo_`WX={`i4-NF%F2 zJM9rN+77~1*bJY{#6$t{Al%k@zV~8Sb^W+5F$r3GH8##6@nrOpbb@)?Ra*?#^P}eE z!53Fr?@;R#M58gAWG&!&z>Ovo<~a*{uHJ9#tnZUuS}vTk5i`gNfT9^sFnBWdG9FzA z!IjBUb1chk=uv;GDf;gyUd%XX-7~f z@eq{_c1wYb_O*b$nl}Tf!3ZzB$v3%ZG=?JE~!Dl;EPIcW$*PJ-qCv zw;HzP{YtB7Y|sifs5s>#_}PSy*;X&`C@c@nQ3RVapaM6&5?=p9>LlbU zXRBaas-Be^eJHTMX!d8()&J?|W$`pLmuxo)b9#U0xbU;#J8Q}-&2biZdTE(}!-wlt zQ6I(*mf>Dr=_cQIqeSDcJZT)gS_n#nS_B!|2f6v-@gBZGfRQ)O1AfCVIMB^Q6Ydoh z+%E0y`aMWi-JT4LTM&4Bgr*81Jg7Y2fD;d^k6nnwE!I0!sM@ zb&CJX=HJkonkF9Jo?bzKItt~lm(uuu+t_)#2YCU?$||UGZ?QViM)*I*vH<5dD9qmj z_!pKUIG4KRwiiGn!T&U}2CV=k6*V;#6*ZK)n$g6oYS4t|KTNE=!#w~M<@4vx)kX{M QgN=Zx;RS<|bJ*Db0{BiNF8}}l literal 0 HcmV?d00001 diff --git a/docs/townhall-history.md b/docs/townhall-history.md index 31b498eec43cb..40162d2b6cd81 100644 --- a/docs/townhall-history.md +++ b/docs/townhall-history.md @@ -2,6 +2,17 @@ A list of previous Town Halls, their planned schedule, and the recording of the meeting. +## 11/19/2021 +[Full YouTube video](https://youtu.be/to80sEDZz7k) + +## Agenda +- Community & Roadmap Updates by Maggie Hays (Acryl Data) +- Project Updates by Shirshanka Das (Acryl Data) +- DataHub Basics -- Lineage 101 by John Joyce & Surya Lanka (Acryl Data) +- Introducing No-Code UI by Gabe Lyons & Shirshanka Das (Acryl Data) +- DataHub API Authentication by John Joyce (Acryl Data) +- Case Study: LinkedIn pilot to extend the OSS UI by Aikepaer Abuduweili & Joshua Shinavier + ## 10/29/2021 [Full YouTube video](https://youtu.be/GrS_uZhYNm0) From 46861600fad443dd535eb0a03b3db07293b18bfe Mon Sep 17 00:00:00 2001 From: Swaroop Jagadish <67564030+swaroopjagadish@users.noreply.github.com> Date: Tue, 30 Nov 2021 18:01:56 -0800 Subject: [PATCH 08/14] feat(build): add incremental builds for python (#3647) --- metadata-ingestion/build.gradle | 39 ++++++++++++++++++++++++++++++-- metadata-ingestion/developing.md | 4 ++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/build.gradle b/metadata-ingestion/build.gradle index e3b3ed03d855d..c3f9740e9168a 100644 --- a/metadata-ingestion/build.gradle +++ b/metadata-ingestion/build.gradle @@ -12,21 +12,31 @@ task checkPythonVersion(type: Exec) { } task environmentSetup(type: Exec, dependsOn: checkPythonVersion) { + inputs.file file('setup.py') + outputs.dir("${venv_name}") commandLine 'bash', '-c', "${python_executable} -m venv ${venv_name} && ${venv_name}/bin/python -m pip install --upgrade pip wheel setuptools==57.5.0" } task installPackage(type: Exec, dependsOn: environmentSetup) { + inputs.file file('setup.py') + outputs.dir("${venv_name}") commandLine "${venv_name}/bin/pip", 'install', '-e', '.' } task codegen(type: Exec, dependsOn: [environmentSetup, installPackage, ':metadata-events:mxe-schemas:build']) { + inputs.files(project.fileTree(dir: "../metadata-events/mxe-schemas/src/", include: "**/*.avsc")) + outputs.dir('src/datahub/metadata') commandLine 'bash', '-c', "source ${venv_name}/bin/activate && ./scripts/codegen.sh" } task install(dependsOn: [installPackage, codegen]) task installDev(type: Exec, dependsOn: [install]) { - commandLine "${venv_name}/bin/pip", 'install', '-e', '.[dev]' + inputs.file file('setup.py') + outputs.dir("${venv_name}") + outputs.file("${venv_name}/.build_install_dev_sentinel") + commandLine 'bash', '-x', '-c', + "${venv_name}/bin/pip install -e .[dev] && touch ${venv_name}/.build_install_dev_sentinel" } task lint(type: Exec, dependsOn: installDev) { /* @@ -48,12 +58,36 @@ task lintFix(type: Exec, dependsOn: installDev) { task testQuick(type: Exec, dependsOn: installDev) { // We can't enforce the coverage requirements if we run a subset of the tests. + inputs.files(project.fileTree(dir: "src/", include: "**/*.py")) + inputs.files(project.fileTree(dir: "tests/")) + outputs.dir("${venv_name}") commandLine 'bash', '-x', '-c', "source ${venv_name}/bin/activate && pytest -m 'not integration' -vv --continue-on-collection-errors --junit-xml=junit.quick.xml" } + task installDevTest(type: Exec, dependsOn: [installDev]) { - commandLine "${venv_name}/bin/pip", 'install', '-e', '.[dev,integration-tests]' + inputs.file file('setup.py') + outputs.dir("${venv_name}") + outputs.file("${venv_name}/.build_install_dev_test_sentinel") + commandLine 'bash', '-x', '-c', + "${venv_name}/bin/pip install -e .[dev,integration-tests] && touch ${venv_name}/.build_install_dev_test_sentinel" } + +def testFile = hasProperty('testFile') ? testFile : 'unknown' +task testSingle(dependsOn: [installDevTest]) { + println "$testFile" + doLast { + if (testFile != 'unknown') { + exec { + commandLine 'bash', '-x', '-c', + "source ${venv_name}/bin/activate && pytest ${testFile}" + } + } else { + throw new GradleException("No file provided. Use -PtestFile=") + } + } +} + task testFull(type: Exec, dependsOn: [testQuick, installDevTest]) { commandLine 'bash', '-x', '-c', "source ${venv_name}/bin/activate && pytest -vv --continue-on-collection-errors --junit-xml=junit.full.xml" @@ -72,5 +106,6 @@ clean { delete venv_name delete 'build' delete 'dist' + delete 'src/datahub/metadata' } clean.dependsOn cleanPythonCache diff --git a/metadata-ingestion/developing.md b/metadata-ingestion/developing.md index 418db258171bd..58b7e6cd603f7 100644 --- a/metadata-ingestion/developing.md +++ b/metadata-ingestion/developing.md @@ -130,4 +130,8 @@ pytest -vv ../gradlew :metadata-ingestion:testQuick ../gradlew :metadata-ingestion:testFull ../gradlew :metadata-ingestion:check +# Run all tests in a single file +../gradlew :metadata-ingestion:testSingle -PtestFile=tests/unit/test_airflow.py +# Run all tests under tests/unit +../gradlew :metadata-ingestion:testSingle -PtestFile=tests/unit ``` From 21f535bce874b52050f1fd6021ba2c02991e0b1e Mon Sep 17 00:00:00 2001 From: Gabe Lyons Date: Tue, 30 Nov 2021 18:09:33 -0800 Subject: [PATCH 09/14] fix(ui): fix issue where markdown links are unclickable (#3646) --- .../profile/schema/components/SchemaDescriptionField.tsx | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaDescriptionField.tsx b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaDescriptionField.tsx index d96a5f374f1ee..dacdef11fb945 100644 --- a/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaDescriptionField.tsx +++ b/datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaDescriptionField.tsx @@ -116,12 +116,7 @@ export default function DescriptionField({ const showAddDescription = editable && !description; return ( - { - e.preventDefault(); - e.stopPropagation(); - }} - > + {expanded ? ( <> {!!description && } From 2f6fca169a2bce24d5c0ccecd5a0bbe177286fdb Mon Sep 17 00:00:00 2001 From: Gabe Lyons Date: Tue, 30 Nov 2021 18:15:46 -0800 Subject: [PATCH 10/14] fix(ui): fix bug where key/value toggle would appear on schema tabs with no fields (#3643) --- .../dataset/profile/__tests__/Schema.test.tsx | 74 ++++++++++++++++++- .../dataset/profile/stories/sampleSchema.ts | 41 ++++++++++ .../shared/tabs/Dataset/Schema/SchemaTab.tsx | 12 ++- 3 files changed, 122 insertions(+), 5 deletions(-) diff --git a/datahub-web-react/src/app/entity/dataset/profile/__tests__/Schema.test.tsx b/datahub-web-react/src/app/entity/dataset/profile/__tests__/Schema.test.tsx index d164c135a673a..e1a30a54e5837 100644 --- a/datahub-web-react/src/app/entity/dataset/profile/__tests__/Schema.test.tsx +++ b/datahub-web-react/src/app/entity/dataset/profile/__tests__/Schema.test.tsx @@ -3,7 +3,13 @@ import { fireEvent, render } from '@testing-library/react'; import { MockedProvider } from '@apollo/client/testing'; import TestPageContainer from '../../../../../utils/test-utils/TestPageContainer'; -import { sampleSchema, sampleSchemaWithPkFk, sampleSchemaWithTags } from '../stories/sampleSchema'; +import { + sampleSchema, + sampleSchemaWithKeyValueFields, + sampleSchemaWithoutFields, + sampleSchemaWithPkFk, + sampleSchemaWithTags, +} from '../stories/sampleSchema'; import { mocks } from '../../../../../Mocks'; import { SchemaTab } from '../../../shared/tabs/Dataset/Schema/SchemaTab'; import EntityContext from '../../../shared/EntityContext'; @@ -212,4 +218,70 @@ describe('Schema', () => { expect(getByText('Foreign Key to')).toBeInTheDocument(); expect(getAllByText('Yet Another Dataset')).toHaveLength(2); }); + + it('renders key/value toggle', () => { + const { getByText, queryByText } = render( + + + + + + + , + ); + expect(getByText('Key')).toBeInTheDocument(); + expect(getByText('Value')).toBeInTheDocument(); + expect(getByText('count')).toBeInTheDocument(); + expect(getByText('cost')).toBeInTheDocument(); + expect(queryByText('id')).not.toBeInTheDocument(); + + const keyButton = getByText('Key'); + fireEvent.click(keyButton); + + expect(getByText('Key')).toBeInTheDocument(); + expect(getByText('Value')).toBeInTheDocument(); + expect(getByText('id')).toBeInTheDocument(); + expect(queryByText('count')).not.toBeInTheDocument(); + expect(queryByText('cost')).not.toBeInTheDocument(); + }); + + it('does not renders key/value toggle when no schema', () => { + const { queryByText } = render( + + + + + + + , + ); + expect(queryByText('Key')).not.toBeInTheDocument(); + expect(queryByText('Value')).not.toBeInTheDocument(); + }); }); diff --git a/datahub-web-react/src/app/entity/dataset/profile/stories/sampleSchema.ts b/datahub-web-react/src/app/entity/dataset/profile/stories/sampleSchema.ts index c00ffd49fec3c..8afacdbcdd01a 100644 --- a/datahub-web-react/src/app/entity/dataset/profile/stories/sampleSchema.ts +++ b/datahub-web-react/src/app/entity/dataset/profile/stories/sampleSchema.ts @@ -313,3 +313,44 @@ export const sampleSchemaWithPkFk: SchemaMetadata = { } as SchemaField, ], }; + +export const sampleSchemaWithoutFields: SchemaMetadata | Schema | null = { + name: 'MockSchema', + platformUrn: 'mock:urn', + version: 1, + hash: '', + fields: [], +}; + +export const sampleSchemaWithKeyValueFields: SchemaMetadata | Schema | null = { + name: 'MockSchema', + platformUrn: 'mock:urn', + version: 1, + hash: '', + fields: [ + { + fieldPath: '[key=True].[version=2.0].id', + nullable: true, + description: 'the number of items in the order', + type: SchemaFieldDataType.Number, + nativeDataType: 'number', + recursive: false, + }, + { + fieldPath: 'count', + nullable: true, + description: 'the number of items in the order', + type: SchemaFieldDataType.Number, + nativeDataType: 'number', + recursive: false, + }, + { + fieldPath: 'cost', + nullable: true, + description: 'the dollar value of the order', + type: SchemaFieldDataType.Number, + nativeDataType: 'number', + recursive: false, + } as SchemaField, + ], +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTab.tsx index d487440d43ffb..2a6a2a74b083e 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTab.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/SchemaTab.tsx @@ -32,15 +32,20 @@ export const SchemaTab = () => { [schemaMetadata], ); const hasKeySchema = useMemo( - () => schemaMetadata?.fields?.findIndex((field) => field.fieldPath.indexOf(KEY_SCHEMA_PREFIX) > -1) !== -1, + () => + (schemaMetadata?.fields?.length || 0) > 0 && + schemaMetadata?.fields?.findIndex((field) => field.fieldPath.indexOf(KEY_SCHEMA_PREFIX) > -1) !== -1, [schemaMetadata], ); + const hasValueSchema = useMemo( - () => schemaMetadata?.fields?.findIndex((field) => field.fieldPath.indexOf(KEY_SCHEMA_PREFIX) === -1) !== -1, + () => + (schemaMetadata?.fields?.length || 0) > 0 && + schemaMetadata?.fields?.findIndex((field) => field.fieldPath.indexOf(KEY_SCHEMA_PREFIX) === -1) !== -1, [schemaMetadata], ); - const [showKeySchema, setShowKeySchema] = useState(!hasValueSchema); + const [showKeySchema, setShowKeySchema] = useState(false); // if there is no value schema, default the selected schema to Key useEffect(() => { @@ -48,7 +53,6 @@ export const SchemaTab = () => { setShowKeySchema(true); } }, [hasValueSchema, hasKeySchema, setShowKeySchema]); - const rows = useMemo(() => { return groupByFieldPath(schemaMetadata?.fields, { showKeySchema }); }, [schemaMetadata, showKeySchema]); From f043b79f4cff35f27c67895ed9ab555273bef9eb Mon Sep 17 00:00:00 2001 From: Tamas Nemeth Date: Wed, 1 Dec 2021 20:16:32 +0100 Subject: [PATCH 11/14] feat(build): Preflight script for metadata ingestion on m1 (#3652) --- metadata-ingestion/build.gradle | 6 +- .../scripts/datahub_preflight.sh | 78 +++++++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) create mode 100755 metadata-ingestion/scripts/datahub_preflight.sh diff --git a/metadata-ingestion/build.gradle b/metadata-ingestion/build.gradle index c3f9740e9168a..454a29c71248f 100644 --- a/metadata-ingestion/build.gradle +++ b/metadata-ingestion/build.gradle @@ -17,7 +17,11 @@ task environmentSetup(type: Exec, dependsOn: checkPythonVersion) { commandLine 'bash', '-c', "${python_executable} -m venv ${venv_name} && ${venv_name}/bin/python -m pip install --upgrade pip wheel setuptools==57.5.0" } -task installPackage(type: Exec, dependsOn: environmentSetup) { +task runPreFlightScript(type: Exec, dependsOn: environmentSetup) { + commandLine "scripts/datahub_preflight.sh" +} + +task installPackage(type: Exec, dependsOn: runPreFlightScript) { inputs.file file('setup.py') outputs.dir("${venv_name}") commandLine "${venv_name}/bin/pip", 'install', '-e', '.' diff --git a/metadata-ingestion/scripts/datahub_preflight.sh b/metadata-ingestion/scripts/datahub_preflight.sh new file mode 100755 index 0000000000000..fea6ebf731ddd --- /dev/null +++ b/metadata-ingestion/scripts/datahub_preflight.sh @@ -0,0 +1,78 @@ +#!/bin/bash -e + +brew_install() { + printf '\nπŸ”Ž Checking if %s installed\n' "${1}" + if brew list "$1" &>/dev/null; then + printf 'βœ… %s is already installed\n' "${1}" + else + brew install "$1" && printf 'βœ… %s is installed\n' "${1}" + fi +} + +arm64_darwin_preflight() { + printf "✨ Creating/activating Virtual Environment" + python3 -m venv venv + source venv/bin/activate + + printf "πŸ”Ž Checking if Scipy installed\n" + if pip list | grep -F scipy; then + printf "βœ… Scipy already installed\n" + else + printf "Scipy not installed\n" + printf "β›… Installing prerequisities for scipy" + brew install openblas + OPENBLAS="$(brew --prefix openblas)" + export OPENBLAS + ##preinstall numpy and pythran from source + pip3 uninstall -y numpy pythran + pip3 install cython pybind11 + pip3 install --no-binary :all: --no-use-pep517 numpy + pip3 install pythran + pip3 install --no-binary :all: --no-use-pep517 scipy + fi + + printf "✨ Setting up librdkafka prerequisities\n" + brew_install "librdkafka" + brew_install "openssl@1.1" + brew install "postgresql" + + printf "\e[38;2;0;255;0mβœ… Done\e[38;2;255;255;255m\n" + + printf "✨ Setting up environment variable:\n" + GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 + export GRPC_PYTHON_BUILD_SYSTEM_OPENSSL + GRPC_PYTHON_BUILD_SYSTEM_ZLIB=1 + export GRPC_PYTHON_BUILD_SYSTEM_ZLIB + CPPFLAGS="-I$(brew --prefix openssl@1.1)/include -I$(brew --prefix librdkafka)/include" + export CPPFLAGS + LDFLAGS="-L$(brew --prefix openssl@1.1)/lib -L$(brew --prefix librdkafka)/lib" + export LDFLAGS + +cat << EOF + export GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 + export GRPC_PYTHON_BUILD_SYSTEM_ZLIB=1 + export CPPFLAGS="-I$(brew --prefix openssl@1.1)/include -I$(brew --prefix librdkafka)/include" + export LDFLAGS="-L$(brew --prefix openssl@1.1)/lib -L$(brew --prefix librdkafka)/lib" +EOF + + printf "✨ Setting up prerequisities\n" + brew install "jq" + + printf "\e[38;2;0;255;0mβœ… Done\e[38;2;255;255;255m\n" +} + + +printf "πŸ”Ž Checking if current directory is metadata-ingestion folder\n" +if [ "$(basename "$(pwd)")" != "metadata-ingestion" ]; then + printf "πŸ’₯ You should run this script in Datahub\'s metadata-ingestion folder but your folder is %s\n" "$(pwd)" + exit 123 +fi +printf 'βœ… Current folder is metadata-ingestion (%s) folder\n' "$(pwd)" +if [[ $(uname -m) == 'arm64' || $(uname) == 'Darwin' ]]; then + printf "πŸ‘Ÿ Running preflight for m1 mac\n" + arm64_darwin_preflight +fi + + +printf "\n\e[38;2;0;255;0mβœ… Preflight was successful\e[38;2;255;255;255m\n" + From d062e14e37b7fa39307221e50323a9828ebce5ca Mon Sep 17 00:00:00 2001 From: John Joyce Date: Wed, 1 Dec 2021 11:17:31 -0800 Subject: [PATCH 12/14] docs(graphql): adding additional GraphQL docs (#3649) --- docs-website/graphql/generateGraphQLSchema.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs-website/graphql/generateGraphQLSchema.sh b/docs-website/graphql/generateGraphQLSchema.sh index 1b14ad94231be..a05d4de26d5ed 100755 --- a/docs-website/graphql/generateGraphQLSchema.sh +++ b/docs-website/graphql/generateGraphQLSchema.sh @@ -6,3 +6,6 @@ echo "# Auto Generated During Docs Build" >> combined.graphql cat ../../datahub-graphql-core/src/main/resources/entity.graphql >> combined.graphql cat ../../datahub-graphql-core/src/main/resources/search.graphql >> combined.graphql cat ../../datahub-graphql-core/src/main/resources/app.graphql >> combined.graphql +cat ../../datahub-graphql-core/src/main/resources/recommendation.graphql >> combined.graphql +cat ../../datahub-graphql-core/src/main/resources/auth.graphql >> combined.graphql + From 790363fea1c0cea9fc05ce26922bee82098304c6 Mon Sep 17 00:00:00 2001 From: bartlomiejolma Date: Wed, 1 Dec 2021 20:18:22 +0100 Subject: [PATCH 13/14] docs: correct title of postgres gms (#3650) Co-authored-by: bartlomiej.olma.kitopi --- docker/postgres/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/postgres/README.md b/docker/postgres/README.md index 3cb187af1e412..299f75c832e9c 100644 --- a/docker/postgres/README.md +++ b/docker/postgres/README.md @@ -1,4 +1,4 @@ -# MySQL +# Postgres DataHub GMS can use PostgreSQL as an alternate storage backend. From fe1d6fe59ae108547502127929de0e7b526d7bad Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Thu, 2 Dec 2021 21:21:07 +0530 Subject: [PATCH 14/14] fix(cli): fix response handling for deletion cli (#3653) --- metadata-ingestion/src/datahub/cli/delete_cli.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/metadata-ingestion/src/datahub/cli/delete_cli.py b/metadata-ingestion/src/datahub/cli/delete_cli.py index 7e537f6d63ece..aca3d948bc203 100644 --- a/metadata-ingestion/src/datahub/cli/delete_cli.py +++ b/metadata-ingestion/src/datahub/cli/delete_cli.py @@ -6,7 +6,6 @@ import click import progressbar -from pydantic import Field from requests import sessions from tabulate import tabulate @@ -32,7 +31,7 @@ class DeletionResult: end_time_millis: int = 0 num_records: int = 0 num_entities: int = 0 - sample_records: List[List[str]] = Field(default_factory=list) + sample_records: Optional[List[List[str]]] = None def start(self) -> None: self.start_time_millis = int(time.time() * 1000.0) @@ -48,7 +47,10 @@ def merge(self, another_result: "DeletionResult") -> None: else UNKNOWN_NUM_RECORDS ) self.num_entities += another_result.num_entities - self.sample_records.extend(another_result.sample_records) + if another_result.sample_records: + if not self.sample_records: + self.sample_records = [] + self.sample_records.extend(another_result.sample_records) def delete_for_registry(