Merge branch 'linkedin:master' into master

README.md

@@ -49,7 +49,7 @@ HOSTED_DOCS_ONLY-->
 [Town Hall](https://datahubproject.io/docs/townhalls)
 
 ---
-> 📣 Next DataHub town hall meeting on Nov 19th, 9am-10am PDT ([convert to your local time](https://greenwichmeantime.com/time/to/pacific-local/))
+> 📣 Next DataHub town hall meeting on Dec 17th, 9am-10am PDT ([convert to your local time](https://greenwichmeantime.com/time/to/pacific-local/))
 >
 > - Topic Proposals: [submit here](https://docs.google.com/forms/d/1v2ynbAXjJlqY97xE_X1DAntNrXDznOFiNfryUkMPtkI/)
 > - Signup to get a calendar invite: [here](https://docs.google.com/forms/d/1r9bObXKS3tgKpISqqO3rw4yQog5zwuaFxg8IrJGUbvQ/)
@@ -58,7 +58,7 @@ HOSTED_DOCS_ONLY-->
 
 > ✨ Latest Update:
 >
-> - Monthly project update: [August 2021 Edition](https://medium.com/datahub-project/datahub-project-updates-7a0b75cae2b7?source=friends_link&sk=307c9c9983a2d0c778d0455fef12e1e9).
+> - Monthly project update: [October 2021 Edition](https://blog.datahubproject.io/datahub-project-update-62adced87ad0).
 > - Bringing The Power Of The DataHub Real-Time Metadata Graph To Everyone At Acryl Data: [Data Engineering Podcast](https://www.dataengineeringpodcast.com/acryl-data-datahub-metadata-graph-episode-230/)
 > - Unleashing Excellent DataOps with LinkedIn DataHub: [DataOps Unleashed Talk](https://www.youtube.com/watch?v=ccsIKK9nVxk).
 > - Latest blog post [DataHub: Popular Metadata Architectures Explained](https://engineering.linkedin.com/blog/2020/datahub-popular-metadata-architectures-explained) @ LinkedIn Engineering Blog.
@@ -115,6 +115,7 @@ Here are the companies that have officially adopted DataHub. Please feel free to
 - [Experius](https://www.experius.nl)
 - [Geotab](https://www.geotab.com)
 - [Grofers](https://grofers.com)
+- [hipages](https://hipages.com.au/)
 - [Klarna](https://www.klarna.com)
 - [LinkedIn](http://linkedin.com)
 - [Peloton](https://www.onepeloton.com)
@@ -128,7 +129,7 @@ Here are the companies that have officially adopted DataHub. Please feel free to
 
 ## Select Articles & Talks
 
-- [DataHub Project Newsletter on Medium](https://medium.com/datahub-project)
+- [DataHub Blog](https://blog.datahubproject.io/)
 - [DataHub YouTube Channel](https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w)
 - [Saxo Bank: Enabling Data Discovery in Data Mesh](https://medium.com/datahub-project/enabling-data-discovery-in-a-data-mesh-the-saxo-journey-451b06969c8f)
 - Bringing The Power Of The DataHub Real-Time Metadata Graph To Everyone At Acryl Data: [Data Engineering Podcast](https://www.dataengineeringpodcast.com/acryl-data-datahub-metadata-graph-episode-230/)

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java

@@ -110,6 +110,7 @@
 import com.linkedin.datahub.graphql.types.usage.UsageType;
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.metadata.entity.EntityService;
+import com.linkedin.metadata.models.registry.EntityRegistry;
 import com.linkedin.metadata.recommendation.RecommendationsService;
 import com.linkedin.metadata.graph.GraphClient;
 import com.linkedin.usage.UsageClient;
@@ -149,6 +150,7 @@ public class GmsGraphQLEngine {
     private final EntityService entityService;
     private final AnalyticsService analyticsService;
     private final RecommendationsService recommendationsService;
+    private final EntityRegistry entityRegistry;
     private final TokenService tokenService;
 
     private final DatasetType datasetType;
@@ -203,6 +205,7 @@ public GmsGraphQLEngine() {
             null,
             null,
             null,
+            null,
             null);
     }
 
@@ -213,7 +216,9 @@ public GmsGraphQLEngine(
         final AnalyticsService analyticsService,
         final EntityService entityService,
         final RecommendationsService recommendationsService,
-        final TokenService tokenService) {
+        final TokenService tokenService,
+        final EntityRegistry entityRegistry
+        ) {
 
         this.entityClient = entityClient;
         this.graphClient = graphClient;
@@ -223,6 +228,7 @@ public GmsGraphQLEngine(
         this.entityService = entityService;
         this.recommendationsService = recommendationsService;
         this.tokenService = tokenService;
+        this.entityRegistry = entityRegistry;
 
         this.datasetType = new DatasetType(entityClient);
         this.corpUserType = new CorpUserType(entityClient);
@@ -559,6 +565,9 @@ private void configureDatasetResolvers(final RuntimeWiring.Builder builder) {
                 .dataFetcher("schemaMetadata", new AuthenticatedResolver<>(
                     new AspectResolver())
                 )
+               .dataFetcher("aspects", new AuthenticatedResolver<>(
+                   new WeaklyTypedAspectsResolver(entityClient, entityRegistry))
+               )
                .dataFetcher("subTypes", new AuthenticatedResolver(new SubTypesResolver(
                    this.entityClient,
                    "dataset",

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/WeaklyTypedAspectsResolver.java

@@ -0,0 +1,84 @@
+package com.linkedin.datahub.graphql;
+
+import com.linkedin.data.DataMap;
+
+import com.linkedin.data.codec.JacksonDataCodec;
+import com.linkedin.datahub.graphql.generated.AspectParams;
+import com.linkedin.datahub.graphql.generated.AspectRenderSpec;
+import com.linkedin.datahub.graphql.generated.Entity;
+import com.linkedin.datahub.graphql.generated.EntityType;
+import com.linkedin.datahub.graphql.generated.RawAspect;
+import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper;
+import com.linkedin.entity.client.EntityClient;
+import com.linkedin.metadata.models.AspectSpec;
+import com.linkedin.metadata.models.EntitySpec;
+import com.linkedin.metadata.models.registry.EntityRegistry;
+import com.linkedin.r2.RemoteInvocationException;
+import graphql.schema.DataFetcher;
+import graphql.schema.DataFetchingEnvironment;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import lombok.AllArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*;
+
+
+@Slf4j
+@AllArgsConstructor
+public class WeaklyTypedAspectsResolver implements DataFetcher<CompletableFuture<List<RawAspect>>> {
+
+    private final EntityClient _entityClient;
+    private final EntityRegistry _entityRegistry;
+    private static final JacksonDataCodec CODEC = new JacksonDataCodec();
+
+    private boolean shouldReturnAspect(AspectSpec aspectSpec, AspectParams params) {
+      return !params.getAutoRenderOnly() || aspectSpec.isAutoRender();
+    }
+
+    @Override
+    public CompletableFuture<List<RawAspect>> get(DataFetchingEnvironment environment) throws Exception {
+        return CompletableFuture.supplyAsync(() -> {
+            List<RawAspect> results = new ArrayList<>();
+
+            final QueryContext context = environment.getContext();
+            final String urn = ((Entity) environment.getSource()).getUrn();
+            final EntityType entityType = ((Entity) environment.getSource()).getType();
+            final String entityTypeName = EntityTypeMapper.getName(entityType);
+            final AspectParams input = bindArgument(environment.getArgument("input"), AspectParams.class);
+
+            EntitySpec entitySpec = _entityRegistry.getEntitySpec(entityTypeName);
+            entitySpec.getAspectSpecs().stream().filter(aspectSpec -> shouldReturnAspect(aspectSpec, input)).forEach(aspectSpec -> {
+                try {
+                    RawAspect result = new RawAspect();
+                    DataMap resolvedAspect =
+                        _entityClient.getRawAspect(urn, aspectSpec.getName(), 0L, context.getAuthentication());
+                    if (resolvedAspect == null || resolvedAspect.keySet().size() != 1) {
+                        return;
+                    }
+
+                    DataMap aspectPayload = resolvedAspect.getDataMap(resolvedAspect.keySet().iterator().next());
+
+                    result.setPayload(CODEC.mapToString(aspectPayload));
+                    result.setAspectName(aspectSpec.getName());
+
+                    DataMap renderSpec = aspectSpec.getRenderSpec();
+
+                    AspectRenderSpec resultRenderSpec = new AspectRenderSpec();
+
+                    resultRenderSpec.setDisplayType(renderSpec.getString("displayType"));
+                    resultRenderSpec.setDisplayName(renderSpec.getString("displayName"));
+                    resultRenderSpec.setKey(renderSpec.getString("key"));
+                    result.setRenderSpec(resultRenderSpec);
+
+                    results.add(result);
+                } catch (IOException | RemoteInvocationException e) {
+                    throw new RuntimeException("Failed to fetch aspect " + aspectSpec.getName() + " for urn " + urn + " ", e);
+                }
+            });
+            return results;
+        });
+    }
+}

datahub-graphql-core/src/main/resources/entity.graphql

@@ -611,6 +611,64 @@ type Dataset implements EntityWithRelationships & Entity {
     View related properties. Only relevant if subtypes field contains VIEW.
     """
     viewProperties: ViewProperties
+
+    """
+    Experimental API.
+    For fetching extra entities that do not have custom UI code yet
+    """
+    aspects(input: AspectParams): [RawAspect!]
+}
+
+
+"""
+Params to configure what list of aspects should be fetched by the aspects property
+"""
+input AspectParams {
+    """
+    Only fetch auto render aspects
+    """
+    autoRenderOnly: Boolean
+}
+
+
+"""
+Payload representing data about a single aspect
+"""
+type RawAspect {
+    """
+    The name of the aspect
+    """
+    aspectName: String!
+
+    """
+    JSON string containing the aspect's payload
+    """
+    payload: String
+
+    """
+    Details for the frontend on how the raw aspect should be rendered
+    """
+    renderSpec: AspectRenderSpec
+}
+
+"""
+Details for the frontend on how the raw aspect should be rendered
+"""
+type AspectRenderSpec {
+    """
+    Format the aspect should be displayed in for the UI. Powered by the renderSpec annotation on the aspect model
+    """
+    displayType: String
+
+    """
+    Name to refer to the aspect type by for the UI. Powered by the renderSpec annotation on the aspect model
+    """
+    displayName: String
+
+    """
+    Field in the aspect payload to index into for rendering.
+    """
+    key: String
 }
 
 """

datahub-web-react/src/Mocks.tsx

@@ -420,6 +420,17 @@ export const dataset3 = {
     ],
     subTypes: null,
     viewProperties: null,
+    autoRenderAspects: [
+        {
+            aspectName: 'autoRenderAspect',
+            payload: '{ "values": [{ "autoField1": "autoValue1", "autoField2": "autoValue2" }] }',
+            renderSpec: {
+                displayType: 'tabular',
+                displayName: 'Auto Render Aspect Custom Tab Name',
+                key: 'values',
+            },
+        },
+    ],
 } as Dataset;
 
 export const dataset4 = {

datahub-web-react/src/app/entity/dataset/profile/__tests__/Schema.test.tsx

@@ -3,7 +3,13 @@ import { fireEvent, render } from '@testing-library/react';
 import { MockedProvider } from '@apollo/client/testing';
 
 import TestPageContainer from '../../../../../utils/test-utils/TestPageContainer';
-import { sampleSchema, sampleSchemaWithPkFk, sampleSchemaWithTags } from '../stories/sampleSchema';
+import {
+    sampleSchema,
+    sampleSchemaWithKeyValueFields,
+    sampleSchemaWithoutFields,
+    sampleSchemaWithPkFk,
+    sampleSchemaWithTags,
+} from '../stories/sampleSchema';
 import { mocks } from '../../../../../Mocks';
 import { SchemaTab } from '../../../shared/tabs/Dataset/Schema/SchemaTab';
 import EntityContext from '../../../shared/EntityContext';
@@ -212,4 +218,70 @@ describe('Schema', () => {
         expect(getByText('Foreign Key to')).toBeInTheDocument();
         expect(getAllByText('Yet Another Dataset')).toHaveLength(2);
     });
+
+    it('renders key/value toggle', () => {
+        const { getByText, queryByText } = render(
+            <MockedProvider mocks={mocks} addTypename={false}>
+                <TestPageContainer>
+                    <EntityContext.Provider
+                        value={{
+                            urn: 'urn:li:dataset:123',
+                            entityType: EntityType.Dataset,
+                            entityData: {
+                                description: 'This is a description',
+                                schemaMetadata: sampleSchemaWithKeyValueFields as SchemaMetadata,
+                            },
+                            baseEntity: {},
+                            updateEntity: jest.fn(),
+                            routeToTab: jest.fn(),
+                            refetch: jest.fn(),
+                        }}
+                    >
+                        <SchemaTab />
+                    </EntityContext.Provider>
+                </TestPageContainer>
+            </MockedProvider>,
+        );
+        expect(getByText('Key')).toBeInTheDocument();
+        expect(getByText('Value')).toBeInTheDocument();
+        expect(getByText('count')).toBeInTheDocument();
+        expect(getByText('cost')).toBeInTheDocument();
+        expect(queryByText('id')).not.toBeInTheDocument();
+
+        const keyButton = getByText('Key');
+        fireEvent.click(keyButton);
+
+        expect(getByText('Key')).toBeInTheDocument();
+        expect(getByText('Value')).toBeInTheDocument();
+        expect(getByText('id')).toBeInTheDocument();
+        expect(queryByText('count')).not.toBeInTheDocument();
+        expect(queryByText('cost')).not.toBeInTheDocument();
+    });
+
+    it('does not renders key/value toggle when no schema', () => {
+        const { queryByText } = render(
+            <MockedProvider mocks={mocks} addTypename={false}>
+                <TestPageContainer>
+                    <EntityContext.Provider
+                        value={{
+                            urn: 'urn:li:dataset:123',
+                            entityType: EntityType.Dataset,
+                            entityData: {
+                                description: 'This is a description',
+                                schemaMetadata: sampleSchemaWithoutFields as SchemaMetadata,
+                            },
+                            baseEntity: {},
+                            updateEntity: jest.fn(),
+                            routeToTab: jest.fn(),
+                            refetch: jest.fn(),
+                        }}
+                    >
+                        <SchemaTab />
+                    </EntityContext.Provider>
+                </TestPageContainer>
+            </MockedProvider>,
+        );
+        expect(queryByText('Key')).not.toBeInTheDocument();
+        expect(queryByText('Value')).not.toBeInTheDocument();
+    });
 });

datahub-web-react/src/app/entity/dataset/profile/schema/components/SchemaDescriptionField.tsx

@@ -116,12 +116,7 @@ export default function DescriptionField({
     const showAddDescription = editable && !description;
 
     return (
-        <DescriptionContainer
-            onClick={(e) => {
-                e.preventDefault();
-                e.stopPropagation();
-            }}
-        >
+        <DescriptionContainer>
             {expanded ? (
                 <>
                     {!!description && <DescriptionText source={description} />}