Merge branch 'linkedin:master' into master

acryldata · Dec 5, 2021 · bbd44ac · bbd44ac
2 parents 19650d8 + a16c432
commit bbd44ac
Show file tree

Hide file tree

Showing 114 changed files with 2,989 additions and 3,380 deletions.
diff --git a/.gitignore b/.gitignore
@@ -40,3 +40,6 @@ MANIFEST
 **/.DS_Store
 
 .vscode
+
+# Metadata Ingestion Generated
+metadata-ingestion/generated/**
diff --git a/...core/src/main/java/com/linkedin/datahub/graphql/resolvers/group/EntityCountsResolver.java b/...core/src/main/java/com/linkedin/datahub/graphql/resolvers/group/EntityCountsResolver.java
@@ -8,6 +8,7 @@
 import com.linkedin.entity.client.EntityClient;
 import graphql.schema.DataFetcher;
 import graphql.schema.DataFetchingEnvironment;
+import io.opentelemetry.extension.annotations.WithSpan;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
@@ -25,6 +26,7 @@ public EntityCountsResolver(final EntityClient entityClient) {
   }
 
   @Override
+  @WithSpan
   public CompletableFuture<EntityCountResults> get(final DataFetchingEnvironment environment) throws Exception {
 
     final QueryContext context = environment.getContext();

diff --git a/...va/com/linkedin/datahub/graphql/resolvers/recommendation/ListRecommendationsResolver.java b/...va/com/linkedin/datahub/graphql/resolvers/recommendation/ListRecommendationsResolver.java
@@ -21,6 +21,7 @@
 import com.linkedin.metadata.recommendation.SearchRequestContext;
 import graphql.schema.DataFetcher;
 import graphql.schema.DataFetchingEnvironment;
+import io.opentelemetry.extension.annotations.WithSpan;
 import java.net.URISyntaxException;
 import java.util.Collections;
 import java.util.List;
@@ -41,6 +42,7 @@ public class ListRecommendationsResolver implements DataFetcher<CompletableFutur
 
   private final RecommendationsService _recommendationsService;
 
+  @WithSpan
   @Override
   public CompletableFuture<ListRecommendationsResult> get(DataFetchingEnvironment environment) {
     final ListRecommendationsInput input =

diff --git a/...phql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchResolver.java b/...phql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchResolver.java
@@ -8,6 +8,7 @@
 import com.linkedin.entity.client.EntityClient;
 import graphql.schema.DataFetcher;
 import graphql.schema.DataFetchingEnvironment;
+import io.opentelemetry.extension.annotations.WithSpan;
 import java.util.concurrent.CompletableFuture;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -28,6 +29,7 @@ public class SearchResolver implements DataFetcher<CompletableFuture<SearchResul
   private final EntityClient _entityClient;
 
   @Override
+  @WithSpan
   public CompletableFuture<SearchResults> get(DataFetchingEnvironment environment) {
     final SearchInput input = bindArgument(environment.getArgument("input"), SearchInput.class);
     final String entityName = EntityTypeMapper.getName(input.getType());

diff --git a/docs-website/build.gradle b/docs-website/build.gradle
@@ -12,7 +12,7 @@ node {
   }
 
   // Version of node to use.
-  version = '14.15.3'
+  version = '16.8.0'
 
   // Version of Yarn to use.
   yarnVersion = '1.22.0'

diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js
@@ -221,6 +221,8 @@ module.exports = {
           // WIP "docs/advanced/entity-hierarchy",
           // WIP "docs/advanced/partial-update",
           // WIP "docs/advanced/pdl-best-practices",
+          // WIP "docs/introducing-metadata-service-authentication"
+          // WIP "metadata-models-custom/README"
         ],
       },
     ],

diff --git a/docs/modeling/metadata-model.md b/docs/modeling/metadata-model.md
@@ -39,6 +39,17 @@ For example, here are helpful links to the most popular entities in DataHub's me
 * Feature Table (a.k.a. MLFeatureTable): [Profile](https://demo.datahubproject.io/dataset/urn:li:dataset:(urn:li:dataPlatform:datahub,MlFeatureTable,PROD)/Schema?is_lineage_mode=false) [Documentation](https://demo.datahubproject.io/dataset/urn:li:dataset:(urn:li:dataPlatform:datahub,MlFeatureTable,PROD)/Documentation?is_lineage_mode=false)
 * For the full list of entities in the metadata model, browse them [here](https://demo.datahubproject.io/browse/dataset/prod/datahub/entities)
 
+### Generating documentation for the Metadata Model
+
+The metadata model documentation can be generated and uploaded into a running DataHub instance using the following command below.
+
+```console
+./gradlew :metadata-ingestion:modelDocUpload
+```
+
+**_NOTE_**: This will upload the model documentation to the DataHub instance running at the environment variable `$DATAHUB_HOST` (http://localhost:8080 by default)
+
+It will also generate a few files under `metadata-ingestion/generated/docs` such as a dot file called `metadata_graph.dot` that you can use to visualize the relationships among the entities.
 
 ## Querying the Metadata Graph 
 

diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/config/Entity.java b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/config/Entity.java
@@ -12,6 +12,7 @@
 @AllArgsConstructor
 public class Entity {
   String name;
+  String doc;
   String keyAspect;
   List<String> aspects;
 }
diff --git a/metadata-events/mxe-registration/src/main/java/com/linkedin/mxe/Configs.java b/metadata-events/mxe-registration/src/main/java/com/linkedin/mxe/Configs.java
@@ -3,8 +3,6 @@
 import com.linkedin.pegasus2avro.mxe.FailedMetadataChangeEvent;
 import com.linkedin.pegasus2avro.mxe.MetadataAuditEvent;
 import com.linkedin.pegasus2avro.mxe.MetadataChangeEvent;
-import com.linkedin.pegasus2avro.mxe.MetadataGraphEvent;
-import com.linkedin.pegasus2avro.mxe.MetadataSearchEvent;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -26,8 +24,6 @@ public class Configs {
       put(Topics.METADATA_AUDIT_EVENT, MetadataAuditEvent.SCHEMA$);
       put(Topics.METADATA_CHANGE_EVENT, MetadataChangeEvent.SCHEMA$);
       put(Topics.FAILED_METADATA_CHANGE_EVENT, FailedMetadataChangeEvent.SCHEMA$);
-      put(Topics.METADATA_GRAPH_EVENT, MetadataGraphEvent.SCHEMA$);
-      put(Topics.METADATA_SEARCH_EVENT, MetadataSearchEvent.SCHEMA$);
 
       put(Topics.DEV_METADATA_AUDIT_EVENT, MetadataAuditEvent.SCHEMA$);
       put(Topics.DEV_METADATA_CHANGE_EVENT, MetadataChangeEvent.SCHEMA$);

diff --git a/metadata-ingestion/build.gradle b/metadata-ingestion/build.gradle
@@ -42,6 +42,18 @@ task installDev(type: Exec, dependsOn: [install]) {
   commandLine 'bash', '-x', '-c',
     "${venv_name}/bin/pip install -e .[dev] && touch ${venv_name}/.build_install_dev_sentinel"
 }
+
+task modelDocGen(type: Exec, dependsOn: [codegen, installDev]) {
+    inputs.files(project.fileTree(dir: "../metadata-events/mxe-schemas/src/", include: "**/*.avsc"))
+    outputs.dir('generated/docs')
+    commandLine 'bash', '-c', "source ${venv_name}/bin/activate && ./scripts/modeldocgen.sh"
+}
+
+task modelDocUpload(type: Exec, dependsOn: [modelDocGen]) {
+    commandLine 'bash', '-c', "source ${venv_name}/bin/activate && ./scripts/modeldocupload.sh"
+}
+
+
 task lint(type: Exec, dependsOn: installDev) {
   /*
     The find/sed combo below is a temporary work-around for the following mypy issue with airflow 2.2.0:
@@ -79,7 +91,6 @@ task installDevTest(type: Exec, dependsOn: [installDev]) {
 
 def testFile = hasProperty('testFile') ? testFile : 'unknown'
 task testSingle(dependsOn: [installDevTest]) {
-  println "$testFile"
   doLast {
   if (testFile != 'unknown')  {
   exec {