Merge branch 'master' into ab-add-undo-soft-delete

datahub-project · Nov 4, 2024 · a18d4fb · a18d4fb
2 parents e4bdf53 + 95f361a
commit a18d4fb
Show file tree

Hide file tree

Showing 236 changed files with 5,499 additions and 696 deletions.
diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql
@@ -645,7 +645,7 @@ type Mutation {
     updateERModelRelationship(
         "The urn of the ERModelRelationship to update"
         urn: String!,
-        "Input required to updat an existing DataHub View"
+        "Input required to update an existing DataHub View"
         input: ERModelRelationshipUpdateInput!): Boolean
 
     """
@@ -795,7 +795,7 @@ type Mutation {
     updateView(
       "The urn of the View to update"
       urn: String!,
-      "Input required to updat an existing DataHub View"
+      "Input required to update an existing DataHub View"
       input: UpdateViewInput!): DataHubView
 
     """

diff --git a/...in/java/com/linkedin/datahub/upgrade/config/restoreindices/PropertyDefinitionsConfig.java b/...in/java/com/linkedin/datahub/upgrade/config/restoreindices/PropertyDefinitionsConfig.java
@@ -0,0 +1,30 @@
+package com.linkedin.datahub.upgrade.config.restoreindices;
+
+import com.linkedin.datahub.upgrade.config.SystemUpdateCondition;
+import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade;
+import com.linkedin.datahub.upgrade.system.restoreindices.structuredproperties.PropertyDefinitions;
+import com.linkedin.metadata.entity.AspectDao;
+import com.linkedin.metadata.entity.EntityService;
+import io.datahubproject.metadata.context.OperationContext;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Conditional;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@Conditional(SystemUpdateCondition.NonBlockingSystemUpdateCondition.class)
+public class PropertyDefinitionsConfig {
+
+  @Bean
+  public NonBlockingSystemUpgrade propertyDefinitions(
+      final OperationContext opContext,
+      final EntityService<?> entityService,
+      final AspectDao aspectDao,
+      @Value("${systemUpdate.propertyDefinitions.enabled}") final boolean enabled,
+      @Value("${systemUpdate.propertyDefinitions.batchSize}") final Integer batchSize,
+      @Value("${systemUpdate.propertyDefinitions.delayMs}") final Integer delayMs,
+      @Value("${systemUpdate.propertyDefinitions.limit}") final Integer limit) {
+    return new PropertyDefinitions(
+        opContext, entityService, aspectDao, enabled, batchSize, delayMs, limit);
+  }
+}
diff --git a/...onfig/ReindexDomainDescriptionConfig.java → ...dices/ReindexDomainDescriptionConfig.java b/...onfig/ReindexDomainDescriptionConfig.java → ...dices/ReindexDomainDescriptionConfig.java
@@ -1,7 +1,8 @@
-package com.linkedin.datahub.upgrade.config;
+package com.linkedin.datahub.upgrade.config.restoreindices;
 
+import com.linkedin.datahub.upgrade.config.SystemUpdateCondition;
 import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade;
-import com.linkedin.datahub.upgrade.system.domaindescription.ReindexDomainDescription;
+import com.linkedin.datahub.upgrade.system.restoreindices.domaindescription.ReindexDomainDescription;
 import com.linkedin.metadata.entity.AspectDao;
 import com.linkedin.metadata.entity.EntityService;
 import io.datahubproject.metadata.context.OperationContext;

diff --git a/...raph/ReindexDataJobViaNodesCLLConfig.java → ...raph/ReindexDataJobViaNodesCLLConfig.java b/...raph/ReindexDataJobViaNodesCLLConfig.java → ...raph/ReindexDataJobViaNodesCLLConfig.java
@@ -1,8 +1,8 @@
-package com.linkedin.datahub.upgrade.config.graph;
+package com.linkedin.datahub.upgrade.config.restoreindices.graph;
 
 import com.linkedin.datahub.upgrade.config.SystemUpdateCondition;
 import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade;
-import com.linkedin.datahub.upgrade.system.graph.vianodes.ReindexDataJobViaNodesCLL;
+import com.linkedin.datahub.upgrade.system.restoreindices.graph.vianodes.ReindexDataJobViaNodesCLL;
 import com.linkedin.metadata.entity.AspectDao;
 import com.linkedin.metadata.entity.EntityService;
 import io.datahubproject.metadata.context.OperationContext;

diff --git a/...config/graph/ReindexEdgeStatusConfig.java → ...ndices/graph/ReindexEdgeStatusConfig.java b/...config/graph/ReindexEdgeStatusConfig.java → ...ndices/graph/ReindexEdgeStatusConfig.java
@@ -1,8 +1,8 @@
-package com.linkedin.datahub.upgrade.config.graph;
+package com.linkedin.datahub.upgrade.config.restoreindices.graph;
 
 import com.linkedin.datahub.upgrade.config.SystemUpdateCondition;
 import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade;
-import com.linkedin.datahub.upgrade.system.graph.edgestatus.ReindexEdgeStatus;
+import com.linkedin.datahub.upgrade.system.restoreindices.graph.edgestatus.ReindexEdgeStatus;
 import com.linkedin.metadata.entity.AspectDao;
 import com.linkedin.metadata.entity.EntityService;
 import io.datahubproject.metadata.context.OperationContext;

diff --git a/...description/ReindexDomainDescription.java → ...description/ReindexDomainDescription.java b/...description/ReindexDomainDescription.java → ...description/ReindexDomainDescription.java
@@ -1,4 +1,4 @@
-package com.linkedin.datahub.upgrade.system.domaindescription;
+package com.linkedin.datahub.upgrade.system.restoreindices.domaindescription;
 
 import com.google.common.collect.ImmutableList;
 import com.linkedin.datahub.upgrade.UpgradeStep;

diff --git a/...ription/ReindexDomainDescriptionStep.java → ...ription/ReindexDomainDescriptionStep.java b/...ription/ReindexDomainDescriptionStep.java → ...ription/ReindexDomainDescriptionStep.java
@@ -1,4 +1,4 @@
-package com.linkedin.datahub.upgrade.system.domaindescription;
+package com.linkedin.datahub.upgrade.system.restoreindices.domaindescription;
 
 import static com.linkedin.metadata.Constants.*;
 

diff --git a/...m/graph/edgestatus/ReindexEdgeStatus.java → ...s/graph/edgestatus/ReindexEdgeStatus.java b/...m/graph/edgestatus/ReindexEdgeStatus.java → ...s/graph/edgestatus/ReindexEdgeStatus.java
@@ -1,4 +1,4 @@
-package com.linkedin.datahub.upgrade.system.graph.edgestatus;
+package com.linkedin.datahub.upgrade.system.restoreindices.graph.edgestatus;
 
 import com.google.common.collect.ImmutableList;
 import com.linkedin.datahub.upgrade.UpgradeStep;

diff --git a/...estatus/ReindexReindexEdgeStatusStep.java → ...estatus/ReindexReindexEdgeStatusStep.java b/...estatus/ReindexReindexEdgeStatusStep.java → ...estatus/ReindexReindexEdgeStatusStep.java
@@ -1,4 +1,4 @@
-package com.linkedin.datahub.upgrade.system.graph.edgestatus;
+package com.linkedin.datahub.upgrade.system.restoreindices.graph.edgestatus;
 
 import static com.linkedin.metadata.Constants.STATUS_ASPECT_NAME;
 

diff --git a/...h/vianodes/ReindexDataJobViaNodesCLL.java → ...h/vianodes/ReindexDataJobViaNodesCLL.java b/...h/vianodes/ReindexDataJobViaNodesCLL.java → ...h/vianodes/ReindexDataJobViaNodesCLL.java
@@ -1,4 +1,4 @@
-package com.linkedin.datahub.upgrade.system.graph.vianodes;
+package com.linkedin.datahub.upgrade.system.restoreindices.graph.vianodes;
 
 import com.google.common.collect.ImmutableList;
 import com.linkedin.datahub.upgrade.UpgradeStep;

diff --git a/...anodes/ReindexDataJobViaNodesCLLStep.java → ...anodes/ReindexDataJobViaNodesCLLStep.java b/...anodes/ReindexDataJobViaNodesCLLStep.java → ...anodes/ReindexDataJobViaNodesCLLStep.java
@@ -1,4 +1,4 @@
-package com.linkedin.datahub.upgrade.system.graph.vianodes;
+package com.linkedin.datahub.upgrade.system.restoreindices.graph.vianodes;
 
 import static com.linkedin.metadata.Constants.*;
 

diff --git a/...kedin/datahub/upgrade/system/restoreindices/structuredproperties/PropertyDefinitions.java b/...kedin/datahub/upgrade/system/restoreindices/structuredproperties/PropertyDefinitions.java
@@ -0,0 +1,49 @@
+package com.linkedin.datahub.upgrade.system.restoreindices.structuredproperties;
+
+import com.google.common.collect.ImmutableList;
+import com.linkedin.datahub.upgrade.UpgradeStep;
+import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade;
+import com.linkedin.metadata.entity.AspectDao;
+import com.linkedin.metadata.entity.EntityService;
+import io.datahubproject.metadata.context.OperationContext;
+import java.util.List;
+import javax.annotation.Nonnull;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * A job that reindexes all domain aspects as part of reindexing descriptions This is required to
+ * fix the analytics for domains
+ */
+@Slf4j
+public class PropertyDefinitions implements NonBlockingSystemUpgrade {
+
+  private final List<UpgradeStep> _steps;
+
+  public PropertyDefinitions(
+      @Nonnull OperationContext opContext,
+      EntityService<?> entityService,
+      AspectDao aspectDao,
+      boolean enabled,
+      Integer batchSize,
+      Integer batchDelayMs,
+      Integer limit) {
+    if (enabled) {
+      _steps =
+          ImmutableList.of(
+              new PropertyDefinitionsStep(
+                  opContext, entityService, aspectDao, batchSize, batchDelayMs, limit));
+    } else {
+      _steps = ImmutableList.of();
+    }
+  }
+
+  @Override
+  public String id() {
+    return this.getClass().getName();
+  }
+
+  @Override
+  public List<UpgradeStep> steps() {
+    return _steps;
+  }
+}
diff --git a/...n/datahub/upgrade/system/restoreindices/structuredproperties/PropertyDefinitionsStep.java b/...n/datahub/upgrade/system/restoreindices/structuredproperties/PropertyDefinitionsStep.java
@@ -0,0 +1,42 @@
+package com.linkedin.datahub.upgrade.system.restoreindices.structuredproperties;
+
+import static com.linkedin.metadata.Constants.*;
+
+import com.linkedin.datahub.upgrade.system.AbstractMCLStep;
+import com.linkedin.metadata.entity.AspectDao;
+import com.linkedin.metadata.entity.EntityService;
+import io.datahubproject.metadata.context.OperationContext;
+import javax.annotation.Nonnull;
+import lombok.extern.slf4j.Slf4j;
+import org.jetbrains.annotations.Nullable;
+
+@Slf4j
+public class PropertyDefinitionsStep extends AbstractMCLStep {
+
+  public PropertyDefinitionsStep(
+      OperationContext opContext,
+      EntityService<?> entityService,
+      AspectDao aspectDao,
+      Integer batchSize,
+      Integer batchDelayMs,
+      Integer limit) {
+    super(opContext, entityService, aspectDao, batchSize, batchDelayMs, limit);
+  }
+
+  @Override
+  public String id() {
+    return "structured-property-definitions-v1";
+  }
+
+  @Nonnull
+  @Override
+  protected String getAspectName() {
+    return STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME;
+  }
+
+  @Nullable
+  @Override
+  protected String getUrnLike() {
+    return "urn:li:" + STRUCTURED_PROPERTY_ENTITY_NAME + ":%";
+  }
+}
diff --git a/...hub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNonBlockingTest.java b/...hub-upgrade/src/test/java/com/linkedin/datahub/upgrade/DatahubUpgradeNonBlockingTest.java
@@ -12,7 +12,7 @@
 import com.linkedin.datahub.upgrade.impl.DefaultUpgradeManager;
 import com.linkedin.datahub.upgrade.system.SystemUpdateNonBlocking;
 import com.linkedin.datahub.upgrade.system.bootstrapmcps.BootstrapMCPStep;
-import com.linkedin.datahub.upgrade.system.graph.vianodes.ReindexDataJobViaNodesCLL;
+import com.linkedin.datahub.upgrade.system.restoreindices.graph.vianodes.ReindexDataJobViaNodesCLL;
 import com.linkedin.metadata.boot.kafka.MockSystemUpdateDeserializer;
 import com.linkedin.metadata.boot.kafka.MockSystemUpdateSerializer;
 import com.linkedin.metadata.config.kafka.KafkaConfiguration;

diff --git a/datahub-upgrade/src/test/resources/bootstrapmcp/datahub-test-mcp.yaml b/datahub-upgrade/src/test/resources/bootstrapmcp/datahub-test-mcp.yaml
@@ -23,7 +23,13 @@
               keep_last_n: {{dataprocess_cleanup.keep_last_n}}{{^dataprocess_cleanup.keep_last_n}}5{{/dataprocess_cleanup.keep_last_n}}
             soft_deleted_entities_cleanup:
               retention_days: {{soft_deleted_entities_cleanup.retention_days}}{{^soft_deleted_entities_cleanup.retention_days}}10{{/soft_deleted_entities_cleanup.retention_days}}
+            execution_request_cleanup:
+              keep_history_min_count: {{execution_request_cleanup.keep_history_min_count}}{{^execution_request_cleanup.keep_history_min_count}}10{{/execution_request_cleanup.keep_history_min_count}}
+              keep_history_max_count: {{execution_request_cleanup.keep_history_max_count}}{{^execution_request_cleanup.keep_history_max_count}}1000{{/execution_request_cleanup.keep_history_max_count}}
+              keep_history_max_days: {{execution_request_cleanup.keep_history_max_days}}{{^execution_request_cleanup.keep_history_max_days}}30{{/execution_request_cleanup.keep_history_max_days}}
+              batch_read_size: {{execution_request_cleanup.batch_read_size}}{{^execution_request_cleanup.batch_read_size}}100{{/execution_request_cleanup.batch_read_size}}
+              enabled: {{execution_request_cleanup.enabled}}{{^execution_request_cleanup.enabled}}false{{/execution_request_cleanup.enabled}}
       extraArgs: {}
       debugMode: false
       executorId: default
-  headers: {}
+  headers: {}
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Incident/components/AddIncidentModal.tsx b/datahub-web-react/src/app/entity/shared/tabs/Incident/components/AddIncidentModal.tsx
@@ -1,14 +1,20 @@
 import React, { useState } from 'react';
 import { message, Modal, Button, Form, Input, Typography, Select } from 'antd';
 import { useApolloClient } from '@apollo/client';
-import TextArea from 'antd/lib/input/TextArea';
+import styled from 'styled-components';
 import analytics, { EventType, EntityActionType } from '../../../../../analytics';
 import { useEntityData } from '../../../EntityContext';
 import { EntityType, IncidentSourceType, IncidentState, IncidentType } from '../../../../../../types.generated';
 import { INCIDENT_DISPLAY_TYPES, PAGE_SIZE, addActiveIncidentToCache } from '../incidentUtils';
 import { useRaiseIncidentMutation } from '../../../../../../graphql/mutations.generated';
 import handleGraphQLError from '../../../../../shared/handleGraphQLError';
 import { useUserContext } from '../../../../../context/useUserContext';
+import { Editor } from '../../Documentation/components/editor/Editor';
+import { ANTD_GRAY } from '../../../constants';
+
+const StyledEditor = styled(Editor)`
+    border: 1px solid ${ANTD_GRAY[4.5]};
+`;
 
 type AddIncidentProps = {
     open: boolean;
@@ -112,6 +118,7 @@ export const AddIncidentModal = ({ open, onClose, refetch }: AddIncidentProps) =
                 open={open}
                 destroyOnClose
                 onCancel={handleClose}
+                width={600}
                 footer={[
                     <Button type="text" onClick={handleClose}>
                         Cancel
@@ -174,7 +181,17 @@ export const AddIncidentModal = ({ open, onClose, refetch }: AddIncidentProps) =
                             },
                         ]}
                     >
-                        <TextArea placeholder="Provide some additional details" />
+                        <StyledEditor
+                            doNotFocus
+                            className="add-incident-description"
+                            onKeyDown={(e) => {
+                                // Preventing the modal from closing when the Enter key is pressed
+                                if (e.key === 'Enter') {
+                                    e.preventDefault();
+                                    e.stopPropagation();
+                                }
+                            }}
+                        />
                     </Form.Item>
                 </Form>
             </Modal>

diff --git a/datahub-web-react/src/app/entity/shared/tabs/Incident/components/IncidentListItem.tsx b/datahub-web-react/src/app/entity/shared/tabs/Incident/components/IncidentListItem.tsx
@@ -14,6 +14,7 @@ import { useUpdateIncidentStatusMutation } from '../../../../../../graphql/mutat
 import { ResolveIncidentModal } from './ResolveIncidentModal';
 import handleGraphQLError from '../../../../../shared/handleGraphQLError';
 import { MenuItemStyle } from '../../../../view/menu/item/styledComponent';
+import MarkdownViewer from '../../../components/legacy/MarkdownViewer';
 
 type Props = {
     incident: any;
@@ -227,7 +228,7 @@ export default function IncidentListItem({ incident, refetch }: Props) {
                             </TitleContainer>
                             <DescriptionContainer>
                                 <IncidentDescriptionLabel>Description</IncidentDescriptionLabel>
-                                <IncidentDescriptionText>{incident?.description}</IncidentDescriptionText>
+                                <MarkdownViewer source={incident?.description} />
                                 {incident.status.state === IncidentState.Resolved ? (
                                     <>
                                         <IncidentDescriptionLabel>Resolution Note</IncidentDescriptionLabel>

diff --git a/docs-website/docusaurus.config.js b/docs-website/docusaurus.config.js
@@ -88,7 +88,26 @@ module.exports = {
       },
       items: [
         {
-          to: "cloud/",
+          type: "dropdown",
+          label: "Solutions",
+          position: "right",
+          items: [
+            {
+              to: "/solutions/discovery",
+              label: "Discovery",
+            },
+            {
+              to: "/solutions/observability",
+              label: "Observability",
+            },
+            {
+              to: "/solutions/governance",
+              label: "Governance",
+            },
+          ]
+        },
+        {
+          to: "/cloud",
           activeBasePath: "cloud",
           label: "Cloud",
           position: "right",

diff --git a/docs-website/package.json b/docs-website/package.json
@@ -35,6 +35,7 @@
     "@octokit/plugin-throttling": "^3.5.1",
     "@octokit/rest": "^18.6.2",
     "@radix-ui/react-visually-hidden": "^1.0.2",
+    "@servicebell/widget": "^0.1.6",
     "@supabase/supabase-js": "^2.33.1",
     "@swc/core": "^1.4.2",
     "antd": "^5.0.7",

diff --git a/docs-website/src/pages/_components/Community/community.module.scss b/docs-website/src/pages/_components/Community/community.module.scss
@@ -221,14 +221,14 @@
 
 .numberContainer {
   display: inline-block;
-  width: 11rem;
+  width: 12rem;
   text-align: right;
 }
 
 .numberChange {
   display: inline-block;
   animation: slideIn 0.5s ease-in-out;
-  width: 11rem;
+  width: 12rem;
 }
 
 

diff --git a/docs-website/src/pages/_components/Community/index.js b/docs-website/src/pages/_components/Community/index.js
@@ -2,7 +2,7 @@ import React, { useState, useRef, useEffect } from "react";
 import styles from "./community.module.scss";
 import useBaseUrl from "@docusaurus/useBaseUrl";
 
-const TARGET_COUNT = 11535;
+const TARGET_COUNT = 12219;
 const INCREMENT = 1;
 
 const Community = () => {

diff --git a/docs-website/src/pages/_components/Hero/hero.module.scss b/docs-website/src/pages/_components/Hero/hero.module.scss
@@ -233,6 +233,9 @@
         }
         .hero__cta {
           margin-top: 12px;
+          .cta__primary {
+            margin-bottom: 4px;
+          }
         }
         .hero__footer_cta {
           margin-top: 12px;

diff --git a/docs-website/src/pages/_components/Hero/index.js b/docs-website/src/pages/_components/Hero/index.js
@@ -69,7 +69,7 @@ const Hero = ({ onOpenTourModal }) => {
             </div>
             <div className="hero__cta">
               <Link className="cta__primary" to="/cloud">
-                Book a Demo
+                Get Cloud
               </Link>
               <a
                 className="cta__secondary"
@@ -80,7 +80,7 @@ const Hero = ({ onOpenTourModal }) => {
               </a>
             </div>
             <Link className="hero__footer_cta" to="/docs">
-              Get started with Core →
+              Get started with Open Source →
             </Link>
           </div>
           <div className="hero__img">