Merge branch 'datahub-project:master' into master

datahub-upgrade/src/test/resources/bootstrapmcp/datahub-test-mcp.yaml

@@ -23,7 +23,13 @@
               keep_last_n: {{dataprocess_cleanup.keep_last_n}}{{^dataprocess_cleanup.keep_last_n}}5{{/dataprocess_cleanup.keep_last_n}}
             soft_deleted_entities_cleanup:
               retention_days: {{soft_deleted_entities_cleanup.retention_days}}{{^soft_deleted_entities_cleanup.retention_days}}10{{/soft_deleted_entities_cleanup.retention_days}}
+            execution_request_cleanup:
+              keep_history_min_count: {{execution_request_cleanup.keep_history_min_count}}{{^execution_request_cleanup.keep_history_min_count}}10{{/execution_request_cleanup.keep_history_min_count}}
+              keep_history_max_count: {{execution_request_cleanup.keep_history_max_count}}{{^execution_request_cleanup.keep_history_max_count}}1000{{/execution_request_cleanup.keep_history_max_count}}
+              keep_history_max_days: {{execution_request_cleanup.keep_history_max_days}}{{^execution_request_cleanup.keep_history_max_days}}30{{/execution_request_cleanup.keep_history_max_days}}
+              batch_read_size: {{execution_request_cleanup.batch_read_size}}{{^execution_request_cleanup.batch_read_size}}100{{/execution_request_cleanup.batch_read_size}}
+              enabled: {{execution_request_cleanup.enabled}}{{^execution_request_cleanup.enabled}}false{{/execution_request_cleanup.enabled}}
       extraArgs: {}
       debugMode: false
       executorId: default
-  headers: {}
+  headers: {}

docs-website/src/pages/_components/Community/community.module.scss

@@ -221,14 +221,14 @@
 
 .numberContainer {
   display: inline-block;
-  width: 11rem;
+  width: 12rem;
   text-align: right;
 }
 
 .numberChange {
   display: inline-block;
   animation: slideIn 0.5s ease-in-out;
-  width: 11rem;
+  width: 12rem;
 }
 
 

docs-website/src/pages/_components/QuickstartContent/quickstartcontent.module.scss

@@ -129,7 +129,7 @@
 
     .quickstart__content {
       display: flex;
-      margin-bottom: 3rem;
+      margin-bottom: 6rem;
       width: 100%;
 
       .quickstart__text {

docs-website/src/pages/solutions/_components/Integrations/index.js

@@ -47,6 +47,7 @@ const Integrations = () => {
               </div>
           </div>
       </div>
+      <a href="/integrations">See all →</a>
     </div>
   );
 };

docs-website/src/pages/solutions/_components/Integrations/integrations.module.scss

@@ -1,6 +1,13 @@
 .container {
   display: flex;
   flex-direction: column;
+  >a {
+    text-decoration: none;
+    text-align: center;
+    margin-top: 1rem;
+    margin-bottom: 1rem;
+    font-size: 1.25rem;
+  }
 
   .section_header {
     color: var(--primitives-text-tex-subtext, #777E99);

docs-website/src/pages/solutions/_components/IntegrationsStatic/index.js

@@ -17,7 +17,7 @@ const Integrations = () => {
                 <div className={styles.slide_track}>
                   {[...Array(1)].map((_, i) => (
                     <React.Fragment key={i}>
-                      {[1, 2, 3, 4, 5, 6].map((item, index) => (
+                      {[1, 2, 3, 4, 5, 6, 7, 8].map((item, index) => (
                         <div className={styles.slide} key={index} style={{ backgroundImage: `url(${useBaseUrl(`${integrationsPath}/logo-integration-${item}.png`)})` }}>
                         </div>
                       ))}

docs-website/src/pages/solutions/_components/IntegrationsStatic/integrations.module.scss

@@ -71,18 +71,22 @@
 
 .slider {
   position: relative;
+  display: flex;
 }
 
 .slide_track {
   display: flex;
-  width: max-content;
+  width: 80%;
   margin: auto;
+  flex-direction: row;
+  align-items: center;
+  justify-content: space-evenly;
 }
 
 .slide {
-  width: 100px;
-  height: 100px;
-  margin: auto 3rem;
+  width: 80px;
+  height: 80px;
+  margin: auto 0;
   display: flex;
   justify-content: space-between;
   overflow: hidden;
@@ -99,9 +103,12 @@
     max-width: 100vw;
     min-width: auto;
   }
+  .slide_track {
+    width: 95%;
+  }
   .slide {
-    width: 80px;
-    height: 80px;
-    margin: auto 1rem;
+    width: 40px;
+    height: 40px;
+    margin: auto 0;
   }
 }

metadata-ingestion/docs/sources/gc/gc_recipe.dhub.yml

@@ -22,3 +22,14 @@ source:
     soft_deleted_entities_cleanup:
       # Delete soft deleted entities which were deleted 10 days ago
       retention_days: 10
+    execution_request_cleanup:
+      # Minimum number of execution requests to keep, per ingestion source
+      keep_history_min_count: 10
+      # Maximum number of execution requests to keep, per ingestion source
+      keep_history_max_count: 1000
+      # Maximum number of days to keep execution requests for, per ingestion source
+      keep_history_max_days: 30
+      # Number of records per read operation
+      batch_read_size: 100
+      # Global switch for this cleanup task 
+      enabled: true

metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py

@@ -24,6 +24,11 @@
     DataProcessCleanupConfig,
     DataProcessCleanupReport,
 )
+from datahub.ingestion.source.gc.execution_request_cleanup import (
+    DatahubExecutionRequestCleanup,
+    DatahubExecutionRequestCleanupConfig,
+    DatahubExecutionRequestCleanupReport,
+)
 from datahub.ingestion.source.gc.soft_deleted_entity_cleanup import (
     SoftDeletedEntitiesCleanup,
     SoftDeletedEntitiesCleanupConfig,
@@ -70,9 +75,18 @@ class DataHubGcSourceConfig(ConfigModel):
         description="Configuration for soft deleted entities cleanup",
     )
 
+    execution_request_cleanup: Optional[DatahubExecutionRequestCleanupConfig] = Field(
+        default=None,
+        description="Configuration for execution request cleanup",
+    )
+
 
 @dataclass
-class DataHubGcSourceReport(DataProcessCleanupReport, SoftDeletedEntitiesReport):
+class DataHubGcSourceReport(
+    DataProcessCleanupReport,
+    SoftDeletedEntitiesReport,
+    DatahubExecutionRequestCleanupReport,
+):
     expired_tokens_revoked: int = 0
 
 
@@ -97,6 +111,7 @@ def __init__(self, ctx: PipelineContext, config: DataHubGcSourceConfig):
         self.graph = ctx.require_graph("The DataHubGc source")
         self.dataprocess_cleanup: Optional[DataProcessCleanup] = None
         self.soft_deleted_entities_cleanup: Optional[SoftDeletedEntitiesCleanup] = None
+        self.execution_request_cleanup: Optional[DatahubExecutionRequestCleanup] = None
 
         if self.config.dataprocess_cleanup:
             self.dataprocess_cleanup = DataProcessCleanup(
@@ -109,6 +124,12 @@ def __init__(self, ctx: PipelineContext, config: DataHubGcSourceConfig):
                 self.report,
                 self.config.dry_run,
             )
+        if self.config.execution_request_cleanup:
+            self.execution_request_cleanup = DatahubExecutionRequestCleanup(
+                config=self.config.execution_request_cleanup,
+                graph=self.graph,
+                report=self.report,
+            )
 
     @classmethod
     def create(cls, config_dict, ctx):
@@ -130,6 +151,8 @@ def get_workunits_internal(
             yield from self.dataprocess_cleanup.get_workunits_internal()
         if self.soft_deleted_entities_cleanup:
             self.soft_deleted_entities_cleanup.cleanup_soft_deleted_entities()
+        if self.execution_request_cleanup:
+            self.execution_request_cleanup.run()
         yield from []
 
     def truncate_indices(self) -> None: