fix(ingest/looker): limit dashboard object lifetime to thread (datahu…

…b-project#9564)
acryldata · Jan 4, 2024 · 20b7dd3 · 20b7dd3
1 parent 8f45d3d
commit 20b7dd3
Showing 1 changed file with 11 additions and 9 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py
@@ -1044,7 +1044,7 @@ def process_dashboard(
         self, dashboard_id: str, fields: List[str]
     ) -> Tuple[
         List[MetadataWorkUnit],
-        Optional[Dashboard],
+        Optional[looker_usage.LookerDashboardForUsage],
         str,
         datetime.datetime,
         datetime.datetime,
@@ -1095,9 +1095,15 @@ def process_dashboard(
         )
         workunits.extend(metric_dim_workunits)
         self.reporter.report_dashboards_scanned()
+
+        # generate usage tracking object
+        dashboard_usage = looker_usage.LookerDashboardForUsage.from_dashboard(
+            dashboard_object
+        )
+
         return (
             workunits,
-            dashboard_object,
+            dashboard_usage,
             dashboard_id,
             start_time,
             datetime.datetime.now(),
@@ -1287,7 +1293,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
             for job in concurrent.futures.as_completed(async_workunits):
                 (
                     work_units,
-                    dashboard_object,
+                    dashboard_usage,
                     dashboard_id,
                     start_time,
                     end_time,
@@ -1299,12 +1305,8 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
                 self.reporter.report_upstream_latency(start_time, end_time)
 
                 yield from work_units
-                if dashboard_object is not None:
-                    looker_dashboards_for_usage.append(
-                        looker_usage.LookerDashboardForUsage.from_dashboard(
-                            dashboard_object
-                        )
-                    )
+                if dashboard_usage is not None:
+                    looker_dashboards_for_usage.append(dashboard_usage)
 
         self.reporter.report_stage_end("dashboard_chart_metadata")