leverage minimal graph to build query builder with datsapaces (#3488)

.changeset/kind-rabbits-draw.md

@@ -0,0 +1,6 @@
+---
+'@finos/legend-extension-dsl-data-space': minor
+'@finos/legend-application-query': minor
+---
+
+Use PMCD returned by mapping analysis to build minimal graph for query

.changeset/twenty-buses-sit.md

@@ -0,0 +1,7 @@
+---
+'@finos/legend-extension-dsl-data-space-studio': patch
+'@finos/legend-application-studio': patch
+'@finos/legend-query-builder': patch
+'@finos/legend-server-depot': patch
+'@finos/legend-graph': patch
+---

packages/legend-application-query/src/application/LegendQueryApplicationConfig.ts

@@ -65,6 +65,8 @@ class LegendQueryApplicationCoreOptions {
   TEMPORARY__serviceRegistrationConfig: ServiceRegistrationEnvironmentConfig[] =
     [];
 
+  TEMPORARY__enableMinimalGraph = false;
+
   /**
    * Config specific to query builder
    */
@@ -78,6 +80,7 @@ class LegendQueryApplicationCoreOptions {
       queryBuilderConfig: optional(
         usingModelSchema(QueryBuilderConfig.serialization.schema),
       ),
+      TEMPORARY__enableMinimalGraph: optional(primitive()),
     }),
   );
 

packages/legend-application-query/src/components/Core_LegendQueryApplicationPlugin.tsx

@@ -15,9 +15,9 @@
  */
 
 import {
+  type QuerySetupActionConfiguration,
   LegendQueryApplicationPlugin,
   QuerySetupActionTag,
-  type QuerySetupActionConfiguration,
 } from '../stores/LegendQueryApplicationPlugin.js';
 import packageJson from '../../package.json' with { type: 'json' };
 import type { QuerySetupLandingPageStore } from '../stores/QuerySetupStore.js';
@@ -50,10 +50,10 @@ import {
   LEGEND_QUERY_ROUTE_PATTERN,
 } from '../__lib__/LegendQueryNavigation.js';
 import {
-  ActionAlertActionType,
-  ActionAlertType,
   type ApplicationPageEntry,
   type LegendApplicationSetup,
+  ActionAlertActionType,
+  ActionAlertType,
 } from '@finos/legend-application';
 import { CloneQueryServiceSetup } from './CloneQueryServiceSetup.js';
 import { QueryProductionizerSetup } from './QueryProductionizerSetup.js';
@@ -65,28 +65,60 @@ import {
   generateDataSpaceQuerySetupRoute,
 } from '../__lib__/DSL_DataSpace_LegendQueryNavigation.js';
 import {
-  QUERY_BUILDER_SUPPORTED_GET_ALL_FUNCTIONS,
+  type QueryBuilderState,
   type QueryBuilderHeaderActionConfiguration,
   type QueryBuilderMenuActionConfiguration,
+  type QueryBuilderPropagateExecutionContextChangeHelper,
+  QUERY_BUILDER_SUPPORTED_GET_ALL_FUNCTIONS,
 } from '@finos/legend-query-builder';
 import {
   ExistingQueryEditorStore,
   QueryBuilderActionConfig_QueryApplication,
 } from '../stores/QueryEditorStore.js';
 import {
   DataSpaceQueryBuilderState,
+  DataSpacesDepotRepository,
   generateDataSpaceTemplateQueryPromotionRoute,
 } from '@finos/legend-extension-dsl-data-space/application';
-import { RuntimePointer } from '@finos/legend-graph';
+import {
+  createGraphBuilderReport,
+  GRAPH_MANAGER_EVENT,
+  LegendSDLC,
+  PackageableElementPointerType,
+  resolvePackagePathAndElementName,
+  RuntimePointer,
+  V1_EngineRuntime,
+  V1_Mapping,
+  V1_PackageableElementPointer,
+  V1_PackageableRuntime,
+  V1_PureGraphManager,
+} from '@finos/legend-graph';
 import { LegendQueryTelemetryHelper } from '../__lib__/LegendQueryTelemetryHelper.js';
-import { StoreProjectData } from '@finos/legend-server-depot';
-import { buildUrl } from '@finos/legend-shared';
+import { resolveVersion, StoreProjectData } from '@finos/legend-server-depot';
+import {
+  ActionState,
+  assertErrorThrown,
+  buildUrl,
+  getNullableFirstEntry,
+  guaranteeNonNullable,
+  guaranteeType,
+  LogEvent,
+  StopWatch,
+  uniq,
+} from '@finos/legend-shared';
 import { parseProjectIdentifier } from '@finos/legend-storage';
 import { QueryEditorExistingQueryHeader } from './QueryEditor.js';
 import { DataSpaceTemplateQueryCreatorStore } from '../stores/data-space/DataSpaceTemplateQueryCreatorStore.js';
 import { createViewSDLCProjectHandler } from '../stores/data-space/DataSpaceQueryBuilderHelper.js';
 import { DataSpaceQueryCreatorStore } from '../stores/data-space/DataSpaceQueryCreatorStore.js';
 import { configureCodeEditorComponent } from '@finos/legend-lego/code-editor';
+import {
+  resolveUsableDataSpaceClasses,
+  V1_DataSpace,
+  V1_DataSpaceExecutionContext,
+} from '@finos/legend-extension-dsl-data-space/graph';
+import { flowResult } from 'mobx';
+import { LEGEND_QUERY_APP_EVENT } from '../__lib__/LegendQueryEvent.js';
 
 export class Core_LegendQueryApplicationPlugin extends LegendQueryApplicationPlugin {
   static NAME = packageJson.extensions.applicationQueryPlugin;
@@ -725,4 +757,246 @@ export class Core_LegendQueryApplicationPlugin extends LegendQueryApplicationPlu
       },
     };
   }
+
+  getExtraQueryBuilderPropagateExecutionContextChangeHelper?(): QueryBuilderPropagateExecutionContextChangeHelper[] {
+    return [
+      (
+        queryBuilderState: QueryBuilderState,
+        isGraphBuildingNotRequired?: boolean,
+      ): (() => Promise<void>) | undefined => {
+        /**
+         * Propagation after changing the execution context:
+         * - The mapping will be updated to the mapping of the execution context
+         * - The runtime will be updated to the default runtime of the execution context
+         * - If no class is chosen, try to choose a compatible class
+         * - If the chosen class is compatible with the new selected execution context mapping, do nothing, otherwise, try to choose a compatible class
+         */
+        const propagateExecutionContextChange = async (): Promise<void> => {
+          const dataSpaceQueryBuilderState = guaranteeType(
+            queryBuilderState,
+            DataSpaceQueryBuilderState,
+          );
+          const mapping =
+            dataSpaceQueryBuilderState.executionContext.mapping.value;
+          const mappingModelCoverageAnalysisResult =
+            dataSpaceQueryBuilderState.dataSpaceAnalysisResult?.mappingToMappingCoverageResult?.get(
+              mapping.path,
+            );
+          const editorStore = (
+            queryBuilderState.workflowState
+              .actionConfig as QueryBuilderActionConfig_QueryApplication
+          ).editorStore;
+          if (
+            dataSpaceQueryBuilderState.dataSpaceAnalysisResult &&
+            mappingModelCoverageAnalysisResult
+          ) {
+            if (
+              !isGraphBuildingNotRequired &&
+              dataSpaceQueryBuilderState.isLightGraphEnabled
+            ) {
+              const supportBuildMinimalGraph =
+                editorStore.applicationStore.config.options
+                  .TEMPORARY__enableMinimalGraph;
+              if (
+                editorStore.enableMinialGraphForDataSpaceLoadingPerformance &&
+                supportBuildMinimalGraph
+              ) {
+                try {
+                  const stopWatch = new StopWatch();
+                  const graph =
+                    dataSpaceQueryBuilderState.graphManagerState.createNewGraph();
+                  const graph_buildReport = createGraphBuilderReport();
+                  const graphManager = guaranteeType(
+                    dataSpaceQueryBuilderState.graphManagerState.graphManager,
+                    V1_PureGraphManager,
+                  );
+                  // Create dummy mappings and runtimes
+                  // TODO?: these stubbed mappings and runtimes are not really useful that useful, so either we should
+                  // simplify the model here or potentially refactor the backend analytics endpoint to return these as model
+                  const mappingModels = uniq(
+                    Array.from(
+                      dataSpaceQueryBuilderState.dataSpaceAnalysisResult.executionContextsIndex.values(),
+                    ).map((context) => context.mapping),
+                  ).map((m) => {
+                    const _mapping = new V1_Mapping();
+                    const [packagePath, name] =
+                      resolvePackagePathAndElementName(m.path);
+                    _mapping.package = packagePath;
+                    _mapping.name = name;
+                    return graphManager.elementProtocolToEntity(_mapping);
+                  });
+                  const runtimeModels = uniq(
+                    Array.from(
+                      dataSpaceQueryBuilderState.dataSpaceAnalysisResult.executionContextsIndex.values(),
+                    )
+                      .map((context) => context.defaultRuntime)
+                      .concat(
+                        Array.from(
+                          dataSpaceQueryBuilderState.dataSpaceAnalysisResult.executionContextsIndex.values(),
+                        ).flatMap((val) => val.compatibleRuntimes),
+                      ),
+                  ).map((r) => {
+                    const runtime = new V1_PackageableRuntime();
+                    const [packagePath, name] =
+                      resolvePackagePathAndElementName(r.path);
+                    runtime.package = packagePath;
+                    runtime.name = name;
+                    runtime.runtimeValue = new V1_EngineRuntime();
+                    return graphManager.elementProtocolToEntity(runtime);
+                  });
+                  // The DataSpace entity is excluded from AnalyticsResult.Json to reduce the JSON size
+                  // because all its information can be found in V1_DataSpaceAnalysisResult.
+                  // Therefore, we are building a simple v1_DataSpace entity based on V1_DataSpaceAnalysisResult.
+                  const dataspaceProtocol = new V1_DataSpace();
+                  dataspaceProtocol.name =
+                    dataSpaceQueryBuilderState.dataSpaceAnalysisResult.name;
+                  dataspaceProtocol.package =
+                    dataSpaceQueryBuilderState.dataSpaceAnalysisResult.package;
+                  dataspaceProtocol.supportInfo =
+                    dataSpaceQueryBuilderState.dataSpaceAnalysisResult.supportInfo;
+                  dataspaceProtocol.executionContexts = Array.from(
+                    dataSpaceQueryBuilderState.dataSpaceAnalysisResult
+                      .executionContextsIndex,
+                  ).map(([key, execContext]) => {
+                    const contextProtocol = new V1_DataSpaceExecutionContext();
+                    contextProtocol.name = execContext.name;
+                    contextProtocol.title = execContext.title;
+                    contextProtocol.description = execContext.description;
+                    contextProtocol.mapping = new V1_PackageableElementPointer(
+                      PackageableElementPointerType.MAPPING,
+                      execContext.mapping.path,
+                    );
+                    contextProtocol.defaultRuntime =
+                      new V1_PackageableElementPointer(
+                        PackageableElementPointerType.RUNTIME,
+                        execContext.defaultRuntime.path,
+                      );
+                    return contextProtocol;
+                  });
+                  dataspaceProtocol.defaultExecutionContext =
+                    dataSpaceQueryBuilderState.dataSpaceAnalysisResult.defaultExecutionContext.name;
+                  dataspaceProtocol.title =
+                    dataSpaceQueryBuilderState.dataSpaceAnalysisResult.title;
+                  dataspaceProtocol.description =
+                    dataSpaceQueryBuilderState.dataSpaceAnalysisResult.description;
+                  const dataspaceEntity =
+                    graphManager.elementProtocolToEntity(dataspaceProtocol);
+
+                  const graphEntities = guaranteeNonNullable(
+                    mappingModelCoverageAnalysisResult.entities,
+                  )
+                    .concat(mappingModels)
+                    .concat(runtimeModels)
+                    .concat(dataspaceEntity)
+                    // NOTE: if an element could be found in the graph already it means it comes from system
+                    // so we could rid of it
+                    .filter(
+                      (el) =>
+                        !graph.getNullableElement(el.path, false) &&
+                        !el.path.startsWith('meta::'),
+                    );
+                  let option;
+                  if (
+                    dataSpaceQueryBuilderState.dataSpaceRepo instanceof
+                    DataSpacesDepotRepository
+                  ) {
+                    option = new LegendSDLC(
+                      dataSpaceQueryBuilderState.dataSpaceRepo.project.groupId,
+                      dataSpaceQueryBuilderState.dataSpaceRepo.project.artifactId,
+                      resolveVersion(
+                        dataSpaceQueryBuilderState.dataSpaceRepo.project
+                          .versionId,
+                      ),
+                    );
+                  }
+                  await dataSpaceQueryBuilderState.graphManagerState.graphManager.buildGraph(
+                    graph,
+                    graphEntities,
+                    ActionState.create(),
+                    option
+                      ? {
+                          origin: option,
+                        }
+                      : {},
+                    graph_buildReport,
+                  );
+                  dataSpaceQueryBuilderState.graphManagerState.graph = graph;
+                  const dependency_buildReport = createGraphBuilderReport();
+                  // report
+                  stopWatch.record(
+                    GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS,
+                  );
+                  const graphBuilderReportData = {
+                    timings:
+                      dataSpaceQueryBuilderState.applicationStore.timeService.finalizeTimingsRecord(
+                        stopWatch,
+                      ),
+                    dependencies: dependency_buildReport,
+                    dependenciesCount:
+                      dataSpaceQueryBuilderState.graphManagerState.graph
+                        .dependencyManager.numberOfDependencies,
+                    graph: graph_buildReport,
+                  };
+                  editorStore.logBuildGraphMetrics(graphBuilderReportData);
+                  dataSpaceQueryBuilderState.applicationStore.logService.info(
+                    LogEvent.create(
+                      GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS,
+                    ),
+                    graphBuilderReportData,
+                  );
+                } catch (error) {
+                  assertErrorThrown(error);
+                  editorStore.applicationStore.logService.error(
+                    LogEvent.create(LEGEND_QUERY_APP_EVENT.GENERIC_FAILURE),
+                    error,
+                  );
+                  editorStore.graphManagerState.graph =
+                    editorStore.graphManagerState.createNewGraph();
+                  await flowResult(editorStore.buildFullGraph());
+                }
+              } else {
+                editorStore.graphManagerState.graph =
+                  editorStore.graphManagerState.createNewGraph();
+                await flowResult(editorStore.buildFullGraph());
+              }
+            }
+            dataSpaceQueryBuilderState.explorerState.mappingModelCoverageAnalysisResult =
+              mappingModelCoverageAnalysisResult;
+          }
+          const compatibleClasses = resolveUsableDataSpaceClasses(
+            dataSpaceQueryBuilderState.dataSpace,
+            mapping,
+            dataSpaceQueryBuilderState.graphManagerState,
+            dataSpaceQueryBuilderState,
+          );
+          dataSpaceQueryBuilderState.changeMapping(mapping);
+          dataSpaceQueryBuilderState.changeRuntime(
+            new RuntimePointer(
+              dataSpaceQueryBuilderState.executionContext.defaultRuntime,
+            ),
+          );
+          // if there is no chosen class or the chosen one is not compatible
+          // with the mapping then pick a compatible class if possible
+          if (
+            !dataSpaceQueryBuilderState.class ||
+            !compatibleClasses.includes(dataSpaceQueryBuilderState.class)
+          ) {
+            const possibleNewClass = getNullableFirstEntry(compatibleClasses);
+            if (possibleNewClass) {
+              dataSpaceQueryBuilderState.changeClass(possibleNewClass);
+            }
+          }
+          dataSpaceQueryBuilderState.explorerState.refreshTreeData();
+        };
+        if (
+          queryBuilderState instanceof DataSpaceQueryBuilderState &&
+          queryBuilderState.workflowState.actionConfig instanceof
+            QueryBuilderActionConfig_QueryApplication
+        ) {
+          return propagateExecutionContextChange;
+        }
+        return undefined;
+      },
+    ];
+  }
 }