Skip to content

Commit

Permalink
datacube: prepare for query persistence
Browse files Browse the repository at this point in the history
  • Loading branch information
@akphi
akphi committed Dec 22, 2024
1 parent 348636c commit 169fd30
Show file tree
Hide file tree
Showing 9 changed files with 221 additions and 81 deletions.
9 changes: 8 additions & 1 deletion ...e-repl-client/src/main/java/org/finos/legend/engine/repl/core/legend/LegendInterface.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,14 @@

public interface LegendInterface
{
PureModelContextData parse(String txt);
default PureModelContextData parse(String txt)
{
return this.parse(txt, true);
}

PureModelContextData parse(String txt, boolean returnSourceInformation);

String render(PureModelContextData model);

PureModel compile(PureModelContextData model);

Expand Down
16 changes: 12 additions & 4 deletions ...l-client/src/main/java/org/finos/legend/engine/repl/core/legend/LocalLegendInterface.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,14 @@

package org.finos.legend.engine.repl.core.legend;

import java.util.concurrent.ForkJoinPool;
import org.eclipse.collections.api.RichIterable;
import org.eclipse.collections.api.tuple.Pair;
import org.finos.legend.engine.language.pure.compiler.Compiler;
import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel;
import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModelProcessParameter;
import org.finos.legend.engine.language.pure.grammar.from.PureGrammarParser;
import org.finos.legend.engine.language.pure.grammar.to.PureGrammarComposer;
import org.finos.legend.engine.language.pure.grammar.to.PureGrammarComposerContext;
import org.finos.legend.engine.plan.generation.PlanGenerator;
import org.finos.legend.engine.plan.platform.PlanPlatform;
import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData;
Expand All @@ -31,6 +32,7 @@
import org.finos.legend.pure.generated.Root_meta_pure_extension_Extension;

import java.net.URL;
import java.util.concurrent.ForkJoinPool;

import static org.finos.legend.engine.repl.shared.ExecutionHelper.REPL_RUN_FUNCTION_QUALIFIED_PATH;

Expand All @@ -39,7 +41,7 @@ public class LocalLegendInterface implements LegendInterface
private final ForkJoinPool forkJoinPool = new ForkJoinPool(Runtime.getRuntime().availableProcessors());

@Override
public PureModelContextData parse(String txt)
public PureModelContextData parse(String txt, boolean returnSourceInformation)
{
// txt = "#>{a::DB.test}#->filter(t|$t.name->startsWith('Dr'))->meta::pure::mapping::from(^meta::core::runtime::Runtime\n" +
// " (\n" +
Expand All @@ -52,7 +54,7 @@ public PureModelContextData parse(String txt)
// " )\n" +
// " )\n" +
// " )";
return PureGrammarParser.newInstance().parseModel(txt);
return PureGrammarParser.newInstance().parseModel(txt, returnSourceInformation);
//
// "" +
// "###Runtime\n" +
Expand All @@ -78,6 +80,12 @@ public PureModelContextData parse(String txt)
// "function a::b::c::d():Any[*]{"+txt+"}");
}

@Override
public String render(PureModelContextData model)
{
return PureGrammarComposer.newInstance(PureGrammarComposerContext.Builder.newInstance().build()).renderPureModelContextData(model);
}

@Override
public PureModel compile(PureModelContextData pureModelContextData)
{
Expand All @@ -87,7 +95,7 @@ public PureModel compile(PureModelContextData pureModelContextData)
@Override
public Root_meta_pure_executionPlan_ExecutionPlan generatePlan(PureModel pureModel, boolean debug)
{
RichIterable<? extends Root_meta_pure_extension_Extension> extensions = PureCoreExtensionLoader.extensions().flatCollect(e -> e.extraPureCoreExtensions(pureModel.getExecutionSupport()));
RichIterable<? extends Root_meta_pure_extension_Extension> extensions = PureCoreExtensionLoader.extensions().flatCollect(e -> e.extraPureCoreExtensions(pureModel.getExecutionSupport()));
Pair<Root_meta_pure_executionPlan_ExecutionPlan, String> res = PlanGenerator.generateExecutionPlanAsPure(pureModel.getConcreteFunctionDefinition_safe(REPL_RUN_FUNCTION_QUALIFIED_PATH), null, pureModel, PlanPlatform.JAVA, "", debug, extensions);
if (debug)
{
Expand Down
163 changes: 134 additions & 29 deletions ...ta-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/REPLServerHelpers.java
View file

Large diffs are not rendered by default.

24 changes: 20 additions & 4 deletions ...ain/java/org/finos/legend/engine/repl/dataCube/server/handler/DataCubeInfrastructure.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
import org.finos.legend.engine.repl.dataCube.server.REPLServer;
import org.finos.legend.engine.repl.dataCube.server.model.DataCubeInfrastructureInfo;
import org.finos.legend.engine.repl.dataCube.shared.DataCubeSampleData;
import org.finos.legend.engine.shared.core.identity.Identity;
import org.finos.legend.engine.shared.core.kerberos.SubjectTools;

import java.io.InputStream;
import java.io.OutputStream;
Expand All @@ -39,14 +41,28 @@ public HttpHandler getHandler(REPLServerState state)
try
{
DataCubeInfrastructureInfo info = new DataCubeInfrastructureInfo();
info.gridClientLicense = System.getProperty("legend.repl.dataCube.gridLicenseKey") == null ? "" : System.getProperty("legend.repl.dataCube.gridLicenseKey");
try
{
Identity identity = Identity.makeIdentity(SubjectTools.getLocalSubject());
if (identity != null)
{
info.currentUser = identity.getName();
}
}
catch (Exception ignored)
{
// do nothing
}
info.gridClientLicense = System.getProperty("legend.repl.dataCube.gridLicenseKey");
info.queryServerBaseUrl = System.getProperty("legend.repl.dataCube.queryServerBaseUrl");
info.hostedApplicationBaseUrl = System.getProperty("legend.repl.dataCube.hostedApplicationBaseUrl");
info.simpleSampleDataTableName = DataCubeSampleData.TREE.tableName;
info.complexSampleDataTableName = DataCubeSampleData.SPORT.tableName;
handleResponse(exchange, 200, state.objectMapper.writeValueAsString(info), state);
handleJSONResponse(exchange, 200, state.objectMapper.writeValueAsString(info), state);
}
catch (Exception e)
{
handleResponse(exchange, 500, e.getMessage(), state);
handleTextResponse(exchange, 500, e.getMessage(), state);
}
}
};
Expand Down Expand Up @@ -93,7 +109,7 @@ else if (resourcePath.endsWith(".css"))
}
catch (Exception e)
{
handleResponse(exchange, 500, e.getMessage(), state);
handleTextResponse(exchange, 500, e.getMessage(), state);
}
}
};
Expand Down
57 changes: 40 additions & 17 deletions .../main/java/org/finos/legend/engine/repl/dataCube/server/handler/DataCubeQueryBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,11 +58,18 @@ public HttpHandler getHandler(REPLServerState state)
String requestBody = bufferReader.lines().collect(Collectors.joining());
DataCubeParseQueryInput input = state.objectMapper.readValue(requestBody, DataCubeParseQueryInput.class);
ValueSpecification result = DataCubeHelpers.parseQuery(input.code, input.returnSourceInformation);
handleResponse(exchange, 200, state.objectMapper.writeValueAsString(result), state);
handleJSONResponse(exchange, 200, state.objectMapper.writeValueAsString(result), state);
}
catch (Exception e)
{
handleResponse(exchange, 400, e instanceof EngineException ? state.objectMapper.writeValueAsString(new DataCubeQueryBuilderError((EngineException) e)) : e.getMessage(), state);
if (e instanceof EngineException)
{
handleJSONResponse(exchange, 400, state.objectMapper.writeValueAsString(new DataCubeQueryBuilderError((EngineException) e)), state);
}
else
{
handleTextResponse(exchange, 500, e.getMessage(), state);
}
}
}
};
Expand All @@ -84,11 +91,11 @@ public HttpHandler getHandler(REPLServerState state)
BufferedReader bufferReader = new BufferedReader(inputStreamReader);
String requestBody = bufferReader.lines().collect(Collectors.joining());
DataCubeGetValueSpecificationCodeInput input = state.objectMapper.readValue(requestBody, DataCubeGetValueSpecificationCodeInput.class);
handleResponse(exchange, 200, DataCubeHelpers.getQueryCode(input.value, input.pretty), state);
handleTextResponse(exchange, 200, DataCubeHelpers.getQueryCode(input.value, input.pretty), state);
}
catch (Exception e)
{
handleResponse(exchange, 400, e.getMessage(), state);
handleTextResponse(exchange, 400, e.getMessage(), state);
}
}
};
Expand Down Expand Up @@ -122,11 +129,11 @@ public HttpHandler getHandler(REPLServerState state)
result.queries.put(key, null);
}
});
handleResponse(exchange, 200, state.objectMapper.writeValueAsString(result), state);
handleJSONResponse(exchange, 200, state.objectMapper.writeValueAsString(result), state);
}
catch (Exception e)
{
handleResponse(exchange, 400, e.getMessage(), state);
handleTextResponse(exchange, 400, e.getMessage(), state);
}
}
};
Expand All @@ -149,11 +156,11 @@ public HttpHandler getHandler(REPLServerState state)
String requestBody = bufferReader.lines().collect(Collectors.joining());
DataCubeQueryTypeaheadInput input = state.objectMapper.readValue(requestBody, DataCubeQueryTypeaheadInput.class);
CompletionResult result = DataCubeHelpers.getCodeTypeahead(input.code, input.baseQuery, input.isolated ? null : state.getCurrentPureModelContextData(), state.client.getCompleterExtensions(), state.legendInterface);
handleResponse(exchange, 200, state.objectMapper.writeValueAsString(result.getCompletion()), state);
handleJSONResponse(exchange, 200, state.objectMapper.writeValueAsString(result.getCompletion()), state);
}
catch (Exception e)
{
handleResponse(exchange, 500, e.getMessage(), state);
handleTextResponse(exchange, 500, e.getMessage(), state);
}
}
};
Expand All @@ -175,11 +182,18 @@ public HttpHandler getHandler(REPLServerState state)
BufferedReader bufferReader = new BufferedReader(inputStreamReader);
String requestBody = bufferReader.lines().collect(Collectors.joining());
DataCubeGetQueryRelationReturnTypeInput input = state.objectMapper.readValue(requestBody, DataCubeGetQueryRelationReturnTypeInput.class);
handleResponse(exchange, 200, state.objectMapper.writeValueAsString(DataCubeHelpers.getRelationReturnType(state.legendInterface, input.query, input.isolated ? null : state.getCurrentPureModelContextData())), state);
handleJSONResponse(exchange, 200, state.objectMapper.writeValueAsString(DataCubeHelpers.getRelationReturnType(state.legendInterface, input.query, input.isolated ? null : state.getCurrentPureModelContextData())), state);
}
catch (Exception e)
{
handleResponse(exchange, 500, e instanceof EngineException ? state.objectMapper.writeValueAsString(new DataCubeQueryBuilderError((EngineException) e)) : e.getMessage(), state);
if (e instanceof EngineException)
{
handleJSONResponse(exchange, 500, state.objectMapper.writeValueAsString(new DataCubeQueryBuilderError((EngineException) e)), state);
}
else
{
handleTextResponse(exchange, 500, e.getMessage(), state);
}
}
}
};
Expand Down Expand Up @@ -224,19 +238,26 @@ public HttpHandler getHandler(REPLServerState state)
{
PureModelContextData data = PureGrammarParser.newInstance().parseModel(graphCode);
RelationType relationType = DataCubeHelpers.getRelationReturnType(state.legendInterface, data);
handleResponse(exchange, 200, state.objectMapper.writeValueAsString(relationType), state);
handleJSONResponse(exchange, 200, state.objectMapper.writeValueAsString(relationType), state);
}
catch (EngineException e)
{
SourceInformation sourceInformation = e.getSourceInformation();
sourceInformation.startLine -= lineOffset;
sourceInformation.endLine -= lineOffset;
handleResponse(exchange, 400, state.objectMapper.writeValueAsString(new DataCubeQueryBuilderError(new EngineException(e.getMessage(), sourceInformation, e.getErrorType()))), state);
handleJSONResponse(exchange, 400, state.objectMapper.writeValueAsString(new DataCubeQueryBuilderError(new EngineException(e.getMessage(), sourceInformation, e.getErrorType()))), state);
}
}
catch (Exception e)
{
handleResponse(exchange, 500, e instanceof EngineException ? state.objectMapper.writeValueAsString(new DataCubeQueryBuilderError((EngineException) e)) : e.getMessage(), state);
if (e instanceof EngineException)
{
handleJSONResponse(exchange, 500, state.objectMapper.writeValueAsString(new DataCubeQueryBuilderError((EngineException) e)), state);
}
else
{
handleTextResponse(exchange, 500, e.getMessage(), state);
}
}
}
};
Expand All @@ -254,14 +275,16 @@ public HttpHandler getHandler(REPLServerState state)
{
try
{
DataCubeQuery query = state.getQuery();
Map<String, ?> source = state.getSource();
String query = state.getQuery();
Map<String, ?> configuration = state.getQueryConfiguration();
Map<String, ?> source = state.getQuerySource();
if (query != null)
{
DataCubeGetBaseQueryResult result = new DataCubeGetBaseQueryResult();
result.query = query;
result.configuration = configuration;
result.source = source;
handleResponse(exchange, 200, state.objectMapper.writeValueAsString(result), state);
handleJSONResponse(exchange, 200, state.objectMapper.writeValueAsString(result), state);
}
else
{
Expand All @@ -270,7 +293,7 @@ public HttpHandler getHandler(REPLServerState state)
}
catch (Exception e)
{
handleResponse(exchange, 500, e.getMessage(), state);
handleTextResponse(exchange, 500, e.getMessage(), state);
}
}
};
Expand Down
4 changes: 2 additions & 2 deletions ...main/java/org/finos/legend/engine/repl/dataCube/server/handler/DataCubeQueryExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,11 @@ public HttpHandler getHandler(REPLServerState state)
Lambda lambda = input.query;
PureModelContextData data = DataCubeHelpers.injectNewFunction(state.getCurrentPureModelContextData(), lambda).getOne();
DataCubeExecutionResult result = executeQuery(state.client, state.legendInterface, state.planExecutor, data, debug);
handleResponse(exchange, 200, state.objectMapper.writeValueAsString(result), state);
handleJSONResponse(exchange, 200, state.objectMapper.writeValueAsString(result), state);
}
catch (Exception e)
{
handleResponse(exchange, 500, e.getMessage(), state);
handleTextResponse(exchange, 500, e.getMessage(), state);
}
}
};
Expand Down
3 changes: 2 additions & 1 deletion ...n/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeGetBaseQueryResult.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

public class DataCubeGetBaseQueryResult
{
public DataCubeQuery query;
public String query;
public Map<String, ?> configuration;
public Map<String, ?> source;
}
3 changes: 3 additions & 0 deletions ...n/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeInfrastructureInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,10 @@

public class DataCubeInfrastructureInfo
{
public String currentUser;
public String gridClientLicense;
public String queryServerBaseUrl;
public String hostedApplicationBaseUrl;
public String simpleSampleDataTableName;
public String complexSampleDataTableName;
}
23 changes: 0 additions & 23 deletions ...-cube/src/main/java/org/finos/legend/engine/repl/dataCube/server/model/DataCubeQuery.java
View file

This file was deleted.

0 comments on commit 169fd30

Please sign in to comment.