Skip to content

Commit

Permalink
Remove executionPlan dependency on xts-authentication (#2743)
Browse files Browse the repository at this point in the history
  • Loading branch information
@pierredebelen
pierredebelen authored Apr 1, 2024
1 parent bca1939 commit caf4623
Show file tree
Hide file tree
Showing 16 changed files with 116 additions and 74 deletions.
6 changes: 4 additions & 2 deletions ...legend-engine-server-http-server/src/main/java/org/finos/legend/engine/server/Server.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@
import org.finos.legend.engine.plan.execution.service.api.ServiceModelingApi;
import org.finos.legend.engine.plan.execution.stores.elasticsearch.v7.plugin.ElasticsearchV7StoreExecutor;
import org.finos.legend.engine.plan.execution.stores.elasticsearch.v7.plugin.ElasticsearchV7StoreExecutorBuilder;
import org.finos.legend.engine.plan.execution.stores.elasticsearch.v7.plugin.ElasticsearchV7StoreExecutorConfiguration;
import org.finos.legend.engine.plan.execution.stores.inMemory.plugin.InMemory;
import org.finos.legend.engine.plan.execution.stores.mongodb.plugin.MongoDBStoreExecutor;
import org.finos.legend.engine.plan.execution.stores.mongodb.plugin.MongoDBStoreExecutorBuilder;
Expand Down Expand Up @@ -283,9 +284,10 @@ public void run(T serverConfiguration, Environment environment)
ServiceStoreExecutor serviceStoreExecutor = (ServiceStoreExecutor) new ServiceStoreExecutorBuilder().build(serviceStoreExecutionConfiguration);

MongoDBStoreExecutorConfiguration mongoDBExecutorConfiguration = MongoDBStoreExecutorConfiguration.newInstance().withCredentialProviderProvider(credentialProviderProvider).build();
MongoDBStoreExecutor mongoDBStoreExecutor = (MongoDBStoreExecutor) new MongoDBStoreExecutorBuilder().build(mongoDBExecutorConfiguration);
MongoDBStoreExecutor mongoDBStoreExecutor = new MongoDBStoreExecutorBuilder().build(mongoDBExecutorConfiguration);

ElasticsearchV7StoreExecutor elasticsearchV7StoreExecutor = (ElasticsearchV7StoreExecutor) new ElasticsearchV7StoreExecutorBuilder().build();
ElasticsearchV7StoreExecutorConfiguration elasticsearchV7StoreExecutorConfiguration = ElasticsearchV7StoreExecutorConfiguration.newInstance().withCredentialProviderProvider(credentialProviderProvider).build();
ElasticsearchV7StoreExecutor elasticsearchV7StoreExecutor = (ElasticsearchV7StoreExecutor) new ElasticsearchV7StoreExecutorBuilder().build(elasticsearchV7StoreExecutorConfiguration);

PlanExecutor planExecutor;
ParallelGraphFetchExecutionExecutorPool parallelGraphFetchExecutionExecutorPool = null;
Expand Down
4 changes: 0 additions & 4 deletions .../legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,10 +74,6 @@
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-executionPlan-dependencies</artifactId>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-authentication-implementation-core</artifactId>
</dependency>
<!-- ENGINE -->

<!-- COMMONS LANG -->
Expand Down
16 changes: 3 additions & 13 deletions ...tionPlan-execution/src/main/java/org/finos/legend/engine/plan/execution/PlanExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import org.eclipse.collections.impl.factory.Maps;
import org.eclipse.collections.impl.utility.Iterate;
import org.eclipse.collections.impl.utility.internal.IterableIterate;
import org.finos.legend.authentication.credentialprovider.CredentialProviderProvider;
import org.finos.legend.engine.plan.execution.concurrent.ConcurrentExecutionNodeExecutorPool;
import org.finos.legend.engine.plan.execution.concurrent.ParallelGraphFetchExecutionExecutorPool;
import org.finos.legend.engine.plan.execution.graphFetch.GraphFetchExecutionConfiguration;
Expand Down Expand Up @@ -78,17 +77,15 @@ public class PlanExecutor
private ParallelGraphFetchExecutionExecutorPool graphFetchExecutionNodeExecutorPool;
private GraphFetchExecutionConfiguration graphFetchExecutionConfiguration;
private BiFunction<Identity, ExecutionState, ExecutionNodeExecutor> executionNodeExecutorBuilder;
private final CredentialProviderProvider credentialProviderProvider;
private final boolean logSQLWithParamValues;


private PlanExecutor(boolean isJavaCompilationAllowed, ImmutableList<StoreExecutor> extraExecutors, CredentialProviderProvider credentialProviderProvider, GraphFetchExecutionConfiguration graphFetchExecutionConfiguration, boolean logSQLWithParamValues)
private PlanExecutor(boolean isJavaCompilationAllowed, ImmutableList<StoreExecutor> extraExecutors, GraphFetchExecutionConfiguration graphFetchExecutionConfiguration, boolean logSQLWithParamValues)
{
EngineUrlStreamHandlerFactory.initialize();
this.isJavaCompilationAllowed = isJavaCompilationAllowed;
this.extraExecutors = extraExecutors;
this.planExecutorInfo = PlanExecutorInfo.fromStoreExecutors(this.extraExecutors);
this.credentialProviderProvider = credentialProviderProvider;
this.graphFetchExecutionConfiguration = graphFetchExecutionConfiguration;
this.logSQLWithParamValues = logSQLWithParamValues;
}
Expand Down Expand Up @@ -373,7 +370,7 @@ public ExecutionState buildDefaultExecutionState(SingleExecutionPlan executionPl

private ExecutionState buildDefaultExecutionState(SingleExecutionPlan executionPlan, Map<String, Result> vars, PlanExecutionContext planExecutionContext)
{
ExecutionState executionState = new ExecutionState(vars, executionPlan.templateFunctions, this.extraExecutors.collect(StoreExecutor::buildStoreExecutionState), this.isJavaCompilationAllowed, null, this.credentialProviderProvider, this.graphFetchExecutionConfiguration, this.logSQLWithParamValues);
ExecutionState executionState = new ExecutionState(vars, executionPlan.templateFunctions, this.extraExecutors.collect(StoreExecutor::buildStoreExecutionState), this.isJavaCompilationAllowed, null, this.graphFetchExecutionConfiguration, this.logSQLWithParamValues);

if (planExecutionContext != null)
{
Expand Down Expand Up @@ -435,7 +432,6 @@ public static class Builder
private boolean isJavaCompilationAllowed = DEFAULT_IS_JAVA_COMPILATION_ALLOWED;
private final MutableList<StoreExecutor> storeExecutors = Lists.mutable.empty();
private GraphFetchExecutionConfiguration graphFetchExecutionConfiguration = new GraphFetchExecutionConfiguration();
private CredentialProviderProvider credentialProviderProvider = CredentialProviderProvider.defaultProviderProvider();
private boolean logSQLWithParamValues = true;

private Builder()
Expand Down Expand Up @@ -474,12 +470,6 @@ public Builder withAvailableStoreExecutors()
return this;
}

public Builder withCredentialProviderProvider(CredentialProviderProvider credentialProviderProvider)
{
this.credentialProviderProvider = credentialProviderProvider;
return this;
}

public Builder logSQLWithParamValues(boolean value)
{
this.logSQLWithParamValues = value;
Expand All @@ -488,7 +478,7 @@ public Builder logSQLWithParamValues(boolean value)

public PlanExecutor build()
{
return new PlanExecutor(this.isJavaCompilationAllowed, this.storeExecutors.toImmutable(), this.credentialProviderProvider, this.graphFetchExecutionConfiguration, this.logSQLWithParamValues);
return new PlanExecutor(this.isJavaCompilationAllowed, this.storeExecutors.toImmutable(), this.graphFetchExecutionConfiguration, this.logSQLWithParamValues);
}
}

Expand Down
33 changes: 12 additions & 21 deletions ...tion/src/main/java/org/finos/legend/engine/plan/execution/nodes/state/ExecutionState.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import org.eclipse.collections.impl.factory.Lists;
import org.eclipse.collections.impl.factory.Maps;
import org.eclipse.collections.impl.utility.ListIterate;
import org.finos.legend.authentication.credentialprovider.CredentialProviderProvider;
import org.finos.legend.engine.plan.execution.concurrent.ParallelGraphFetchExecutionExecutorPool;
import org.finos.legend.engine.plan.execution.graphFetch.GraphFetchExecutionConfiguration;
import org.finos.legend.engine.plan.execution.cache.graphFetch.GraphFetchCache;
Expand Down Expand Up @@ -80,7 +79,6 @@ public class ExecutionState
public final List<Function3<ExecutionNode,Identity, ExecutionState, Result>> extraNodeExecutors;
public final List<Function3<ExecutionNode, Identity, ExecutionState, Result>> extraSequenceNodeExecutors;
public RequestContext requestContext;
private final CredentialProviderProvider credentialProviderProvider;

public ExecutionState(ExecutionState state)
{
Expand Down Expand Up @@ -108,44 +106,43 @@ public ExecutionState(ExecutionState state)
this.extraNodeExecutors = ListIterate.flatCollect(extensions, ExecutionExtension::getExtraNodeExecutors);
this.extraSequenceNodeExecutors = ListIterate.flatCollect(extensions, ExecutionExtension::getExtraSequenceNodeExecutors);
this.requestContext = state.requestContext;
this.credentialProviderProvider = state.credentialProviderProvider;
this.logSQLWithParamValues = state.logSQLWithParamValues;
}

@Deprecated
public ExecutionState(Map<String, Result> res, List<? extends String> templateFunctions, Iterable<? extends StoreExecutionState> extraStates, boolean isJavaCompilationAllowed, long graphFetchBatchMemoryLimit)
{
this(res, templateFunctions, extraStates, isJavaCompilationAllowed, graphFetchBatchMemoryLimit, new RequestContext(), null);
this(res, templateFunctions, extraStates, isJavaCompilationAllowed, graphFetchBatchMemoryLimit, new RequestContext());
}

public ExecutionState(Map<String, Result> res, List<? extends String> templateFunctions, Iterable<? extends StoreExecutionState> extraStates, boolean isJavaCompilationAllowed, GraphFetchExecutionConfiguration graphFetchExecutionConfiguration)
{
this(res, templateFunctions, extraStates, isJavaCompilationAllowed, new RequestContext(), null, graphFetchExecutionConfiguration);
this(res, templateFunctions, extraStates, isJavaCompilationAllowed, new RequestContext(), graphFetchExecutionConfiguration);
}

public ExecutionState(Map<String, Result> res, List<? extends String> templateFunctions, Iterable<? extends StoreExecutionState> extraStates, boolean isJavaCompilationAllowed, RequestContext requestContext, CredentialProviderProvider credentialProviderProvider)
public ExecutionState(Map<String, Result> res, List<? extends String> templateFunctions, Iterable<? extends StoreExecutionState> extraStates, boolean isJavaCompilationAllowed, RequestContext requestContext)
{
this(res, templateFunctions, extraStates, isJavaCompilationAllowed, requestContext, credentialProviderProvider, new GraphFetchExecutionConfiguration());
this(res, templateFunctions, extraStates, isJavaCompilationAllowed, requestContext, new GraphFetchExecutionConfiguration());
}

@Deprecated
public ExecutionState(Map<String, Result> res, List<? extends String> templateFunctions, Iterable<? extends StoreExecutionState> extraStates, boolean isJavaCompilationAllowed, long graphFetchBatchMemoryLimit, RequestContext requestContext, CredentialProviderProvider credentialProviderProvider)
public ExecutionState(Map<String, Result> res, List<? extends String> templateFunctions, Iterable<? extends StoreExecutionState> extraStates, boolean isJavaCompilationAllowed, long graphFetchBatchMemoryLimit, RequestContext requestContext)
{
this(res, templateFunctions, extraStates, isJavaCompilationAllowed, graphFetchBatchMemoryLimit, requestContext, credentialProviderProvider, true);
this(res, templateFunctions, extraStates, isJavaCompilationAllowed, graphFetchBatchMemoryLimit, requestContext, true);
}

public ExecutionState(Map<String, Result> res, List<? extends String> templateFunctions, Iterable<? extends StoreExecutionState> extraStates, boolean isJavaCompilationAllowed, RequestContext requestContext, CredentialProviderProvider credentialProviderProvider, GraphFetchExecutionConfiguration graphFetchExecutionConfiguration)
public ExecutionState(Map<String, Result> res, List<? extends String> templateFunctions, Iterable<? extends StoreExecutionState> extraStates, boolean isJavaCompilationAllowed, RequestContext requestContext, GraphFetchExecutionConfiguration graphFetchExecutionConfiguration)
{
this(res, templateFunctions, extraStates, isJavaCompilationAllowed, requestContext, credentialProviderProvider, graphFetchExecutionConfiguration, true);
this(res, templateFunctions, extraStates, isJavaCompilationAllowed, requestContext, graphFetchExecutionConfiguration, true);
}

@Deprecated
public ExecutionState(Map<String, Result> res, List<? extends String> templateFunctions, Iterable<? extends StoreExecutionState> extraStates, boolean isJavaCompilationAllowed, long graphFetchBatchMemoryLimit, RequestContext requestContext, CredentialProviderProvider credentialProviderProvider, boolean logSQLWithParamValues)
public ExecutionState(Map<String, Result> res, List<? extends String> templateFunctions, Iterable<? extends StoreExecutionState> extraStates, boolean isJavaCompilationAllowed, long graphFetchBatchMemoryLimit, RequestContext requestContext, boolean logSQLWithParamValues)
{
this(res, templateFunctions, extraStates, isJavaCompilationAllowed, requestContext, credentialProviderProvider, new GraphFetchExecutionConfiguration(graphFetchBatchMemoryLimit), logSQLWithParamValues);
this(res, templateFunctions, extraStates, isJavaCompilationAllowed, requestContext, new GraphFetchExecutionConfiguration(graphFetchBatchMemoryLimit), logSQLWithParamValues);
}

public ExecutionState(Map<String, Result> res, List<? extends String> templateFunctions, Iterable<? extends StoreExecutionState> extraStates, boolean isJavaCompilationAllowed, RequestContext requestContext, CredentialProviderProvider credentialProviderProvider, GraphFetchExecutionConfiguration graphFetchExecutionConfiguration, boolean logSQLWithParamValues)
public ExecutionState(Map<String, Result> res, List<? extends String> templateFunctions, Iterable<? extends StoreExecutionState> extraStates, boolean isJavaCompilationAllowed, RequestContext requestContext, GraphFetchExecutionConfiguration graphFetchExecutionConfiguration, boolean logSQLWithParamValues)
{
this.inAllocation = false;
this.inLake = false;
Expand All @@ -160,7 +157,6 @@ public ExecutionState(Map<String, Result> res, List<? extends String> templateFu
this.extraNodeExecutors = ListIterate.flatCollect(extensions, ExecutionExtension::getExtraNodeExecutors);
this.extraSequenceNodeExecutors = ListIterate.flatCollect(extensions, ExecutionExtension::getExtraSequenceNodeExecutors);
this.requestContext = requestContext;
this.credentialProviderProvider = credentialProviderProvider;
this.logSQLWithParamValues = logSQLWithParamValues;
}

Expand All @@ -179,7 +175,7 @@ public ExecutionState copy()
Map<String, Result> resCopy = Maps.mutable.ofMap(this.res);
List<? extends String> templateFunctionsCopy = Lists.mutable.ofAll(this.templateFunctions);
List<? extends StoreExecutionState> extraStatesCopy = this.states.values().stream().map(StoreExecutionState::copy).collect(Collectors.toList());
ExecutionState copy = new ExecutionState(resCopy, templateFunctionsCopy, extraStatesCopy, this.isJavaCompilationAllowed, this.requestContext, this.credentialProviderProvider, this.graphFetchExecutionConfiguration, this.logSQLWithParamValues);
ExecutionState copy = new ExecutionState(resCopy, templateFunctionsCopy, extraStatesCopy, this.isJavaCompilationAllowed, this.requestContext, this.graphFetchExecutionConfiguration, this.logSQLWithParamValues);

copy.activities = Lists.mutable.withAll(this.activities);
copy.allocationNodeName = this.allocationNodeName;
Expand Down Expand Up @@ -358,11 +354,6 @@ public void setRequestContext(RequestContext requestContext)
this.requestContext = requestContext;
}

public CredentialProviderProvider getCredentialProviderProvider()
{
return this.credentialProviderProvider;
}

public boolean logSQLWithParamValues()
{
return this.logSQLWithParamValues;
Expand Down
6 changes: 3 additions & 3 deletions ...Plan-execution/src/test/java/org/finos/legend/engine/plan/execution/TestPlanExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,17 +40,17 @@ public void testStateWithReferralHeaderPlaceHolder()
res.put("userId", new ConstantResult("anumam"));
Identity identity = IdentityFactoryProvider.getInstance().getAnonymousIdentity();

ExecutionState state1 = new ExecutionState(res, Lists.mutable.empty(), Lists.mutable.empty(), false, 52_428_800L, new RequestContext(session, "https://allo'y'.site.gs.com/"), null);
ExecutionState state1 = new ExecutionState(res, Lists.mutable.empty(), Lists.mutable.empty(), false, 52_428_800L, new RequestContext(session, "https://allo'y'.site.gs.com/"));
PlanExecutor.setUpState(new SingleExecutionPlan(), state1, identity, "anumam");
String sqlQuery1 = FreeMarkerExecutor.process("ALTER SESSION SET QUERY_TAG = '{\"executionTraceID\" : \"${execID}\", \"engineUser\" : \"${userId}\", \"referer\" : \"${referer}\"}';", state1, "snowflake", null);
Assert.assertEquals("ALTER SESSION SET QUERY_TAG = '{\"executionTraceID\" : \"b26973f8-8857-4ece-bfdc-107176c9da8b\", \"engineUser\" : \"anumam\", \"referer\" : \"https://allo''y''.site.gs.com/\"}';", sqlQuery1);

ExecutionState state2 = new ExecutionState(res, Lists.mutable.empty(), Lists.mutable.empty(), false, 52_428_800L, new RequestContext(session, null), null);
ExecutionState state2 = new ExecutionState(res, Lists.mutable.empty(), Lists.mutable.empty(), false, 52_428_800L, new RequestContext(session, null));
PlanExecutor.setUpState(new SingleExecutionPlan(), state2, identity, "anumam");
String sqlQuery2 = FreeMarkerExecutor.process("ALTER SESSION SET QUERY_TAG = '{\"executionTraceID\" : \"${execID}\", \"engineUser\" : \"${userId}\", \"referer\" : \"${referer}\"}';", state2, "snowflake", null);
Assert.assertEquals("ALTER SESSION SET QUERY_TAG = '{\"executionTraceID\" : \"b26973f8-8857-4ece-bfdc-107176c9da8b\", \"engineUser\" : \"anumam\", \"referer\" : \"null\"}';", sqlQuery2);

ExecutionState state3 = new ExecutionState(res, Lists.mutable.empty(), Lists.mutable.empty(), false, 52_428_800L, null, null);
ExecutionState state3 = new ExecutionState(res, Lists.mutable.empty(), Lists.mutable.empty(), false, 52_428_800L, null);
PlanExecutor.setUpState(new SingleExecutionPlan(), state3, identity, "anumam");
String sqlQuery3 = FreeMarkerExecutor.process("ALTER SESSION SET QUERY_TAG = '{\"executionTraceID\" : \"${execID}\", \"engineUser\" : \"${userId}\", \"referer\" : \"${referer}\"}';", state3, "snowflake", null);
Assert.assertEquals("ALTER SESSION SET QUERY_TAG = '{\"executionTraceID\" : \"b26973f8-8857-4ece-bfdc-107176c9da8b\", \"engineUser\" : \"anumam\", \"referer\" : \"null\"}';", sqlQuery3);
Expand Down
4 changes: 2 additions & 2 deletions ...e/plan/execution/stores/elasticsearch/v7/plugin/ElasticsearchV7ExecutionNodeExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import org.apache.http.client.protocol.HttpClientContext;
import org.finos.legend.engine.plan.execution.nodes.state.ExecutionState;
import org.finos.legend.engine.plan.execution.result.Result;
import org.finos.legend.engine.plan.execution.stores.StoreType;
import org.finos.legend.engine.plan.execution.stores.elasticsearch.v7.connection.ElasticsearchHttpContextUtil;
import org.finos.legend.engine.plan.execution.stores.elasticsearch.v7.result.ExecutionRequestVisitor;
import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.nodes.ExecutionNode;
Expand Down Expand Up @@ -51,8 +52,7 @@ public Result visit(ExecutionNode executionNode)
Elasticsearch7RequestExecutionNode esNode = (Elasticsearch7RequestExecutionNode) executionNode;
Elasticsearch7StoreConnection connection = esNode.connection;

HttpClientContext httpClientContext = ElasticsearchHttpContextUtil.authToHttpContext(this.identity, this.executionState.getCredentialProviderProvider(), connection.authSpec, this.state.getProviders());

HttpClientContext httpClientContext = ElasticsearchHttpContextUtil.authToHttpContext(this.identity, ((ElasticsearchV7StoreExecutionState)this.executionState.getStoreExecutionState(StoreType.ESv7)).getStoreExecutionConfiguration().getCredentialProviderProvider(), connection.authSpec, this.state.getProviders());
RequestBase request = null;
try
{
Expand Down
Loading

0 comments on commit caf4623

Please sign in to comment.