xt-sql: allow passing PAT when fetching PMCD from SDLC server (#2433)

finos · Nov 3, 2023 · c93d69a · c93d69a
1 parent be3ff70
commit c93d69a
Show file tree

Hide file tree

Showing 9 changed files with 131 additions and 20 deletions.
diff --git a/legend-engine-config/legend-engine-server/pom.xml b/legend-engine-config/legend-engine-server/pom.xml
@@ -78,6 +78,10 @@
         <!-- PURE -->
 
         <!-- ENGINE -->
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-server-support-core</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.finos.legend.engine</groupId>
             <artifactId>legend-engine-pure-code-compiled-core</artifactId>

diff --git a/...gine-config/legend-engine-server/src/main/java/org/finos/legend/engine/server/Server.java b/...gine-config/legend-engine-server/src/main/java/org/finos/legend/engine/server/Server.java
@@ -31,6 +31,7 @@
 import org.eclipse.collections.api.list.MutableList;
 import org.eclipse.collections.impl.list.mutable.FastList;
 import org.eclipse.collections.impl.utility.Iterate;
+import org.eclipse.collections.impl.utility.LazyIterate;
 import org.eclipse.collections.impl.utility.ListIterate;
 import org.eclipse.jetty.server.session.SessionHandler;
 import org.eclipse.jetty.servlets.CrossOriginFilter;
@@ -104,8 +105,6 @@
 import org.finos.legend.engine.plan.execution.api.ExecutePlanStrategic;
 import org.finos.legend.engine.plan.execution.api.concurrent.ConcurrentExecutionNodeExecutorPoolInfo;
 import org.finos.legend.engine.plan.execution.api.concurrent.ParallelGraphFetchExecutionExecutorPoolInfo;
-import org.finos.legend.engine.plan.execution.api.request.RequestContextHelper;
-import org.finos.legend.engine.plan.execution.api.result.ResultManager;
 import org.finos.legend.engine.plan.execution.concurrent.ParallelGraphFetchExecutionExecutorPool;
 import org.finos.legend.engine.plan.execution.graphFetch.GraphFetchExecutionConfiguration;
 import org.finos.legend.engine.plan.execution.service.api.ServiceModelingApi;
@@ -385,7 +384,7 @@ public void run(T serverConfiguration, Environment environment)
         environment.jersey().register(new ExecutePlanLegacy(planExecutor));
 
         // Function Activator
-        environment.jersey().register(new FunctionActivatorAPI(modelManager,Lists.mutable.empty(), Lists.mutable.with(new SnowflakeAppService(planExecutor), new HostedServiceService()), routerExtensions));
+        environment.jersey().register(new FunctionActivatorAPI(modelManager, Lists.mutable.empty(), Lists.mutable.with(new SnowflakeAppService(planExecutor), new HostedServiceService()), routerExtensions));
 
         // GraphQL
         environment.jersey().register(new GraphQLGrammar());
@@ -432,7 +431,7 @@ public void run(T serverConfiguration, Environment environment)
         //TestData Generation
         environment.jersey().register(new TestDataGeneration(modelManager));
 
-        enableCors(environment);
+        enableCors(environment, serverConfiguration);
     }
 
     // TODO: @akphi - this is temporary, rework when we find a better way to handle the initialization of connection factory from config or some external source.
@@ -491,13 +490,24 @@ public void shutDown() throws Exception
         CollectorRegistry.defaultRegistry.clear();
     }
 
-    private void enableCors(Environment environment)
+    private void enableCors(Environment environment, ServerConfiguration configuration)
     {
+        // Enable CORS
         FilterRegistration.Dynamic corsFilter = environment.servlets().addFilter("CORS", CrossOriginFilter.class);
         corsFilter.setInitParameter(CrossOriginFilter.ALLOWED_METHODS_PARAM, "GET,PUT,POST,DELETE,OPTIONS");
         corsFilter.setInitParameter(CrossOriginFilter.ALLOWED_ORIGINS_PARAM, "*");
         corsFilter.setInitParameter(CrossOriginFilter.ALLOWED_TIMING_ORIGINS_PARAM, "*");
-        corsFilter.setInitParameter(CrossOriginFilter.ALLOWED_HEADERS_PARAM, "X-Requested-With,Content-Type,Accept,Origin,Access-Control-Allow-Credentials,x-b3-parentspanid,x-b3-sampled,x-b3-spanid,x-b3-traceid," + RequestContextHelper.LEGEND_REQUEST_ID + "," + RequestContextHelper.LEGEND_USE_PLAN_CACHE + "," + ResultManager.LEGEND_RESPONSE_FORMAT);
+
+        if (configuration.cors != null && configuration.cors.getAllowedHeaders() != null)
+        {
+            corsFilter.setInitParameter(CrossOriginFilter.ALLOWED_HEADERS_PARAM, LazyIterate.adapt(configuration.cors.getAllowedHeaders()).makeString(","));
+        }
+        else
+        {
+            // NOTE: this set of headers are kept as default for backward compatibility, the headers starting with prefix `x-` are meant for Zipkin
+            // client using SDLC server. We should consider using the CORS configuration and remove those from this default list.
+            corsFilter.setInitParameter(CrossOriginFilter.ALLOWED_HEADERS_PARAM, "X-Requested-With,Content-Type,Accept,Origin,Access-Control-Allow-Credentials,x-b3-parentspanid,x-b3-sampled,x-b3-spanid,x-b3-traceid");
+        }
         corsFilter.setInitParameter(CrossOriginFilter.CHAIN_PREFLIGHT_PARAM, "false");
         corsFilter.addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), false, "*");
     }

diff --git a/...egend-engine-server/src/main/java/org/finos/legend/engine/server/ServerConfiguration.java b/...egend-engine-server/src/main/java/org/finos/legend/engine/server/ServerConfiguration.java
@@ -23,6 +23,7 @@
 import org.finos.legend.engine.server.core.configuration.DeploymentConfiguration;
 import org.finos.legend.engine.server.core.configuration.ErrorHandlingConfiguration;
 import org.finos.legend.engine.server.core.configuration.OpenTracingConfiguration;
+import org.finos.legend.engine.server.support.server.config.CORSConfiguration;
 import org.finos.legend.engine.shared.core.vault.VaultConfiguration;
 import org.finos.legend.server.pac4j.LegendPac4jConfiguration;
 
@@ -45,6 +46,7 @@ public class ServerConfiguration extends Configuration
     public GraphFetchExecutionConfiguration graphFetchExecutionConfiguration;
     public ErrorHandlingConfiguration errorhandlingconfiguration = new ErrorHandlingConfiguration();
     public List<org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentConfiguration> activatorConfiguration;
+    public CORSConfiguration cors;
 
     /*
         This configuration has been deprecated in favor of the 'temporarytestdb' in RelationalExecutionConfiguration

diff --git a/...dlc/src/main/java/org/finos/legend/engine/language/pure/modelManager/sdlc/SDLCLoader.java b/...dlc/src/main/java/org/finos/legend/engine/language/pure/modelManager/sdlc/SDLCLoader.java
@@ -19,16 +19,11 @@
 import io.opentracing.Span;
 import io.opentracing.tag.Tags;
 import io.opentracing.util.GlobalTracer;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.function.Function;
-import java.util.function.Supplier;
-import javax.security.auth.Subject;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpStatus;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpRequestBase;
 import org.apache.http.impl.client.BasicCookieStore;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.util.EntityUtils;
@@ -59,6 +54,13 @@
 import org.pac4j.core.profile.CommonProfile;
 import org.slf4j.Logger;
 
+import javax.security.auth.Subject;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.function.Supplier;
+
 import static io.opentracing.propagation.Format.Builtin.HTTP_HEADERS;
 import static org.finos.legend.engine.shared.core.kerberos.ExecSubject.exec;
 
@@ -234,6 +236,11 @@ public static PureModelContextData loadMetadataFromHTTPURL(MutableList<CommonPro
     }
 
     public static PureModelContextData loadMetadataFromHTTPURL(MutableList<CommonProfile> pm, LoggingEventType startEvent, LoggingEventType stopEvent, String url, Function<MutableList<CommonProfile>, CloseableHttpClient> httpClientProvider)
+    {
+        return loadMetadataFromHTTPURL(pm, startEvent, stopEvent, url, httpClientProvider, null);
+    }
+
+    public static PureModelContextData loadMetadataFromHTTPURL(MutableList<CommonProfile> pm, LoggingEventType startEvent, LoggingEventType stopEvent, String url, Function<MutableList<CommonProfile>, CloseableHttpClient> httpClientProvider, Function<String, HttpRequestBase> httpRequestProvider)
     {
         Scope scope = GlobalTracer.get().scopeManager().active();
         CloseableHttpClient httpclient;
@@ -260,15 +267,24 @@ public static PureModelContextData loadMetadataFromHTTPURL(MutableList<CommonPro
         }
         LOGGER.info(new LogInfo(pm, LoggingEventType.METADATA_LOAD_FROM_URL, "Loading from URL " + url).toString());
 
-        HttpGet httpGet = new HttpGet(url);
+        HttpRequestBase httpRequest;
+        if (httpRequestProvider != null)
+        {
+            httpRequest = httpRequestProvider.apply(url);
+        }
+        else
+        {
+            httpRequest = new HttpGet(url);
+        }
+
         if (span != null)
         {
-            GlobalTracer.get().inject(scope.span().context(), HTTP_HEADERS, new HttpRequestHeaderMap(httpGet));
+            GlobalTracer.get().inject(scope.span().context(), HTTP_HEADERS, new HttpRequestHeaderMap(httpRequest));
         }
 
         try
         {
-            HttpEntity entity1 = execHttpRequest(span, httpclient, httpGet);
+            HttpEntity entity1 = execHttpRequest(span, httpclient, httpRequest);
             PureModelContextData modelContextData = objectMapper.readValue(entity1.getContent(), PureModelContextData.class);
             Assert.assertTrue(modelContextData.getSerializer() != null, () -> "Engine was unable to load information from the Pure SDLC <a href='" + url + "'>link</a>");
             LOGGER.info(new LogInfo(pm, stopEvent, (double) System.currentTimeMillis() - start).toString());
@@ -298,14 +314,14 @@ public static PureModelContextData loadMetadataFromHTTPURL(MutableList<CommonPro
             HttpStatus.SC_GATEWAY_TIMEOUT
     );
 
-    private static HttpEntity execHttpRequest(Span span, CloseableHttpClient client, HttpGet httpGet) throws Exception
+    private static HttpEntity execHttpRequest(Span span, CloseableHttpClient client, HttpRequestBase httpRequest) throws Exception
     {
         int statusCode = -1;
         CloseableHttpResponse response = null;
         int i = 0;
         while (i++ < 5)
         {
-            response = client.execute(httpGet);
+            response = client.execute(httpRequest);
             statusCode = response.getStatusLine().getStatusCode();
             if (!HTTP_RESPONSE_CODE_TO_RETRY.contains(statusCode))
             {
@@ -333,7 +349,7 @@ private static HttpEntity execHttpRequest(Span span, CloseableHttpClient client,
         {
             String msg = EntityUtils.toString(entity);
             response.close();
-            throw new EngineException("Error response from " + httpGet.getURI() + ", HTTP" + statusCode + "\n" + msg);
+            throw new EngineException("Error response from " + httpRequest.getURI() + ", HTTP" + statusCode + "\n" + msg);
         }
 
         return entity;

diff --git a/...ngine/language/pure/modelManager/sdlc/configuration/MetadataServerPac4jConfiguration.java b/...ngine/language/pure/modelManager/sdlc/configuration/MetadataServerPac4jConfiguration.java
@@ -0,0 +1,26 @@
+// Copyright 2023 Goldman Sachs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.finos.legend.engine.language.pure.modelManager.sdlc.configuration;
+
+import com.fasterxml.jackson.annotation.JsonSubTypes;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "_type")
+@JsonSubTypes({
+        @JsonSubTypes.Type(value = MetadataServerPrivateAccessTokenConfiguration.class, name = "privateAccessToken"),
+})
+public abstract class MetadataServerPac4jConfiguration
+{
+}
diff --git a/...e/pure/modelManager/sdlc/configuration/MetadataServerPrivateAccessTokenConfiguration.java b/...e/pure/modelManager/sdlc/configuration/MetadataServerPrivateAccessTokenConfiguration.java
@@ -0,0 +1,20 @@
+// Copyright 2023 Goldman Sachs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.finos.legend.engine.language.pure.modelManager.sdlc.configuration;
+
+public class MetadataServerPrivateAccessTokenConfiguration extends MetadataServerPac4jConfiguration
+{
+    public String accessTokenHeaderName;
+}
diff --git a/...d/engine/language/pure/modelManager/sdlc/configuration/ServerConnectionConfiguration.java b/...d/engine/language/pure/modelManager/sdlc/configuration/ServerConnectionConfiguration.java
@@ -26,6 +26,7 @@ public class ServerConnectionConfiguration
     public String host;
     public Integer port;
     public String prefix = "";
+    public MetadataServerPac4jConfiguration pac4j;
 
     public ServerConnectionConfiguration()
     {

diff --git a/...gine-xts-sql/legend-engine-xt-sql-providers/legend-engine-xt-sql-providers-shared/pom.xml b/...gine-xts-sql/legend-engine-xt-sql-providers/legend-engine-xt-sql-providers-shared/pom.xml
@@ -154,6 +154,10 @@
             <artifactId>commons-io</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.finos.legend.shared</groupId>
+            <artifactId>legend-shared-pac4j-gitlab</artifactId>
+        </dependency>
         <!-- TEST -->
     </dependencies>
 </project>
diff --git a/...a/org/finos/legend/engine/query/sql/providers/shared/project/ProjectCoordinateLoader.java b/...a/org/finos/legend/engine/query/sql/providers/shared/project/ProjectCoordinateLoader.java
@@ -19,16 +19,19 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.http.HttpRequest;
 import org.apache.http.StatusLine;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.impl.client.BasicCookieStore;
 import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.message.BasicHeader;
 import org.apache.http.util.EntityUtils;
 import org.eclipse.collections.api.block.function.Function;
 import org.eclipse.collections.api.list.MutableList;
 import org.finos.legend.engine.language.pure.modelManager.ModelManager;
 import org.finos.legend.engine.language.pure.modelManager.sdlc.SDLCLoader;
+import org.finos.legend.engine.language.pure.modelManager.sdlc.configuration.MetadataServerPrivateAccessTokenConfiguration;
 import org.finos.legend.engine.language.pure.modelManager.sdlc.configuration.ServerConnectionConfiguration;
 import org.finos.legend.engine.protocol.pure.PureClientVersions;
 import org.finos.legend.engine.protocol.pure.v1.model.context.AlloySDLC;
@@ -40,10 +43,12 @@
 import org.finos.legend.engine.shared.core.kerberos.ProfileManagerHelper;
 import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException;
 import org.finos.legend.engine.shared.core.operational.logs.LoggingEventType;
+import org.finos.legend.server.pac4j.gitlab.GitlabPersonalAccessTokenProfile;
 import org.pac4j.core.profile.CommonProfile;
 
 import javax.security.auth.Subject;
 import javax.ws.rs.core.Response;
+import java.net.URI;
 import java.security.PrivilegedAction;
 import java.util.List;
 import java.util.Optional;
@@ -133,7 +138,21 @@ private PureModelContextData loadProjectPureModelContextData(String project, Str
         return doAs(ProfileManagerHelper.extractSubject(profiles), () ->
         {
             String url = String.format("%s/api/projects/%s/%s/%s/pureModelContextData", sdlcServerConfig.getBaseUrl(), project, isGroup ? "groupWorkspaces" : "workspaces", workspace);
-            PureModelContextData projectPMCD = SDLCLoader.loadMetadataFromHTTPURL(profiles, LoggingEventType.METADATA_REQUEST_ALLOY_PROJECT_START, LoggingEventType.METADATA_REQUEST_ALLOY_PROJECT_STOP, url, httpClientProvider);
+            PureModelContextData projectPMCD = SDLCLoader.loadMetadataFromHTTPURL(profiles, LoggingEventType.METADATA_REQUEST_ALLOY_PROJECT_START, LoggingEventType.METADATA_REQUEST_ALLOY_PROJECT_STOP, url, httpClientProvider, (String _url) ->
+            {
+                HttpGet httpRequest = new HttpGet(_url);
+                if (this.sdlcServerConfig.pac4j != null && this.sdlcServerConfig.pac4j instanceof MetadataServerPrivateAccessTokenConfiguration)
+                {
+                    String patHeaderName = ((MetadataServerPrivateAccessTokenConfiguration) this.sdlcServerConfig.pac4j).accessTokenHeaderName;
+                    MutableList<GitlabPersonalAccessTokenProfile> patProfiles = profiles.selectInstancesOf(GitlabPersonalAccessTokenProfile.class);
+                    if (patProfiles.getFirst() != null)
+                    {
+                        httpRequest = new HttpGet(String.format("%s?client_name=%s", _url, patProfiles.getFirst().getClientName()));
+                        httpRequest.addHeader(new BasicHeader(patHeaderName, patProfiles.getFirst().getPersonalAccessToken()));
+                    }
+                }
+                return httpRequest;
+            });
             PureModelContextData dependencyPMCD = getSDLCDependenciesPMCD(project, workspace, isGroup, profiles);
 
             return projectPMCD.combine(dependencyPMCD);
@@ -161,7 +180,16 @@ private PureModelContextData getSDLCDependenciesPMCD(String project, String work
                         isGroup ? "groupWorkspaces" : "workspaces",
                         workspace);
 
-                try (CloseableHttpResponse response = client.execute(new HttpGet(url)))
+                HttpGet httpRequest = new HttpGet(url);
+                String patHeaderName = ((MetadataServerPrivateAccessTokenConfiguration) this.sdlcServerConfig.pac4j).accessTokenHeaderName;
+                MutableList<GitlabPersonalAccessTokenProfile> patProfiles = profiles.selectInstancesOf(GitlabPersonalAccessTokenProfile.class);
+                if (patProfiles.getFirst() != null)
+                {
+                    httpRequest = new HttpGet(String.format("%s?client_name=%s", url, patProfiles.getFirst().getClientName()));
+                    httpRequest.addHeader(new BasicHeader(patHeaderName, patProfiles.getFirst().getPersonalAccessToken()));
+                }
+
+                try (CloseableHttpResponse response = client.execute(httpRequest))
                 {
                     StatusLine status = response.getStatusLine();
                     if (!Response.Status.Family.familyOf(status.getStatusCode()).equals(Response.Status.Family.SUCCESSFUL))