From 3a9243b0bedf4395accfd923a8f2203903d97d8f Mon Sep 17 00:00:00 2001 From: Jordan Zimmerman Date: Sat, 14 Sep 2024 07:04:43 +0100 Subject: [PATCH] Create a test-container for this project Includes methods to add to the Airlift config and add Trino AWS Proxy plugins. TODO - will benefit from its own base image but, for now, it's constructed via a startup script that installs Python and uploads the main tarball. --- pom.xml | 21 ++ trino-aws-proxy-test-container/pom.xml | 113 +++++++++ .../TrinoAwsProxyServerTestContainer.java | 216 ++++++++++++++++++ .../main/template/project-version.properties | 2 + .../test/container/TestTestContainer.java | 58 +++++ trino-aws-proxy/pom.xml | 2 + 6 files changed, 412 insertions(+) create mode 100644 trino-aws-proxy-test-container/pom.xml create mode 100644 trino-aws-proxy-test-container/src/main/java/io/trino/aws/proxy/server/test/container/TrinoAwsProxyServerTestContainer.java create mode 100644 trino-aws-proxy-test-container/src/main/template/project-version.properties create mode 100644 trino-aws-proxy-test-container/src/test/java/io/trino/aws/proxy/server/test/container/TestTestContainer.java diff --git a/pom.xml b/pom.xml index bffee3c9..7e960e5b 100644 --- a/pom.xml +++ b/pom.xml @@ -31,6 +31,7 @@ trino-aws-proxy-spark3 trino-aws-proxy-spark4 trino-aws-proxy-spi + trino-aws-proxy-test-container @@ -58,6 +59,7 @@ 8.5.9 3.3.6 4.1.1 + 1.24.0 @@ -90,6 +92,13 @@ ${project.version} + + ${project.groupId} + trino-aws-proxy + ${project.version} + tar.gz + + ${project.groupId} trino-aws-proxy-spark3 @@ -108,6 +117,12 @@ ${project.version} + + ${project.groupId} + trino-aws-proxy-test-container + ${project.version} + + com.github.docker-java docker-java-api @@ -126,6 +141,12 @@ ${dep.minio.version} + + org.apache.commons + commons-compress + ${dep.commons-compress.version} + + org.awaitility awaitility diff --git a/trino-aws-proxy-test-container/pom.xml b/trino-aws-proxy-test-container/pom.xml new file mode 100644 index 00000000..8c5b3e0c --- /dev/null +++ b/trino-aws-proxy-test-container/pom.xml @@ -0,0 +1,113 @@ + + + 4.0.0 + + io.trino + trino-aws-proxy-root + 2-SNAPSHOT + + ../pom.xml + + + trino-aws-proxy-test-container + + + ${project.parent.basedir} + + + + + ${project.groupId} + trino-aws-proxy + tar.gz + + + + com.google.guava + guava + + + + org.apache.commons + commons-compress + + + + org.testcontainers + testcontainers + + + + org.assertj + assertj-core + test + + + + org.junit.jupiter + junit-jupiter-api + test + + + + + + + true + ${project.basedir}/src/main/template + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + ${project.groupId}:trino-aws-proxy + + + + + get-trino-aws-proxy + + copy + + generate-sources + + + + ${project.groupId} + trino-aws-proxy + ${project.version} + tar.gz + + + ${project.build.outputDirectory} + + + + + get-trino-aws-proxy-test + + copy + + generate-sources + + + + ${project.groupId} + trino-aws-proxy + ${project.version} + tests + test-jar + + + ${project.build.directory} + + + + + + + diff --git a/trino-aws-proxy-test-container/src/main/java/io/trino/aws/proxy/server/test/container/TrinoAwsProxyServerTestContainer.java b/trino-aws-proxy-test-container/src/main/java/io/trino/aws/proxy/server/test/container/TrinoAwsProxyServerTestContainer.java new file mode 100644 index 00000000..a149aac3 --- /dev/null +++ b/trino-aws-proxy-test-container/src/main/java/io/trino/aws/proxy/server/test/container/TrinoAwsProxyServerTestContainer.java @@ -0,0 +1,216 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.test.container; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.io.Resources; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; +import org.testcontainers.images.builder.Transferable; +import org.testcontainers.utility.DockerImageName; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.util.Properties; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.io.ByteStreams.exhaust; +import static java.util.Objects.requireNonNull; + +/** + * Notes: + *

+ * I couldn't find a base image with both Java 22 and Python 3 so I use Temurin and install Python in a script. + * TODO build a base image that does this. + *

+ *

+ * This container relies on the Maven build to work properly. See the {@code maven-dependency-plugin} of this module's + * POM file. It copies the launcher tarball from the main module (trino-aws-proxy). The launcher + * tarball is bundled into this module's JAR (i.e. it's copied to target/classes) as we need it at runtime. + * The constructor of {@code TrinoAwsProxyServerTestContainer} copies the tarball into the Temurin container + * via {@code withCopyToContainer()}, {@code startScript} (which is also added to the container) untars it, + * installs Python, adds symlinks to any additional JARs and runs the launcher. + *

+ */ +public class TrinoAwsProxyServerTestContainer + extends GenericContainer +{ + private static final String startScript = """ + #!/usr/bin/env bash + + set -euo pipefail + + cd /opt/app + tar xf app.tar.gz + + set -xeu + apt-get update -q + apt-get install -q -y python3 net-tools lsof + update-alternatives --install /usr/bin/python python /usr/bin/python3 1 + + cd /opt/app/trino* + mkdir etc + cp ../config.properties etc/config.properties + touch etc/jvm.config + + ADDITIONAL_LIBS=($ADDITIONAL_LIB$) + + if (( ${#ADDITIONAL_LIBS[@]} != 0 )); then + for lib in "${ADDITIONAL_LIBS[@]}"; do + name="${lib##*/}"\s + ln -s "$lib" "./lib/$name" + done + fi + + bin/launcher run + """; + + private static final Properties projectProperties; + + static { + URL resource = Resources.getResource("project-version.properties"); + projectProperties = new Properties(); + try (InputStream inputStream = resource.openStream()) { + projectProperties.load(inputStream); + } + catch (IOException e) { + throw new RuntimeException("Could not read project-version file", e); + } + } + + private final ImmutableMap.Builder configFile = ImmutableMap.builder(); + private final ImmutableSet.Builder additionalJars = ImmutableSet.builder(); + + public TrinoAwsProxyServerTestContainer() + { + super(DockerImageName.parse("eclipse-temurin").withTag("%s-jdk-jammy".formatted(javaVersion()))); + + withCopyToContainer(tarballTransferable(), "/opt/app/app.tar.gz") + .withCommand("/opt/app/startup.sh") + .withExposedPorts(8080) + .waitingFor(new LogMessageWaitStrategy().withRegEx(".*SERVER STARTED.*")); + + configFile.put("node.environment", "test"); + } + + public TrinoAwsProxyServerTestContainer withConfig(String name, String value) + { + assertNotStarted(); + + configFile.put(name, value); + return this; + } + + public TrinoAwsProxyServerTestContainer withAdditionalLib(String containerPath) + { + assertNotStarted(); + + additionalJars.add(containerPath); + return this; + } + + @Override + @SuppressWarnings("OctalInteger") + public void start() + { + String configFileContents = configFile.build() + .entrySet() + .stream() + .map(entry -> "%s=%s".formatted(entry.getKey(), entry.getValue())) + .collect(Collectors.joining("\n")); + + Set additionalJarsList = additionalJars.build(); + String additionalLib = additionalJarsList.isEmpty() ? "" : additionalJarsList.stream() + .collect(Collectors.joining(" ", "\"", "\"")); + String adjustedScript = startScript.replace("$ADDITIONAL_LIB$", additionalLib); + + withCopyToContainer(Transferable.of(configFileContents), "/opt/app/config.properties") + .withCopyToContainer(Transferable.of(adjustedScript, 0744), "/opt/app/startup.sh"); + + super.start(); + } + + public static String projectVersion() + { + return projectProperties.getProperty("project.version"); + } + + public static String javaVersion() + { + return projectProperties.getProperty("java.version"); + } + + private void assertNotStarted() + { + checkState(getContainerId() == null, "Container has already been started"); + } + + private static Transferable tarballTransferable() + { + URL resource = Resources.getResource(tarball()); + return new UrlTransferable(resource); + } + + private static String tarball() + { + return "trino-aws-proxy-%s.tar.gz".formatted(projectVersion()); + } + + private static class UrlTransferable + implements Transferable + { + private final URL url; + + private UrlTransferable(URL url) + { + this.url = requireNonNull(url, "url is null"); + } + + @Override + public long getSize() + { + // not used + return 0; + } + + @Override + public void transferTo(TarArchiveOutputStream tarArchiveOutputStream, String destination) + { + try { + long size; + try (InputStream countStream = url.openStream()) { + size = exhaust(countStream); + } + + TarArchiveEntry tarEntry = new TarArchiveEntry(destination); + tarEntry.setMode(getFileMode()); + tarEntry.setSize(size); + tarArchiveOutputStream.putArchiveEntry(tarEntry); + try (InputStream inputStream = url.openStream()) { + inputStream.transferTo(tarArchiveOutputStream); + } + tarArchiveOutputStream.closeArchiveEntry(); + } + catch (IOException e) { + throw new RuntimeException("Can't transfer from %s to %s".formatted(url, destination), e); + } + } + } +} diff --git a/trino-aws-proxy-test-container/src/main/template/project-version.properties b/trino-aws-proxy-test-container/src/main/template/project-version.properties new file mode 100644 index 00000000..710cb1ae --- /dev/null +++ b/trino-aws-proxy-test-container/src/main/template/project-version.properties @@ -0,0 +1,2 @@ +project.version=${project.version} +java.version=${project.build.targetJdk} diff --git a/trino-aws-proxy-test-container/src/test/java/io/trino/aws/proxy/server/test/container/TestTestContainer.java b/trino-aws-proxy-test-container/src/test/java/io/trino/aws/proxy/server/test/container/TestTestContainer.java new file mode 100644 index 00000000..9aad8fe1 --- /dev/null +++ b/trino-aws-proxy-test-container/src/test/java/io/trino/aws/proxy/server/test/container/TestTestContainer.java @@ -0,0 +1,58 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.test.container; + +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.net.HttpURLConnection; +import java.net.URI; + +import static io.trino.aws.proxy.server.test.container.TrinoAwsProxyServerTestContainer.projectVersion; +import static org.assertj.core.api.Assertions.assertThat; +import static org.testcontainers.containers.BindMode.READ_ONLY; + +public class TestTestContainer +{ + @Test + public void testBasicContainerStart() + throws Exception + { + try (TrinoAwsProxyServerTestContainer testContainer = new TrinoAwsProxyServerTestContainer()) { + testContainer.start(); + + URI uri = URI.create("http://localhost:%s/".formatted(testContainer.getFirstMappedPort())); + HttpURLConnection urlConnection = (HttpURLConnection) uri.toURL().openConnection(); + int responseCode = urlConnection.getResponseCode(); + assertThat(responseCode).isEqualTo(404); + } + } + + @Test + // note this tests relies on this module's maven POM to copy the main module's test-jar into + // this module's target directory. The main module's test-jar has MockPlugin. + public void testPlugin() + { + String jarName = "trino-aws-proxy-%s-tests.jar".formatted(projectVersion()); + File targetDirectory = new File(getClass().getProtectionDomain().getCodeSource().getLocation().getFile()).getParentFile(); + File jarFile = new File(targetDirectory, jarName); + + try (TrinoAwsProxyServerTestContainer testContainer = new TrinoAwsProxyServerTestContainer()) { + testContainer.withFileSystemBind(jarFile.getAbsolutePath(), "/opt/mock.jar", READ_ONLY) + .withAdditionalLib("/opt/mock.jar") + .start(); + assertThat(testContainer.getLogs()).contains("Loading plugin: MockPlugin"); + } + } +} diff --git a/trino-aws-proxy/pom.xml b/trino-aws-proxy/pom.xml index a1531302..07618ac1 100644 --- a/trino-aws-proxy/pom.xml +++ b/trino-aws-proxy/pom.xml @@ -11,6 +11,8 @@ ${project.parent.basedir} + README.md + io.trino.aws.proxy.server.TrinoAwsProxyServer