[8.17] Revert Replace cloud-ess docker image with wolfi-ess (#114413)

.buildkite/pipelines/pull-request/packaging-tests-unix.yml

@@ -5,7 +5,7 @@ steps:
     steps:
       - label: "{{matrix.image}} / docker / packaging-tests-unix"
         key: "packaging-tests-unix-docker"
-        command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.docker-cloud-ess
+        command: ./.ci/scripts/packaging-test.sh destructiveDistroTest.docker
         timeout_in_minutes: 300
         matrix:
           setup:

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -21,6 +21,10 @@ public enum DockerBase {
     // The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build
     IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank", "yum"),
 
+    // Based on CLOUD above, with more extras. We don't set a base image because
+    // we programmatically extend from the Cloud image.
+    CLOUD_ESS(null, "-cloud-ess", "apt-get"),
+
     // Chainguard based wolfi image with latest jdk
     // This is usually updated via renovatebot
     // spotless:off
@@ -31,8 +35,8 @@ public enum DockerBase {
     // spotless:on
 
     // Based on WOLFI above, with more extras. We don't set a base image because
-    // we programmatically extend from the wolfi image.
-    CLOUD_ESS(null, "-cloud-ess", "apk");
+    // we programmatically extend from the Wolfi image.
+    WOLFI_ESS(null, "-wolfi-ess", "apk");
 
     private final String image;
     private final String suffix;

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java

@@ -185,6 +185,9 @@ private static String distributionProjectName(ElasticsearchDistribution distribu
         if (distribution.getType() == InternalElasticsearchDistributionTypes.DOCKER_WOLFI) {
             return projectName + "wolfi-docker" + archString + "-export";
         }
+        if (distribution.getType() == InternalElasticsearchDistributionTypes.DOCKER_WOLFI_ESS) {
+            return projectName + "wolfi-ess-docker" + archString + "-export";
+        }
         return projectName + distribution.getType().getName();
     }
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerWolfiEssElasticsearchDistributionType.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.distribution;
+
+import org.elasticsearch.gradle.ElasticsearchDistributionType;
+
+public class DockerWolfiEssElasticsearchDistributionType implements ElasticsearchDistributionType {
+
+    DockerWolfiEssElasticsearchDistributionType() {}
+
+    @Override
+    public String getName() {
+        return "dockerWolfiEss";
+    }
+
+    @Override
+    public boolean isDocker() {
+        return true;
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java

@@ -21,6 +21,7 @@ public class InternalElasticsearchDistributionTypes {
     public static ElasticsearchDistributionType DOCKER_IRONBANK = new DockerIronBankElasticsearchDistributionType();
     public static ElasticsearchDistributionType DOCKER_CLOUD_ESS = new DockerCloudEssElasticsearchDistributionType();
     public static ElasticsearchDistributionType DOCKER_WOLFI = new DockerWolfiElasticsearchDistributionType();
+    public static ElasticsearchDistributionType DOCKER_WOLFI_ESS = new DockerWolfiEssElasticsearchDistributionType();
 
     public static List<ElasticsearchDistributionType> ALL_INTERNAL = List.of(
         DEB,
@@ -29,6 +30,7 @@ public class InternalElasticsearchDistributionTypes {
         DOCKER_UBI,
         DOCKER_IRONBANK,
         DOCKER_CLOUD_ESS,
-        DOCKER_WOLFI
+        DOCKER_WOLFI,
+        DOCKER_WOLFI_ESS
     );
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java

@@ -53,6 +53,7 @@
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_IRONBANK;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_UBI;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_WOLFI;
+import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_WOLFI_ESS;
 import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.RPM;
 import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams;
 
@@ -152,6 +153,7 @@ private static Map<ElasticsearchDistributionType, TaskProvider<?>> lifecycleTask
         lifecyleTasks.put(DOCKER_IRONBANK, project.getTasks().register(taskPrefix + ".docker-ironbank"));
         lifecyleTasks.put(DOCKER_CLOUD_ESS, project.getTasks().register(taskPrefix + ".docker-cloud-ess"));
         lifecyleTasks.put(DOCKER_WOLFI, project.getTasks().register(taskPrefix + ".docker-wolfi"));
+        lifecyleTasks.put(DOCKER_WOLFI_ESS, project.getTasks().register(taskPrefix + ".docker-wolfi-ess"));
         lifecyleTasks.put(ARCHIVE, project.getTasks().register(taskPrefix + ".archives"));
         lifecyleTasks.put(DEB, project.getTasks().register(taskPrefix + ".packages"));
         lifecyleTasks.put(RPM, lifecyleTasks.get(DEB));

distribution/docker/README.md

@@ -7,7 +7,7 @@ the [DockerBase] enum.
    * UBI - the same as the default image, but based upon [RedHat's UBI
      images][ubi], specifically their minimal flavour.
    * Wolfi - the same as the default image, but based upon [Wolfi](https://github.com/wolfi-dev)
-   * Cloud ESS - this directly extends the Wolfi image, and adds all ES plugins
+   * Wolfi ESS - this directly extends the Wolfi image, and adds all ES plugins
      that the ES build generates in an archive directory. It also sets an
      environment variable that points at this directory. This allows plugins to
      be installed from the archive instead of the internet, speeding up
@@ -23,13 +23,20 @@ the [DockerBase] enum.
      software (FOSS) and Commercial off-the-shelf (COTS). In practice, this is
      another UBI build, this time on the regular UBI image, with extra
      hardening. See below for more details.
+
    * Cloud - this is mostly the same as the default image, with some notable differences:
       * `filebeat` and `metricbeat` are included
       * `wget` is included
       * The `ENTRYPOINT` is just `/bin/tini`, and the `CMD` is
         `/app/elasticsearch.sh`. In normal use this file would be bind-mounted
         in, but the image ships a stub version of this file so that the image
         can still be tested.
+   * Cloud ESS - this directly extends the Cloud image, and adds all ES plugins
+     that the ES build generates in an archive directory. It also sets an
+     environment variable that points at this directory. This allows plugins to
+     be installed from the archive instead of the internet, speeding up
+     deployment times.
+
 The long-term goal is for both Cloud images to be retired in favour of the
 default image.
 

distribution/docker/build.gradle

@@ -366,7 +366,7 @@ private static List<String> generateTags(DockerBase base, Architecture architect
   String image = "elasticsearch${base.suffix}"
 
   String namespace = 'elasticsearch'
-  if (base == base == DockerBase.CLOUD_ESS) {
+  if (base == DockerBase.CLOUD_ESS || base == DockerBase.WOLFI_ESS) {
     namespace += '-ci'
   }
 
@@ -431,8 +431,7 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) {
   }
 }
 
-void addBuildEssDockerImageTask(Architecture architecture) {
-  DockerBase dockerBase = DockerBase.CLOUD_ESS
+void addBuildEssDockerImageTask(Architecture architecture, DockerBase dockerBase) {
   String arch = architecture == Architecture.AARCH64 ? '-aarch64' : ''
   String contextDir = "${project.buildDir}/docker-context/elasticsearch${dockerBase.suffix}-${VersionProperties.elasticsearch}-docker-build-context${arch}"
 
@@ -446,20 +445,22 @@ void addBuildEssDockerImageTask(Architecture architecture) {
         from configurations.allPlugins
       }
 
-      // If we're performing a release build, but `build.id` hasn't been set, we can
-      // infer that we're not at the Docker building stage of the build, and therefore
-      // we should skip the beats part of the build.
-      String buildId = providers.systemProperty('build.id').getOrNull()
-      boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null || useDra
+      if (dockerBase == DockerBase.WOLFI_ESS) {
+        // If we're performing a release build, but `build.id` hasn't been set, we can
+        // infer that we're not at the Docker building stage of the build, and therefore
+        // we should skip the beats part of the build.
+        String buildId = providers.systemProperty('build.id').getOrNull()
+        boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null || useDra
 
-      if (includeBeats) {
-        from configurations.getByName("filebeat_${architecture.classifier}")
-        from configurations.getByName("metricbeat_${architecture.classifier}")
+        if (includeBeats) {
+          from configurations.getByName("filebeat_${architecture.classifier}")
+          from configurations.getByName("metricbeat_${architecture.classifier}")
+        }
+        // For some reason, the artifact name can differ depending on what repository we used.
+        rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz"
       }
-      // For some reason, the artifact name can differ depending on what repository we used.
-      rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz"
 
-      String baseSuffix = DockerBase.WOLFI.suffix
+      String baseSuffix = dockerBase == DockerBase.CLOUD_ESS ? DockerBase.CLOUD.suffix : DockerBase.WOLFI.suffix
       from(projectDir.resolve("src/docker/Dockerfile.ess")) {
         expand(
           [
@@ -477,7 +478,7 @@ void addBuildEssDockerImageTask(Architecture architecture) {
   final TaskProvider<DockerBuildTask> buildDockerImageTask =
     tasks.register(taskName("build", architecture, dockerBase, "DockerImage"), DockerBuildTask) {
 
-      DockerBase base = DockerBase.WOLFI
+      DockerBase base = dockerBase == DockerBase.CLOUD_ESS ? DockerBase.CLOUD : DockerBase.WOLFI
 
       TaskProvider<DockerBuildTask> buildBaseTask = tasks.named(taskName("build", architecture, base, "DockerImage"))
       inputs.files(buildBaseTask)
@@ -503,15 +504,16 @@ void addBuildEssDockerImageTask(Architecture architecture) {
 
 for (final Architecture architecture : Architecture.values()) {
   for (final DockerBase base : DockerBase.values()) {
-    if (base == DockerBase.CLOUD_ESS) {
+    if (base == DockerBase.CLOUD_ESS || base == DockerBase.WOLFI_ESS) {
       continue
     }
     addBuildDockerContextTask(architecture, base)
     addTransformDockerContextTask(architecture, base)
     addBuildDockerImageTask(architecture, base)
   }
 
-  addBuildEssDockerImageTask(architecture)
+  addBuildEssDockerImageTask(architecture, DockerBase.CLOUD_ESS)
+  addBuildEssDockerImageTask(architecture, DockerBase.WOLFI_ESS)
 }
 
 def exportDockerImages = tasks.register("exportDockerImages")
@@ -542,7 +544,8 @@ subprojects { Project subProject ->
       (base == DockerBase.IRON_BANK ? 'ironbank.tar' :
           (base == DockerBase.CLOUD_ESS ? 'cloud-ess.tar' :
             (base == DockerBase.WOLFI ? 'wolfi.tar' :
-                'docker.tar')))
+              (base == DockerBase.WOLFI_ESS ? 'wolfi-ess.tar' :
+                'docker.tar'))));
     final String artifactName = "elasticsearch${arch}${base.suffix}_test"
 
     final String exportTaskName = taskName("export", architecture, base, 'DockerImage')

distribution/docker/src/docker/Dockerfile.ess

@@ -2,31 +2,34 @@ FROM ${base_image} AS builder
 
 USER root
 
-# Add plugins infrastructure
-RUN mkdir -p /opt/plugins/archive
-RUN chmod -R 0555 /opt/plugins
-
-COPY filebeat-${version}.tar.gz metricbeat-${version}.tar.gz /tmp/
-RUN set -eux ; \\
-    for beat in filebeat metricbeat ; do \\
-      if [ ! -s /tmp/\$beat-${version}.tar.gz ]; then \\
-        echo "/tmp/\$beat-${version}.tar.gz is empty - cannot uncompress" 2>&1 ; \\
-        exit 1 ; \\
-      fi ; \\
-      if ! tar tf /tmp/\$beat-${version}.tar.gz >/dev/null; then \\
-        echo "/tmp/\$beat-${version}.tar.gz is corrupt - cannot uncompress" 2>&1 ; \\
-        exit 1 ; \\
-      fi ; \\
-      mkdir -p /opt/\$beat ; \\
-      tar xf /tmp/\$beat-${version}.tar.gz -C /opt/\$beat --strip-components=1 ; \\
-    done
+<% if (docker_base == "wolfi_ess") { %>
+    # Add plugins infrastructure
+    RUN mkdir -p /opt/plugins/archive
+    RUN chmod -R 0555 /opt/plugins
+
+    COPY filebeat-${version}.tar.gz metricbeat-${version}.tar.gz /tmp/
+    RUN set -eux ; \\
+        for beat in filebeat metricbeat ; do \\
+          if [ ! -s /tmp/\$beat-${version}.tar.gz ]; then \\
+            echo "/tmp/\$beat-${version}.tar.gz is empty - cannot uncompress" 2>&1 ; \\
+            exit 1 ; \\
+          fi ; \\
+          if ! tar tf /tmp/\$beat-${version}.tar.gz >/dev/null; then \\
+            echo "/tmp/\$beat-${version}.tar.gz is corrupt - cannot uncompress" 2>&1 ; \\
+            exit 1 ; \\
+          fi ; \\
+          mkdir -p /opt/\$beat ; \\
+          tar xf /tmp/\$beat-${version}.tar.gz -C /opt/\$beat --strip-components=1 ; \\
+        done
+<% } %>
 
 COPY plugins/*.zip /opt/plugins/archive/
 
 RUN chown 1000:1000 /opt/plugins/archive/*
 RUN chmod 0444 /opt/plugins/archive/*
 
 FROM ${base_image}
+<% if (docker_base == "wolfi_ess") { %>
 USER root
 
 RUN <%= retry.loop("apk", "export DEBIAN_FRONTEND=noninteractive && apk update && apk update && apk add --no-cache wget") %>
@@ -41,4 +44,8 @@ RUN mkdir /app && \\
 
 COPY --from=builder --chown=0:0 /opt /opt
 USER 1000:0
+<% } else { %>
+COPY --from=builder /opt/plugins /opt/plugins
+<% } %>
+
 ENV ES_PLUGIN_ARCHIVE_DIR /opt/plugins/archive

qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java

@@ -99,7 +99,6 @@
  *     <li>The default image with a custom, small base image</li>
  *     <li>A UBI-based image</li>
  *     <li>Another UBI image for Iron Bank</li>
- *     <li>A WOLFI-based image</li>
  *     <li>Images for Cloud</li>
  * </ul>
  */
@@ -169,7 +168,10 @@ public void test012SecurityCanBeDisabled() throws Exception {
      * Checks that no plugins are initially active.
      */
     public void test020PluginsListWithNoPlugins() {
-        assumeTrue("Only applies to non-Cloud images", distribution().packaging != Packaging.DOCKER_CLOUD_ESS);
+        assumeTrue(
+            "Only applies to non-Cloud images",
+            distribution().packaging != Packaging.DOCKER_CLOUD_ESS && distribution().packaging != Packaging.DOCKER_WOLFI_ESS
+        );
 
         final Installation.Executables bin = installation.executables();
         final Result r = sh.run(bin.pluginTool + " list");
@@ -199,14 +201,15 @@ public void test021InstallPlugin() {
      * Checks that ESS images can install plugins from the local archive.
      */
     public void test022InstallPluginsFromLocalArchive() {
-        assumeTrue("Only ESS images have a local archive", distribution().packaging == Packaging.DOCKER_CLOUD_ESS);
+        assumeTrue(
+            "Only ESS images have a local archive",
+            distribution().packaging == Packaging.DOCKER_CLOUD_ESS || distribution().packaging == Packaging.DOCKER_WOLFI_ESS
+        );
 
         final String plugin = "analysis-icu";
         final Installation.Executables bin = installation.executables();
 
-        listPluginArchive().forEach(System.out::println);
         assertThat("Expected " + plugin + " to not be installed", listPlugins(), not(hasItems(plugin)));
-        assertThat("Expected " + plugin + " available in archive", listPluginArchive(), hasSize(16));
 
         // Stuff the proxy settings with garbage, so any attempt to go out to the internet would fail
         sh.getEnv()
@@ -254,7 +257,10 @@ public void test023InstallPluginUsingConfigFile() {
      * Checks that ESS images can manage plugins from the local archive by deploying a plugins config file.
      */
     public void test024InstallPluginFromArchiveUsingConfigFile() {
-        assumeTrue("Only ESS image has a plugin archive", distribution().packaging == Packaging.DOCKER_CLOUD_ESS);
+        assumeTrue(
+            "Only ESS image has a plugin archive",
+            distribution().packaging == Packaging.DOCKER_CLOUD_ESS || distribution().packaging == Packaging.DOCKER_WOLFI_ESS
+        );
 
         final String filename = "elasticsearch-plugins.yml";
         append(tempDir.resolve(filename), """
@@ -386,7 +392,7 @@ public void test040JavaUsesTheOsProvidedKeystore() {
         if (distribution.packaging == Packaging.DOCKER_UBI || distribution.packaging == Packaging.DOCKER_IRON_BANK) {
             // In these images, the `cacerts` file ought to be a symlink here
             assertThat(path, equalTo("/etc/pki/ca-trust/extracted/java/cacerts"));
-        } else if (distribution.packaging == Packaging.DOCKER_WOLFI || distribution.packaging == Packaging.DOCKER_CLOUD_ESS) {
+        } else if (distribution.packaging == Packaging.DOCKER_WOLFI || distribution.packaging == Packaging.DOCKER_WOLFI_ESS) {
             // In these images, the `cacerts` file ought to be a symlink here
             assertThat(path, equalTo("/etc/ssl/certs/java/cacerts"));
         } else {
@@ -1113,8 +1119,8 @@ public void test170DefaultShellIsBash() {
      */
     public void test171AdditionalCliOptionsAreForwarded() throws Exception {
         assumeTrue(
-            "Does not apply to Cloud ESS images, because they don't use the default entrypoint",
-            distribution().packaging != Packaging.DOCKER_CLOUD_ESS
+            "Does not apply to Cloud and wolfi ess images, because they don't use the default entrypoint",
+            distribution().packaging != Packaging.DOCKER_CLOUD_ESS && distribution().packaging != Packaging.DOCKER_WOLFI_ESS
         );
 
         runContainer(distribution(), builder().runArgs("bin/elasticsearch", "-Ecluster.name=kimchy").envVar("ELASTIC_PASSWORD", PASSWORD));
@@ -1201,7 +1207,7 @@ public void test310IronBankImageHasNoAdditionalLabels() throws Exception {
      * Check that the Cloud image contains the required Beats
      */
     public void test400CloudImageBundlesBeats() {
-        assumeTrue(distribution.packaging == Packaging.DOCKER_CLOUD_ESS);
+        assumeTrue(distribution.packaging == Packaging.DOCKER_CLOUD_ESS || distribution.packaging == Packaging.DOCKER_WOLFI_ESS);
 
         final List<String> contents = listContents("/opt");
         assertThat("Expected beats in /opt", contents, hasItems("filebeat", "metricbeat"));
@@ -1219,10 +1225,6 @@ private List<String> listPlugins() {
         return sh.run(bin.pluginTool + " list").stdout().lines().collect(Collectors.toList());
     }
 
-    private List<String> listPluginArchive() {
-        return sh.run("ls -lh /opt/plugins/archive").stdout().lines().collect(Collectors.toList());
-    }
-
     /**
      * Check that readiness listener works
      */

qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java

@@ -436,7 +436,10 @@ private void verifyKeystorePermissions() {
         switch (distribution.packaging) {
             case TAR, ZIP -> assertThat(keystore, file(File, ARCHIVE_OWNER, ARCHIVE_OWNER, p660));
             case DEB, RPM -> assertThat(keystore, file(File, "root", "elasticsearch", p660));
-            case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> assertThat(keystore, DockerFileMatcher.file(p660));
+            case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD_ESS, DOCKER_WOLFI, DOCKER_WOLFI_ESS -> assertThat(
+                keystore,
+                DockerFileMatcher.file(p660)
+            );
             default -> throw new IllegalStateException("Unknown Elasticsearch packaging type.");
         }
     }