Add QA test module for Lucene N-2 version (elastic#118363)

This change introduces a new QA project to test Lucene 
support for reading indices created in version N-2.

The test suite is inspired from the various full-cluster 
restart suites we already have. It creates a cluster in 
version N-2 (today 7.17.25), then upgrades the cluster 
to N-1 (today 8.18.0) and finally upgrades the cluster 
to the current version (today 9.0), allowing to execute 
test methods after every upgrade.

The test suite has two variants: one for searchable 
snapshots and one for snapshot restore. The suites 
demonstrates that Elasticsearch does not allow 
reading indices written in version N-2 but we hope 
to make this feasible. Also, the tests can be used for 
investigation and debug with the command 
`./gradlew ":qa:lucene-index-compatibility:check" --debug-jvm-server`

Relates ES-10274

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcVersions.java

@@ -252,6 +252,20 @@ private List<Version> getReleased() {
             .toList();
     }
 
+    public List<Version> getReadOnlyIndexCompatible() {
+        // Lucene can read indices in version N-2
+        int compatibleMajor = currentVersion.getMajor() - 2;
+        return versions.stream().filter(v -> v.getMajor() == compatibleMajor).sorted(Comparator.naturalOrder()).toList();
+    }
+
+    public void withLatestReadOnlyIndexCompatible(Consumer<Version> versionAction) {
+        var compatibleVersions = getReadOnlyIndexCompatible();
+        if (compatibleVersions == null || compatibleVersions.isEmpty()) {
+            throw new IllegalStateException("No read-only compatible version found.");
+        }
+        versionAction.accept(compatibleVersions.getLast());
+    }
+
     /**
      * Return versions of Elasticsearch which are index compatible with the current version.
      */

qa/lucene-index-compatibility/build.gradle

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.internal-test-artifact'
+apply plugin: 'elasticsearch.bwc-test'
+
+buildParams.bwcVersions.withLatestReadOnlyIndexCompatible { bwcVersion ->
+  tasks.named("javaRestTest").configure {
+    systemProperty("tests.minimum.index.compatible", bwcVersion)
+    usesBwcDistribution(bwcVersion)
+    enabled = true
+  }
+}
+
+tasks.withType(Test).configureEach {
+  // CI doesn't like it when there's multiple clusters running at once
+  maxParallelForks = 1
+}
+

qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractLuceneIndexCompatibilityTestCase.java

@@ -0,0 +1,141 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.lucene;
+
+import com.carrotsearch.randomizedtesting.TestMethodAndParams;
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.cluster.util.Version;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TemporaryFolder;
+import org.junit.rules.TestRule;
+
+import java.util.Comparator;
+import java.util.Locale;
+import java.util.stream.Stream;
+
+import static org.elasticsearch.test.cluster.util.Version.CURRENT;
+import static org.elasticsearch.test.cluster.util.Version.fromString;
+import static org.elasticsearch.test.rest.ObjectPath.createFromResponse;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+
+/**
+ * Test suite for Lucene indices backward compatibility with N-2 versions. The test suite creates a cluster in N-2 version, then upgrades it
+ * to N-1 version and finally upgrades it to the current version. Test methods are executed after each upgrade.
+ */
+@TestCaseOrdering(AbstractLuceneIndexCompatibilityTestCase.TestCaseOrdering.class)
+public abstract class AbstractLuceneIndexCompatibilityTestCase extends ESRestTestCase {
+
+    protected static final Version VERSION_MINUS_2 = fromString(System.getProperty("tests.minimum.index.compatible"));
+    protected static final Version VERSION_MINUS_1 = fromString(System.getProperty("tests.minimum.wire.compatible"));
+    protected static final Version VERSION_CURRENT = CURRENT;
+
+    protected static TemporaryFolder REPOSITORY_PATH = new TemporaryFolder();
+
+    protected static LocalClusterConfigProvider clusterConfig = c -> {};
+    private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .version(VERSION_MINUS_2)
+        .nodes(2)
+        .setting("path.repo", () -> REPOSITORY_PATH.getRoot().getPath())
+        .setting("xpack.security.enabled", "false")
+        .setting("xpack.ml.enabled", "false")
+        .setting("path.repo", () -> REPOSITORY_PATH.getRoot().getPath())
+        .apply(() -> clusterConfig)
+        .build();
+
+    @ClassRule
+    public static TestRule ruleChain = RuleChain.outerRule(REPOSITORY_PATH).around(cluster);
+
+    private static boolean upgradeFailed = false;
+
+    private final Version clusterVersion;
+
+    public AbstractLuceneIndexCompatibilityTestCase(@Name("cluster") Version clusterVersion) {
+        this.clusterVersion = clusterVersion;
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() {
+        return Stream.of(VERSION_MINUS_2, VERSION_MINUS_1, CURRENT).map(v -> new Object[] { v }).toList();
+    }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    @Override
+    protected boolean preserveClusterUponCompletion() {
+        return true;
+    }
+
+    @Before
+    public void maybeUpgrade() throws Exception {
+        // We want to use this test suite for the V9 upgrade, but we are not fully committed to necessarily having N-2 support
+        // in V10, so we add a check here to ensure we'll revisit this decision once V10 exists.
+        assertThat("Explicit check that N-2 version is Elasticsearch 7", VERSION_MINUS_2.getMajor(), equalTo(7));
+
+        var currentVersion = clusterVersion();
+        if (currentVersion.before(clusterVersion)) {
+            try {
+                cluster.upgradeToVersion(clusterVersion);
+                closeClients();
+                initClient();
+            } catch (Exception e) {
+                upgradeFailed = true;
+                throw e;
+            }
+        }
+
+        // Skip remaining tests if upgrade failed
+        assumeFalse("Cluster upgrade failed", upgradeFailed);
+    }
+
+    protected String suffix(String name) {
+        return name + '-' + getTestName().split(" ")[0].toLowerCase(Locale.ROOT);
+    }
+
+    protected static Version clusterVersion() throws Exception {
+        var response = assertOK(client().performRequest(new Request("GET", "/")));
+        var responseBody = createFromResponse(response);
+        var version = Version.fromString(responseBody.evaluate("version.number").toString());
+        assertThat("Failed to retrieve cluster version", version, notNullValue());
+        return version;
+    }
+
+    protected static Version indexLuceneVersion(String indexName) throws Exception {
+        var response = assertOK(client().performRequest(new Request("GET", "/" + indexName + "/_settings")));
+        int id = Integer.parseInt(createFromResponse(response).evaluate(indexName + ".settings.index.version.created"));
+        return new Version((byte) ((id / 1000000) % 100), (byte) ((id / 10000) % 100), (byte) ((id / 100) % 100));
+    }
+
+    /**
+     * Execute the test suite with the parameters provided by the {@link #parameters()} in version order.
+     */
+    public static class TestCaseOrdering implements Comparator<TestMethodAndParams> {
+        @Override
+        public int compare(TestMethodAndParams o1, TestMethodAndParams o2) {
+            var version1 = (Version) o1.getInstanceArguments().get(0);
+            var version2 = (Version) o2.getInstanceArguments().get(0);
+            return version1.compareTo(version2);
+        }
+    }
+}

qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/LuceneCompatibilityIT.java

@@ -0,0 +1,114 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.lucene;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.repositories.fs.FsRepository;
+import org.elasticsearch.test.cluster.util.Version;
+
+import java.util.stream.IntStream;
+
+import static org.elasticsearch.test.rest.ObjectPath.createFromResponse;
+import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+
+public class LuceneCompatibilityIT extends AbstractLuceneIndexCompatibilityTestCase {
+
+    static {
+        clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial");
+    }
+
+    public LuceneCompatibilityIT(Version version) {
+        super(version);
+    }
+
+    public void testRestoreIndex() throws Exception {
+        final String repository = suffix("repository");
+        final String snapshot = suffix("snapshot");
+        final String index = suffix("index");
+        final int numDocs = 1234;
+
+        logger.debug("--> registering repository [{}]", repository);
+        registerRepository(
+            client(),
+            repository,
+            FsRepository.TYPE,
+            true,
+            Settings.builder().put("location", REPOSITORY_PATH.getRoot().getPath()).build()
+        );
+
+        if (VERSION_MINUS_2.equals(clusterVersion())) {
+            logger.debug("--> creating index [{}]", index);
+            createIndex(
+                client(),
+                index,
+                Settings.builder()
+                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                    .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                    .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
+                    .build()
+            );
+
+            logger.debug("--> indexing [{}] docs in [{}]", numDocs, index);
+            final var bulks = new StringBuilder();
+            IntStream.range(0, numDocs).forEach(n -> bulks.append(Strings.format("""
+                {"index":{"_id":"%s","_index":"%s"}}
+                {"test":"test"}
+                """, n, index)));
+
+            var bulkRequest = new Request("POST", "/_bulk");
+            bulkRequest.setJsonEntity(bulks.toString());
+            var bulkResponse = client().performRequest(bulkRequest);
+            assertOK(bulkResponse);
+            assertThat(entityAsMap(bulkResponse).get("errors"), allOf(notNullValue(), is(false)));
+
+            logger.debug("--> creating snapshot [{}]", snapshot);
+            createSnapshot(client(), repository, snapshot, true);
+            return;
+        }
+
+        if (VERSION_MINUS_1.equals(clusterVersion())) {
+            ensureGreen(index);
+
+            assertThat(indexLuceneVersion(index), equalTo(VERSION_MINUS_2));
+            assertDocCount(client(), index, numDocs);
+
+            logger.debug("--> deleting index [{}]", index);
+            deleteIndex(index);
+            return;
+        }
+
+        if (VERSION_CURRENT.equals(clusterVersion())) {
+            var restoredIndex = suffix("index-restored");
+            logger.debug("--> restoring index [{}] as archive [{}]", index, restoredIndex);
+
+            // Restoring the archive will fail as Elasticsearch does not support reading N-2 yet
+            var request = new Request("POST", "/_snapshot/" + repository + "/" + snapshot + "/_restore");
+            request.addParameter("wait_for_completion", "true");
+            request.setJsonEntity(Strings.format("""
+                {
+                  "indices": "%s",
+                  "include_global_state": false,
+                  "rename_pattern": "(.+)",
+                  "rename_replacement": "%s",
+                  "include_aliases": false
+                }""", index, restoredIndex));
+            var responseBody = createFromResponse(client().performRequest(request));
+            assertThat(responseBody.evaluate("snapshot.shards.total"), equalTo((int) responseBody.evaluate("snapshot.shards.failed")));
+            assertThat(responseBody.evaluate("snapshot.shards.successful"), equalTo(0));
+        }
+    }
+}