diff --git a/docs/changelog/118599.yaml b/docs/changelog/118599.yaml new file mode 100644 index 0000000000000..b410ddf5c5d19 --- /dev/null +++ b/docs/changelog/118599.yaml @@ -0,0 +1,5 @@ +pr: 118599 +summary: Archive-Index upgrade compatibility +area: Search +type: enhancement +issues: [] diff --git a/x-pack/qa/repository-old-versions-compatibility/build.gradle b/x-pack/qa/repository-old-versions-compatibility/build.gradle new file mode 100644 index 0000000000000..37e5eea85a08b --- /dev/null +++ b/x-pack/qa/repository-old-versions-compatibility/build.gradle @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ +apply plugin: 'elasticsearch.internal-java-rest-test' +apply plugin: 'elasticsearch.internal-test-artifact' +apply plugin: 'elasticsearch.bwc-test' + +buildParams.bwcVersions.withLatestReadOnlyIndexCompatible { bwcVersion -> + tasks.named("javaRestTest").configure { + systemProperty("tests.minimum.index.compatible", bwcVersion) + usesBwcDistribution(bwcVersion) + enabled = true + } +} + +tasks.withType(Test).configureEach { + // CI doesn't like it when there's multiple clusters running at once + maxParallelForks = 1 +} + diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/AbstractUpgradeCompatibilityTestCase.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/AbstractUpgradeCompatibilityTestCase.java new file mode 100644 index 0000000000000..4ff2b80aa29cc --- /dev/null +++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/AbstractUpgradeCompatibilityTestCase.java @@ -0,0 +1,211 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.oldrepos; + +import com.carrotsearch.randomizedtesting.TestMethodAndParams; +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering; + +import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.repositories.fs.FsRepository; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.cluster.util.Version; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.TestRule; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Comparator; +import java.util.Objects; +import java.util.stream.Stream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import static org.elasticsearch.test.cluster.util.Version.CURRENT; +import static org.elasticsearch.test.cluster.util.Version.fromString; +import static org.elasticsearch.test.rest.ObjectPath.createFromResponse; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; + +@TestCaseOrdering(AbstractUpgradeCompatibilityTestCase.TestCaseOrdering.class) +public abstract class AbstractUpgradeCompatibilityTestCase extends ESRestTestCase { + + protected static final Version VERSION_MINUS_2 = fromString(System.getProperty("tests.minimum.index.compatible")); + protected static final Version VERSION_MINUS_1 = fromString(System.getProperty("tests.minimum.wire.compatible")); + protected static final Version VERSION_CURRENT = CURRENT; + + protected static TemporaryFolder REPOSITORY_PATH = new TemporaryFolder(); + + protected static LocalClusterConfigProvider clusterConfig = c -> {}; + private static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .version(VERSION_MINUS_1) + .nodes(2) + .setting("xpack.security.enabled", "false") + .setting("xpack.ml.enabled", "false") + .setting("path.repo", () -> REPOSITORY_PATH.getRoot().getPath()) + .apply(() -> clusterConfig) + .build(); + + @ClassRule + public static TestRule ruleChain = RuleChain.outerRule(REPOSITORY_PATH).around(cluster); + + private static boolean upgradeFailed = false; + + private final Version clusterVersion; + + public AbstractUpgradeCompatibilityTestCase(@Name("cluster") Version clusterVersion) { + this.clusterVersion = clusterVersion; + } + + @ParametersFactory + public static Iterable parameters() { + return Stream.of(VERSION_MINUS_1, CURRENT).map(v -> new Object[] { v }).toList(); + } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @Override + protected boolean preserveClusterUponCompletion() { + return true; + } + + /** + * This method verifies the currentVersion against the clusterVersion and performs a "full cluster restart" upgrade if the current + * is before clusterVersion. The cluster version is fetched externally and is controlled by the gradle setup. + * + * @throws Exception + */ + @Before + public void maybeUpgrade() throws Exception { + // We want to use this test suite for the V9 upgrade, but we are not fully committed to necessarily having N-2 support + // in V10, so we add a check here to ensure we'll revisit this decision once V10 exists. + assertThat("Explicit check that N-2 version is Elasticsearch 7", VERSION_MINUS_2.getMajor(), equalTo(7)); + + var currentVersion = clusterVersion(); + if (currentVersion.before(clusterVersion)) { + try { + cluster.upgradeToVersion(clusterVersion); + closeClients(); + initClient(); + } catch (Exception e) { + upgradeFailed = true; + throw e; + } + } + + // Skip remaining tests if upgrade failed + assumeFalse("Cluster upgrade failed", upgradeFailed); + } + + protected static Version clusterVersion() throws Exception { + var response = assertOK(client().performRequest(new Request("GET", "/"))); + var responseBody = createFromResponse(response); + var version = Version.fromString(responseBody.evaluate("version.number").toString()); + assertThat("Failed to retrieve cluster version", version, notNullValue()); + return version; + } + + /** + * Execute the test suite with the parameters provided by the {@link #parameters()} in version order. + */ + public static class TestCaseOrdering implements Comparator { + @Override + public int compare(TestMethodAndParams o1, TestMethodAndParams o2) { + var version1 = (Version) o1.getInstanceArguments().get(0); + var version2 = (Version) o2.getInstanceArguments().get(0); + return version1.compareTo(version2); + } + } + + public final void verifyCompatibility(String version) throws Exception { + final String repository = "repository"; + final String snapshot = "snapshot"; + final String index = "index"; + final int numDocs = 5; + + String repositoryPath = REPOSITORY_PATH.getRoot().getPath(); + + if (VERSION_MINUS_1.equals(clusterVersion())) { + assertEquals(VERSION_MINUS_1, clusterVersion()); + assertTrue(getIndices(client()).isEmpty()); + + // Copy a snapshot of an index with 5 documents + copySnapshotFromResources(repositoryPath, version); + registerRepository(client(), repository, FsRepository.TYPE, true, Settings.builder().put("location", repositoryPath).build()); + recover(client(), repository, snapshot, index); + + assertTrue(getIndices(client()).contains(index)); + assertDocCount(client(), index, numDocs); + + return; + } + + if (VERSION_CURRENT.equals(clusterVersion())) { + assertEquals(VERSION_CURRENT, clusterVersion()); + assertTrue(getIndices(client()).contains(index)); + assertDocCount(client(), index, numDocs); + } + } + + public abstract void recover(RestClient restClient, String repository, String snapshot, String index) throws Exception; + + private static String getIndices(RestClient client) throws IOException { + final Request request = new Request("GET", "_cat/indices"); + Response response = client.performRequest(request); + return EntityUtils.toString(response.getEntity()); + } + + private static void copySnapshotFromResources(String repositoryPath, String version) throws IOException, URISyntaxException { + Path zipFilePath = Paths.get( + Objects.requireNonNull(AbstractUpgradeCompatibilityTestCase.class.getClassLoader().getResource("snapshot_v" + version + ".zip")) + .toURI() + ); + unzip(zipFilePath, Paths.get(repositoryPath)); + } + + private static void unzip(Path zipFilePath, Path outputDir) throws IOException { + try (ZipInputStream zipIn = new ZipInputStream(Files.newInputStream(zipFilePath))) { + ZipEntry entry; + while ((entry = zipIn.getNextEntry()) != null) { + Path outputPath = outputDir.resolve(entry.getName()); + if (entry.isDirectory()) { + Files.createDirectories(outputPath); + } else { + Files.createDirectories(outputPath.getParent()); + try (OutputStream out = Files.newOutputStream(outputPath)) { + byte[] buffer = new byte[1024]; + int len; + while ((len = zipIn.read(buffer)) > 0) { + out.write(buffer, 0, len); + } + } + } + zipIn.closeEntry(); + } + } + } +} diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/ArchiveIndexTestCase.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/ArchiveIndexTestCase.java new file mode 100644 index 0000000000000..17bdb76e0eae5 --- /dev/null +++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/ArchiveIndexTestCase.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.oldrepos.archiveindex; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.oldrepos.AbstractUpgradeCompatibilityTestCase; +import org.elasticsearch.test.cluster.util.Version; + +import static org.elasticsearch.test.rest.ObjectPath.createFromResponse; + +/** + * Test suite for Archive indices backward compatibility with N-2 versions. + * The test suite creates a cluster in the N-1 version, where N is the current version. + * Restores snapshots from old-clusters (version 5/6) and upgrades it to the current version. + * Test methods are executed after each upgrade. + * + * For example the test suite creates a cluster of version 8, then restores a snapshot of an index created + * when deployed ES version 5/6. The cluster then upgrades to version 9, verifying that the archive index + * is successfully restored. + */ +public class ArchiveIndexTestCase extends AbstractUpgradeCompatibilityTestCase { + + static { + clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial"); + } + + public ArchiveIndexTestCase(Version version) { + super(version); + } + + /** + * Overrides the snapshot-restore operation for archive-indices scenario. + */ + @Override + public void recover(RestClient client, String repository, String snapshot, String index) throws Exception { + var request = new Request("POST", "/_snapshot/" + repository + "/" + snapshot + "/_restore"); + request.addParameter("wait_for_completion", "true"); + request.setJsonEntity(Strings.format(""" + { + "indices": "%s", + "include_global_state": false, + "rename_pattern": "(.+)", + "include_aliases": false + }""", index)); + createFromResponse(client.performRequest(request)); + } +} diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion5IT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion5IT.java new file mode 100644 index 0000000000000..9f62d65592a37 --- /dev/null +++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion5IT.java @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.oldrepos.archiveindex; + +import org.elasticsearch.test.cluster.util.Version; + +public class RestoreFromVersion5IT extends ArchiveIndexTestCase { + + public RestoreFromVersion5IT(Version version) { + super(version); + } + + public void testArchiveIndex() throws Exception { + verifyCompatibility("5"); + } +} diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion6IT.java b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion6IT.java new file mode 100644 index 0000000000000..b3cca45c205f6 --- /dev/null +++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion6IT.java @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.oldrepos.archiveindex; + +import org.elasticsearch.test.cluster.util.Version; + +public class RestoreFromVersion6IT extends ArchiveIndexTestCase { + + public RestoreFromVersion6IT(Version version) { + super(version); + } + + public void testArchiveIndex() throws Exception { + verifyCompatibility("6"); + } +} diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/README.md b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/README.md new file mode 100644 index 0000000000000..c937448e97236 --- /dev/null +++ b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/README.md @@ -0,0 +1,147 @@ + +### Create data structure and config file +``` +mkdir /tmp/sharedESData +mkdir /tmp/sharedESData/config +mkdir /tmp/sharedESData/data +mkdir /tmp/sharedESData/snapshots +``` + +``` +touch /tmp/sharedESData/config/elasticsearch.yml + +cat <> /tmp/sharedESData/config/elasticsearch.yml +cluster.name: "archive-indides-test" +node.name: "node-1" +path.repo: ["/usr/share/elasticsearch/snapshots"] +network.host: 0.0.0.0 +http.port: 9200 + +discovery.type: single-node +xpack.security.enabled: false +EOF +``` + +### Define path +``` +SHARED_FOLDER=/tmp/sharedESData +``` + +### Deploy container +``` +docker run -d --name es \ +-p 9200:9200 -p 9300:9300 \ +-v ${SHARED_FOLDER}/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml \ +-v ${SHARED_FOLDER}/data:/usr/share/elasticsearch/data \ +-v ${SHARED_FOLDER}/snapshots:/usr/share/elasticsearch/snapshots \ +--env "discovery.type=single-node" \ +docker.elastic.co/elasticsearch/elasticsearch:5.6.16 + +// Version 6 +docker.elastic.co/elasticsearch/elasticsearch:6.8.23 +``` + +### Create Index Version 5 +``` +PUT /index +{ + "settings": { + "number_of_shards": 1, + "number_of_replicas": 1 + }, + "mappings": { + "my_type": { + "properties": { + "title": { + "type": "text" + }, + "created_at": { + "type": "date" + }, + "views": { + "type": "integer" + } + } + } + } +} +``` + +### Create Index Version 6 +``` +PUT /index +{ + "settings": { + "number_of_shards": 1, + "number_of_replicas": 1 + }, + "mappings": { + "_doc": { + "properties": { + "title": { + "type": "text" + }, + "content": { + "type": "text" + }, + "created_at": { + "type": "date" + } + } + } + } +} +``` + +### Add documents Version 5 +``` +POST /index/my_type +{ + "title": "Title 5", + "content": "Elasticsearch is a powerful search engine.", + "created_at": "2024-12-16" +} +``` + +### Add documents Version 6 +``` +POST /index/_doc +{ + "title": "Title 5", + "content": "Elasticsearch is a powerful search engine.", + "created_at": "2024-12-16" +} +``` + +### Register repository +``` +PUT /_snapshot/repository +{ + "type": "fs", + "settings": { + "location": "/usr/share/elasticsearch/snapshots", + "compress": true + } +} +``` + +### Create a snapshot +``` +PUT /_snapshot/repository/snapshot +{ + "indices": "index", + "ignore_unavailable": "true", + "include_global_state": false +} +``` + +### Create zip file +``` +zip -r snapshot.zip /tmp/sharedESData/snapshots/* +``` + +### Cleanup +``` +docker rm -f es +rm -rf /tmp/sharedESData/ +``` diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v5.zip b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v5.zip new file mode 100644 index 0000000000000..54dcf4f6182cc Binary files /dev/null and b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v5.zip differ diff --git a/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v6.zip b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v6.zip new file mode 100644 index 0000000000000..d83152fb71c62 Binary files /dev/null and b/x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/resources/snapshot_v6.zip differ