diff --git a/docs/changelog/118941.yaml b/docs/changelog/118941.yaml new file mode 100644 index 0000000000000..4f0099bb32704 --- /dev/null +++ b/docs/changelog/118941.yaml @@ -0,0 +1,5 @@ +pr: 118941 +summary: Allow archive and searchable snapshots indices in N-2 version +area: Recovery +type: enhancement +issues: [] diff --git a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractLuceneIndexCompatibilityTestCase.java b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractLuceneIndexCompatibilityTestCase.java index c42e879f84892..1865da06e20c5 100644 --- a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractLuceneIndexCompatibilityTestCase.java +++ b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractLuceneIndexCompatibilityTestCase.java @@ -15,6 +15,8 @@ import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering; import org.elasticsearch.client.Request; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Strings; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider; import org.elasticsearch.test.cluster.local.distribution.DistributionType; @@ -28,12 +30,15 @@ import java.util.Comparator; import java.util.Locale; +import java.util.stream.IntStream; import java.util.stream.Stream; import static org.elasticsearch.test.cluster.util.Version.CURRENT; import static org.elasticsearch.test.cluster.util.Version.fromString; import static org.elasticsearch.test.rest.ObjectPath.createFromResponse; +import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; /** @@ -113,6 +118,12 @@ protected String suffix(String name) { return name + '-' + getTestName().split(" ")[0].toLowerCase(Locale.ROOT); } + protected Settings repositorySettings() { + return Settings.builder() + .put("location", REPOSITORY_PATH.getRoot().toPath().resolve(suffix("location")).toFile().getPath()) + .build(); + } + protected static Version clusterVersion() throws Exception { var response = assertOK(client().performRequest(new Request("GET", "/"))); var responseBody = createFromResponse(response); @@ -121,12 +132,56 @@ protected static Version clusterVersion() throws Exception { return version; } - protected static Version indexLuceneVersion(String indexName) throws Exception { + protected static Version indexVersion(String indexName) throws Exception { var response = assertOK(client().performRequest(new Request("GET", "/" + indexName + "/_settings"))); int id = Integer.parseInt(createFromResponse(response).evaluate(indexName + ".settings.index.version.created")); return new Version((byte) ((id / 1000000) % 100), (byte) ((id / 10000) % 100), (byte) ((id / 100) % 100)); } + protected static void indexDocs(String indexName, int numDocs) throws Exception { + var request = new Request("POST", "/_bulk"); + var docs = new StringBuilder(); + IntStream.range(0, numDocs).forEach(n -> docs.append(Strings.format(""" + {"index":{"_id":"%s","_index":"%s"}} + {"test":"test"} + """, n, indexName))); + request.setJsonEntity(docs.toString()); + var response = assertOK(client().performRequest(request)); + assertThat(entityAsMap(response).get("errors"), allOf(notNullValue(), is(false))); + } + + protected static void mountIndex(String repository, String snapshot, String indexName, boolean partial, String renamedIndexName) + throws Exception { + var request = new Request("POST", "/_snapshot/" + repository + "/" + snapshot + "/_mount"); + request.addParameter("wait_for_completion", "true"); + var storage = partial ? "shared_cache" : "full_copy"; + request.addParameter("storage", storage); + request.setJsonEntity(Strings.format(""" + { + "index": "%s", + "renamed_index": "%s" + }""", indexName, renamedIndexName)); + var responseBody = createFromResponse(client().performRequest(request)); + assertThat(responseBody.evaluate("snapshot.shards.total"), equalTo((int) responseBody.evaluate("snapshot.shards.successful"))); + assertThat(responseBody.evaluate("snapshot.shards.failed"), equalTo(0)); + } + + protected static void restoreIndex(String repository, String snapshot, String indexName, String renamedIndexName) throws Exception { + var request = new Request("POST", "/_snapshot/" + repository + "/" + snapshot + "/_restore"); + request.addParameter("wait_for_completion", "true"); + request.setJsonEntity(org.elasticsearch.common.Strings.format(""" + { + "indices": "%s", + "include_global_state": false, + "rename_pattern": "(.+)", + "rename_replacement": "%s", + "include_aliases": false + }""", indexName, renamedIndexName)); + var responseBody = createFromResponse(client().performRequest(request)); + assertThat(responseBody.evaluate("snapshot.shards.total"), equalTo((int) responseBody.evaluate("snapshot.shards.failed"))); + assertThat(responseBody.evaluate("snapshot.shards.successful"), equalTo(0)); + } + /** * Execute the test suite with the parameters provided by the {@link #parameters()} in version order. */ diff --git a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/LuceneCompatibilityIT.java b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/LuceneCompatibilityIT.java index d6dd949b843d6..655e30f069f18 100644 --- a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/LuceneCompatibilityIT.java +++ b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/LuceneCompatibilityIT.java @@ -10,20 +10,18 @@ package org.elasticsearch.lucene; import org.elasticsearch.client.Request; +import org.elasticsearch.client.ResponseException; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.repositories.fs.FsRepository; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.cluster.util.Version; -import java.util.stream.IntStream; - -import static org.elasticsearch.test.rest.ObjectPath.createFromResponse; +import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.notNullValue; public class LuceneCompatibilityIT extends AbstractLuceneIndexCompatibilityTestCase { @@ -35,22 +33,19 @@ public LuceneCompatibilityIT(Version version) { super(version); } + /** + * Creates an index and a snapshot on N-2, then restores the snapshot on N. + */ public void testRestoreIndex() throws Exception { final String repository = suffix("repository"); final String snapshot = suffix("snapshot"); final String index = suffix("index"); final int numDocs = 1234; - logger.debug("--> registering repository [{}]", repository); - registerRepository( - client(), - repository, - FsRepository.TYPE, - true, - Settings.builder().put("location", REPOSITORY_PATH.getRoot().getPath()).build() - ); - if (VERSION_MINUS_2.equals(clusterVersion())) { + logger.debug("--> registering repository [{}]", repository); + registerRepository(client(), repository, FsRepository.TYPE, true, repositorySettings()); + logger.debug("--> creating index [{}]", index); createIndex( client(), @@ -63,17 +58,7 @@ public void testRestoreIndex() throws Exception { ); logger.debug("--> indexing [{}] docs in [{}]", numDocs, index); - final var bulks = new StringBuilder(); - IntStream.range(0, numDocs).forEach(n -> bulks.append(Strings.format(""" - {"index":{"_id":"%s","_index":"%s"}} - {"test":"test"} - """, n, index))); - - var bulkRequest = new Request("POST", "/_bulk"); - bulkRequest.setJsonEntity(bulks.toString()); - var bulkResponse = client().performRequest(bulkRequest); - assertOK(bulkResponse); - assertThat(entityAsMap(bulkResponse).get("errors"), allOf(notNullValue(), is(false))); + indexDocs(index, numDocs); logger.debug("--> creating snapshot [{}]", snapshot); createSnapshot(client(), repository, snapshot, true); @@ -83,7 +68,7 @@ public void testRestoreIndex() throws Exception { if (VERSION_MINUS_1.equals(clusterVersion())) { ensureGreen(index); - assertThat(indexLuceneVersion(index), equalTo(VERSION_MINUS_2)); + assertThat(indexVersion(index), equalTo(VERSION_MINUS_2)); assertDocCount(client(), index, numDocs); logger.debug("--> deleting index [{}]", index); @@ -93,9 +78,9 @@ public void testRestoreIndex() throws Exception { if (VERSION_CURRENT.equals(clusterVersion())) { var restoredIndex = suffix("index-restored"); - logger.debug("--> restoring index [{}] as archive [{}]", index, restoredIndex); + logger.debug("--> restoring index [{}] as [{}]", index, restoredIndex); - // Restoring the archive will fail as Elasticsearch does not support reading N-2 yet + // Restoring the index will fail as Elasticsearch does not support reading N-2 yet var request = new Request("POST", "/_snapshot/" + repository + "/" + snapshot + "/_restore"); request.addParameter("wait_for_completion", "true"); request.setJsonEntity(Strings.format(""" @@ -106,9 +91,20 @@ public void testRestoreIndex() throws Exception { "rename_replacement": "%s", "include_aliases": false }""", index, restoredIndex)); - var responseBody = createFromResponse(client().performRequest(request)); - assertThat(responseBody.evaluate("snapshot.shards.total"), equalTo((int) responseBody.evaluate("snapshot.shards.failed"))); - assertThat(responseBody.evaluate("snapshot.shards.successful"), equalTo(0)); + + var responseException = expectThrows(ResponseException.class, () -> client().performRequest(request)); + assertEquals(RestStatus.INTERNAL_SERVER_ERROR.getStatus(), responseException.getResponse().getStatusLine().getStatusCode()); + assertThat( + responseException.getMessage(), + allOf( + containsString("cannot restore index [[" + index), + containsString("because it cannot be upgraded"), + containsString("has current compatibility version [" + VERSION_MINUS_2 + '-' + VERSION_MINUS_1.getMajor() + ".0.0]"), + containsString("but the minimum compatible version is [" + VERSION_MINUS_1.getMajor() + ".0.0]."), + containsString("It should be re-indexed in Elasticsearch " + VERSION_MINUS_1.getMajor() + ".x"), + containsString("before upgrading to " + VERSION_CURRENT) + ) + ); } } } diff --git a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/SearchableSnapshotCompatibilityIT.java b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/SearchableSnapshotCompatibilityIT.java index 4f348b7fb122f..d5db17f257b0c 100644 --- a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/SearchableSnapshotCompatibilityIT.java +++ b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/SearchableSnapshotCompatibilityIT.java @@ -9,21 +9,13 @@ package org.elasticsearch.lucene; -import org.elasticsearch.client.Request; import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.test.cluster.util.Version; -import java.util.stream.IntStream; - -import static org.elasticsearch.test.rest.ObjectPath.createFromResponse; -import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.notNullValue; public class SearchableSnapshotCompatibilityIT extends AbstractLuceneIndexCompatibilityTestCase { @@ -37,24 +29,19 @@ public SearchableSnapshotCompatibilityIT(Version version) { super(version); } - // TODO Add a test to mount the N-2 index on N-1 and then search it on N - + /** + * Creates an index and a snapshot on N-2, then mounts the snapshot on N. + */ public void testSearchableSnapshot() throws Exception { final String repository = suffix("repository"); final String snapshot = suffix("snapshot"); final String index = suffix("index"); final int numDocs = 1234; - logger.debug("--> registering repository [{}]", repository); - registerRepository( - client(), - repository, - FsRepository.TYPE, - true, - Settings.builder().put("location", REPOSITORY_PATH.getRoot().getPath()).build() - ); - if (VERSION_MINUS_2.equals(clusterVersion())) { + logger.debug("--> registering repository [{}]", repository); + registerRepository(client(), repository, FsRepository.TYPE, true, repositorySettings()); + logger.debug("--> creating index [{}]", index); createIndex( client(), @@ -67,17 +54,7 @@ public void testSearchableSnapshot() throws Exception { ); logger.debug("--> indexing [{}] docs in [{}]", numDocs, index); - final var bulks = new StringBuilder(); - IntStream.range(0, numDocs).forEach(n -> bulks.append(Strings.format(""" - {"index":{"_id":"%s","_index":"%s"}} - {"test":"test"} - """, n, index))); - - var bulkRequest = new Request("POST", "/_bulk"); - bulkRequest.setJsonEntity(bulks.toString()); - var bulkResponse = client().performRequest(bulkRequest); - assertOK(bulkResponse); - assertThat(entityAsMap(bulkResponse).get("errors"), allOf(notNullValue(), is(false))); + indexDocs(index, numDocs); logger.debug("--> creating snapshot [{}]", snapshot); createSnapshot(client(), repository, snapshot, true); @@ -87,7 +64,7 @@ public void testSearchableSnapshot() throws Exception { if (VERSION_MINUS_1.equals(clusterVersion())) { ensureGreen(index); - assertThat(indexLuceneVersion(index), equalTo(VERSION_MINUS_2)); + assertThat(indexVersion(index), equalTo(VERSION_MINUS_2)); assertDocCount(client(), index, numDocs); logger.debug("--> deleting index [{}]", index); @@ -98,20 +75,75 @@ public void testSearchableSnapshot() throws Exception { if (VERSION_CURRENT.equals(clusterVersion())) { var mountedIndex = suffix("index-mounted"); logger.debug("--> mounting index [{}] as [{}]", index, mountedIndex); + mountIndex(repository, snapshot, index, randomBoolean(), mountedIndex); + + ensureGreen(mountedIndex); + + assertThat(indexVersion(mountedIndex), equalTo(VERSION_MINUS_2)); + assertDocCount(client(), mountedIndex, numDocs); + + logger.debug("--> adding replica to test peer-recovery"); + updateIndexSettings(mountedIndex, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)); + ensureGreen(mountedIndex); + } + } + + /** + * Creates an index and a snapshot on N-2, mounts the snapshot on N -1 and then upgrades to N. + */ + public void testSearchableSnapshotUpgrade() throws Exception { + final String mountedIndex = suffix("index-mounted"); + final String repository = suffix("repository"); + final String snapshot = suffix("snapshot"); + final String index = suffix("index"); + final int numDocs = 4321; + + if (VERSION_MINUS_2.equals(clusterVersion())) { + logger.debug("--> registering repository [{}]", repository); + registerRepository(client(), repository, FsRepository.TYPE, true, repositorySettings()); + + logger.debug("--> creating index [{}]", index); + createIndex( + client(), + index, + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) + .build() + ); + + logger.debug("--> indexing [{}] docs in [{}]", numDocs, index); + indexDocs(index, numDocs); + + logger.debug("--> creating snapshot [{}]", snapshot); + createSnapshot(client(), repository, snapshot, true); + + logger.debug("--> deleting index [{}]", index); + deleteIndex(index); + return; + } + + if (VERSION_MINUS_1.equals(clusterVersion())) { + logger.debug("--> mounting index [{}] as [{}]", index, mountedIndex); + mountIndex(repository, snapshot, index, randomBoolean(), mountedIndex); + + ensureGreen(mountedIndex); + + assertThat(indexVersion(mountedIndex), equalTo(VERSION_MINUS_2)); + assertDocCount(client(), mountedIndex, numDocs); + return; + } + + if (VERSION_CURRENT.equals(clusterVersion())) { + ensureGreen(mountedIndex); + + assertThat(indexVersion(mountedIndex), equalTo(VERSION_MINUS_2)); + assertDocCount(client(), mountedIndex, numDocs); - // Mounting the index will fail as Elasticsearch does not support reading N-2 yet - var request = new Request("POST", "/_snapshot/" + repository + "/" + snapshot + "/_mount"); - request.addParameter("wait_for_completion", "true"); - var storage = randomBoolean() ? "shared_cache" : "full_copy"; - request.addParameter("storage", storage); - request.setJsonEntity(Strings.format(""" - { - "index": "%s", - "renamed_index": "%s" - }""", index, mountedIndex)); - var responseBody = createFromResponse(client().performRequest(request)); - assertThat(responseBody.evaluate("snapshot.shards.total"), equalTo((int) responseBody.evaluate("snapshot.shards.failed"))); - assertThat(responseBody.evaluate("snapshot.shards.successful"), equalTo(0)); + logger.debug("--> adding replica to test peer-recovery"); + updateIndexSettings(mountedIndex, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)); + ensureGreen(mountedIndex); } } } diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/NodeJoinExecutor.java b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeJoinExecutor.java index 74a8dc7851c89..916a192d53871 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/NodeJoinExecutor.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeJoinExecutor.java @@ -49,6 +49,7 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static org.elasticsearch.cluster.metadata.IndexMetadataVerifier.isReadOnlySupportedVersion; import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK; public class NodeJoinExecutor implements ClusterStateTaskExecutor { @@ -179,7 +180,12 @@ public ClusterState execute(BatchExecutionContext batchExecutionContex Set newNodeEffectiveFeatures = enforceNodeFeatureBarrier(node, effectiveClusterFeatures, features); // we do this validation quite late to prevent race conditions between nodes joining and importing dangling indices // we have to reject nodes that don't support all indices we have in this cluster - ensureIndexCompatibility(node.getMinIndexVersion(), node.getMaxIndexVersion(), initialState.getMetadata()); + ensureIndexCompatibility( + node.getMinIndexVersion(), + node.getMinReadOnlyIndexVersion(), + node.getMaxIndexVersion(), + initialState.getMetadata() + ); nodesBuilder.add(node); compatibilityVersionsMap.put(node.getId(), compatibilityVersions); @@ -394,9 +400,15 @@ private Set calculateEffectiveClusterFeatures(DiscoveryNodes nodes, Map< * will not be created with a newer version of elasticsearch as well as that all indices are newer or equal to the minimum index * compatibility version. * @see IndexVersions#MINIMUM_COMPATIBLE + * @see IndexVersions#MINIMUM_READONLY_COMPATIBLE * @throws IllegalStateException if any index is incompatible with the given version */ - public static void ensureIndexCompatibility(IndexVersion minSupportedVersion, IndexVersion maxSupportedVersion, Metadata metadata) { + public static void ensureIndexCompatibility( + IndexVersion minSupportedVersion, + IndexVersion minReadOnlySupportedVersion, + IndexVersion maxSupportedVersion, + Metadata metadata + ) { // we ensure that all indices in the cluster we join are compatible with us no matter if they are // closed or not we can't read mappings of these indices so we need to reject the join... for (IndexMetadata idxMetadata : metadata) { @@ -411,14 +423,17 @@ public static void ensureIndexCompatibility(IndexVersion minSupportedVersion, In ); } if (idxMetadata.getCompatibilityVersion().before(minSupportedVersion)) { - throw new IllegalStateException( - "index " - + idxMetadata.getIndex() - + " version not supported: " - + idxMetadata.getCompatibilityVersion().toReleaseVersion() - + " minimum compatible index version is: " - + minSupportedVersion.toReleaseVersion() - ); + boolean isReadOnlySupported = isReadOnlySupportedVersion(idxMetadata, minSupportedVersion, minReadOnlySupportedVersion); + if (isReadOnlySupported == false) { + throw new IllegalStateException( + "index " + + idxMetadata.getIndex() + + " version not supported: " + + idxMetadata.getCompatibilityVersion().toReleaseVersion() + + " minimum compatible index version is: " + + minSupportedVersion.toReleaseVersion() + ); + } } } } @@ -542,7 +557,12 @@ public static Collection> addBuiltInJoin final Collection> validators = new ArrayList<>(); validators.add((node, state) -> { ensureNodesCompatibility(node.getVersion(), state.getNodes()); - ensureIndexCompatibility(node.getMinIndexVersion(), node.getMaxIndexVersion(), state.getMetadata()); + ensureIndexCompatibility( + node.getMinIndexVersion(), + node.getMinReadOnlyIndexVersion(), + node.getMaxIndexVersion(), + state.getMetadata() + ); }); validators.addAll(onJoinValidators); return Collections.unmodifiableCollection(validators); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java index 0b8095c24519f..be2563c4732b7 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java @@ -88,8 +88,12 @@ public IndexMetadataVerifier( * If the index does not need upgrade it returns the index metadata unchanged, otherwise it returns a modified index metadata. If index * cannot be updated the method throws an exception. */ - public IndexMetadata verifyIndexMetadata(IndexMetadata indexMetadata, IndexVersion minimumIndexCompatibilityVersion) { - checkSupportedVersion(indexMetadata, minimumIndexCompatibilityVersion); + public IndexMetadata verifyIndexMetadata( + IndexMetadata indexMetadata, + IndexVersion minimumIndexCompatibilityVersion, + IndexVersion minimumReadOnlyIndexCompatibilityVersion + ) { + checkSupportedVersion(indexMetadata, minimumIndexCompatibilityVersion, minimumReadOnlyIndexCompatibilityVersion); // First convert any shared_cache searchable snapshot indices to only use _tier_preference: data_frozen IndexMetadata newMetadata = convertSharedCacheTierPreference(indexMetadata); @@ -105,26 +109,81 @@ public IndexMetadata verifyIndexMetadata(IndexMetadata indexMetadata, IndexVersi } /** - * Check that the index version is compatible. Elasticsearch does not support indices created before the - * previous major version. + * Check that the index version is compatible. Elasticsearch supports reading and writing indices created in the current version ("N") + + as well as the previous major version ("N-1"). Elasticsearch only supports reading indices created down to the penultimate version + + ("N-2") and does not support reading nor writing any version below that. */ - private static void checkSupportedVersion(IndexMetadata indexMetadata, IndexVersion minimumIndexCompatibilityVersion) { - boolean isSupportedVersion = indexMetadata.getCompatibilityVersion().onOrAfter(minimumIndexCompatibilityVersion); - if (isSupportedVersion == false) { - throw new IllegalStateException( - "The index " - + indexMetadata.getIndex() - + " has current compatibility version [" - + indexMetadata.getCompatibilityVersion().toReleaseVersion() - + "] but the minimum compatible version is [" - + minimumIndexCompatibilityVersion.toReleaseVersion() - + "]. It should be re-indexed in Elasticsearch " - + (Version.CURRENT.major - 1) - + ".x before upgrading to " - + Build.current().version() - + "." - ); + private static void checkSupportedVersion( + IndexMetadata indexMetadata, + IndexVersion minimumIndexCompatibilityVersion, + IndexVersion minimumReadOnlyIndexCompatibilityVersion + ) { + if (isFullySupportedVersion(indexMetadata, minimumIndexCompatibilityVersion)) { + return; + } + if (isReadOnlySupportedVersion(indexMetadata, minimumIndexCompatibilityVersion, minimumReadOnlyIndexCompatibilityVersion)) { + return; + } + throw new IllegalStateException( + "The index " + + indexMetadata.getIndex() + + " has current compatibility version [" + + indexMetadata.getCompatibilityVersion().toReleaseVersion() + + "] but the minimum compatible version is [" + + minimumIndexCompatibilityVersion.toReleaseVersion() + + "]. It should be re-indexed in Elasticsearch " + + (Version.CURRENT.major - 1) + + ".x before upgrading to " + + Build.current().version() + + "." + ); + } + + private static boolean isFullySupportedVersion(IndexMetadata indexMetadata, IndexVersion minimumIndexCompatibilityVersion) { + return indexMetadata.getCompatibilityVersion().onOrAfter(minimumIndexCompatibilityVersion); + } + + /** + * Returns {@code true} if the index version is compatible in read-only mode. As of today, only searchable snapshots and archive indices + * in version N-2 with a write block are read-only compatible. This method throws an {@link IllegalStateException} if the index is + * either a searchable snapshot or an archive index with a read-only compatible version but is missing the write block. + * + * @param indexMetadata the index metadata + * @param minimumIndexCompatibilityVersion the min. index compatible version for reading and writing indices (used in assertion) + * @param minReadOnlyIndexCompatibilityVersion the min. index compatible version for only reading indices + * + * @return {@code true} if the index version is compatible in read-only mode, {@code false} otherwise. + * @throws IllegalStateException if the index is read-only compatible but has no write block in place. + */ + public static boolean isReadOnlySupportedVersion( + IndexMetadata indexMetadata, + IndexVersion minimumIndexCompatibilityVersion, + IndexVersion minReadOnlyIndexCompatibilityVersion + ) { + boolean isReadOnlySupportedVersion = indexMetadata.getCompatibilityVersion().onOrAfter(minReadOnlyIndexCompatibilityVersion); + assert isFullySupportedVersion(indexMetadata, minimumIndexCompatibilityVersion) == false; + + if (isReadOnlySupportedVersion + && (indexMetadata.isSearchableSnapshot() || indexMetadata.getCreationVersion().isLegacyIndexVersion())) { + boolean isReadOnly = IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.get(indexMetadata.getSettings()); + if (isReadOnly == false) { + throw new IllegalStateException( + "The index " + + indexMetadata.getIndex() + + " created in version [" + + indexMetadata.getCreationVersion() + + "] with current compatibility version [" + + indexMetadata.getCompatibilityVersion().toReleaseVersion() + + "] must be marked as read-only using the setting [" + + IndexMetadata.SETTING_BLOCKS_WRITE + + "] set to [true] before upgrading to " + + Build.current().version() + + '.' + ); + } + return true; } + return false; } /** diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java index 0c33878b01229..95d1c37ec41ae 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java @@ -1120,6 +1120,7 @@ private ClusterState openIndices(final Index[] indices, final ClusterState curre final Metadata.Builder metadata = Metadata.builder(currentState.metadata()); final ClusterBlocks.Builder blocks = ClusterBlocks.builder(currentState.blocks()); final IndexVersion minIndexCompatibilityVersion = currentState.getNodes().getMinSupportedIndexVersion(); + final IndexVersion minReadOnlyIndexCompatibilityVersion = currentState.getNodes().getMinReadOnlySupportedIndexVersion(); for (IndexMetadata indexMetadata : indicesToOpen) { final Index index = indexMetadata.getIndex(); @@ -1137,7 +1138,11 @@ private ClusterState openIndices(final Index[] indices, final ClusterState curre // The index might be closed because we couldn't import it due to an old incompatible // version, so we need to verify its compatibility. - newIndexMetadata = indexMetadataVerifier.verifyIndexMetadata(newIndexMetadata, minIndexCompatibilityVersion); + newIndexMetadata = indexMetadataVerifier.verifyIndexMetadata( + newIndexMetadata, + minIndexCompatibilityVersion, + minReadOnlyIndexCompatibilityVersion + ); try { indicesService.verifyIndexMetadata(newIndexMetadata, newIndexMetadata); } catch (Exception e) { diff --git a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java index a57b8b4d23cdb..bd48572a8bc11 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java @@ -70,6 +70,8 @@ import org.elasticsearch.common.util.iterable.Iterables; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -87,6 +89,8 @@ import java.util.Map; import java.util.Objects; +import static org.apache.lucene.util.Version.LUCENE_10_0_0; + public class Lucene { public static final String LATEST_CODEC = "Lucene100"; @@ -109,13 +113,6 @@ public class Lucene { private Lucene() {} - /** - * Reads the segments infos, failing if it fails to load - */ - public static SegmentInfos readSegmentInfos(Directory directory) throws IOException { - return SegmentInfos.readLatestCommit(directory); - } - /** * Returns an iterable that allows to iterate over all files in this segments info */ @@ -139,21 +136,45 @@ public static int getNumDocs(SegmentInfos info) { return numDocs; } + /** + * Reads the segments infos, failing if it fails to load + */ + public static SegmentInfos readSegmentInfos(Directory directory) throws IOException { + return SegmentInfos.readLatestCommit(directory, IndexVersions.MINIMUM_READONLY_COMPATIBLE.luceneVersion().major); + } + /** * Reads the segments infos from the given commit, failing if it fails to load */ public static SegmentInfos readSegmentInfos(IndexCommit commit) throws IOException { - // Using commit.getSegmentsFileName() does NOT work here, have to - // manually create the segment filename + // Using commit.getSegmentsFileName() does NOT work here, have to manually create the segment filename String filename = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", commit.getGeneration()); - return SegmentInfos.readCommit(commit.getDirectory(), filename); + return readSegmentInfos(filename, commit.getDirectory()); } /** * Reads the segments infos from the given segments file name, failing if it fails to load */ private static SegmentInfos readSegmentInfos(String segmentsFileName, Directory directory) throws IOException { - return SegmentInfos.readCommit(directory, segmentsFileName); + // TODO Use readCommit(Directory directory, String segmentFileName, int minSupportedMajorVersion) once Lucene 10.1 is available + // and remove the try-catch block for IndexFormatTooOldException + assert IndexVersion.current().luceneVersion().equals(LUCENE_10_0_0) : "remove the try-catch block below"; + try { + return SegmentInfos.readCommit(directory, segmentsFileName); + } catch (IndexFormatTooOldException e) { + try { + // Temporary workaround until Lucene 10.1 is available: try to leverage min. read-only compatibility to read the last commit + // and then check if this is the commit we want. This should always work for the case we are interested in (archive and + // searchable snapshots indices in N-2 version) as no newer commit should be ever written. + var segmentInfos = readSegmentInfos(directory); + if (segmentsFileName.equals(segmentInfos.getSegmentsFileName())) { + return segmentInfos; + } + } catch (Exception suppressed) { + e.addSuppressed(suppressed); + } + throw e; + } } /** diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index bf2387453145d..6038a83130db5 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -295,7 +295,11 @@ static Metadata upgradeMetadata(Metadata metadata, IndexMetadataVerifier indexMe boolean changed = false; final Metadata.Builder upgradedMetadata = Metadata.builder(metadata); for (IndexMetadata indexMetadata : metadata) { - IndexMetadata newMetadata = indexMetadataVerifier.verifyIndexMetadata(indexMetadata, IndexVersions.MINIMUM_COMPATIBLE); + IndexMetadata newMetadata = indexMetadataVerifier.verifyIndexMetadata( + indexMetadata, + IndexVersions.MINIMUM_COMPATIBLE, + IndexVersions.MINIMUM_READONLY_COMPATIBLE + ); changed |= indexMetadata != newMetadata; upgradedMetadata.put(newMetadata, false); } diff --git a/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java b/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java index a15fef653dabe..9359f6e377ef4 100644 --- a/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java +++ b/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java @@ -125,6 +125,7 @@ public ClusterState execute(ClusterState currentState) { currentState.routingTable() ); IndexVersion minIndexCompatibilityVersion = currentState.nodes().getMinSupportedIndexVersion(); + IndexVersion minReadOnlyIndexCompatibilityVersion = currentState.nodes().getMinReadOnlySupportedIndexVersion(); IndexVersion maxIndexCompatibilityVersion = currentState.nodes().getMaxDataNodeCompatibleIndexVersion(); boolean importNeeded = false; StringBuilder sb = new StringBuilder(); @@ -176,7 +177,11 @@ public ClusterState execute(ClusterState currentState) { try { // The dangled index might be from an older version, we need to make sure it's compatible // with the current version. - newIndexMetadata = indexMetadataVerifier.verifyIndexMetadata(indexMetadata, minIndexCompatibilityVersion); + newIndexMetadata = indexMetadataVerifier.verifyIndexMetadata( + indexMetadata, + minIndexCompatibilityVersion, + minReadOnlyIndexCompatibilityVersion + ); newIndexMetadata = IndexMetadata.builder(newIndexMetadata) .settings( Settings.builder() diff --git a/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java b/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java index 1d032c1f400ef..c3ab2ee910805 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java @@ -220,7 +220,7 @@ protected DirectoryReader open(IndexCommit commit) throws IOException { assert Transports.assertNotTransportThread("opening index commit of a read-only engine"); DirectoryReader directoryReader = DirectoryReader.open( commit, - org.apache.lucene.util.Version.MIN_SUPPORTED_MAJOR, + IndexVersions.MINIMUM_READONLY_COMPATIBLE.luceneVersion().major, engineConfig.getLeafSorter() ); if (lazilyLoadSoftDeletes) { @@ -575,7 +575,8 @@ public void advanceMaxSeqNoOfUpdatesOrDeletes(long maxSeqNoOfUpdatesOnPrimary) { protected DirectoryReader openDirectory(Directory directory) throws IOException { assert Transports.assertNotTransportThread("opening directory reader of a read-only engine"); - final DirectoryReader reader = DirectoryReader.open(directory); + var commit = Lucene.getIndexCommit(Lucene.readSegmentInfos(directory), directory); + final DirectoryReader reader = DirectoryReader.open(commit, IndexVersions.MINIMUM_READONLY_COMPATIBLE.luceneVersion().major, null); if (lazilyLoadSoftDeletes) { return new LazySoftDeletesDirectoryReaderWrapper(reader, Lucene.SOFT_DELETES_FIELD); } else { diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 966764d2797c9..2c9741a87c335 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -2206,7 +2206,7 @@ private Engine createEngine(EngineConfig config) { * Asserts that the latest Lucene commit contains expected information about sequence numbers or ES version. */ private boolean assertLastestCommitUserData() throws IOException { - final SegmentInfos segmentCommitInfos = SegmentInfos.readLatestCommit(store.directory()); + final SegmentInfos segmentCommitInfos = store.readLastCommittedSegmentsInfo(); final Map userData = segmentCommitInfos.getUserData(); // Ensure sequence numbers are present in commit data assert userData.containsKey(SequenceNumbers.LOCAL_CHECKPOINT_KEY) : "commit point doesn't contains a local checkpoint"; diff --git a/server/src/main/java/org/elasticsearch/index/store/Store.java b/server/src/main/java/org/elasticsearch/index/store/Store.java index 322064f09cf77..64bbd15198b4b 100644 --- a/server/src/main/java/org/elasticsearch/index/store/Store.java +++ b/server/src/main/java/org/elasticsearch/index/store/Store.java @@ -819,12 +819,12 @@ public record MetadataSnapshot(Map fileMetadataMap, M public static final MetadataSnapshot EMPTY = new MetadataSnapshot(emptyMap(), emptyMap(), 0L); - static MetadataSnapshot loadFromIndexCommit(IndexCommit commit, Directory directory, Logger logger) throws IOException { + static MetadataSnapshot loadFromIndexCommit(@Nullable IndexCommit commit, Directory directory, Logger logger) throws IOException { final long numDocs; final Map metadataByFile = new HashMap<>(); final Map commitUserData; try { - final SegmentInfos segmentCommitInfos = Store.readSegmentsInfo(commit, directory); + final SegmentInfos segmentCommitInfos = readSegmentsInfo(commit, directory); numDocs = Lucene.getNumDocs(segmentCommitInfos); commitUserData = Map.copyOf(segmentCommitInfos.getUserData()); // we don't know which version was used to write so we take the max version. @@ -1449,7 +1449,6 @@ public void bootstrapNewHistory() throws IOException { * @see SequenceNumbers#MAX_SEQ_NO */ public void bootstrapNewHistory(long localCheckpoint, long maxSeqNo) throws IOException { - assert indexSettings.getIndexMetadata().isSearchableSnapshot() == false; metadataLock.writeLock().lock(); try (IndexWriter writer = newTemporaryAppendingIndexWriter(directory, null)) { final Map map = new HashMap<>(); @@ -1573,7 +1572,6 @@ private IndexWriter newTemporaryEmptyIndexWriter(final Directory dir, final Vers } private IndexWriterConfig newTemporaryIndexWriterConfig() { - assert indexSettings.getIndexMetadata().isSearchableSnapshot() == false; // this config is only used for temporary IndexWriter instances, used to initialize the index or update the commit data, // so we don't want any merges to happen var iwc = indexWriterConfigWithNoMerging(null).setSoftDeletesField(Lucene.SOFT_DELETES_FIELD).setCommitOnClose(false); diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index ddb1e3d384fbe..5aec8e0e3253c 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -1348,6 +1348,7 @@ public ClusterState execute(ClusterState currentState) { final Map shards = new HashMap<>(); final IndexVersion minIndexCompatibilityVersion = currentState.getNodes().getMinSupportedIndexVersion(); + final IndexVersion minReadOnlyIndexCompatibilityVersion = currentState.getNodes().getMinReadOnlySupportedIndexVersion(); final String localNodeId = clusterService.state().nodes().getLocalNodeId(); for (Map.Entry indexEntry : indicesToRestore.entrySet()) { final IndexId index = indexEntry.getValue(); @@ -1360,12 +1361,16 @@ public ClusterState execute(ClusterState currentState) { request.indexSettings(), request.ignoreIndexSettings() ); - if (snapshotIndexMetadata.getCompatibilityVersion().before(minIndexCompatibilityVersion)) { + if (snapshotIndexMetadata.getCompatibilityVersion().isLegacyIndexVersion()) { // adapt index metadata so that it can be understood by current version snapshotIndexMetadata = convertLegacyIndex(snapshotIndexMetadata, currentState, indicesService); } try { - snapshotIndexMetadata = indexMetadataVerifier.verifyIndexMetadata(snapshotIndexMetadata, minIndexCompatibilityVersion); + snapshotIndexMetadata = indexMetadataVerifier.verifyIndexMetadata( + snapshotIndexMetadata, + minIndexCompatibilityVersion, + minReadOnlyIndexCompatibilityVersion + ); } catch (Exception ex) { throw new SnapshotRestoreException(snapshot, "cannot restore index [" + index + "] because it cannot be upgraded", ex); } diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinExecutorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinExecutorTests.java index 34add82e66557..270315f23a53c 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinExecutorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinExecutorTests.java @@ -39,6 +39,7 @@ import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; +import org.elasticsearch.snapshots.SearchableSnapshotsSettings; import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.MockLog; @@ -58,6 +59,7 @@ import static org.elasticsearch.cluster.metadata.DesiredNodesTestCase.assertDesiredNodesStatusIsCorrect; import static org.elasticsearch.cluster.metadata.DesiredNodesTestCase.randomDesiredNode; +import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING; import static org.elasticsearch.test.VersionUtils.maxCompatibleVersion; import static org.elasticsearch.test.VersionUtils.randomCompatibleVersion; import static org.elasticsearch.test.VersionUtils.randomVersion; @@ -89,12 +91,18 @@ public void testPreventJoinClusterWithNewerIndices() { .build(); metaBuilder.put(indexMetadata, false); Metadata metadata = metaBuilder.build(); - NodeJoinExecutor.ensureIndexCompatibility(IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current(), metadata); + NodeJoinExecutor.ensureIndexCompatibility( + IndexVersions.MINIMUM_COMPATIBLE, + IndexVersions.MINIMUM_READONLY_COMPATIBLE, + IndexVersion.current(), + metadata + ); expectThrows( IllegalStateException.class, () -> NodeJoinExecutor.ensureIndexCompatibility( IndexVersions.MINIMUM_COMPATIBLE, + IndexVersions.MINIMUM_READONLY_COMPATIBLE, IndexVersionUtils.getPreviousVersion(IndexVersion.current()), metadata ) @@ -113,10 +121,136 @@ public void testPreventJoinClusterWithUnsupportedIndices() { Metadata metadata = metaBuilder.build(); expectThrows( IllegalStateException.class, - () -> NodeJoinExecutor.ensureIndexCompatibility(IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current(), metadata) + () -> NodeJoinExecutor.ensureIndexCompatibility( + IndexVersions.MINIMUM_COMPATIBLE, + IndexVersions.MINIMUM_READONLY_COMPATIBLE, + IndexVersion.current(), + metadata + ) ); } + public void testJoinClusterWithReadOnlyCompatibleIndices() { + { + var indexMetadata = IndexMetadata.builder("searchable-snapshot") + .settings( + Settings.builder() + .put(INDEX_STORE_TYPE_SETTING.getKey(), SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOT_STORE_TYPE) + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersions.MINIMUM_READONLY_COMPATIBLE) + .put(IndexMetadata.SETTING_BLOCKS_WRITE, true) + ) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + + NodeJoinExecutor.ensureIndexCompatibility( + IndexVersions.MINIMUM_COMPATIBLE, + IndexVersions.MINIMUM_READONLY_COMPATIBLE, + IndexVersion.current(), + Metadata.builder().put(indexMetadata, false).build() + ); + } + { + var indexMetadata = IndexMetadata.builder("searchable-snapshot-no-write-block") + .settings( + Settings.builder() + .put(INDEX_STORE_TYPE_SETTING.getKey(), SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOT_STORE_TYPE) + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersions.MINIMUM_READONLY_COMPATIBLE) + ) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + + expectThrows( + IllegalStateException.class, + () -> NodeJoinExecutor.ensureIndexCompatibility( + IndexVersions.MINIMUM_COMPATIBLE, + IndexVersions.MINIMUM_READONLY_COMPATIBLE, + IndexVersion.current(), + Metadata.builder().put(indexMetadata, false).build() + ) + ); + } + { + var indexMetadata = IndexMetadata.builder("archive") + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.fromId(randomFrom(5000099, 6000099))) + .put(IndexMetadata.SETTING_VERSION_COMPATIBILITY, IndexVersions.MINIMUM_READONLY_COMPATIBLE) + .put(IndexMetadata.SETTING_BLOCKS_WRITE, true) + ) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + + NodeJoinExecutor.ensureIndexCompatibility( + IndexVersions.MINIMUM_COMPATIBLE, + IndexVersions.MINIMUM_READONLY_COMPATIBLE, + IndexVersion.current(), + Metadata.builder().put(indexMetadata, false).build() + ); + } + { + var indexMetadata = IndexMetadata.builder("archive-no-write-block") + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.fromId(randomFrom(5000099, 6000099))) + .put(IndexMetadata.SETTING_VERSION_COMPATIBILITY, IndexVersions.MINIMUM_READONLY_COMPATIBLE) + ) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + + expectThrows( + IllegalStateException.class, + () -> NodeJoinExecutor.ensureIndexCompatibility( + IndexVersions.MINIMUM_COMPATIBLE, + IndexVersions.MINIMUM_READONLY_COMPATIBLE, + IndexVersion.current(), + Metadata.builder().put(indexMetadata, false).build() + ) + ); + } + { + var indexMetadata = IndexMetadata.builder("legacy") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.fromId(randomFrom(5000099, 6000099)))) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + + expectThrows( + IllegalStateException.class, + () -> NodeJoinExecutor.ensureIndexCompatibility( + IndexVersions.MINIMUM_COMPATIBLE, + IndexVersions.MINIMUM_READONLY_COMPATIBLE, + IndexVersion.current(), + Metadata.builder().put(indexMetadata, false).build() + ) + ); + } + { + var indexMetadata = IndexMetadata.builder("read-only-compatible-but-unsupported") + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersions.MINIMUM_READONLY_COMPATIBLE) + .put(IndexMetadata.SETTING_BLOCKS_WRITE, true) + ) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + + expectThrows( + IllegalStateException.class, + () -> NodeJoinExecutor.ensureIndexCompatibility( + IndexVersions.MINIMUM_COMPATIBLE, + IndexVersions.MINIMUM_READONLY_COMPATIBLE, + IndexVersion.current(), + Metadata.builder().put(indexMetadata, false).build() + ) + ); + } + } + public void testPreventJoinClusterWithUnsupportedNodeVersions() { DiscoveryNodes.Builder builder = DiscoveryNodes.builder(); final Version version = randomCompatibleVersion(random(), Version.CURRENT); @@ -467,7 +601,13 @@ public void testSuccess() { .build(); metaBuilder.put(indexMetadata, false); Metadata metadata = metaBuilder.build(); - NodeJoinExecutor.ensureIndexCompatibility(IndexVersions.MINIMUM_READONLY_COMPATIBLE, IndexVersion.current(), metadata); + NodeJoinExecutor.ensureIndexCompatibility( + // randomCompatibleVersionSettings() can set a version as low as MINIMUM_READONLY_COMPATIBLE + IndexVersions.MINIMUM_READONLY_COMPATIBLE, + IndexVersions.MINIMUM_READONLY_COMPATIBLE, + IndexVersion.current(), + metadata + ); } public static Settings.Builder randomCompatibleVersionSettings() { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java index 6ee86470861b4..417ae89da0a69 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java @@ -18,11 +18,13 @@ import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.plugins.MapperPlugin; +import org.elasticsearch.snapshots.SearchableSnapshotsSettings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.index.IndexVersionUtils; import java.util.Collections; +import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING; import static org.hamcrest.Matchers.equalTo; public class IndexMetadataVerifierTests extends ESTestCase { @@ -97,7 +99,8 @@ public void testCustomSimilarity() { .put("index.similarity.my_similarity.after_effect", "l") .build() ); - service.verifyIndexMetadata(src, IndexVersions.MINIMUM_READONLY_COMPATIBLE); + // The random IndexMetadata.SETTING_VERSION_CREATED in IndexMetadata can be as low as MINIMUM_READONLY_COMPATIBLE + service.verifyIndexMetadata(src, IndexVersions.MINIMUM_READONLY_COMPATIBLE, IndexVersions.MINIMUM_READONLY_COMPATIBLE); } public void testIncompatibleVersion() { @@ -110,7 +113,7 @@ public void testIncompatibleVersion() { ); String message = expectThrows( IllegalStateException.class, - () -> service.verifyIndexMetadata(metadata, IndexVersions.MINIMUM_COMPATIBLE) + () -> service.verifyIndexMetadata(metadata, IndexVersions.MINIMUM_COMPATIBLE, IndexVersions.MINIMUM_READONLY_COMPATIBLE) ).getMessage(); assertThat( message, @@ -132,7 +135,80 @@ public void testIncompatibleVersion() { indexCreated = IndexVersionUtils.randomVersionBetween(random(), minCompat, IndexVersion.current()); IndexMetadata goodMeta = newIndexMeta("foo", Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, indexCreated).build()); - service.verifyIndexMetadata(goodMeta, IndexVersions.MINIMUM_COMPATIBLE); + service.verifyIndexMetadata(goodMeta, IndexVersions.MINIMUM_COMPATIBLE, IndexVersions.MINIMUM_READONLY_COMPATIBLE); + } + + public void testReadOnlyVersionCompatibility() { + var service = getIndexMetadataVerifier(); + var indexCreated = IndexVersions.MINIMUM_READONLY_COMPATIBLE; + { + var idxMetadata = newIndexMeta( + "not-searchable-snapshot", + Settings.builder() + .put(IndexMetadata.SETTING_BLOCKS_WRITE, randomBoolean()) + .put(IndexMetadata.SETTING_VERSION_CREATED, indexCreated) + .build() + ); + String message = expectThrows( + IllegalStateException.class, + () -> service.verifyIndexMetadata(idxMetadata, IndexVersions.MINIMUM_COMPATIBLE, IndexVersions.MINIMUM_READONLY_COMPATIBLE) + ).getMessage(); + assertThat( + message, + equalTo( + "The index [not-searchable-snapshot/" + + idxMetadata.getIndexUUID() + + "] has current compatibility version [" + + indexCreated.toReleaseVersion() + + "] " + + "but the minimum compatible version is [" + + IndexVersions.MINIMUM_COMPATIBLE.toReleaseVersion() + + "]. It should be re-indexed in Elasticsearch " + + (Version.CURRENT.major - 1) + + ".x before upgrading to " + + Build.current().version() + + "." + ) + ); + } + { + var idxMetadata = newIndexMeta( + "not-read-only", + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, indexCreated) + .put(INDEX_STORE_TYPE_SETTING.getKey(), SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOT_STORE_TYPE) + .build() + ); + String message = expectThrows( + IllegalStateException.class, + () -> service.verifyIndexMetadata(idxMetadata, IndexVersions.MINIMUM_COMPATIBLE, IndexVersions.MINIMUM_READONLY_COMPATIBLE) + ).getMessage(); + assertThat( + message, + equalTo( + "The index [not-read-only/" + + idxMetadata.getIndexUUID() + + "] created in version [" + + indexCreated + + "] with current compatibility version [" + + indexCreated.toReleaseVersion() + + "] must be marked as read-only using the setting [index.blocks.write] set to [true] before upgrading to " + + Build.current().version() + + "." + ) + ); + } + { + var idxMetadata = newIndexMeta( + "good", + Settings.builder() + .put(IndexMetadata.SETTING_BLOCKS_WRITE, true) + .put(IndexMetadata.SETTING_VERSION_CREATED, indexCreated) + .put(INDEX_STORE_TYPE_SETTING.getKey(), SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOT_STORE_TYPE) + .build() + ); + service.verifyIndexMetadata(idxMetadata, IndexVersions.MINIMUM_COMPATIBLE, IndexVersions.MINIMUM_READONLY_COMPATIBLE); + } } private IndexMetadataVerifier getIndexMetadataVerifier() { diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java index a161794e35b91..fd0718e5280fe 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java @@ -264,7 +264,11 @@ private static class MockIndexMetadataVerifier extends IndexMetadataVerifier { } @Override - public IndexMetadata verifyIndexMetadata(IndexMetadata indexMetadata, IndexVersion minimumIndexCompatibilityVersion) { + public IndexMetadata verifyIndexMetadata( + IndexMetadata indexMetadata, + IndexVersion minimumIndexCompatibilityVersion, + IndexVersion minimumReadOnlyIndexCompatibilityVersion + ) { return upgrade ? IndexMetadata.builder(indexMetadata).build() : indexMetadata; } } diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java index 82154848ea039..39c327ddee228 100644 --- a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java +++ b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java @@ -252,7 +252,11 @@ public Transport.Connection getConnection(DiscoveryNode node) { ) { // metadata upgrader should do nothing @Override - public IndexMetadata verifyIndexMetadata(IndexMetadata indexMetadata, IndexVersion minimumIndexCompatibilityVersion) { + public IndexMetadata verifyIndexMetadata( + IndexMetadata indexMetadata, + IndexVersion minimumIndexCompatibilityVersion, + IndexVersion minimumReadOnlyIndexCompatibilityVersion + ) { return indexMetadata; } };