mirrors
/
elasticsearch
mirror of https://mirror.osredm.com/root/elasticsearch.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support recovery for closed shard in N-2 version (#121141) 

Allow shard recovery for shards in version N-2 that have been verified before being closed, but not verified as read-only, in 7.x or 8.x. Reopening such closed indices automatically adds an index.blocks.write.

Requires #120595 for the 8.x changes.

Closes ES-10320
Closes #121170
Closes #121171
This commit is contained in:
Tanguy Leroux 2025-01-29 15:43:26 +01:00 committed by GitHub
parent 49a20c149c
commit 7b7cd1f87c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 184 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
muted-tests.yml
View File

@ -305,12 +305,6 @@ tests:
- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
method: testProfileIndexAutoCreation
issue: https://github.com/elastic/elasticsearch/issues/120987
- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase
method: testRestoreIndex {p0=[9.0.0, 9.0.0, 8.18.0]}
issue: https://github.com/elastic/elasticsearch/issues/121170
- class: org.elasticsearch.lucene.RollingUpgradeLuceneIndexCompatibilityTestCase
method: testRestoreIndex {p0=[9.0.0, 8.18.0, 8.18.0]}
issue: https://github.com/elastic/elasticsearch/issues/121171
- class: org.elasticsearch.xpack.security.FileSettingsRoleMappingsRestartIT
method: testFileSettingsReprocessedOnRestartWithoutVersionChange
issue: https://github.com/elastic/elasticsearch/issues/120964

98
qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartLuceneIndexCompatibilityIT.java
View File

@ -9,10 +9,10 @@
package org.elasticsearch.lucene;
import org.elasticsearch.client.ResponseException;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.translog.Translog;
import org.elasticsearch.repositories.fs.FsRepository;
import org.elasticsearch.test.cluster.util.Version;
@ -22,10 +22,10 @@ import static org.elasticsearch.cluster.metadata.MetadataIndexStateService.INDEX
import static org.elasticsearch.cluster.metadata.MetadataIndexStateService.VERIFIED_BEFORE_CLOSE_SETTING;
import static org.elasticsearch.cluster.metadata.MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.either;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.not;
public class FullClusterRestartLuceneIndexCompatibilityIT extends FullClusterRestartIndexCompatibilityTestCase {
@ -51,7 +51,6 @@ public class FullClusterRestartLuceneIndexCompatibilityIT extends FullClusterRes
Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, randomInt(2))
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
.build()
);
indexDocs(index, numDocs);
@ -111,12 +110,10 @@ public class FullClusterRestartLuceneIndexCompatibilityIT extends FullClusterRes
.putNull(IndexMetadata.APIBlock.WRITE.settingName())
.putNull(IndexMetadata.APIBlock.READ_ONLY.settingName())
);
logger.debug("--> but attempts to re-opening [{}] should fail due to the missing block", index);
var ex = expectThrows(ResponseException.class, () -> openIndex(index));
assertThat(ex.getMessage(), containsString("must be marked as read-only"));
// TODO this could be randomized once we support recovering verified-before-close closed indices with no write/ro block
addIndexBlock(index, IndexMetadata.APIBlock.WRITE);
assertThat(indexBlocks(index), contains(INDEX_CLOSED_BLOCK));
assertIndexSetting(index, VERIFIED_BEFORE_CLOSE_SETTING, is(true));
assertIndexSetting(index, VERIFIED_READ_ONLY_SETTING, is(true));
}
var block = indexBlocks(index).stream().filter(c -> c.equals(INDEX_WRITE_BLOCK) || c.equals(INDEX_READ_ONLY_BLOCK)).findFirst();
@ -128,11 +125,11 @@ public class FullClusterRestartLuceneIndexCompatibilityIT extends FullClusterRes
.putNull(IndexMetadata.APIBlock.READ_ONLY.settingName())
.put(IndexMetadata.APIBlock.WRITE.settingName(), true)
);
}
assertThat(indexBlocks(index), isClosed ? contains(INDEX_CLOSED_BLOCK, INDEX_WRITE_BLOCK) : contains(INDEX_WRITE_BLOCK));
assertIndexSetting(index, VERIFIED_BEFORE_CLOSE_SETTING, is(isClosed));
assertIndexSetting(index, VERIFIED_READ_ONLY_SETTING, is(true));
assertThat(indexBlocks(index), isClosed ? contains(INDEX_CLOSED_BLOCK, INDEX_WRITE_BLOCK) : contains(INDEX_WRITE_BLOCK));
assertIndexSetting(index, VERIFIED_BEFORE_CLOSE_SETTING, is(isClosed));
assertIndexSetting(index, VERIFIED_READ_ONLY_SETTING, is(true));
}
var numberOfReplicas = getNumberOfReplicas(index);
if (0 < numberOfReplicas) {
@ -173,6 +170,71 @@ public class FullClusterRestartLuceneIndexCompatibilityIT extends FullClusterRes
}
}
/**
* Creates an index on N-2, closes it on N-1 (without marking it as read-only), then upgrades to N.
*/
public void testClosedIndexUpgrade() throws Exception {
final String index = suffix("index");
final int numDocs = 2437;
if (isFullyUpgradedTo(VERSION_MINUS_2)) {
createIndex(
client(),
index,
Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, randomInt(2))
.put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), randomFrom(Translog.Durability.values()))
.build()
);
indexDocs(index, numDocs);
return;
}
assertThat(indexVersion(index), equalTo(VERSION_MINUS_2));
ensureGreen(index);
if (isIndexClosed(index) == false) {
assertDocCount(client(), index, numDocs);
}
if (isFullyUpgradedTo(VERSION_MINUS_1)) {
logger.debug("--> [{}] closing index before upgrade without adding a read_only/write block", index);
closeIndex(index);
assertThat(indexBlocks(index), contains(INDEX_CLOSED_BLOCK));
assertThat(indexBlocks(index), not(contains(INDEX_WRITE_BLOCK)));
assertIndexSetting(index, VERIFIED_BEFORE_CLOSE_SETTING, is(true));
assertIndexSetting(index, VERIFIED_READ_ONLY_SETTING, is(false));
return;
}
if (isFullyUpgradedTo(VERSION_CURRENT)) {
assertThat(indexBlocks(index), contains(INDEX_CLOSED_BLOCK));
assertIndexSetting(index, VERIFIED_BEFORE_CLOSE_SETTING, is(true));
assertIndexSetting(index, VERIFIED_READ_ONLY_SETTING, is(false));
logger.debug("--> re-opening index [{}] will add a write block", index);
openIndex(index);
ensureGreen(index);
assertThat(indexBlocks(index), contains(INDEX_WRITE_BLOCK));
assertIndexSetting(index, VERIFIED_BEFORE_CLOSE_SETTING, is(false));
assertIndexSetting(index, VERIFIED_READ_ONLY_SETTING, is(true));
assertDocCount(client(), index, numDocs);
logger.debug("--> closing index [{}]", index);
closeIndex(index);
ensureGreen(index);
assertThat(indexBlocks(index), contains(INDEX_CLOSED_BLOCK, INDEX_WRITE_BLOCK));
assertIndexSetting(index, VERIFIED_BEFORE_CLOSE_SETTING, is(true));
assertIndexSetting(index, VERIFIED_READ_ONLY_SETTING, is(true));
deleteIndex(index);
}
}
/**
* Creates an index on N-2, marks as read-only on N-1 and creates a snapshot, then restores the snapshot on N.
*/
@ -190,11 +252,7 @@ public class FullClusterRestartLuceneIndexCompatibilityIT extends FullClusterRes
createIndex(
client(),
index,
Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
.build()
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
);
logger.debug("--> indexing [{}] docs in [{}]", numDocs, index);
@ -272,11 +330,7 @@ public class FullClusterRestartLuceneIndexCompatibilityIT extends FullClusterRes
createIndex(
client(),
index,
Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
.build()
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
);
logger.debug("--> indexing [{}] docs in [{}]", numDocs, index);

13
qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartSearchableSnapshotIndexCompatibilityIT.java
View File

@ -11,7 +11,6 @@ package org.elasticsearch.lucene;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.repositories.fs.FsRepository;
import org.elasticsearch.test.cluster.util.Version;
@ -46,11 +45,7 @@ public class FullClusterRestartSearchableSnapshotIndexCompatibilityIT extends Fu
createIndex(
client(),
index,
Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
.build()
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
);
logger.debug("--> indexing [{}] docs in [{}]", numDocs, index);
@ -125,11 +120,7 @@ public class FullClusterRestartSearchableSnapshotIndexCompatibilityIT extends Fu
createIndex(
client(),
index,
Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
.build()
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
);
logger.debug("--> indexing [{}] docs in [{}]", numDocs, index);

100
qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/RollingUpgradeLuceneIndexCompatibilityTestCase.java
View File

@ -14,6 +14,7 @@ import org.elasticsearch.client.ResponseException;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.translog.Translog;
import org.elasticsearch.repositories.fs.FsRepository;
import org.elasticsearch.test.cluster.util.Version;
@ -54,11 +55,7 @@ public class RollingUpgradeLuceneIndexCompatibilityTestCase extends RollingUpgra
createIndex(
client(),
index,
Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
.build()
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
);
indexDocs(index, numDocs);
return;
@ -181,6 +178,75 @@ public class RollingUpgradeLuceneIndexCompatibilityTestCase extends RollingUpgra
}
}
/**
* Creates an index on N-2, closes it on N-1 and then upgrades the cluster.
*/
public void testClosedIndexUpgrade() throws Exception {
final String index = suffix("closed-rolling-upgraded");
final int numDocs = 1543;
if (isFullyUpgradedTo(VERSION_MINUS_2)) {
createIndex(
client(),
index,
Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), randomFrom(Translog.Durability.values()))
.build()
);
indexDocs(index, numDocs);
return;
}
assertThat(indexVersion(index), equalTo(VERSION_MINUS_2));
ensureGreen(index);
if (isIndexClosed(index) == false) {
assertDocCount(client(), index, numDocs);
}
if (isFullyUpgradedTo(VERSION_MINUS_1)) {
logger.debug("--> closing index [{}]", index);
closeIndex(index);
assertThat(indexBlocks(index), contains(INDEX_CLOSED_BLOCK));
assertIndexSetting(index, VERIFIED_BEFORE_CLOSE_SETTING, is(true));
assertIndexSetting(index, VERIFIED_READ_ONLY_SETTING, is(false));
return;
}
if (nodesVersions().values().stream().anyMatch(v -> v.onOrAfter(VERSION_CURRENT))) {
long upgradedNodes = nodesVersions().values().stream().filter(v -> v.onOrAfter(VERSION_CURRENT)).count();
if (upgradedNodes == 1) {
// Mixed cluster with 1 of the 3 nodes upgraded: the index hasn't been reopened yet
assertThat(indexBlocks(index), contains(INDEX_CLOSED_BLOCK));
assertIndexSetting(index, VERIFIED_BEFORE_CLOSE_SETTING, is(true));
assertIndexSetting(index, VERIFIED_READ_ONLY_SETTING, is(false));
} else {
// Index has been reopened at least once, it should have an additional write block and the verified-read-only setting
assertThat(indexBlocks(index), contains(INDEX_CLOSED_BLOCK, INDEX_WRITE_BLOCK));
assertIndexSetting(index, VERIFIED_BEFORE_CLOSE_SETTING, is(true));
assertIndexSetting(index, VERIFIED_READ_ONLY_SETTING, is(true));
}
openIndex(index);
ensureGreen(index);
assertThat(indexBlocks(index), contains(INDEX_WRITE_BLOCK));
assertIndexSetting(index, VERIFIED_BEFORE_CLOSE_SETTING, is(false));
assertIndexSetting(index, VERIFIED_READ_ONLY_SETTING, is(true));
assertDocCount(client(), index, numDocs);
updateRandomIndexSettings(index);
updateRandomMappings(index);
closeIndex(index);
ensureGreen(index);
}
}
/**
* Creates an index on N-2, marks as read-only on N-1 and creates a snapshot, then restores the snapshot during rolling upgrades to N.
*/
@ -198,11 +264,7 @@ public class RollingUpgradeLuceneIndexCompatibilityTestCase extends RollingUpgra
createIndex(
client(),
index,
Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
.build()
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
);
logger.debug("--> indexing [{}] docs in [{}]", numDocs, index);
@ -253,19 +315,29 @@ public class RollingUpgradeLuceneIndexCompatibilityTestCase extends RollingUpgra
closeIndex(restoredIndex);
ensureGreen(restoredIndex);
assertThat(indexBlocks(restoredIndex), contains(INDEX_CLOSED_BLOCK, INDEX_WRITE_BLOCK));
assertIndexSetting(restoredIndex, VERIFIED_BEFORE_CLOSE_SETTING, is(true));
assertIndexSetting(restoredIndex, VERIFIED_READ_ONLY_SETTING, is(true));
logger.debug("--> write API block can be removed on a closed index: INDEX_CLOSED_BLOCK already blocks writes");
updateIndexSettings(restoredIndex, Settings.builder().putNull(IndexMetadata.APIBlock.WRITE.settingName()));
logger.debug("--> but attempts to re-opening [{}] should fail due to the missing block", restoredIndex);
ex = expectThrows(ResponseException.class, () -> openIndex(restoredIndex));
assertThat(ex.getMessage(), containsString("must be marked as read-only"));
assertThat(indexBlocks(restoredIndex), contains(INDEX_CLOSED_BLOCK));
assertIndexSetting(restoredIndex, VERIFIED_BEFORE_CLOSE_SETTING, is(true));
assertIndexSetting(restoredIndex, VERIFIED_READ_ONLY_SETTING, is(true));
addIndexBlock(restoredIndex, IndexMetadata.APIBlock.WRITE);
if (randomBoolean()) {
addIndexBlock(restoredIndex, IndexMetadata.APIBlock.WRITE);
}
logger.debug("--> re-opening restored index [{}]", restoredIndex);
openIndex(restoredIndex);
ensureGreen(restoredIndex);
assertThat(indexBlocks(restoredIndex), contains(INDEX_WRITE_BLOCK));
assertIndexSetting(restoredIndex, VERIFIED_BEFORE_CLOSE_SETTING, is(false));
assertIndexSetting(restoredIndex, VERIFIED_READ_ONLY_SETTING, is(true));
assertDocCount(client(), restoredIndex, numDocs);
logger.debug("--> deleting restored index [{}]", restoredIndex);

13
qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/RollingUpgradeSearchableSnapshotIndexCompatibilityIT.java
View File

@ -13,7 +13,6 @@ import org.elasticsearch.client.Request;
import org.elasticsearch.client.ResponseException;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.repositories.fs.FsRepository;
import org.elasticsearch.test.cluster.util.Version;
@ -51,11 +50,7 @@ public class RollingUpgradeSearchableSnapshotIndexCompatibilityIT extends Rollin
createIndex(
client(),
index,
Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
.build()
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
);
logger.debug("--> indexing [{}] docs in [{}]", numDocs, index);
@ -122,11 +117,7 @@ public class RollingUpgradeSearchableSnapshotIndexCompatibilityIT extends Rollin
createIndex(
client(),
index,
Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
.build()
Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
);
logger.debug("--> indexing [{}] docs in [{}]", numDocs, index);

9
server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java
View File

@ -39,6 +39,7 @@ import java.util.Collections;
import java.util.Map;
import java.util.Set;
import static org.elasticsearch.cluster.metadata.MetadataIndexStateService.isIndexVerifiedBeforeClosed;
import static org.elasticsearch.core.Strings.format;
/**
@ -164,7 +165,7 @@ public class IndexMetadataVerifier {
) {
if (isReadOnlyCompatible(indexMetadata, minimumCompatible, minimumReadOnlyCompatible)) {
assert isFullySupportedVersion(indexMetadata, minimumCompatible) == false : indexMetadata;
final boolean isReadOnly = hasIndexWritesBlock(indexMetadata);
final boolean isReadOnly = hasReadOnlyBlocks(indexMetadata) || isIndexVerifiedBeforeClosed(indexMetadata);
if (isReadOnly == false) {
throw new IllegalStateException(
"The index "
@ -185,7 +186,7 @@ public class IndexMetadataVerifier {
return false;
}
private static boolean isReadOnlyCompatible(
public static boolean isReadOnlyCompatible(
IndexMetadata indexMetadata,
IndexVersion minimumCompatible,
IndexVersion minimumReadOnlyCompatible
@ -208,7 +209,7 @@ public class IndexMetadataVerifier {
return false;
}
private static boolean hasIndexWritesBlock(IndexMetadata indexMetadata) {
static boolean hasReadOnlyBlocks(IndexMetadata indexMetadata) {
var indexSettings = indexMetadata.getSettings();
if (IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.get(indexSettings) || IndexMetadata.INDEX_READ_ONLY_SETTING.get(indexSettings)) {
return indexMetadata.isSearchableSnapshot()
@ -220,7 +221,7 @@ public class IndexMetadataVerifier {
public static boolean isReadOnlyVerified(IndexMetadata indexMetadata) {
if (isReadOnlyCompatible(indexMetadata, IndexVersions.MINIMUM_COMPATIBLE, IndexVersions.MINIMUM_READONLY_COMPATIBLE)) {
return hasIndexWritesBlock(indexMetadata);
return hasReadOnlyBlocks(indexMetadata);
}
return false;
}

13
server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java
View File

@ -90,6 +90,7 @@ import java.util.function.Consumer;
import java.util.stream.Collectors;
import static java.util.stream.Collectors.joining;
import static org.elasticsearch.cluster.metadata.IndexMetadataVerifier.hasReadOnlyBlocks;
import static org.elasticsearch.core.Strings.format;
/**
@ -1185,6 +1186,18 @@ public class MetadataIndexStateService {
final Settings.Builder updatedSettings = Settings.builder().put(indexMetadata.getSettings());
updatedSettings.remove(VERIFIED_BEFORE_CLOSE_SETTING.getKey());
// Reopening a read-only compatible index that has not been marked as read-only is possible if the index was
// verified-before-close in the first place.
var compatibilityVersion = indexMetadata.getCompatibilityVersion();
if (compatibilityVersion.before(minIndexCompatibilityVersion) && hasReadOnlyBlocks(indexMetadata) == false) {
if (isIndexVerifiedBeforeClosed(indexMetadata)) {
updatedSettings.put(VERIFIED_READ_ONLY_SETTING.getKey(), true);
// at least set a write block if the index was verified-before-close at the time the cluster was upgraded
blocks.addIndexBlock(index.getName(), APIBlock.WRITE.block);
updatedSettings.put(APIBlock.WRITE.settingName(), true);
} // or else, the following indexMetadataVerifier.verifyIndexMetadata() should throw.
}
IndexMetadata newIndexMetadata = IndexMetadata.builder(indexMetadata)
.state(IndexMetadata.State.OPEN)
.settingsVersion(indexMetadata.getSettingsVersion() + 1)