mirrors
/
elasticsearch
mirror of https://mirror.osredm.com/root/elasticsearch.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove obsolete Metadata BWC for repositories (#129685) 

When migrating RepositoriesMetadata from cluster custom to project
custom (#125398), we needed temporary BWC handling for clusters running
on a version that is before this change but after the initial MP change.
Such a cluster can only exist in the serverless environment which has
progressed way past any applicable versions. Therefore we no longer need
the BWC handling and this PR removes it.

Relates: #125398
This commit is contained in:
Yang Wang 2025-06-20 11:34:48 +10:00 committed by GitHub
parent 4ce06d1aa2
commit 0e83b19de7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 29 additions and 257 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
server/src/main/java/org/elasticsearch/TransportVersions.java
View File

@ -244,7 +244,6 @@ public class TransportVersions {
public static final TransportVersion RESCORE_VECTOR_ALLOW_ZERO = def(9_039_0_00);
public static final TransportVersion PROJECT_ID_IN_SNAPSHOT = def(9_040_0_00);
public static final TransportVersion INDEX_STATS_AND_METADATA_INCLUDE_PEAK_WRITE_LOAD = def(9_041_0_00);
public static final TransportVersion REPOSITORIES_METADATA_AS_PROJECT_CUSTOM = def(9_042_0_00);
public static final TransportVersion BATCHED_QUERY_PHASE_VERSION = def(9_043_0_00);
public static final TransportVersion REMOTE_EXCEPTION = def(9_044_0_00);
public static final TransportVersion ESQL_REMOVE_AGGREGATE_TYPE = def(9_045_0_00);

222
server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java
View File

@ -11,7 +11,6 @@ package org.elasticsearch.cluster.metadata;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.util.SetOnce;
import org.elasticsearch.TransportVersions;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.Diff;
@ -53,7 +52,6 @@ import org.elasticsearch.xcontent.XContentParser;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
@ -997,12 +995,11 @@ public class Metadata implements Diffable<Metadata>, ChunkedToXContent {
multiProject = null;
} else {
fromNodeBeforeMultiProjectsSupport = false;
// Repositories metadata is sent as Metadata#customs diff from old node. We need to
// 1. Split it from the Metadata#customs diff
// 2. Merge it into the default project's ProjectMetadataDiff
final var bwcCustoms = maybeReadBwcCustoms(in);
clusterCustoms = bwcCustoms.v1();
final var defaultProjectCustoms = bwcCustoms.v2();
clusterCustoms = DiffableUtils.readImmutableOpenMapDiff(
in,
DiffableUtils.getStringKeySerializer(),
CLUSTER_CUSTOM_VALUE_SERIALIZER
);
reservedStateMetadata = DiffableUtils.readImmutableOpenMapDiff(
in,
@ -1011,61 +1008,15 @@ public class Metadata implements Diffable<Metadata>, ChunkedToXContent {
);
singleProject = null;
multiProject = readMultiProjectDiffs(in, defaultProjectCustoms);
}
}
private static
Tuple<
MapDiff<String, ClusterCustom, ImmutableOpenMap<String, ClusterCustom>>,
MapDiff<String, ProjectCustom, ImmutableOpenMap<String, ProjectCustom>>>
maybeReadBwcCustoms(StreamInput in) throws IOException {
if (in.getTransportVersion().before(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM)) {
return readBwcCustoms(in);
} else {
return new Tuple<>(
DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), CLUSTER_CUSTOM_VALUE_SERIALIZER),
null
multiProject = DiffableUtils.readJdkMapDiff(
in,
PROJECT_ID_SERIALIZER,
ProjectMetadata::readFrom,
ProjectMetadata.ProjectMetadataDiff::new
);
}
}
@SuppressWarnings("unchecked")
private static MapDiff<ProjectId, ProjectMetadata, Map<ProjectId, ProjectMetadata>> readMultiProjectDiffs(
StreamInput in,
MapDiff<String, ProjectCustom, ImmutableOpenMap<String, ProjectCustom>> defaultProjectCustoms
) throws IOException {
final var multiProject = DiffableUtils.readJdkMapDiff(
in,
PROJECT_ID_SERIALIZER,
ProjectMetadata::readFrom,
ProjectMetadata.ProjectMetadataDiff::new
);
// If the defaultProjectCustoms has content, the diff is read from an old node. We need to merge it into the
// default project's ProjectMetadataDiff
if (defaultProjectCustoms != null && defaultProjectCustoms.isEmpty() == false) {
return DiffableUtils.updateDiffsAndUpserts(multiProject, ProjectId.DEFAULT::equals, (k, v) -> {
assert ProjectId.DEFAULT.equals(k) : k;
assert v instanceof ProjectMetadata.ProjectMetadataDiff : v;
final var projectMetadataDiff = (ProjectMetadata.ProjectMetadataDiff) v;
return projectMetadataDiff.withCustoms(
DiffableUtils.merge(
projectMetadataDiff.customs(),
defaultProjectCustoms,
DiffableUtils.getStringKeySerializer(),
BWC_CUSTOM_VALUE_SERIALIZER
)
);
}, (k, v) -> {
assert ProjectId.DEFAULT.equals(k) : k;
return ProjectMetadata.builder(v).clearCustoms().customs(defaultProjectCustoms.apply(v.customs())).build();
});
} else {
return multiProject;
}
}
@SuppressWarnings("unchecked")
private static
Tuple<
@ -1108,104 +1059,15 @@ public class Metadata implements Diffable<Metadata>, ChunkedToXContent {
singleProject.templates().writeTo(out);
buildUnifiedCustomDiff().writeTo(out);
buildUnifiedReservedStateMetadataDiff().writeTo(out);
} else {
final var multiProjectToWrite = multiProject != null
? multiProject
: DiffableUtils.singleEntryDiff(DEFAULT_PROJECT_ID, singleProject, PROJECT_ID_SERIALIZER);
if (out.getTransportVersion().before(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM)) {
writeDiffWithRepositoriesMetadataAsClusterCustom(out, clusterCustoms, multiProjectToWrite, reservedStateMetadata);
} else {
clusterCustoms.writeTo(out);
reservedStateMetadata.writeTo(out);
multiProjectToWrite.writeTo(out);
}
}
}
@SuppressWarnings({ "rawtypes", "unchecked" })
private static void writeDiffWithRepositoriesMetadataAsClusterCustom(
StreamOutput out,
MapDiff<String, ClusterCustom, ImmutableOpenMap<String, ClusterCustom>> clusterCustoms,
MapDiff<ProjectId, ProjectMetadata, Map<ProjectId, ProjectMetadata>> multiProject,
MapDiff<String, ReservedStateMetadata, ImmutableOpenMap<String, ReservedStateMetadata>> reservedStateMetadata
) throws IOException {
assert out.getTransportVersion().onOrAfter(TransportVersions.MULTI_PROJECT)
&& out.getTransportVersion().before(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM) : out.getTransportVersion();
// For old nodes, RepositoriesMetadata needs to be sent as a cluster custom. This is possible when (a) the repositories
// are defined only for the default project or (b) no repositories at all. What we need to do are:
// 1. Iterate through the multi-project's MapDiff to extract the RepositoriesMetadata of the default project
// 2. Throws if any repositories are found for non-default projects
// 3. Merge default project's RepositoriesMetadata into Metadata#customs
final var combineClustersCustoms = new SetOnce<MapDiff<String, MetadataCustom, Map<String, MetadataCustom>>>();
final var updatedMultiProject = DiffableUtils.updateDiffsAndUpserts(multiProject, ignore -> true, (k, v) -> {
assert v instanceof ProjectMetadata.ProjectMetadataDiff : v;
final var projectMetadataDiff = (ProjectMetadata.ProjectMetadataDiff) v;
final var bwcCustoms = DiffableUtils.split(
projectMetadataDiff.customs(),
RepositoriesMetadata.TYPE::equals,
PROJECT_CUSTOM_VALUE_SERIALIZER,
type -> RepositoriesMetadata.TYPE.equals(type) == false,
PROJECT_CUSTOM_VALUE_SERIALIZER
);
// Simply return if RepositoriesMetadata is not found
if (bwcCustoms.v1().isEmpty()) {
return projectMetadataDiff;
}
// RepositoriesMetadata can only be defined for the default project. Otherwise throw exception.
if (ProjectId.DEFAULT.equals(k) == false) {
throwForVersionBeforeRepositoriesMetadataMigration(out);
}
// RepositoriesMetadata is found for the default project as a diff, merge it into the Metadata#customs
combineClustersCustoms.set(
DiffableUtils.<String, MetadataCustom, ClusterCustom, ProjectCustom, Map<String, MetadataCustom>>merge(
clusterCustoms,
bwcCustoms.v1(),
DiffableUtils.getStringKeySerializer()
)
);
return projectMetadataDiff.withCustoms(bwcCustoms.v2());
}, (k, v) -> {
final ProjectCustom projectCustom = v.customs().get(RepositoriesMetadata.TYPE);
// Simply return if RepositoriesMetadata is not found
if (projectCustom == null) {
return v;
}
// RepositoriesMetadata can only be defined for the default project. Otherwise throw exception.
if (ProjectId.DEFAULT.equals(k) == false) {
throwForVersionBeforeRepositoriesMetadataMigration(out);
}
// RepositoriesMetadata found for the default project as an upsert, package it as MapDiff and merge into Metadata#customs
combineClustersCustoms.set(
DiffableUtils.<String, MetadataCustom, ClusterCustom, ProjectCustom, Map<String, MetadataCustom>>merge(
clusterCustoms,
DiffableUtils.singleUpsertDiff(RepositoriesMetadata.TYPE, projectCustom, DiffableUtils.getStringKeySerializer()),
DiffableUtils.getStringKeySerializer()
)
);
return ProjectMetadata.builder(v).removeCustom(RepositoriesMetadata.TYPE).build();
});
if (combineClustersCustoms.get() != null) {
combineClustersCustoms.get().writeTo(out);
} else {
clusterCustoms.writeTo(out);
reservedStateMetadata.writeTo(out);
if (multiProject != null) {
multiProject.writeTo(out);
} else {
DiffableUtils.singleEntryDiff(DEFAULT_PROJECT_ID, singleProject, PROJECT_ID_SERIALIZER).writeTo(out);
}
}
reservedStateMetadata.writeTo(out);
updatedMultiProject.writeTo(out);
}
private static void throwForVersionBeforeRepositoriesMetadataMigration(StreamOutput out) {
assert out.getTransportVersion().before(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM) : out.getTransportVersion();
throw new UnsupportedOperationException(
"Serialize a diff with repositories defined for multiple projects requires version on or after ["
+ TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM
+ "], but got ["
+ out.getTransportVersion()
+ "]"
);
}
@SuppressWarnings("unchecked")
@ -1361,17 +1223,7 @@ public class Metadata implements Diffable<Metadata>, ChunkedToXContent {
builder.put(ReservedStateMetadata.readFrom(in));
}
} else {
List<ProjectCustom> defaultProjectCustoms = List.of();
if (in.getTransportVersion().before(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM)) {
// Extract the default project's repositories metadata from the Metadata#customs from an old node
defaultProjectCustoms = new ArrayList<>();
readBwcCustoms(in, builder, defaultProjectCustoms::add);
assert defaultProjectCustoms.size() <= 1
: "expect only a single default project custom for repository metadata, but got "
+ defaultProjectCustoms.stream().map(ProjectCustom::getWriteableName).toList();
} else {
readClusterCustoms(in, builder);
}
readClusterCustoms(in, builder);
int reservedStateSize = in.readVInt();
for (int i = 0; i < reservedStateSize; i++) {
@ -1379,16 +1231,11 @@ public class Metadata implements Diffable<Metadata>, ChunkedToXContent {
}
builder.projectMetadata(in.readMap(ProjectId::readFrom, ProjectMetadata::readFrom));
defaultProjectCustoms.forEach(c -> builder.getProject(ProjectId.DEFAULT).putCustom(c.getWriteableName(), c));
}
return builder.build();
}
private static void readBwcCustoms(StreamInput in, Builder builder) throws IOException {
readBwcCustoms(in, builder, projectCustom -> builder.putProjectCustom(projectCustom.getWriteableName(), projectCustom));
}
private static void readBwcCustoms(StreamInput in, Builder builder, Consumer<ProjectCustom> projectCustomConsumer) throws IOException {
final Set<String> clusterScopedNames = in.namedWriteableRegistry().getReaders(ClusterCustom.class).keySet();
final Set<String> projectScopedNames = in.namedWriteableRegistry().getReaders(ProjectCustom.class).keySet();
final int count = in.readVInt();
@ -1404,9 +1251,9 @@ public class Metadata implements Diffable<Metadata>, ChunkedToXContent {
if (custom instanceof PersistentTasksCustomMetadata persistentTasksCustomMetadata) {
final var tuple = persistentTasksCustomMetadata.split();
builder.putCustom(tuple.v1().getWriteableName(), tuple.v1());
projectCustomConsumer.accept(tuple.v2());
builder.putProjectCustom(tuple.v2().getWriteableName(), tuple.v2());
} else {
projectCustomConsumer.accept(custom);
builder.putProjectCustom(custom.getWriteableName(), custom);
}
} else {
throw new IllegalArgumentException("Unknown custom name [" + name + "]");
@ -1473,42 +1320,13 @@ public class Metadata implements Diffable<Metadata>, ChunkedToXContent {
combinedMetadata.addAll(singleProject.reservedStateMetadata().values());
out.writeCollection(combinedMetadata);
} else {
if (out.getTransportVersion().before(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM)) {
if (isSingleProject() || hasNoNonDefaultProjectRepositories(projects().values())) {
// Repositories metadata must be sent as Metadata#customs for old nodes
final List<VersionedNamedWriteable> combinedCustoms = new ArrayList<>(customs.size() + 1);
combinedCustoms.addAll(customs.values());
final ProjectCustom custom = getProject(ProjectId.DEFAULT).custom(RepositoriesMetadata.TYPE);
if (custom != null) {
combinedCustoms.add(custom);
}
VersionedNamedWriteable.writeVersionedWriteables(out, combinedCustoms);
} else {
throw new UnsupportedOperationException(
"Serialize metadata with repositories defined for multiple projects requires version on or after ["
+ TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM
+ "], but got ["
+ out.getTransportVersion()
+ "]"
);
}
} else {
VersionedNamedWriteable.writeVersionedWriteables(out, customs.values());
}
VersionedNamedWriteable.writeVersionedWriteables(out, customs.values());
out.writeCollection(reservedStateMetadata.values());
out.writeMap(projectMetadata, StreamOutput::writeWriteable, StreamOutput::writeWriteable);
}
}
/**
* @return {@code true} iff no repositories are defined for non-default-projects.
*/
private static boolean hasNoNonDefaultProjectRepositories(Collection<ProjectMetadata> projects) {
return projects.stream()
.allMatch(project -> ProjectId.DEFAULT.equals(project.id()) || project.custom(RepositoriesMetadata.TYPE) == null);
}
public static Builder builder() {
return new Builder();
}

23
server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java
View File

@ -1530,11 +1530,6 @@ public class ProjectMetadata implements Iterable<IndexMetadata>, Diffable<Projec
return this;
}
public Builder clearCustoms() {
customs.clear();
return this;
}
public Builder customs(Map<String, Metadata.ProjectCustom> customs) {
customs.forEach((key, value) -> Objects.requireNonNull(value, key));
this.customs.putAllFromMap(customs);
@ -2266,17 +2261,7 @@ public class ProjectMetadata implements Iterable<IndexMetadata>, Diffable<Projec
indexMetadata.writeTo(out, true);
}
out.writeCollection(templates.values());
Collection<Metadata.ProjectCustom> filteredCustoms = customs.values();
if (out.getTransportVersion().before(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM)) {
// RepositoriesMetadata is sent as part of Metadata#customs for version before RepositoriesMetadata migration
// So we exclude it from the project level customs
if (custom(RepositoriesMetadata.TYPE) != null) {
assert ProjectId.DEFAULT.equals(id)
: "Only default project can have repositories metadata. Otherwise the code should have thrown before it reaches here";
filteredCustoms = filteredCustoms.stream().filter(custom -> custom instanceof RepositoriesMetadata == false).toList();
}
}
VersionedNamedWriteable.writeVersionedWriteables(out, filteredCustoms);
VersionedNamedWriteable.writeVersionedWriteables(out, customs.values());
out.writeCollection(reservedStateMetadata.values());
if (out.getTransportVersion().onOrAfter(TransportVersions.PROJECT_METADATA_SETTINGS)) {
@ -2409,12 +2394,6 @@ public class ProjectMetadata implements Iterable<IndexMetadata>, Diffable<Projec
builder.settings = settingsDiff.apply(part.settings);
return builder.build(true);
}
ProjectMetadataDiff withCustoms(
DiffableUtils.MapDiff<String, Metadata.ProjectCustom, ImmutableOpenMap<String, Metadata.ProjectCustom>> customs
) {
return new ProjectMetadataDiff(indices, templates, customs, reservedStateMetadata, settingsDiff);
}
}
@Override

40
server/src/test/java/org/elasticsearch/cluster/metadata/MetadataRepositoriesMetadataTests.java
View File

@ -96,22 +96,10 @@ public class MetadataRepositoriesMetadataTests extends ESTestCase {
}
public void testRepositoriesMetadataSerializationBwc() throws IOException {
{
final var oldVersion = TransportVersionUtils.randomVersionBetween(
random(),
TransportVersions.MULTI_PROJECT,
TransportVersionUtils.getPreviousVersion(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM)
);
final Metadata orig = randomMetadata(between(0, 5), -1);
doTestRepositoriesMetadataSerializationBwc(orig, oldVersion);
}
{
final var oldVersion = TransportVersionUtils.getPreviousVersion(TransportVersions.MULTI_PROJECT);
// Before multi-project, BWC is possible for a single project
final Metadata orig = randomMetadata(0, -1);
doTestRepositoriesMetadataSerializationBwc(orig, oldVersion);
}
final var oldVersion = TransportVersionUtils.getPreviousVersion(TransportVersions.MULTI_PROJECT);
// Before multi-project, BWC is possible for a single project
final Metadata orig = randomMetadata(0, -1);
doTestRepositoriesMetadataSerializationBwc(orig, oldVersion);
}
private void doTestRepositoriesMetadataSerializationBwc(Metadata orig, TransportVersion oldVersion) throws IOException {
@ -138,22 +126,10 @@ public class MetadataRepositoriesMetadataTests extends ESTestCase {
}
public void testRepositoriesMetadataDiffSerializationBwc() throws IOException {
{
final var oldVersion = TransportVersionUtils.randomVersionBetween(
random(),
TransportVersions.MULTI_PROJECT,
TransportVersionUtils.getPreviousVersion(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM)
);
final Tuple<Metadata, Metadata> tuple = randomMetadataAndUpdate(between(0, 5), -1);
doTestRepositoriesMetadataDiffSerializationBwc(tuple, oldVersion);
}
{
final var oldVersion = TransportVersionUtils.getPreviousVersion(TransportVersions.MULTI_PROJECT);
// Before multi-project, BWC is possible for a single project
final Tuple<Metadata, Metadata> tuple = randomMetadataAndUpdate(0, -1);
doTestRepositoriesMetadataDiffSerializationBwc(tuple, oldVersion);
}
final var oldVersion = TransportVersionUtils.getPreviousVersion(TransportVersions.MULTI_PROJECT);
// Before multi-project, BWC is possible for a single project
final Tuple<Metadata, Metadata> tuple = randomMetadataAndUpdate(0, -1);
doTestRepositoriesMetadataDiffSerializationBwc(tuple, oldVersion);
}
private void doTestRepositoriesMetadataDiffSerializationBwc(Tuple<Metadata, Metadata> tuple, TransportVersion oldVersion)