Upload all the metadata attributes in parallel

Signed-off-by: Shivansh Arora <[email protected]>

server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java

@@ -752,6 +752,10 @@ public String getIndexUUID() {
         return index.getUUID();
     }
 
+    public String getIndexName() {
+        return index.getName();
+    }
+
     /**
      * Test whether the current index UUID is the same as the given one. Returns true if either are _na_
      */

server/src/main/java/org/opensearch/cluster/metadata/Metadata.java

@@ -948,7 +948,7 @@ public Iterator<IndexMetadata> iterator() {
     }
 
     public static boolean isGlobalStateEquals(Metadata metadata1, Metadata metadata2) {
-        if (!metadata1.coordinationMetadata.equals(metadata2.coordinationMetadata)) {
+        if (!isCoordinationMetadataEqual(metadata1, metadata2)) {
             return false;
         }
         if (!metadata1.hashesOfConsistentSettings.equals(metadata2.hashesOfConsistentSettings)) {
@@ -967,13 +967,29 @@ public static boolean isGlobalStateEquals(Metadata metadata1, Metadata metadata2
      * Compares Metadata entities persisted in Remote Store.
      */
     public static boolean isGlobalResourcesMetadataEquals(Metadata metadata1, Metadata metadata2) {
-        if (!metadata1.persistentSettings.equals(metadata2.persistentSettings)) {
+        if (!isSettingsMetadataEqual(metadata1, metadata2)) {
             return false;
         }
-        if (!metadata1.templates.equals(metadata2.templates)) {
+        if (!isTemplatesMetadataEqual(metadata1, metadata2)) {
             return false;
         }
         // Check if any persistent metadata needs to be saved
+        return isCustomMetadataEqual(metadata1, metadata2);
+    }
+
+    public static boolean isCoordinationMetadataEqual(Metadata metadata1, Metadata metadata2) {
+        return metadata1.coordinationMetadata.equals(metadata2.coordinationMetadata);
+    }
+
+    public static boolean isSettingsMetadataEqual(Metadata metadata1, Metadata metadata2) {
+        return metadata1.persistentSettings.equals(metadata2.persistentSettings);
+    }
+
+    public static boolean isTemplatesMetadataEqual(Metadata metadata1, Metadata metadata2) {
+        return metadata1.templates.equals(metadata2.templates);
+    }
+
+    public static boolean isCustomMetadataEqual(Metadata metadata1, Metadata metadata2) {
         int customCount1 = 0;
         for (Map.Entry<String, Custom> cursor : metadata1.customs.entrySet()) {
             if (cursor.getValue().context().contains(XContentContext.GATEWAY)) {
@@ -987,8 +1003,7 @@ public static boolean isGlobalResourcesMetadataEquals(Metadata metadata1, Metada
                 customCount2++;
             }
         }
-        if (customCount1 != customCount2) return false;
-        return true;
+        return customCount1 == customCount2;
     }
 
     @Override
@@ -1037,7 +1052,11 @@ private static class MetadataDiff implements Diff<Metadata> {
             persistentSettings = after.persistentSettings;
             hashesOfConsistentSettings = after.hashesOfConsistentSettings.diff(before.hashesOfConsistentSettings);
             indices = DiffableUtils.diff(before.indices, after.indices, DiffableUtils.getStringKeySerializer());
-            templates = DiffableUtils.diff(before.templates.getTemplates(), after.templates.getTemplates(), DiffableUtils.getStringKeySerializer());
+            templates = DiffableUtils.diff(
+                before.templates.getTemplates(),
+                after.templates.getTemplates(),
+                DiffableUtils.getStringKeySerializer()
+            );
             customs = DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER);
         }
 
@@ -1765,7 +1784,9 @@ public static void toXContent(Metadata metadata, XContentBuilder builder, ToXCon
                 builder.endObject();
             }
 
+            builder.startObject("templates");
             metadata.templatesMetadata().toXContent(builder, params);
+            builder.endObject();
 
             if (context == XContentContext.API) {
                 builder.startObject("indices");

server/src/main/java/org/opensearch/cluster/metadata/TemplatesMetadata.java

@@ -9,9 +9,6 @@
 package org.opensearch.cluster.metadata;
 
 import org.opensearch.cluster.AbstractDiffable;
-import org.opensearch.cluster.coordination.CoordinationMetadata;
-import org.opensearch.common.annotation.PublicApi;
-import org.opensearch.common.settings.Settings;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.xcontent.ToXContentFragment;
 import org.opensearch.core.xcontent.XContentBuilder;
@@ -66,7 +63,7 @@ public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
 
-        TemplatesMetadata  that = (TemplatesMetadata) o;
+        TemplatesMetadata that = (TemplatesMetadata) o;
 
         return Objects.equals(templates, that.templates);
     }
@@ -111,11 +108,11 @@ public TemplatesMetadata build() {
         }
 
         public static void toXContent(TemplatesMetadata templates, XContentBuilder builder, Params params) throws IOException {
-//            builder.startObject("templates-metadata");
-            for(IndexTemplateMetadata cursor : templates.getTemplates().values()) {
+            // builder.startObject("templates-metadata");
+            for (IndexTemplateMetadata cursor : templates.getTemplates().values()) {
                 IndexTemplateMetadata.Builder.toXContentWithTypes(cursor, builder, params);
             }
-//            builder.endObject();
+            // builder.endObject();
         }
 
         public static TemplatesMetadata fromXContent(XContentParser parser) throws IOException {

server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java

@@ -24,6 +24,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -210,10 +211,26 @@ private static void declareParser(ConstructingObjectParser<ClusterMetadataManife
             parser.declareString(ConstructingObjectParser.constructorArg(), GLOBAL_METADATA_FIELD);
         } else if (codec_version == CODEC_V2) {
             parser.declareInt(ConstructingObjectParser.constructorArg(), CODEC_VERSION_FIELD);
-            parser.declareNamedObject(ConstructingObjectParser.constructorArg(), UploadedMetadataAttribute.PARSER, UPLOADED_COORDINATOR_METADATA);
-            parser.declareNamedObject(ConstructingObjectParser.constructorArg(), UploadedMetadataAttribute.PARSER, UPLOADED_SETTINGS_METADATA);
-            parser.declareNamedObject(ConstructingObjectParser.constructorArg(), UploadedMetadataAttribute.PARSER, UPLOADED_TEMPLATES_METADATA);
-            parser.declareNamedObjects(ConstructingObjectParser.constructorArg(), UploadedMetadataAttribute.PARSER, UPLOADED_CUSTOM_METADATA);
+            parser.declareNamedObject(
+                ConstructingObjectParser.optionalConstructorArg(),
+                UploadedMetadataAttribute.PARSER,
+                UPLOADED_COORDINATOR_METADATA
+            );
+            parser.declareNamedObject(
+                ConstructingObjectParser.optionalConstructorArg(),
+                UploadedMetadataAttribute.PARSER,
+                UPLOADED_SETTINGS_METADATA
+            );
+            parser.declareNamedObject(
+                ConstructingObjectParser.optionalConstructorArg(),
+                UploadedMetadataAttribute.PARSER,
+                UPLOADED_TEMPLATES_METADATA
+            );
+            parser.declareNamedObjects(
+                ConstructingObjectParser.constructorArg(),
+                UploadedMetadataAttribute.PARSER,
+                UPLOADED_CUSTOM_METADATA
+            );
         }
     }
 
@@ -299,7 +316,10 @@ public Map<String, UploadedMetadataAttribute> getCustomMetadataMap() {
     }
 
     public boolean hasMetadataAttributesFiles() {
-        return uploadedCoordinationMetadata != null || uploadedSettingsMetadata != null || uploadedTemplatesMetadata != null || uploadedCustomMetadataMap != null;
+        return uploadedCoordinationMetadata != null
+            || uploadedSettingsMetadata != null
+            || uploadedTemplatesMetadata != null
+            || !uploadedCustomMetadataMap.isEmpty();
     }
 
     public ClusterMetadataManifest(
@@ -369,7 +389,9 @@ public ClusterMetadataManifest(
         this.uploadedCoordinationMetadata = uploadedCoordinationMetadata;
         this.uploadedSettingsMetadata = uploadedSettingsMetadata;
         this.uploadedTemplatesMetadata = uploadedTemplatesMetadata;
-        this.uploadedCustomMetadataMap = Collections.unmodifiableMap(uploadedCustomMetadataMap);
+        this.uploadedCustomMetadataMap = Collections.unmodifiableMap(
+            uploadedCustomMetadataMap != null ? uploadedCustomMetadataMap : new HashMap<>()
+        );
     }
 
     public ClusterMetadataManifest(StreamInput in) throws IOException {
@@ -388,7 +410,9 @@ public ClusterMetadataManifest(StreamInput in) throws IOException {
             this.uploadedCoordinationMetadata = new UploadedMetadataAttribute(in);
             this.uploadedSettingsMetadata = new UploadedMetadataAttribute(in);
             this.uploadedTemplatesMetadata = new UploadedMetadataAttribute(in);
-            this.uploadedCustomMetadataMap = Collections.unmodifiableMap(in.readMap(StreamInput::readString, UploadedMetadataAttribute::new));
+            this.uploadedCustomMetadataMap = Collections.unmodifiableMap(
+                in.readMap(StreamInput::readString, UploadedMetadataAttribute::new)
+            );
             this.globalMetadataFileName = null;
         } else if (in.getVersion().onOrAfter(Version.V_2_12_0)) {
             this.codecVersion = in.readInt();
@@ -435,17 +459,23 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field(CLUSTER_UUID_COMMITTED.getPreferredName(), isClusterUUIDCommitted());
         if (onOrAfterCodecVersion(CODEC_V2)) {
             builder.field(CODEC_VERSION_FIELD.getPreferredName(), getCodecVersion());
-            builder.startObject(UPLOADED_COORDINATOR_METADATA.getPreferredName());
-            getCoordinationMetadata().toXContent(builder, params);
-            builder.endObject();
-            builder.startObject(UPLOADED_SETTINGS_METADATA.getPreferredName());
-            getSettingsMetadata().toXContent(builder, params);
-            builder.endObject();
-            builder.startObject(UPLOADED_TEMPLATES_METADATA.getPreferredName());
-            getTemplatesMetadata().toXContent(builder, params);
-            builder.endObject();
+            if (getCoordinationMetadata() != null) {
+                builder.startObject(UPLOADED_COORDINATOR_METADATA.getPreferredName());
+                getCoordinationMetadata().toXContent(builder, params);
+                builder.endObject();
+            }
+            if (getSettingsMetadata() != null) {
+                builder.startObject(UPLOADED_SETTINGS_METADATA.getPreferredName());
+                getSettingsMetadata().toXContent(builder, params);
+                builder.endObject();
+            }
+            if (getTemplatesMetadata() != null) {
+                builder.startObject(UPLOADED_TEMPLATES_METADATA.getPreferredName());
+                getTemplatesMetadata().toXContent(builder, params);
+                builder.endObject();
+            }
             builder.startObject(UPLOADED_CUSTOM_METADATA.getPreferredName());
-            for (UploadedMetadataAttribute attribute: getCustomMetadataMap().values()) {
+            for (UploadedMetadataAttribute attribute : getCustomMetadataMap().values()) {
                 attribute.toXContent(builder, params);
             }
             builder.endObject();
@@ -534,6 +564,10 @@ public static ClusterMetadataManifest fromXContentV0(XContentParser parser) thro
         return PARSER_V0.parse(parser, null);
     }
 
+    public static ClusterMetadataManifest fromXContentV1(XContentParser parser) throws IOException {
+        return PARSER_V1.parse(parser, null);
+    }
+
     public static ClusterMetadataManifest fromXContent(XContentParser parser) throws IOException {
         return CURRENT_PARSER.parse(parser, null);
     }
@@ -546,6 +580,10 @@ public static ClusterMetadataManifest fromXContent(XContentParser parser) throws
     public static class Builder {
 
         private String globalMetadataFileName;
+        private UploadedMetadataAttribute coordinationMetadata;
+        private UploadedMetadataAttribute settingsMetadata;
+        private UploadedMetadataAttribute templatesMetadata;
+        private Map<String, UploadedMetadataAttribute> customMetadataMap;
         private int codecVersion;
         private List<UploadedIndexMetadata> indices;
         private long clusterTerm;
@@ -573,6 +611,31 @@ public Builder globalMetadataFileName(String globalMetadataFileName) {
             return this;
         }
 
+        public Builder coordinationMetadata(UploadedMetadataAttribute coordinationMetadata) {
+            this.coordinationMetadata = coordinationMetadata;
+            return this;
+        }
+
+        public Builder settingMetadata(UploadedMetadataAttribute settingsMetadata) {
+            this.settingsMetadata = settingsMetadata;
+            return this;
+        }
+
+        public Builder templatesMetadata(UploadedMetadataAttribute templatesMetadata) {
+            this.templatesMetadata = templatesMetadata;
+            return this;
+        }
+
+        public Builder customMetadataMap(Map<String, UploadedMetadataAttribute> customMetadataMap) {
+            this.customMetadataMap = customMetadataMap;
+            return this;
+        }
+
+        public Builder put(String custom, UploadedMetadataAttribute customMetadata) {
+            this.customMetadataMap.put(custom, customMetadata);
+            return this;
+        }
+
         public Builder clusterTerm(long clusterTerm) {
             this.clusterTerm = clusterTerm;
             return this;
@@ -624,6 +687,7 @@ public Builder clusterUUIDCommitted(boolean clusterUUIDCommitted) {
 
         public Builder() {
             indices = new ArrayList<>();
+            customMetadataMap = new HashMap<>();
         }
 
         public Builder(ClusterMetadataManifest manifest) {
@@ -635,6 +699,10 @@ public Builder(ClusterMetadataManifest manifest) {
             this.nodeId = manifest.nodeId;
             this.committed = manifest.committed;
             this.globalMetadataFileName = manifest.globalMetadataFileName;
+            this.coordinationMetadata = manifest.uploadedCoordinationMetadata;
+            this.settingsMetadata = manifest.uploadedSettingsMetadata;
+            this.templatesMetadata = manifest.uploadedTemplatesMetadata;
+            this.customMetadataMap = manifest.uploadedCustomMetadataMap;
             this.codecVersion = manifest.codecVersion;
             this.indices = new ArrayList<>(manifest.indices);
             this.previousClusterUUID = manifest.previousClusterUUID;
@@ -654,18 +722,28 @@ public ClusterMetadataManifest build() {
                 globalMetadataFileName,
                 indices,
                 previousClusterUUID,
-                clusterUUIDCommitted
+                clusterUUIDCommitted,
+                coordinationMetadata,
+                settingsMetadata,
+                templatesMetadata,
+                customMetadataMap
             );
         }
 
     }
 
+    public static interface UploadedMetadata {
+        String getComponent();
+
+        String getUploadedFilename();
+    }
+
     /**
      * Metadata for uploaded index metadata
      *
      * @opensearch.internal
      */
-    public static class UploadedIndexMetadata implements Writeable, ToXContentFragment {
+    public static class UploadedIndexMetadata implements UploadedMetadata, Writeable, ToXContentFragment {
 
         private static final ParseField INDEX_NAME_FIELD = new ParseField("index_name");
         private static final ParseField INDEX_UUID_FIELD = new ParseField("index_uuid");
@@ -714,6 +792,11 @@ public String getUploadedFilePath() {
             return uploadedFilename;
         }
 
+        @Override
+        public String getComponent() {
+            return getIndexName();
+        }
+
         public String getUploadedFilename() {
             String[] splitPath = uploadedFilename.split("/");
             return splitPath[splitPath.length - 1];
@@ -772,7 +855,7 @@ public static UploadedIndexMetadata fromXContent(XContentParser parser) throws I
         }
     }
 
-    public static class UploadedMetadataAttribute implements Writeable, ToXContentFragment {
+    public static class UploadedMetadataAttribute implements UploadedMetadata, Writeable, ToXContentFragment {
         private static final ParseField UPLOADED_FILENAME_FIELD = new ParseField("uploaded_filename");
 
         private static final ObjectParser.NamedObjectParser<UploadedMetadataAttribute, Void> PARSER;
@@ -807,6 +890,11 @@ public String getAttributeName() {
             return attributeName;
         }
 
+        @Override
+        public String getComponent() {
+            return getAttributeName();
+        }
+
         public String getUploadedFilename() {
             return uploadedFilename;
         }
@@ -830,10 +918,14 @@ public static UploadedMetadataAttribute fromXContent(XContentParser parser) thro
 
         @Override
         public String toString() {
-            return "UploadedMetadataAttribute{" +
-                "attributeName='" + attributeName + '\'' +
-                ", uploadedFilename='" + uploadedFilename + '\'' +
-                '}';
+            return "UploadedMetadataAttribute{"
+                + "attributeName='"
+                + attributeName
+                + '\''
+                + ", uploadedFilename='"
+                + uploadedFilename
+                + '\''
+                + '}';
         }
     }
 }