From a441fe2d7ada0ecdcfdaa4d7ab39989fcce6aa9d Mon Sep 17 00:00:00 2001 From: Dhwanil Patel Date: Tue, 24 Oct 2023 19:52:11 +0530 Subject: [PATCH 1/4] Improved logging around remote cluster state Signed-off-by: Dhwanil Patel --- .../remote/RemoteClusterStateService.java | 26 ++++++++++++++----- .../recovery/RemoteStoreRestoreService.java | 1 + 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index ae4a3fab9852d..b40eaeb9d7afd 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -243,7 +243,7 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri } else { // todo change to debug logger.info( - "writing cluster state took [{}ms]; " + "wrote full state with [{}] indices", + "writing cluster state took [{}ms]; " + "wrote full state with [{}] indices and global metadata", durationMillis, allUploadedIndexMetadata.size() ); @@ -282,6 +282,7 @@ public ClusterMetadataManifest writeIncrementalMetadata( if (updateGlobalMetadata || previousManifest.getGlobalMetadataFileName() == null) { globalMetadataFile = writeGlobalMetadata(clusterState); } else { + logger.debug("Global metadata has not updated in cluster state, skipping upload of it"); globalMetadataFile = previousManifest.getGlobalMetadataFileName(); } @@ -337,18 +338,22 @@ public ClusterMetadataManifest writeIncrementalMetadata( if (durationMillis >= slowWriteLoggingThreshold.getMillis()) { logger.warn( "writing cluster state took [{}ms] which is above the warn threshold of [{}]; " - + "wrote metadata for [{}] indices and skipped [{}] unchanged indices", + + "wrote metadata for [{}] indices and skipped [{}] unchanged indices, global metadata updated : [{}]", durationMillis, slowWriteLoggingThreshold, numIndicesUpdated, - numIndicesUnchanged + numIndicesUnchanged, + updateGlobalMetadata ); } else { - logger.trace( - "writing cluster state took [{}ms]; " + "wrote metadata for [{}] indices and skipped [{}] unchanged indices", + logger.info( + "writing cluster state for version [{}] took [{}ms]; " + + "wrote metadata for [{}] indices and skipped [{}] unchanged indices, global metadata updated : [{}]", + manifest.getStateVersion(), durationMillis, numIndicesUpdated, - numIndicesUnchanged + numIndicesUnchanged, + updateGlobalMetadata ); } return manifest; @@ -476,6 +481,7 @@ private List writeIndexMetadataParallel(ClusterState clus ) ); exceptionList.forEach(exception::addSuppressed); + logger.error("Exception in transferring index metadata to remote store", exception); throw exception; } return result; @@ -596,6 +602,11 @@ private void writeMetadataManifest(String clusterName, String clusterUUID, Clust blobStoreRepository.getCompressor(), FORMAT_PARAMS ); + logger.debug( + "Metadata manifest file [{}] written during [{}] phase. ", + fileName, + uploadManifest.isCommitted() ? "commit" : "publish" + ); } private String fetchPreviousClusterUUID(String clusterName, String clusterUUID) { @@ -903,6 +914,7 @@ private List createClusterChain(final Map { indexMetadataMap.put(indexMetadata.getIndex().getName(), new Tuple<>(true, indexMetadata)); From 18ec182bd582009c5fa99a863c704e9bd6208e93 Mon Sep 17 00:00:00 2001 From: Dhwanil Patel Date: Wed, 25 Oct 2023 20:17:25 +0530 Subject: [PATCH 2/4] Incorporated comments Signed-off-by: Dhwanil Patel --- .../gateway/remote/RemoteClusterStateService.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 03676097302db..55d090052fe67 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -306,7 +306,7 @@ public ClusterMetadataManifest writeIncrementalMetadata( for (final IndexMetadata indexMetadata : clusterState.metadata().indices().values()) { final Long previousVersion = previousStateIndexMetadataVersionByName.get(indexMetadata.getIndex().getName()); if (previousVersion == null || indexMetadata.getVersion() != previousVersion) { - logger.trace( + logger.debug( "updating metadata for [{}], changing version from [{}] to [{}]", indexMetadata.getIndex(), previousVersion, @@ -416,6 +416,13 @@ private String writeGlobalMetadata(ClusterState clusterState) throws IOException throw exception; } if (exceptionReference.get() != null) { + logger.error( + () -> new ParameterizedMessage( + "Exception during transfer of GlobalMetadata to Remote {}", + exceptionReference.get().getMessage() + ), + exceptionReference.get() + ); throw new GlobalMetadataTransferException(exceptionReference.get().getMessage(), exceptionReference.get()); } return result.get(); From 1cec3dbb67a3e0d1c9e6d23cc6d08b2f35db73a1 Mon Sep 17 00:00:00 2001 From: Dhwanil Patel Date: Wed, 25 Oct 2023 22:21:15 +0530 Subject: [PATCH 3/4] Incorporated comments Signed-off-by: Dhwanil Patel --- .../gateway/remote/RemoteClusterStateService.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 55d090052fe67..f05c5b4595729 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -244,7 +244,6 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri allUploadedIndexMetadata.size() ); } else { - // todo change to debug logger.info( "writing cluster state took [{}ms]; " + "wrote full state with [{}] indices and global metadata", durationMillis, @@ -416,13 +415,6 @@ private String writeGlobalMetadata(ClusterState clusterState) throws IOException throw exception; } if (exceptionReference.get() != null) { - logger.error( - () -> new ParameterizedMessage( - "Exception during transfer of GlobalMetadata to Remote {}", - exceptionReference.get().getMessage() - ), - exceptionReference.get() - ); throw new GlobalMetadataTransferException(exceptionReference.get().getMessage(), exceptionReference.get()); } return result.get(); @@ -497,7 +489,6 @@ private List writeIndexMetadataParallel(ClusterState clus ) ); exceptionList.forEach(exception::addSuppressed); - logger.error("Exception in transferring index metadata to remote store", exception); throw exception; } return result; From 16c266c6d5958c29957253064cd7b98d88694f59 Mon Sep 17 00:00:00 2001 From: Dhwanil Patel Date: Thu, 26 Oct 2023 10:47:08 +0530 Subject: [PATCH 4/4] made uuid chain log as info Signed-off-by: Dhwanil Patel --- .../opensearch/gateway/remote/RemoteClusterStateService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index f05c5b4595729..fa4eef79c46dc 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -921,7 +921,7 @@ private List createClusterChain(final Map