Perform unreferenced file cleanup for any operation failure and mute flaky test

CHANGELOG.md

@@ -85,6 +85,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Update supported version for max_shard_size parameter in Shrink API ([#11439](https://github.com/opensearch-project/OpenSearch/pull/11439))
 - Fix typo in API annotation check message ([11836](https://github.com/opensearch-project/OpenSearch/pull/11836))
 - Update supported version for must_exist parameter in update aliases API ([#11872](https://github.com/opensearch-project/OpenSearch/pull/11872))
+- Fix unreferenced file cleanup on any operations failure ([#12054](https://github.com/opensearch-project/OpenSearch/issues/12054))
 - [Bug] Check phase name before SearchRequestOperationsListener onPhaseStart ([#12035](https://github.com/opensearch-project/OpenSearch/pull/12035))
 
 ### Security

server/src/internalClusterTest/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java

@@ -219,6 +219,7 @@ public void testIndexCreateBlockNotAppliedWhenAnyNodesBelowHighWatermark() throw
         assertFalse(state.blocks().hasGlobalBlockWithId(Metadata.CLUSTER_CREATE_INDEX_BLOCK.id()));
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/6445")
     public void testIndexCreateBlockIsRemovedWhenAnyNodesNotExceedHighWatermarkWithAutoReleaseEnabled() throws Exception {
         final Settings settings = Settings.builder()
             .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), false)

server/src/main/java/org/opensearch/index/engine/Engine.java

@@ -1317,11 +1317,11 @@ public void failEngine(String reason, @Nullable Exception failure) {
                         }
                     }
 
-                    // If cleanup of unreferenced flag is enabled and force merge or regular merge failed due to IOException,
-                    // clean all unreferenced files on best effort basis created during failed merge and reset the
-                    // shard state back to last Lucene Commit.
-                    if (shouldCleanupUnreferencedFiles() && isMergeFailureDueToIOException(failure, reason)) {
-                        logger.info("Cleaning up unreferenced files as merge failed due to: {}", reason);
+                    // If cleanup of unreferenced file flag is enabled and any operation failed due to IOException,
+                    // clean up all unreferenced files on best effort basis created during failed merge and reset the
+                    // shard state back to last Lucene Commit
+                    if (shouldCleanupUnreferencedFiles() && isOperationFailureDueToIOException(failure)) {
+                        logger.info("Cleaning up unreferenced files created during failed merge due to: {}", reason);
                         cleanUpUnreferencedFiles();
                     }
 
@@ -1365,10 +1365,9 @@ private void cleanUpUnreferencedFiles() {
         }
     }
 
-    /** Check whether the merge failure happened due to IOException. */
-    private boolean isMergeFailureDueToIOException(Exception failure, String reason) {
-        return (reason.equals(FORCE_MERGE) || reason.equals(MERGE_FAILED))
-            && ExceptionsHelper.unwrap(failure, IOException.class) instanceof IOException;
+    /** Check whether an operation failed due to IOException. */
+    private boolean isOperationFailureDueToIOException(Exception failure) {
+        return ExceptionsHelper.unwrap(failure, IOException.class) instanceof IOException;
     }
 
     /** Check whether the engine should be failed */

server/src/test/java/org/opensearch/monitor/fs/FsHealthServiceTests.java

@@ -184,6 +184,7 @@ public void testLoggingOnHungIO() throws Exception {
         }
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/7557")
     public void testFailsHealthOnHungIOBeyondHealthyTimeout() throws Exception {
         long healthyTimeoutThreshold = randomLongBetween(500, 1000);
         long refreshInterval = randomLongBetween(500, 1000);