-
Notifications
You must be signed in to change notification settings - Fork 1.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Abstract out remote segemnt garbage collection logic
Signed-off-by: Sachin Kale <[email protected]>
- Loading branch information
Sachin Kale
committed
Apr 16, 2024
1 parent
3d1d5e7
commit cc805f9
Showing
4 changed files
with
156 additions
and
20 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
63 changes: 63 additions & 0 deletions
63
...rc/main/java/org/opensearch/index/shard/CommitTriggeredRemoteSegmentGarbageCollector.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,63 @@ | ||
/* | ||
* SPDX-License-Identifier: Apache-2.0 | ||
* | ||
* The OpenSearch Contributors require contributions made to | ||
* this file be licensed under the Apache-2.0 license or a | ||
* compatible open source license. | ||
*/ | ||
|
||
package org.opensearch.index.shard; | ||
|
||
import org.apache.logging.log4j.Logger; | ||
import org.apache.lucene.store.Directory; | ||
import org.opensearch.common.logging.Loggers; | ||
import org.opensearch.core.action.ActionListener; | ||
import org.opensearch.core.index.shard.ShardId; | ||
import org.opensearch.index.remote.RemoteStoreUtils; | ||
import org.opensearch.index.store.RemoteSegmentStoreDirectory; | ||
import org.opensearch.indices.RemoteStoreSettings; | ||
|
||
public class CommitTriggeredRemoteSegmentGarbageCollector implements RemoteSegmentGarbageCollector { | ||
private final Logger logger; | ||
private final Directory storeDirectory; | ||
private final RemoteSegmentStoreDirectory remoteDirectory; | ||
|
||
private final RemoteStoreSettings remoteStoreSettings; | ||
|
||
public CommitTriggeredRemoteSegmentGarbageCollector( | ||
ShardId shardId, | ||
Directory storeDirectory, | ||
RemoteSegmentStoreDirectory remoteDirectory, | ||
RemoteStoreSettings remoteStoreSettings | ||
) { | ||
logger = Loggers.getLogger(getClass(), shardId); | ||
this.storeDirectory = storeDirectory; | ||
this.remoteDirectory = remoteDirectory; | ||
this.remoteStoreSettings = remoteStoreSettings; | ||
} | ||
|
||
@Override | ||
public void deleteStaleSegments(Listener listener) { | ||
// if a new segments_N file is present in local that is not uploaded to remote store yet, it | ||
// is considered as a first refresh post commit. A cleanup of stale commit files is triggered. | ||
// This is done to avoid triggering delete post each refresh. | ||
if (RemoteStoreUtils.isLatestSegmentInfosUploadedToRemote(storeDirectory, remoteDirectory) == false) { | ||
remoteDirectory.deleteStaleSegmentsAsync(remoteStoreSettings.getMinRemoteSegmentMetadataFiles(), new ActionListener<Void>() { | ||
@Override | ||
public void onResponse(Void unused) { | ||
listener.onSuccess(unused); | ||
} | ||
|
||
@Override | ||
public void onFailure(Exception e) { | ||
listener.onFailure(e); | ||
} | ||
}); | ||
} else { | ||
String skipReason = | ||
"Skipping garbage collection, CommitTriggeredRemoteSegmentGarbageCollector only triggers deletion on commit"; | ||
logger.debug(skipReason); | ||
listener.onSkip(skipReason); | ||
} | ||
} | ||
} |
46 changes: 46 additions & 0 deletions
46
server/src/main/java/org/opensearch/index/shard/RemoteSegmentGarbageCollector.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,46 @@ | ||
/* | ||
* SPDX-License-Identifier: Apache-2.0 | ||
* | ||
* The OpenSearch Contributors require contributions made to | ||
* this file be licensed under the Apache-2.0 license or a | ||
* compatible open source license. | ||
*/ | ||
|
||
package org.opensearch.index.shard; | ||
|
||
public interface RemoteSegmentGarbageCollector { | ||
|
||
interface Listener { | ||
|
||
/** | ||
* Called after the successful deletion of stale segments from remote store | ||
*/ | ||
void onSuccess(Void response); | ||
|
||
/** | ||
* Called on exception while deleting stale segments from remote store | ||
*/ | ||
void onFailure(Exception e); | ||
|
||
/** | ||
* Called on skipping the garbage collection | ||
*/ | ||
void onSkip(String reason); | ||
} | ||
|
||
Listener DEFAULT_NOOP_LISTENER = new Listener() { | ||
@Override | ||
public void onSuccess(Void response) {} | ||
|
||
@Override | ||
public void onFailure(Exception e) {} | ||
|
||
@Override | ||
public void onSkip(String reason) {} | ||
}; | ||
|
||
/** | ||
* Deletes stale segments from remote store and notifies the listener. | ||
*/ | ||
void deleteStaleSegments(Listener listener); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters