Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/1.x' into 1.x-za
Browse files Browse the repository at this point in the history
Signed-off-by: Bukhtawar Khan <[email protected]>
  • Loading branch information
Bukhtawar committed Sep 29, 2021
2 parents 43052f4 + a80979c commit 0f12b99
Show file tree
Hide file tree
Showing 45 changed files with 361 additions and 45 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ class InternalDistributionArchiveSetupPluginFuncTest extends AbstractGradleFuncT

where:
buildTaskName | expectedOutputArchivePath
"buildDarwinTar" | "darwin-tar/build/distributions/opensearch.tar.gz"
"buildDarwinTar" | "darwin-tar/build/distributions/opensearch-min.tar.gz"
}

def "applies defaults to zip tasks"() {
Expand All @@ -91,7 +91,7 @@ class InternalDistributionArchiveSetupPluginFuncTest extends AbstractGradleFuncT

where:
buildTaskName | expectedOutputArchivePath
"buildDarwinZip" | "darwin-zip/build/distributions/opensearch.zip"
"buildDarwinZip" | "darwin-zip/build/distributions/opensearch-min.zip"
}

def "registered distribution provides archives and directory variant"() {
Expand Down Expand Up @@ -148,8 +148,8 @@ class InternalDistributionArchiveSetupPluginFuncTest extends AbstractGradleFuncT
then: "tar task executed and target folder contains plain tar"
result.task(':buildProducerTar').outcome == TaskOutcome.SUCCESS
result.task(':consumer:copyArchive').outcome == TaskOutcome.SUCCESS
file("producer-tar/build/distributions/opensearch.tar.gz").exists()
file("consumer/build/archives/opensearch.tar.gz").exists()
file("producer-tar/build/distributions/opensearch-min.tar.gz").exists()
file("consumer/build/archives/opensearch-min.tar.gz").exists()

when:
result = gradleRunner("copyDir", "-Pversion=1.0").build()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ private void configureGeneralTaskDefaults(Project project) {
project.getTasks().withType(AbstractArchiveTask.class).configureEach(t -> {
String subdir = archiveTaskToSubprojectName(t.getName());
t.getDestinationDirectory().set(project.file(subdir + "/build/distributions"));
t.getArchiveBaseName().set("opensearch");
t.getArchiveBaseName().set("opensearch-min");
});
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -239,10 +239,17 @@ private static class DistributionProject {
DistributionProject(String name, String baseDir, Version version, String classifier, String extension, File checkoutDir) {
this.name = name;
this.projectPath = baseDir + "/" + name;
this.distFile = new File(
checkoutDir,
baseDir + "/" + name + "/build/distributions/opensearch-" + version + "-SNAPSHOT" + classifier + "." + extension
);
if (version.onOrAfter("1.1.0")) {
this.distFile = new File(
checkoutDir,
baseDir + "/" + name + "/build/distributions/opensearch-min-" + version + "-SNAPSHOT" + classifier + "." + extension
);
} else {
this.distFile = new File(
checkoutDir,
baseDir + "/" + name + "/build/distributions/opensearch-" + version + "-SNAPSHOT" + classifier + "." + extension
);
}
// we only ported this down to the 7.x branch.
if (version.onOrAfter("7.10.0") && (name.endsWith("zip") || name.endsWith("tar"))) {
this.expandedDistDir = new File(checkoutDir, baseDir + "/" + name + "/build/install");
Expand Down
2 changes: 1 addition & 1 deletion buildSrc/version.properties
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ checkstyle = 8.29
# optional dependencies
spatial4j = 0.7
jts = 1.15.0
jackson = 2.11.4
jackson = 2.12.5
snakeyaml = 1.26
icu4j = 62.1
supercsv = 2.4.0
Expand Down
1 change: 0 additions & 1 deletion client/sniffer/licenses/jackson-core-2.11.4.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions client/sniffer/licenses/jackson-core-2.12.5.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
725e364cc71b80e60fa450bd06d75cdea7fb2d59
2 changes: 1 addition & 1 deletion distribution/docker/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ ext.expansions = { Architecture architecture, DockerBase base, boolean local ->
classifier = "linux-\$(arch)"
}

final String opensearch = "opensearch-${VersionProperties.getOpenSearch()}-${classifier}.tar.gz"
final String opensearch = "opensearch-min-${VersionProperties.getOpenSearch()}-${classifier}.tar.gz"

/* Both the following Dockerfile commands put the resulting artifact at
* the same location, regardless of classifier, so that the commands that
Expand Down
4 changes: 2 additions & 2 deletions distribution/packages/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -136,9 +136,9 @@ Closure commonPackageConfig(String type, boolean jdk, String architecture) {
// SystemPackagingTask overrides default archive task convention mappings, but doesn't provide a setter so we have to override the convention mapping itself
// Deb convention uses a '_' for final separator before architecture, rpm uses a '.'
if (type == 'deb') {
conventionMapping.archiveFile = { objects.fileProperty().fileValue(file("${destinationDirectory.get()}/${packageName}_${project.version}${jdkString}_${archString}.${type}")) }
conventionMapping.archiveFile = { objects.fileProperty().fileValue(file("${destinationDirectory.get()}/${packageName}-min_${project.version}${jdkString}_${archString}.${type}")) }
} else {
conventionMapping.archiveFile = { objects.fileProperty().fileValue(file("${destinationDirectory.get()}/${packageName}-${project.version}${jdkString}.${archString}.${type}")) }
conventionMapping.archiveFile = { objects.fileProperty().fileValue(file("${destinationDirectory.get()}/${packageName}-min-${project.version}${jdkString}.${archString}.${type}")) }
}

String packagingFiles = "build/packaging/${jdk ? '' : 'no-jdk-'}${type}"
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
52d929d5bb21d0186fe24c09624cc3ee4bafc3b3

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
b064cf057f23d3d35390328c5030847efeffedde
1 change: 0 additions & 1 deletion libs/x-content/licenses/jackson-core-2.11.4.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions libs/x-content/licenses/jackson-core-2.12.5.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
725e364cc71b80e60fa450bd06d75cdea7fb2d59

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
2b6f24ee5ac7cde7f5a4e574bd0af4a72ecb55f6

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
4233326c74c6601fbbeea11c103c011859cc687d

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
89ec27e5f422b0749b1133137c1b36debbb5f3bc

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
52d929d5bb21d0186fe24c09624cc3ee4bafc3b3

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
b064cf057f23d3d35390328c5030847efeffedde

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
52d929d5bb21d0186fe24c09624cc3ee4bafc3b3

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
b064cf057f23d3d35390328c5030847efeffedde

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
52d929d5bb21d0186fe24c09624cc3ee4bafc3b3

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
b064cf057f23d3d35390328c5030847efeffedde
Original file line number Diff line number Diff line change
Expand Up @@ -661,6 +661,33 @@ public void testNumberOfReplicasSettingsVersion() {
assertThat(newSettingsVersion, equalTo(1 + settingsVersion));
}

public void testTranslogPruningSetting() {
createIndex("test");
ensureGreen("test");
final String REPLICATION_TRANSLOG_SETTING = "index.plugins.replication.translog.retention_lease.pruning.enabled";
final long settingsVersion =
client().admin().cluster().prepareState().get().getState().metadata().index("test").getSettingsVersion();
GetSettingsResponse settingsResponse = client().admin().indices().prepareGetSettings("test").get();
boolean shouldPruneTranslogByRetentionLease = Boolean.parseBoolean(
settingsResponse.getSetting("test", REPLICATION_TRANSLOG_SETTING)
);

assertFalse(shouldPruneTranslogByRetentionLease);
assertAcked(client().admin().indices()
.prepareUpdateSettings("test")
.setSettings(Settings.builder()
.put("index.plugins.replication.translog.retention_lease.pruning.enabled", true)
));
final long newSettingsVersion =
client().admin().cluster().prepareState().get().getState().metadata().index("test").getSettingsVersion();
assertThat(newSettingsVersion, equalTo(1 + settingsVersion));
settingsResponse = client().admin().indices().prepareGetSettings("test").get();
shouldPruneTranslogByRetentionLease = Boolean.parseBoolean(
settingsResponse.getSetting("test", REPLICATION_TRANSLOG_SETTING)
);
assertTrue(shouldPruneTranslogByRetentionLease);
}

/*
* Test that we are able to set the setting index.number_of_replicas to the default.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1258,11 +1258,19 @@ public static int calculateNumRoutingShards(int numShards, Version indexVersionC
}

public static void validateTranslogRetentionSettings(Settings indexSettings) {
if (IndexSettings.INDEX_SOFT_DELETES_SETTING.get(indexSettings) &&
(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.exists(indexSettings)
|| IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.exists(indexSettings))) {
DEPRECATION_LOGGER.deprecate("translog_retention", "Translog retention settings [index.translog.retention.age] "
+ "and [index.translog.retention.size] are deprecated and effectively ignored. They will be removed in a future version.");
if (IndexSettings.INDEX_SOFT_DELETES_SETTING.get(indexSettings)) {
if(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.exists(indexSettings)
|| IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.exists(indexSettings)) {
DEPRECATION_LOGGER.deprecate("translog_retention", "Translog retention settings " +
"[index.translog.retention.age] "
+ "and [index.translog.retention.size] are deprecated and effectively ignored. " +
"They will be removed in a future version.");
} else if(IndexSettings.INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING.exists(indexSettings)) {
DEPRECATION_LOGGER.deprecate("translog_pruning_retention_lease",
"[index.plugins.replication.translog.retention_lease.pruning.enabled] " +
"setting was deprecated in OpenSearch and will be removed in a future release! " +
"See the breaking changes documentation for the next major version.");
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING,
EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING,
IndexSettings.INDEX_FLUSH_AFTER_MERGE_THRESHOLD_SIZE_SETTING,
IndexSettings.INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING,
IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING,
IndexSettings.INDEX_TRANSLOG_GENERATION_THRESHOLD_SIZE_SETTING,
IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING,
Expand Down
61 changes: 58 additions & 3 deletions server/src/main/java/org/opensearch/index/IndexSettings.java
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,15 @@ public final class IndexSettings {
settings -> Boolean.toString(IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(LegacyESVersion.V_7_0_0)),
Property.IndexScope, Property.Final);

/**
* Specifies if the index translog should prune based on retention leases.
* @deprecated EXPERT: this setting is specific to CCR and will be moved to a plugin in the next release
*/
@Deprecated
public static final Setting<Boolean> INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING =
Setting.boolSetting("index.plugins.replication.translog.retention_lease.pruning.enabled", false,
Property.IndexScope, Property.Dynamic, Property.Deprecated);

/**
* Controls how many soft-deleted documents will be kept around before being merged away. Keeping more deleted
* documents increases the chance of operation-based recoveries and allows querying a longer history of documents.
Expand All @@ -286,9 +295,12 @@ public final class IndexSettings {
* the chance of ops based recoveries for indices with soft-deletes disabled.
* This setting will be ignored if soft-deletes is used in peer recoveries (default in 7.4).
**/
@Deprecated
private static final ByteSizeValue DEFAULT_TRANSLOG_RETENTION_SIZE = new ByteSizeValue(512, ByteSizeUnit.MB);

public static final Setting<ByteSizeValue> INDEX_TRANSLOG_RETENTION_SIZE_SETTING =
Setting.byteSizeSetting("index.translog.retention.size",
settings -> shouldDisableTranslogRetention(settings) ? "-1" : "512MB",
settings -> DEFAULT_TRANSLOG_RETENTION_SIZE.getStringRep(),
Property.Dynamic, Property.IndexScope);

/**
Expand Down Expand Up @@ -389,6 +401,8 @@ public final class IndexSettings {
private final IndexScopedSettings scopedSettings;
private long gcDeletesInMillis = DEFAULT_GC_DELETES.millis();
private final boolean softDeleteEnabled;
@Deprecated
private volatile boolean translogPruningByRetentionLease;
private volatile long softDeleteRetentionOperations;

private volatile long retentionLeaseMillis;
Expand Down Expand Up @@ -525,6 +539,10 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
mergeSchedulerConfig = new MergeSchedulerConfig(this);
gcDeletesInMillis = scopedSettings.get(INDEX_GC_DELETES_SETTING).getMillis();
softDeleteEnabled = version.onOrAfter(LegacyESVersion.V_6_5_0) && scopedSettings.get(INDEX_SOFT_DELETES_SETTING);
// @todo move to CCR plugin
this.translogPruningByRetentionLease = version.onOrAfter(Version.V_1_1_0) &&
softDeleteEnabled &&
scopedSettings.get(INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING);
softDeleteRetentionOperations = scopedSettings.get(INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING);
retentionLeaseMillis = scopedSettings.get(INDEX_SOFT_DELETES_RETENTION_LEASE_PERIOD_SETTING).millis();
warmerEnabled = scopedSettings.get(INDEX_WARMER_ENABLED_SETTING);
Expand Down Expand Up @@ -593,6 +611,8 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
this::setGenerationThresholdSize);
scopedSettings.addSettingsUpdateConsumer(INDEX_TRANSLOG_RETENTION_AGE_SETTING, this::setTranslogRetentionAge);
scopedSettings.addSettingsUpdateConsumer(INDEX_TRANSLOG_RETENTION_SIZE_SETTING, this::setTranslogRetentionSize);
scopedSettings.addSettingsUpdateConsumer(INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING,
this::setTranslogPruningByRetentionLease);
scopedSettings.addSettingsUpdateConsumer(INDEX_REFRESH_INTERVAL_SETTING, this::setRefreshInterval);
scopedSettings.addSettingsUpdateConsumer(MAX_REFRESH_LISTENERS_PER_SHARD, this::setMaxRefreshListeners);
scopedSettings.addSettingsUpdateConsumer(MAX_ANALYZED_OFFSET_SETTING, this::setHighlightMaxAnalyzedOffset);
Expand Down Expand Up @@ -623,8 +643,20 @@ private void setFlushAfterMergeThresholdSize(ByteSizeValue byteSizeValue) {
this.flushAfterMergeThresholdSize = byteSizeValue;
}

/**
* Enable or disable translog pruning by retention lease requirement
*
* @deprecated EXPERT: this setting is specific to CCR and will be moved to a plugin in the next release
*/
@Deprecated
private void setTranslogPruningByRetentionLease(boolean enabled) {
this.translogPruningByRetentionLease = INDEX_SOFT_DELETES_SETTING.get(settings) && enabled;
}

private void setTranslogRetentionSize(ByteSizeValue byteSizeValue) {
if (shouldDisableTranslogRetention(settings) && byteSizeValue.getBytes() >= 0) {
if (shouldDisableTranslogRetention(settings) &&
shouldPruneTranslogByRetentionLease(settings) == false &&
byteSizeValue.getBytes() >= 0) {
// ignore the translog retention settings if soft-deletes enabled
this.translogRetentionSize = new ByteSizeValue(-1);
} else {
Expand Down Expand Up @@ -826,7 +858,12 @@ public TimeValue getRefreshInterval() {
* Returns the transaction log retention size which controls how much of the translog is kept around to allow for ops based recoveries
*/
public ByteSizeValue getTranslogRetentionSize() {
assert shouldDisableTranslogRetention(settings) == false || translogRetentionSize.getBytes() == -1L : translogRetentionSize;
if(shouldDisableTranslogRetention(settings) && shouldPruneTranslogByRetentionLease(settings) == false) {
return new ByteSizeValue(-1);
}
else if(shouldPruneTranslogByRetentionLease(settings) && translogRetentionSize.getBytes() == -1) {
return DEFAULT_TRANSLOG_RETENTION_SIZE;
}
return translogRetentionSize;
}

Expand Down Expand Up @@ -1071,6 +1108,24 @@ public void setRequiredPipeline(final String requiredPipeline) {
this.requiredPipeline = requiredPipeline;
}

/**
* Returns <code>true</code> if translog ops should be pruned based on retention lease
* @deprecated EXPERT: this setting is specific to CCR and will be moved to a plugin in the next release
*/
@Deprecated
public boolean shouldPruneTranslogByRetentionLease() {
return translogPruningByRetentionLease;
}

/**
* Returns <code>true</code> if translog ops should be pruned based on retention lease
* @deprecated EXPERT: this setting is specific to CCR and will be moved to a plugin in the next release
*/
@Deprecated
public static boolean shouldPruneTranslogByRetentionLease(Settings settings) {
return INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING.get(settings);
}

/**
* Returns <code>true</code> if soft-delete is enabled.
*/
Expand Down
11 changes: 11 additions & 0 deletions server/src/main/java/org/opensearch/index/engine/Engine.java
Original file line number Diff line number Diff line change
Expand Up @@ -1858,7 +1858,18 @@ public IndexCommit getIndexCommit() {
}

public void onSettingsChanged(TimeValue translogRetentionAge, ByteSizeValue translogRetentionSize, long softDeletesRetentionOps) {
onSettingsChanged(translogRetentionAge, translogRetentionSize, softDeletesRetentionOps, false);
}

/**
* callback when index settings change
*
* @deprecated EXPERT: this method is specific to CCR and will be moved to a plugin in the next release
*/
@Deprecated
public void onSettingsChanged(TimeValue translogRetentionAge, ByteSizeValue translogRetentionSize,
long softDeletesRetentionOps, boolean translogPruningByRetentionLease) {
// @todo this is overly silent; make this abstract and force derived classes to noop in the next release
}

/**
Expand Down
Loading

0 comments on commit 0f12b99

Please sign in to comment.