From 3376b35cc5d279c36da357c4ce790180194cedbd Mon Sep 17 00:00:00 2001 From: Xue Zhou <85715413+xuezhou25@users.noreply.github.com> Date: Wed, 22 Sep 2021 11:08:50 +0000 Subject: [PATCH 1/6] Rename artifact produced by the build to include -min (#1267) Signed-off-by: Xue Zhou --- .../InternalDistributionArchiveSetupPluginFuncTest.groovy | 8 ++++---- .../internal/InternalDistributionArchiveSetupPlugin.java | 2 +- distribution/docker/build.gradle | 2 +- distribution/packages/build.gradle | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/buildSrc/src/integTest/groovy/org/opensearch/gradle/internal/InternalDistributionArchiveSetupPluginFuncTest.groovy b/buildSrc/src/integTest/groovy/org/opensearch/gradle/internal/InternalDistributionArchiveSetupPluginFuncTest.groovy index a27377c84fc92..fa36bd36ed031 100644 --- a/buildSrc/src/integTest/groovy/org/opensearch/gradle/internal/InternalDistributionArchiveSetupPluginFuncTest.groovy +++ b/buildSrc/src/integTest/groovy/org/opensearch/gradle/internal/InternalDistributionArchiveSetupPluginFuncTest.groovy @@ -70,7 +70,7 @@ class InternalDistributionArchiveSetupPluginFuncTest extends AbstractGradleFuncT where: buildTaskName | expectedOutputArchivePath - "buildDarwinTar" | "darwin-tar/build/distributions/opensearch.tar.gz" + "buildDarwinTar" | "darwin-tar/build/distributions/opensearch-min.tar.gz" } def "applies defaults to zip tasks"() { @@ -91,7 +91,7 @@ class InternalDistributionArchiveSetupPluginFuncTest extends AbstractGradleFuncT where: buildTaskName | expectedOutputArchivePath - "buildDarwinZip" | "darwin-zip/build/distributions/opensearch.zip" + "buildDarwinZip" | "darwin-zip/build/distributions/opensearch-min.zip" } def "registered distribution provides archives and directory variant"() { @@ -148,8 +148,8 @@ class InternalDistributionArchiveSetupPluginFuncTest extends AbstractGradleFuncT then: "tar task executed and target folder contains plain tar" result.task(':buildProducerTar').outcome == TaskOutcome.SUCCESS result.task(':consumer:copyArchive').outcome == TaskOutcome.SUCCESS - file("producer-tar/build/distributions/opensearch.tar.gz").exists() - file("consumer/build/archives/opensearch.tar.gz").exists() + file("producer-tar/build/distributions/opensearch-min.tar.gz").exists() + file("consumer/build/archives/opensearch-min.tar.gz").exists() when: result = gradleRunner("copyDir", "-Pversion=1.0").build() diff --git a/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java index d5905b7da2647..6153d5f6fe4a9 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java @@ -121,7 +121,7 @@ private void configureGeneralTaskDefaults(Project project) { project.getTasks().withType(AbstractArchiveTask.class).configureEach(t -> { String subdir = archiveTaskToSubprojectName(t.getName()); t.getDestinationDirectory().set(project.file(subdir + "/build/distributions")); - t.getArchiveBaseName().set("opensearch"); + t.getArchiveBaseName().set("opensearch-min"); }); } diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index bce24b8f59b32..72d44a17b4eaa 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -50,7 +50,7 @@ ext.expansions = { Architecture architecture, DockerBase base, boolean local -> classifier = "linux-\$(arch)" } - final String opensearch = "opensearch-${VersionProperties.getOpenSearch()}-${classifier}.tar.gz" + final String opensearch = "opensearch-min-${VersionProperties.getOpenSearch()}-${classifier}.tar.gz" /* Both the following Dockerfile commands put the resulting artifact at * the same location, regardless of classifier, so that the commands that diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle index 5b6b41db3ceab..921439f750205 100644 --- a/distribution/packages/build.gradle +++ b/distribution/packages/build.gradle @@ -136,9 +136,9 @@ Closure commonPackageConfig(String type, boolean jdk, String architecture) { // SystemPackagingTask overrides default archive task convention mappings, but doesn't provide a setter so we have to override the convention mapping itself // Deb convention uses a '_' for final separator before architecture, rpm uses a '.' if (type == 'deb') { - conventionMapping.archiveFile = { objects.fileProperty().fileValue(file("${destinationDirectory.get()}/${packageName}_${project.version}${jdkString}_${archString}.${type}")) } + conventionMapping.archiveFile = { objects.fileProperty().fileValue(file("${destinationDirectory.get()}/${packageName}-min_${project.version}${jdkString}_${archString}.${type}")) } } else { - conventionMapping.archiveFile = { objects.fileProperty().fileValue(file("${destinationDirectory.get()}/${packageName}-${project.version}${jdkString}.${archString}.${type}")) } + conventionMapping.archiveFile = { objects.fileProperty().fileValue(file("${destinationDirectory.get()}/${packageName}-min-${project.version}${jdkString}.${archString}.${type}")) } } String packagingFiles = "build/packaging/${jdk ? '' : 'no-jdk-'}${type}" From 3187e63d6202a5f6a632b6aedea680f632943cfd Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Wed, 22 Sep 2021 10:52:55 -0400 Subject: [PATCH 2/6] Update Jackson to 2.12.5 (#1247) (#1270) Signed-off-by: Andriy Redko --- buildSrc/version.properties | 2 +- client/sniffer/licenses/jackson-core-2.11.4.jar.sha1 | 1 - client/sniffer/licenses/jackson-core-2.12.5.jar.sha1 | 1 + .../upgrade-cli/licenses/jackson-annotations-2.11.4.jar.sha1 | 1 - .../upgrade-cli/licenses/jackson-annotations-2.12.5.jar.sha1 | 1 + .../tools/upgrade-cli/licenses/jackson-databind-2.11.4.jar.sha1 | 1 - .../tools/upgrade-cli/licenses/jackson-databind-2.12.5.jar.sha1 | 1 + libs/x-content/licenses/jackson-core-2.11.4.jar.sha1 | 1 - libs/x-content/licenses/jackson-core-2.12.5.jar.sha1 | 1 + libs/x-content/licenses/jackson-dataformat-cbor-2.11.4.jar.sha1 | 1 - libs/x-content/licenses/jackson-dataformat-cbor-2.12.5.jar.sha1 | 1 + .../x-content/licenses/jackson-dataformat-smile-2.11.4.jar.sha1 | 1 - .../x-content/licenses/jackson-dataformat-smile-2.12.5.jar.sha1 | 1 + libs/x-content/licenses/jackson-dataformat-yaml-2.11.4.jar.sha1 | 1 - libs/x-content/licenses/jackson-dataformat-yaml-2.12.5.jar.sha1 | 1 + .../ingest-geoip/licenses/jackson-annotations-2.11.4.jar.sha1 | 1 - .../ingest-geoip/licenses/jackson-annotations-2.12.5.jar.sha1 | 1 + modules/ingest-geoip/licenses/jackson-databind-2.11.4.jar.sha1 | 1 - modules/ingest-geoip/licenses/jackson-databind-2.12.5.jar.sha1 | 1 + .../discovery-ec2/licenses/jackson-annotations-2.11.4.jar.sha1 | 1 - .../discovery-ec2/licenses/jackson-annotations-2.12.5.jar.sha1 | 1 + plugins/discovery-ec2/licenses/jackson-databind-2.11.4.jar.sha1 | 1 - plugins/discovery-ec2/licenses/jackson-databind-2.12.5.jar.sha1 | 1 + .../repository-s3/licenses/jackson-annotations-2.11.4.jar.sha1 | 1 - .../repository-s3/licenses/jackson-annotations-2.12.5.jar.sha1 | 1 + plugins/repository-s3/licenses/jackson-databind-2.11.4.jar.sha1 | 1 - plugins/repository-s3/licenses/jackson-databind-2.12.5.jar.sha1 | 1 + 27 files changed, 14 insertions(+), 14 deletions(-) delete mode 100644 client/sniffer/licenses/jackson-core-2.11.4.jar.sha1 create mode 100644 client/sniffer/licenses/jackson-core-2.12.5.jar.sha1 delete mode 100644 distribution/tools/upgrade-cli/licenses/jackson-annotations-2.11.4.jar.sha1 create mode 100644 distribution/tools/upgrade-cli/licenses/jackson-annotations-2.12.5.jar.sha1 delete mode 100644 distribution/tools/upgrade-cli/licenses/jackson-databind-2.11.4.jar.sha1 create mode 100644 distribution/tools/upgrade-cli/licenses/jackson-databind-2.12.5.jar.sha1 delete mode 100644 libs/x-content/licenses/jackson-core-2.11.4.jar.sha1 create mode 100644 libs/x-content/licenses/jackson-core-2.12.5.jar.sha1 delete mode 100644 libs/x-content/licenses/jackson-dataformat-cbor-2.11.4.jar.sha1 create mode 100644 libs/x-content/licenses/jackson-dataformat-cbor-2.12.5.jar.sha1 delete mode 100644 libs/x-content/licenses/jackson-dataformat-smile-2.11.4.jar.sha1 create mode 100644 libs/x-content/licenses/jackson-dataformat-smile-2.12.5.jar.sha1 delete mode 100644 libs/x-content/licenses/jackson-dataformat-yaml-2.11.4.jar.sha1 create mode 100644 libs/x-content/licenses/jackson-dataformat-yaml-2.12.5.jar.sha1 delete mode 100644 modules/ingest-geoip/licenses/jackson-annotations-2.11.4.jar.sha1 create mode 100644 modules/ingest-geoip/licenses/jackson-annotations-2.12.5.jar.sha1 delete mode 100644 modules/ingest-geoip/licenses/jackson-databind-2.11.4.jar.sha1 create mode 100644 modules/ingest-geoip/licenses/jackson-databind-2.12.5.jar.sha1 delete mode 100644 plugins/discovery-ec2/licenses/jackson-annotations-2.11.4.jar.sha1 create mode 100644 plugins/discovery-ec2/licenses/jackson-annotations-2.12.5.jar.sha1 delete mode 100644 plugins/discovery-ec2/licenses/jackson-databind-2.11.4.jar.sha1 create mode 100644 plugins/discovery-ec2/licenses/jackson-databind-2.12.5.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/jackson-annotations-2.11.4.jar.sha1 create mode 100644 plugins/repository-s3/licenses/jackson-annotations-2.12.5.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/jackson-databind-2.11.4.jar.sha1 create mode 100644 plugins/repository-s3/licenses/jackson-databind-2.12.5.jar.sha1 diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 57fa170b43fb9..c61dfbe7d2443 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -9,7 +9,7 @@ checkstyle = 8.29 # optional dependencies spatial4j = 0.7 jts = 1.15.0 -jackson = 2.11.4 +jackson = 2.12.5 snakeyaml = 1.26 icu4j = 62.1 supercsv = 2.4.0 diff --git a/client/sniffer/licenses/jackson-core-2.11.4.jar.sha1 b/client/sniffer/licenses/jackson-core-2.11.4.jar.sha1 deleted file mode 100644 index aca9116a02d5b..0000000000000 --- a/client/sniffer/licenses/jackson-core-2.11.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -593f7b18bab07a76767f181e2a2336135ce82cc4 \ No newline at end of file diff --git a/client/sniffer/licenses/jackson-core-2.12.5.jar.sha1 b/client/sniffer/licenses/jackson-core-2.12.5.jar.sha1 new file mode 100644 index 0000000000000..ed27d8a96bf20 --- /dev/null +++ b/client/sniffer/licenses/jackson-core-2.12.5.jar.sha1 @@ -0,0 +1 @@ +725e364cc71b80e60fa450bd06d75cdea7fb2d59 \ No newline at end of file diff --git a/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.11.4.jar.sha1 b/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.11.4.jar.sha1 deleted file mode 100644 index d1ddeaf8a6127..0000000000000 --- a/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.11.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2c3f5c079330f3a01726686a078979420f547ae4 \ No newline at end of file diff --git a/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.12.5.jar.sha1 b/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.12.5.jar.sha1 new file mode 100644 index 0000000000000..797bcf2b161d4 --- /dev/null +++ b/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.12.5.jar.sha1 @@ -0,0 +1 @@ +52d929d5bb21d0186fe24c09624cc3ee4bafc3b3 \ No newline at end of file diff --git a/distribution/tools/upgrade-cli/licenses/jackson-databind-2.11.4.jar.sha1 b/distribution/tools/upgrade-cli/licenses/jackson-databind-2.11.4.jar.sha1 deleted file mode 100644 index dd4ebcc7051e3..0000000000000 --- a/distribution/tools/upgrade-cli/licenses/jackson-databind-2.11.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5d9f3d441f99d721b957e3497f0a6465c764fad4 \ No newline at end of file diff --git a/distribution/tools/upgrade-cli/licenses/jackson-databind-2.12.5.jar.sha1 b/distribution/tools/upgrade-cli/licenses/jackson-databind-2.12.5.jar.sha1 new file mode 100644 index 0000000000000..ca1bd46bc3cd3 --- /dev/null +++ b/distribution/tools/upgrade-cli/licenses/jackson-databind-2.12.5.jar.sha1 @@ -0,0 +1 @@ +b064cf057f23d3d35390328c5030847efeffedde \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-core-2.11.4.jar.sha1 b/libs/x-content/licenses/jackson-core-2.11.4.jar.sha1 deleted file mode 100644 index aca9116a02d5b..0000000000000 --- a/libs/x-content/licenses/jackson-core-2.11.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -593f7b18bab07a76767f181e2a2336135ce82cc4 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-core-2.12.5.jar.sha1 b/libs/x-content/licenses/jackson-core-2.12.5.jar.sha1 new file mode 100644 index 0000000000000..ed27d8a96bf20 --- /dev/null +++ b/libs/x-content/licenses/jackson-core-2.12.5.jar.sha1 @@ -0,0 +1 @@ +725e364cc71b80e60fa450bd06d75cdea7fb2d59 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-cbor-2.11.4.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-cbor-2.11.4.jar.sha1 deleted file mode 100644 index 7b741c3059c04..0000000000000 --- a/libs/x-content/licenses/jackson-dataformat-cbor-2.11.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -67fa6a00bdc31029bf841ee97d993ef2bb530aa0 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-cbor-2.12.5.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-cbor-2.12.5.jar.sha1 new file mode 100644 index 0000000000000..f992f732a7f91 --- /dev/null +++ b/libs/x-content/licenses/jackson-dataformat-cbor-2.12.5.jar.sha1 @@ -0,0 +1 @@ +2b6f24ee5ac7cde7f5a4e574bd0af4a72ecb55f6 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-smile-2.11.4.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-smile-2.11.4.jar.sha1 deleted file mode 100644 index 37d8854642720..0000000000000 --- a/libs/x-content/licenses/jackson-dataformat-smile-2.11.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -10c1faac0b0bd8545eff02599b48a149202de066 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-smile-2.12.5.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-smile-2.12.5.jar.sha1 new file mode 100644 index 0000000000000..9e2c3222014c2 --- /dev/null +++ b/libs/x-content/licenses/jackson-dataformat-smile-2.12.5.jar.sha1 @@ -0,0 +1 @@ +4233326c74c6601fbbeea11c103c011859cc687d \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-yaml-2.11.4.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-yaml-2.11.4.jar.sha1 deleted file mode 100644 index f6c3c12a61a5d..0000000000000 --- a/libs/x-content/licenses/jackson-dataformat-yaml-2.11.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ba01014ab0228449be401975b1a7af2f3cdaf1d7 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-yaml-2.12.5.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-yaml-2.12.5.jar.sha1 new file mode 100644 index 0000000000000..6eba2dad4947f --- /dev/null +++ b/libs/x-content/licenses/jackson-dataformat-yaml-2.12.5.jar.sha1 @@ -0,0 +1 @@ +89ec27e5f422b0749b1133137c1b36debbb5f3bc \ No newline at end of file diff --git a/modules/ingest-geoip/licenses/jackson-annotations-2.11.4.jar.sha1 b/modules/ingest-geoip/licenses/jackson-annotations-2.11.4.jar.sha1 deleted file mode 100644 index d1ddeaf8a6127..0000000000000 --- a/modules/ingest-geoip/licenses/jackson-annotations-2.11.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2c3f5c079330f3a01726686a078979420f547ae4 \ No newline at end of file diff --git a/modules/ingest-geoip/licenses/jackson-annotations-2.12.5.jar.sha1 b/modules/ingest-geoip/licenses/jackson-annotations-2.12.5.jar.sha1 new file mode 100644 index 0000000000000..797bcf2b161d4 --- /dev/null +++ b/modules/ingest-geoip/licenses/jackson-annotations-2.12.5.jar.sha1 @@ -0,0 +1 @@ +52d929d5bb21d0186fe24c09624cc3ee4bafc3b3 \ No newline at end of file diff --git a/modules/ingest-geoip/licenses/jackson-databind-2.11.4.jar.sha1 b/modules/ingest-geoip/licenses/jackson-databind-2.11.4.jar.sha1 deleted file mode 100644 index dd4ebcc7051e3..0000000000000 --- a/modules/ingest-geoip/licenses/jackson-databind-2.11.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5d9f3d441f99d721b957e3497f0a6465c764fad4 \ No newline at end of file diff --git a/modules/ingest-geoip/licenses/jackson-databind-2.12.5.jar.sha1 b/modules/ingest-geoip/licenses/jackson-databind-2.12.5.jar.sha1 new file mode 100644 index 0000000000000..ca1bd46bc3cd3 --- /dev/null +++ b/modules/ingest-geoip/licenses/jackson-databind-2.12.5.jar.sha1 @@ -0,0 +1 @@ +b064cf057f23d3d35390328c5030847efeffedde \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/jackson-annotations-2.11.4.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-annotations-2.11.4.jar.sha1 deleted file mode 100644 index d1ddeaf8a6127..0000000000000 --- a/plugins/discovery-ec2/licenses/jackson-annotations-2.11.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2c3f5c079330f3a01726686a078979420f547ae4 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/jackson-annotations-2.12.5.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-annotations-2.12.5.jar.sha1 new file mode 100644 index 0000000000000..797bcf2b161d4 --- /dev/null +++ b/plugins/discovery-ec2/licenses/jackson-annotations-2.12.5.jar.sha1 @@ -0,0 +1 @@ +52d929d5bb21d0186fe24c09624cc3ee4bafc3b3 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/jackson-databind-2.11.4.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-databind-2.11.4.jar.sha1 deleted file mode 100644 index dd4ebcc7051e3..0000000000000 --- a/plugins/discovery-ec2/licenses/jackson-databind-2.11.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5d9f3d441f99d721b957e3497f0a6465c764fad4 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/jackson-databind-2.12.5.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-databind-2.12.5.jar.sha1 new file mode 100644 index 0000000000000..ca1bd46bc3cd3 --- /dev/null +++ b/plugins/discovery-ec2/licenses/jackson-databind-2.12.5.jar.sha1 @@ -0,0 +1 @@ +b064cf057f23d3d35390328c5030847efeffedde \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jackson-annotations-2.11.4.jar.sha1 b/plugins/repository-s3/licenses/jackson-annotations-2.11.4.jar.sha1 deleted file mode 100644 index d1ddeaf8a6127..0000000000000 --- a/plugins/repository-s3/licenses/jackson-annotations-2.11.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2c3f5c079330f3a01726686a078979420f547ae4 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jackson-annotations-2.12.5.jar.sha1 b/plugins/repository-s3/licenses/jackson-annotations-2.12.5.jar.sha1 new file mode 100644 index 0000000000000..797bcf2b161d4 --- /dev/null +++ b/plugins/repository-s3/licenses/jackson-annotations-2.12.5.jar.sha1 @@ -0,0 +1 @@ +52d929d5bb21d0186fe24c09624cc3ee4bafc3b3 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jackson-databind-2.11.4.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.11.4.jar.sha1 deleted file mode 100644 index dd4ebcc7051e3..0000000000000 --- a/plugins/repository-s3/licenses/jackson-databind-2.11.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5d9f3d441f99d721b957e3497f0a6465c764fad4 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jackson-databind-2.12.5.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.12.5.jar.sha1 new file mode 100644 index 0000000000000..ca1bd46bc3cd3 --- /dev/null +++ b/plugins/repository-s3/licenses/jackson-databind-2.12.5.jar.sha1 @@ -0,0 +1 @@ +b064cf057f23d3d35390328c5030847efeffedde \ No newline at end of file From dbc3066e980055e87fe352d35d7942288fb1d080 Mon Sep 17 00:00:00 2001 From: Xue Zhou <85715413+xuezhou25@users.noreply.github.com> Date: Fri, 24 Sep 2021 19:07:25 +0000 Subject: [PATCH 3/6] fix gradle check fail due to renameing -min in #1094 (#1289) (#1291) Signed-off-by: Xue Zhou --- .../InternalDistributionBwcSetupPlugin.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionBwcSetupPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionBwcSetupPlugin.java index 8934ce03c0ef0..9e1211711ecf8 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionBwcSetupPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/internal/InternalDistributionBwcSetupPlugin.java @@ -239,10 +239,17 @@ private static class DistributionProject { DistributionProject(String name, String baseDir, Version version, String classifier, String extension, File checkoutDir) { this.name = name; this.projectPath = baseDir + "/" + name; - this.distFile = new File( - checkoutDir, - baseDir + "/" + name + "/build/distributions/opensearch-" + version + "-SNAPSHOT" + classifier + "." + extension - ); + if (version.onOrAfter("1.1.0")) { + this.distFile = new File( + checkoutDir, + baseDir + "/" + name + "/build/distributions/opensearch-min-" + version + "-SNAPSHOT" + classifier + "." + extension + ); + } else { + this.distFile = new File( + checkoutDir, + baseDir + "/" + name + "/build/distributions/opensearch-" + version + "-SNAPSHOT" + classifier + "." + extension + ); + } // we only ported this down to the 7.x branch. if (version.onOrAfter("7.10.0") && (name.endsWith("zip") || name.endsWith("tar"))) { this.expandedDistDir = new File(checkoutDir, baseDir + "/" + name + "/build/install"); From 294cee41d6c6f3bef74e7d0a0ec362654a74d3e9 Mon Sep 17 00:00:00 2001 From: Sai Date: Sun, 26 Sep 2021 08:31:48 +0530 Subject: [PATCH 4/6] [Backport] Support for translog pruning based on retention leases (#1226) Support for translog pruning based on retention leases (#1038) Co-authored-by: Nicholas Walter Knize Signed-off-by: Sai Kumar --- .../indices/settings/UpdateSettingsIT.java | 27 ++++++ .../metadata/MetadataCreateIndexService.java | 18 ++-- .../common/settings/IndexScopedSettings.java | 1 + .../org/opensearch/index/IndexSettings.java | 61 ++++++++++++- .../org/opensearch/index/engine/Engine.java | 11 +++ .../index/engine/InternalEngine.java | 7 +- .../opensearch/index/shard/IndexShard.java | 6 +- .../translog/TranslogDeletionPolicy.java | 68 +++++++++++++- .../MetadataCreateIndexServiceTests.java | 13 +++ .../opensearch/index/IndexSettingsTests.java | 33 +++++++ .../translog/TranslogDeletionPolicyTests.java | 90 ++++++++++++++++++- .../index/translog/TranslogTests.java | 4 +- 12 files changed, 321 insertions(+), 18 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/settings/UpdateSettingsIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/settings/UpdateSettingsIT.java index dbe2724bc5f31..224ba0af90c21 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/settings/UpdateSettingsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/settings/UpdateSettingsIT.java @@ -661,6 +661,33 @@ public void testNumberOfReplicasSettingsVersion() { assertThat(newSettingsVersion, equalTo(1 + settingsVersion)); } + public void testTranslogPruningSetting() { + createIndex("test"); + ensureGreen("test"); + final String REPLICATION_TRANSLOG_SETTING = "index.plugins.replication.translog.retention_lease.pruning.enabled"; + final long settingsVersion = + client().admin().cluster().prepareState().get().getState().metadata().index("test").getSettingsVersion(); + GetSettingsResponse settingsResponse = client().admin().indices().prepareGetSettings("test").get(); + boolean shouldPruneTranslogByRetentionLease = Boolean.parseBoolean( + settingsResponse.getSetting("test", REPLICATION_TRANSLOG_SETTING) + ); + + assertFalse(shouldPruneTranslogByRetentionLease); + assertAcked(client().admin().indices() + .prepareUpdateSettings("test") + .setSettings(Settings.builder() + .put("index.plugins.replication.translog.retention_lease.pruning.enabled", true) + )); + final long newSettingsVersion = + client().admin().cluster().prepareState().get().getState().metadata().index("test").getSettingsVersion(); + assertThat(newSettingsVersion, equalTo(1 + settingsVersion)); + settingsResponse = client().admin().indices().prepareGetSettings("test").get(); + shouldPruneTranslogByRetentionLease = Boolean.parseBoolean( + settingsResponse.getSetting("test", REPLICATION_TRANSLOG_SETTING) + ); + assertTrue(shouldPruneTranslogByRetentionLease); + } + /* * Test that we are able to set the setting index.number_of_replicas to the default. */ diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index e40361cde6af4..e57d38a6bced7 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -1258,11 +1258,19 @@ public static int calculateNumRoutingShards(int numShards, Version indexVersionC } public static void validateTranslogRetentionSettings(Settings indexSettings) { - if (IndexSettings.INDEX_SOFT_DELETES_SETTING.get(indexSettings) && - (IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.exists(indexSettings) - || IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.exists(indexSettings))) { - DEPRECATION_LOGGER.deprecate("translog_retention", "Translog retention settings [index.translog.retention.age] " - + "and [index.translog.retention.size] are deprecated and effectively ignored. They will be removed in a future version."); + if (IndexSettings.INDEX_SOFT_DELETES_SETTING.get(indexSettings)) { + if(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.exists(indexSettings) + || IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING.exists(indexSettings)) { + DEPRECATION_LOGGER.deprecate("translog_retention", "Translog retention settings " + + "[index.translog.retention.age] " + + "and [index.translog.retention.size] are deprecated and effectively ignored. " + + "They will be removed in a future version."); + } else if(IndexSettings.INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING.exists(indexSettings)) { + DEPRECATION_LOGGER.deprecate("translog_pruning_retention_lease", + "[index.plugins.replication.translog.retention_lease.pruning.enabled] " + + "setting was deprecated in OpenSearch and will be removed in a future release! " + + "See the breaking changes documentation for the next major version."); + } } } } diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java index 9e73d6fdd9c4f..a6ed4bce83dee 100644 --- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java @@ -156,6 +156,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING, EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING, IndexSettings.INDEX_FLUSH_AFTER_MERGE_THRESHOLD_SIZE_SETTING, + IndexSettings.INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING, IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING, IndexSettings.INDEX_TRANSLOG_GENERATION_THRESHOLD_SIZE_SETTING, IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING, diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java index 10aaaf821cc15..15d723ee1e33f 100644 --- a/server/src/main/java/org/opensearch/index/IndexSettings.java +++ b/server/src/main/java/org/opensearch/index/IndexSettings.java @@ -260,6 +260,15 @@ public final class IndexSettings { settings -> Boolean.toString(IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(LegacyESVersion.V_7_0_0)), Property.IndexScope, Property.Final); + /** + * Specifies if the index translog should prune based on retention leases. + * @deprecated EXPERT: this setting is specific to CCR and will be moved to a plugin in the next release + */ + @Deprecated + public static final Setting INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING = + Setting.boolSetting("index.plugins.replication.translog.retention_lease.pruning.enabled", false, + Property.IndexScope, Property.Dynamic); + /** * Controls how many soft-deleted documents will be kept around before being merged away. Keeping more deleted * documents increases the chance of operation-based recoveries and allows querying a longer history of documents. @@ -286,9 +295,12 @@ public final class IndexSettings { * the chance of ops based recoveries for indices with soft-deletes disabled. * This setting will be ignored if soft-deletes is used in peer recoveries (default in 7.4). **/ + @Deprecated + private static final ByteSizeValue DEFAULT_TRANSLOG_RETENTION_SIZE = new ByteSizeValue(512, ByteSizeUnit.MB); + public static final Setting INDEX_TRANSLOG_RETENTION_SIZE_SETTING = Setting.byteSizeSetting("index.translog.retention.size", - settings -> shouldDisableTranslogRetention(settings) ? "-1" : "512MB", + settings -> DEFAULT_TRANSLOG_RETENTION_SIZE.getStringRep(), Property.Dynamic, Property.IndexScope); /** @@ -389,6 +401,8 @@ public final class IndexSettings { private final IndexScopedSettings scopedSettings; private long gcDeletesInMillis = DEFAULT_GC_DELETES.millis(); private final boolean softDeleteEnabled; + @Deprecated + private volatile boolean translogPruningByRetentionLease; private volatile long softDeleteRetentionOperations; private volatile long retentionLeaseMillis; @@ -525,6 +539,10 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti mergeSchedulerConfig = new MergeSchedulerConfig(this); gcDeletesInMillis = scopedSettings.get(INDEX_GC_DELETES_SETTING).getMillis(); softDeleteEnabled = version.onOrAfter(LegacyESVersion.V_6_5_0) && scopedSettings.get(INDEX_SOFT_DELETES_SETTING); + // @todo move to CCR plugin + this.translogPruningByRetentionLease = version.onOrAfter(Version.V_1_1_0) && + softDeleteEnabled && + scopedSettings.get(INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING); softDeleteRetentionOperations = scopedSettings.get(INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING); retentionLeaseMillis = scopedSettings.get(INDEX_SOFT_DELETES_RETENTION_LEASE_PERIOD_SETTING).millis(); warmerEnabled = scopedSettings.get(INDEX_WARMER_ENABLED_SETTING); @@ -593,6 +611,8 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti this::setGenerationThresholdSize); scopedSettings.addSettingsUpdateConsumer(INDEX_TRANSLOG_RETENTION_AGE_SETTING, this::setTranslogRetentionAge); scopedSettings.addSettingsUpdateConsumer(INDEX_TRANSLOG_RETENTION_SIZE_SETTING, this::setTranslogRetentionSize); + scopedSettings.addSettingsUpdateConsumer(INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING, + this::setTranslogPruningByRetentionLease); scopedSettings.addSettingsUpdateConsumer(INDEX_REFRESH_INTERVAL_SETTING, this::setRefreshInterval); scopedSettings.addSettingsUpdateConsumer(MAX_REFRESH_LISTENERS_PER_SHARD, this::setMaxRefreshListeners); scopedSettings.addSettingsUpdateConsumer(MAX_ANALYZED_OFFSET_SETTING, this::setHighlightMaxAnalyzedOffset); @@ -623,8 +643,20 @@ private void setFlushAfterMergeThresholdSize(ByteSizeValue byteSizeValue) { this.flushAfterMergeThresholdSize = byteSizeValue; } + /** + * Enable or disable translog pruning by retention lease requirement + * + * @deprecated EXPERT: this setting is specific to CCR and will be moved to a plugin in the next release + */ + @Deprecated + private void setTranslogPruningByRetentionLease(boolean enabled) { + this.translogPruningByRetentionLease = INDEX_SOFT_DELETES_SETTING.get(settings) && enabled; + } + private void setTranslogRetentionSize(ByteSizeValue byteSizeValue) { - if (shouldDisableTranslogRetention(settings) && byteSizeValue.getBytes() >= 0) { + if (shouldDisableTranslogRetention(settings) && + shouldPruneTranslogByRetentionLease(settings) == false && + byteSizeValue.getBytes() >= 0) { // ignore the translog retention settings if soft-deletes enabled this.translogRetentionSize = new ByteSizeValue(-1); } else { @@ -826,7 +858,12 @@ public TimeValue getRefreshInterval() { * Returns the transaction log retention size which controls how much of the translog is kept around to allow for ops based recoveries */ public ByteSizeValue getTranslogRetentionSize() { - assert shouldDisableTranslogRetention(settings) == false || translogRetentionSize.getBytes() == -1L : translogRetentionSize; + if(shouldDisableTranslogRetention(settings) && shouldPruneTranslogByRetentionLease(settings) == false) { + return new ByteSizeValue(-1); + } + else if(shouldPruneTranslogByRetentionLease(settings) && translogRetentionSize.getBytes() == -1) { + return DEFAULT_TRANSLOG_RETENTION_SIZE; + } return translogRetentionSize; } @@ -1071,6 +1108,24 @@ public void setRequiredPipeline(final String requiredPipeline) { this.requiredPipeline = requiredPipeline; } + /** + * Returns true if translog ops should be pruned based on retention lease + * @deprecated EXPERT: this setting is specific to CCR and will be moved to a plugin in the next release + */ + @Deprecated + public boolean shouldPruneTranslogByRetentionLease() { + return translogPruningByRetentionLease; + } + + /** + * Returns true if translog ops should be pruned based on retention lease + * @deprecated EXPERT: this setting is specific to CCR and will be moved to a plugin in the next release + */ + @Deprecated + public static boolean shouldPruneTranslogByRetentionLease(Settings settings) { + return INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING.get(settings); + } + /** * Returns true if soft-delete is enabled. */ diff --git a/server/src/main/java/org/opensearch/index/engine/Engine.java b/server/src/main/java/org/opensearch/index/engine/Engine.java index 9497a0ffb7b71..e836e2dc14496 100644 --- a/server/src/main/java/org/opensearch/index/engine/Engine.java +++ b/server/src/main/java/org/opensearch/index/engine/Engine.java @@ -1858,7 +1858,18 @@ public IndexCommit getIndexCommit() { } public void onSettingsChanged(TimeValue translogRetentionAge, ByteSizeValue translogRetentionSize, long softDeletesRetentionOps) { + onSettingsChanged(translogRetentionAge, translogRetentionSize, softDeletesRetentionOps, false); + } + /** + * callback when index settings change + * + * @deprecated EXPERT: this method is specific to CCR and will be moved to a plugin in the next release + */ + @Deprecated + public void onSettingsChanged(TimeValue translogRetentionAge, ByteSizeValue translogRetentionSize, + long softDeletesRetentionOps, boolean translogPruningByRetentionLease) { + // @todo this is overly silent; make this abstract and force derived classes to noop in the next release } /** diff --git a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java index c13e009510c7d..0df1747156672 100644 --- a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java @@ -227,7 +227,8 @@ public InternalEngine(EngineConfig engineConfig) { final TranslogDeletionPolicy translogDeletionPolicy = new TranslogDeletionPolicy( engineConfig.getIndexSettings().getTranslogRetentionSize().getBytes(), engineConfig.getIndexSettings().getTranslogRetentionAge().getMillis(), - engineConfig.getIndexSettings().getTranslogRetentionTotalFiles() + engineConfig.getIndexSettings().getTranslogRetentionTotalFiles(), + engineConfig.retentionLeasesSupplier() ); store.incRef(); IndexWriter writer = null; @@ -2572,7 +2573,8 @@ final void ensureCanFlush() { } @Override - public void onSettingsChanged(TimeValue translogRetentionAge, ByteSizeValue translogRetentionSize, long softDeletesRetentionOps) { + public void onSettingsChanged(TimeValue translogRetentionAge, ByteSizeValue translogRetentionSize, + long softDeletesRetentionOps, boolean translogPruningByRetentionLease) { mergeScheduler.refreshConfig(); // config().isEnableGcDeletes() or config.getGcDeletesInMillis() may have changed: maybePruneDeletes(); @@ -2585,6 +2587,7 @@ public void onSettingsChanged(TimeValue translogRetentionAge, ByteSizeValue tran final TranslogDeletionPolicy translogDeletionPolicy = translog.getDeletionPolicy(); translogDeletionPolicy.setRetentionAgeInMillis(translogRetentionAge.millis()); translogDeletionPolicy.setRetentionSizeInBytes(translogRetentionSize.getBytes()); + translogDeletionPolicy.shouldPruneTranslogByRetentionLease(translogPruningByRetentionLease); softDeletesPolicy.setRetentionOperations(softDeletesRetentionOps); } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 1f81b3caa3006..53977ca2c600d 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -1961,8 +1961,10 @@ public void onSettingsChanged() { final boolean disableTranslogRetention = indexSettings.isSoftDeleteEnabled() && useRetentionLeasesInPeerRecovery; engineOrNull.onSettingsChanged( disableTranslogRetention ? TimeValue.MINUS_ONE : indexSettings.getTranslogRetentionAge(), - disableTranslogRetention ? new ByteSizeValue(-1) : indexSettings.getTranslogRetentionSize(), - indexSettings.getSoftDeleteRetentionOperations() + disableTranslogRetention && !indexSettings.shouldPruneTranslogByRetentionLease() ? + new ByteSizeValue(-1) : indexSettings.getTranslogRetentionSize(), + indexSettings.getSoftDeleteRetentionOperations(), + indexSettings.shouldPruneTranslogByRetentionLease() ); } } diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogDeletionPolicy.java b/server/src/main/java/org/opensearch/index/translog/TranslogDeletionPolicy.java index 42f3893fd98ad..0b0bf9c87ba38 100644 --- a/server/src/main/java/org/opensearch/index/translog/TranslogDeletionPolicy.java +++ b/server/src/main/java/org/opensearch/index/translog/TranslogDeletionPolicy.java @@ -35,6 +35,8 @@ import org.apache.lucene.util.Counter; import org.opensearch.Assertions; import org.opensearch.common.lease.Releasable; +import org.opensearch.index.seqno.RetentionLease; +import org.opensearch.index.seqno.RetentionLeases; import org.opensearch.index.seqno.SequenceNumbers; import java.io.IOException; @@ -43,10 +45,16 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; public class TranslogDeletionPolicy { private final Map openTranslogRef; + /** + * @deprecated EXPERT: this supplier is specific to CCR and will be moved to a plugin in the next release + */ + @Deprecated + private Supplier retentionLeasesSupplier; public void assertNoOpenTranslogRefs() { if (openTranslogRef.isEmpty() == false) { @@ -69,6 +77,12 @@ public void assertNoOpenTranslogRefs() { private int retentionTotalFiles; + /** + * @deprecated EXPERT: this variable is specific to CCR and will be moved to a plugin in the next release + */ + @Deprecated + private boolean shouldPruneTranslogByRetentionLease; + public TranslogDeletionPolicy(long retentionSizeInBytes, long retentionAgeInMillis, int retentionTotalFiles) { this.retentionSizeInBytes = retentionSizeInBytes; this.retentionAgeInMillis = retentionAgeInMillis; @@ -80,6 +94,17 @@ public TranslogDeletionPolicy(long retentionSizeInBytes, long retentionAgeInMill } } + /** + * Construct a TranslogDeletionPolicy to include pruning by retention leases + * @deprecated EXPERT: this ctor is specific to CCR and will be moved to a plugin in the next release + */ + @Deprecated + public TranslogDeletionPolicy(long retentionSizeInBytes, long retentionAgeInMillis, int retentionTotalFiles, + Supplier retentionLeasesSupplier) { + this(retentionSizeInBytes, retentionAgeInMillis, retentionTotalFiles); + this.retentionLeasesSupplier = retentionLeasesSupplier; + } + public synchronized void setLocalCheckpointOfSafeCommit(long newCheckpoint) { if (newCheckpoint < this.localCheckpointOfSafeCommit) { throw new IllegalArgumentException("local checkpoint of the safe commit can't go backwards: " + @@ -100,6 +125,15 @@ synchronized void setRetentionTotalFiles(int retentionTotalFiles) { this.retentionTotalFiles = retentionTotalFiles; } + /** + * Should the translog be pruned by the retention lease heuristic + * @deprecated EXPERT: this setting is specific to CCR and will be moved to a plugin in the next release + */ + @Deprecated + public synchronized void shouldPruneTranslogByRetentionLease(boolean translogPruneByRetentionLease) { + this.shouldPruneTranslogByRetentionLease = translogPruneByRetentionLease; + } + /** * acquires the basis generation for a new snapshot. Any translog generation above, and including, the returned generation * will not be deleted until the returned {@link Releasable} is closed. @@ -157,6 +191,12 @@ synchronized long minTranslogGenRequired(List readers, TranslogW long minByLocks = getMinTranslogGenRequiredByLocks(); long minByAge = getMinTranslogGenByAge(readers, writer, retentionAgeInMillis, currentTime()); long minBySize = getMinTranslogGenBySize(readers, writer, retentionSizeInBytes); + long minByRetentionLeasesAndSize = Long.MAX_VALUE; + if(shouldPruneTranslogByRetentionLease) { + // If retention size is specified, size takes precedence. + long minByRetentionLeases = getMinTranslogGenByRetentionLease(readers, writer, retentionLeasesSupplier); + minByRetentionLeasesAndSize = Math.max(minBySize, minByRetentionLeases); + } final long minByAgeAndSize; if (minBySize == Long.MIN_VALUE && minByAge == Long.MIN_VALUE) { // both size and age are disabled; @@ -165,7 +205,33 @@ synchronized long minTranslogGenRequired(List readers, TranslogW minByAgeAndSize = Math.max(minByAge, minBySize); } long minByNumFiles = getMinTranslogGenByTotalFiles(readers, writer, retentionTotalFiles); - return Math.min(Math.max(minByAgeAndSize, minByNumFiles), minByLocks); + long minByTranslogGenSettings = Math.min(Math.max(minByAgeAndSize, minByNumFiles), minByLocks); + return Math.min(minByTranslogGenSettings, minByRetentionLeasesAndSize); + } + + /** + * Find the minimum translog generation by minimum retaining sequence number + * @deprecated EXPERT: this configuration is specific to CCR and will be moved to a plugin in the next release + */ + @Deprecated + static long getMinTranslogGenByRetentionLease(List readers, TranslogWriter writer, + Supplier retentionLeasesSupplier) { + long minGen = writer.getGeneration(); + final long minimumRetainingSequenceNumber = retentionLeasesSupplier.get() + .leases() + .stream() + .mapToLong(RetentionLease::retainingSequenceNumber) + .min() + .orElse(Long.MAX_VALUE); + + for (int i = readers.size() - 1; i >= 0; i--) { + final TranslogReader reader = readers.get(i); + if(reader.getCheckpoint().minSeqNo <= minimumRetainingSequenceNumber && + reader.getCheckpoint().maxSeqNo >= minimumRetainingSequenceNumber) { + minGen = Math.min(minGen, reader.getGeneration()); + } + } + return minGen; } static long getMinTranslogGenBySize(List readers, TranslogWriter writer, long retentionSizeInBytes) { diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java index faaa404b577cf..ecb46ade1db34 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -1001,6 +1001,19 @@ public void testDeprecatedSimpleFSStoreSettings() { "or better performance, or other file systems instead."); } + public void testTranslogPruningBasedOnRetentionLeaseDeprecation() { + request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); + request.settings(Settings.builder() + .put(INDEX_SOFT_DELETES_SETTING.getKey(), true) + .put(IndexSettings.INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING.getKey(), true).build()); + aggregateIndexSettings(ClusterState.EMPTY_STATE, request, Settings.EMPTY, + null, Settings.EMPTY, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, randomShardLimitService(), + Collections.emptySet()); + assertWarnings("[index.plugins.replication.translog.retention_lease.pruning.enabled] setting " + + "was deprecated in OpenSearch and will be removed in a future release! " + + "See the breaking changes documentation for the next major version."); + } + private IndexTemplateMetadata addMatchingTemplate(Consumer configurator) { IndexTemplateMetadata.Builder builder = templateMetadataBuilder("template1", "te*"); configurator.accept(builder); diff --git a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java index d2c285c06a5dc..130f553c890ac 100644 --- a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java @@ -655,4 +655,37 @@ public void testUpdateTranslogRetentionSettingsWithSoftDeletesDisabled() { assertThat(indexSettings.getTranslogRetentionAge(), equalTo(ageSetting)); assertThat(indexSettings.getTranslogRetentionSize(), equalTo(sizeSetting)); } + + public void testTranslogPruningSettingsWithSoftDeletesEnabled() { + Settings.Builder settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.V_1_1_0); + + ByteSizeValue retentionSize = new ByteSizeValue(512, ByteSizeUnit.MB); + boolean translogPruningEnabled = randomBoolean(); + settings.put(IndexSettings.INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING.getKey(), + translogPruningEnabled); + IndexMetadata metadata = newIndexMeta("index", settings.build()); + IndexSettings indexSettings = new IndexSettings(metadata, Settings.EMPTY); + if(translogPruningEnabled) { + assertTrue(indexSettings.shouldPruneTranslogByRetentionLease()); + assertThat(indexSettings.getTranslogRetentionSize().getBytes(), equalTo(retentionSize.getBytes())); + } else { + assertFalse(indexSettings.shouldPruneTranslogByRetentionLease()); + assertThat(indexSettings.getTranslogRetentionSize().getBytes(), equalTo(-1L)); + } + } + + public void testTranslogPruningSettingsWithSoftDeletesDisabled() { + Settings.Builder settings = Settings.builder() + .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false) + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT); + boolean translogPruningEnabled = randomBoolean(); + ByteSizeValue retentionSize = new ByteSizeValue(512, ByteSizeUnit.MB); + settings.put(IndexSettings.INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING.getKey(), + translogPruningEnabled); + IndexMetadata metadata = newIndexMeta("index", settings.build()); + IndexSettings indexSettings = new IndexSettings(metadata, Settings.EMPTY); + assertFalse(indexSettings.shouldPruneTranslogByRetentionLease()); + assertThat(indexSettings.getTranslogRetentionSize().getBytes(), equalTo(retentionSize.getBytes())); + } } diff --git a/server/src/test/java/org/opensearch/index/translog/TranslogDeletionPolicyTests.java b/server/src/test/java/org/opensearch/index/translog/TranslogDeletionPolicyTests.java index 44aceab0445ad..c8fbb33483fef 100644 --- a/server/src/test/java/org/opensearch/index/translog/TranslogDeletionPolicyTests.java +++ b/server/src/test/java/org/opensearch/index/translog/TranslogDeletionPolicyTests.java @@ -40,6 +40,8 @@ import org.opensearch.common.lease.Releasable; import org.opensearch.common.util.BigArrays; import org.opensearch.core.internal.io.IOUtils; +import org.opensearch.index.seqno.RetentionLease; +import org.opensearch.index.seqno.RetentionLeases; import org.opensearch.index.shard.ShardId; import org.opensearch.test.OpenSearchTestCase; import org.mockito.Mockito; @@ -49,7 +51,9 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.LinkedList; import java.util.List; +import java.util.function.Supplier; import static java.lang.Math.min; import static org.hamcrest.Matchers.equalTo; @@ -57,6 +61,8 @@ public class TranslogDeletionPolicyTests extends OpenSearchTestCase { + private static long TOTAL_OPS_IN_GEN = 10L; + public void testNoRetention() throws IOException { long now = System.currentTimeMillis(); Tuple, TranslogWriter> readersAndWriter = createReadersAndWriter(now); @@ -90,6 +96,33 @@ public void testBytesRetention() throws IOException { } } + public void testWithRetentionLease() throws IOException { + long now = System.currentTimeMillis(); + Tuple, TranslogWriter> readersAndWriter = createReadersAndWriter(now); + List allGens = new ArrayList<>(readersAndWriter.v1()); + allGens.add(readersAndWriter.v2()); + Supplier retentionLeasesSupplier = createRetentionLeases(now, 0L, + allGens.size() * TOTAL_OPS_IN_GEN - 1); + try { + final long minimumRetainingSequenceNumber = retentionLeasesSupplier.get() + .leases() + .stream() + .mapToLong(RetentionLease::retainingSequenceNumber) + .min() + .orElse(Long.MAX_VALUE); + + final long selectedReader = (minimumRetainingSequenceNumber/TOTAL_OPS_IN_GEN); + final long selectedGen = allGens.get((int) selectedReader).generation; + assertThat(TranslogDeletionPolicy + .getMinTranslogGenByRetentionLease(readersAndWriter.v1(), readersAndWriter.v2(), retentionLeasesSupplier), + equalTo(selectedGen)); + + } finally { + IOUtils.close(readersAndWriter.v1()); + IOUtils.close(readersAndWriter.v2()); + } + } + public void testAgeRetention() throws IOException { long now = System.currentTimeMillis(); Tuple, TranslogWriter> readersAndWriter = createReadersAndWriter(now); @@ -128,6 +161,38 @@ public void testTotalFilesRetention() throws Exception { } } + public void testBySizeAndRetentionLease() throws Exception { + long now = System.currentTimeMillis(); + Tuple, TranslogWriter> readersAndWriter = createReadersAndWriter(now); + List allGens = new ArrayList<>(readersAndWriter.v1()); + allGens.add(readersAndWriter.v2()); + try { + int selectedReader = randomIntBetween(0, allGens.size() - 1); + final long selectedGeneration = allGens.get(selectedReader).generation; + // Retaining seqno is part of lower gen + long size = allGens.stream().skip(selectedReader).map(BaseTranslogReader::sizeInBytes).reduce(Long::sum).get(); + Supplier retentionLeasesSupplier = createRetentionLeases(now, 0L, + selectedGeneration * TOTAL_OPS_IN_GEN - 1); + TranslogDeletionPolicy deletionPolicy = new MockDeletionPolicy(now, size, Integer.MAX_VALUE, + Integer.MAX_VALUE, retentionLeasesSupplier); + assertThat(deletionPolicy + .minTranslogGenRequired(readersAndWriter.v1(), readersAndWriter.v2()), equalTo(selectedGeneration)); + assertThat(TranslogDeletionPolicy + .getMinTranslogGenByAge(readersAndWriter.v1(), readersAndWriter.v2(), 100L, System.currentTimeMillis()), + equalTo(readersAndWriter.v2().generation)); + + // Retention lease is part of higher gen + retentionLeasesSupplier = createRetentionLeases(now, selectedGeneration * TOTAL_OPS_IN_GEN, + allGens.size() * TOTAL_OPS_IN_GEN + TOTAL_OPS_IN_GEN - 1); + deletionPolicy = new MockDeletionPolicy(now, size, Long.MIN_VALUE, + Integer.MAX_VALUE, retentionLeasesSupplier); + assertThat(deletionPolicy.minTranslogGenRequired(readersAndWriter.v1(), readersAndWriter.v2()), equalTo(selectedGeneration)); + } finally { + IOUtils.close(readersAndWriter.v1()); + IOUtils.close(readersAndWriter.v2()); + } + } + /** * Tests that age trumps size but recovery trumps both. */ @@ -207,19 +272,32 @@ private Tuple, TranslogWriter> createReadersAndWriter(final () -> 1L, randomNonNegativeLong(), new TragicExceptionHolder(), seqNo -> {}, BigArrays.NON_RECYCLING_INSTANCE); writer = Mockito.spy(writer); Mockito.doReturn(now - (numberOfReaders - gen + 1) * 1000).when(writer).getLastModifiedTime(); - byte[] bytes = new byte[4]; ByteArrayDataOutput out = new ByteArrayDataOutput(bytes); - for (int ops = randomIntBetween(0, 20); ops > 0; ops--) { + final long startSeqNo = (gen - 1) * TOTAL_OPS_IN_GEN; + final long endSeqNo = startSeqNo + TOTAL_OPS_IN_GEN - 1; + for (long ops = endSeqNo; ops >= startSeqNo; ops--) { out.reset(bytes); - out.writeInt(ops); + out.writeInt((int) ops); writer.add(ReleasableBytesReference.wrap(new BytesArray(bytes)), ops); } } return new Tuple<>(readers, writer); } + private Supplier createRetentionLeases(final Long now, final Long lowestSeqNo, + final Long highestSeqNo) throws IOException { + LinkedList leases = new LinkedList<>(); + final int numberOfLeases = randomIntBetween(1, 5); + for(int i=0 ;i new RetentionLeases(1L, 1L, leases); + } + private static class MockDeletionPolicy extends TranslogDeletionPolicy { long now; @@ -229,6 +307,12 @@ private static class MockDeletionPolicy extends TranslogDeletionPolicy { this.now = now; } + MockDeletionPolicy(long now, long retentionSizeInBytes, long maxRetentionAgeInMillis, + int maxRetentionTotalFiles, Supplier retentionLeasesSupplier) { + super(retentionSizeInBytes, maxRetentionAgeInMillis, maxRetentionTotalFiles, retentionLeasesSupplier); + this.now = now; + } + @Override protected long currentTime() { return now; diff --git a/server/src/test/java/org/opensearch/index/translog/TranslogTests.java b/server/src/test/java/org/opensearch/index/translog/TranslogTests.java index 221a2e65bfba7..d3f843889ad00 100644 --- a/server/src/test/java/org/opensearch/index/translog/TranslogTests.java +++ b/server/src/test/java/org/opensearch/index/translog/TranslogTests.java @@ -780,7 +780,7 @@ public void testFullRangeSnapshot() throws Exception { // Successful snapshot long nextSeqNo = populateTranslogOps(false); long fromSeqNo = 0L; - long toSeqNo = Math.max(nextSeqNo - 1, fromSeqNo + 15); + long toSeqNo = Math.min(nextSeqNo - 1, fromSeqNo + 15); try (Translog.Snapshot snapshot = translog.newSnapshot(fromSeqNo, toSeqNo, true)) { int totOps = 0; for (Translog.Operation op = snapshot.next(); op != null; op = snapshot.next()) { @@ -793,7 +793,7 @@ public void testFullRangeSnapshot() throws Exception { public void testFullRangeSnapshotWithFailures() throws Exception { long nextSeqNo = populateTranslogOps(true); long fromSeqNo = 0L; - long toSeqNo = Math.max(nextSeqNo-1, fromSeqNo + 15); + long toSeqNo = Math.min(nextSeqNo-1, fromSeqNo + 15); try (Translog.Snapshot snapshot = translog.newSnapshot(fromSeqNo, toSeqNo, true)) { int totOps = 0; for (Translog.Operation op = snapshot.next(); op != null; op = snapshot.next()) { From 3a9ee8142a0be094a3caf22dd2540743d4803bbe Mon Sep 17 00:00:00 2001 From: Nick Knize Date: Sat, 25 Sep 2021 23:04:26 -0500 Subject: [PATCH 5/6] [Revert] Translog Pruning Setting Deprecation Removal (#1294) INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING was deprecated, then deprecation was removed. This adds deprecation back so that the setting can be moved to the plugin in the next minor release. Signed-off-by: Nicholas Walter Knize --- .../src/main/java/org/opensearch/index/IndexSettings.java | 2 +- .../test/java/org/opensearch/index/IndexSettingsTests.java | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java index 15d723ee1e33f..74583d3e67a17 100644 --- a/server/src/main/java/org/opensearch/index/IndexSettings.java +++ b/server/src/main/java/org/opensearch/index/IndexSettings.java @@ -267,7 +267,7 @@ public final class IndexSettings { @Deprecated public static final Setting INDEX_PLUGINS_REPLICATION_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING = Setting.boolSetting("index.plugins.replication.translog.retention_lease.pruning.enabled", false, - Property.IndexScope, Property.Dynamic); + Property.IndexScope, Property.Dynamic, Property.Deprecated); /** * Controls how many soft-deleted documents will be kept around before being merged away. Keeping more deleted diff --git a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java index 130f553c890ac..c3540d314741e 100644 --- a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java @@ -673,6 +673,9 @@ public void testTranslogPruningSettingsWithSoftDeletesEnabled() { assertFalse(indexSettings.shouldPruneTranslogByRetentionLease()); assertThat(indexSettings.getTranslogRetentionSize().getBytes(), equalTo(-1L)); } + assertWarnings("[index.plugins.replication.translog.retention_lease.pruning.enabled] setting " + + "was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation " + + "for the next major version."); } public void testTranslogPruningSettingsWithSoftDeletesDisabled() { @@ -687,5 +690,8 @@ public void testTranslogPruningSettingsWithSoftDeletesDisabled() { IndexSettings indexSettings = new IndexSettings(metadata, Settings.EMPTY); assertFalse(indexSettings.shouldPruneTranslogByRetentionLease()); assertThat(indexSettings.getTranslogRetentionSize().getBytes(), equalTo(retentionSize.getBytes())); + assertWarnings("[index.plugins.replication.translog.retention_lease.pruning.enabled] setting " + + "was deprecated in OpenSearch and will be removed in a future release! See the breaking changes documentation " + + "for the next major version."); } } From a80979c68d3892a320231cbf45e6ff3dff4dd948 Mon Sep 17 00:00:00 2001 From: Nicholas Walter Knize Date: Sun, 26 Sep 2021 00:00:35 -0500 Subject: [PATCH 6/6] Switch to newSnapshot testing method to use helper Signed-off-by: Nicholas Walter Knize --- .../src/main/java/org/opensearch/index/translog/Translog.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/index/translog/Translog.java b/server/src/main/java/org/opensearch/index/translog/Translog.java index 3151e36785342..2c31105d3c7c9 100644 --- a/server/src/main/java/org/opensearch/index/translog/Translog.java +++ b/server/src/main/java/org/opensearch/index/translog/Translog.java @@ -615,7 +615,7 @@ final Checkpoint getLastSyncedCheckpoint() { // for testing public Snapshot newSnapshot() throws IOException { - return newSnapshot(0, Long.MAX_VALUE, false); + return newSnapshot(0, Long.MAX_VALUE); } public Snapshot newSnapshot(long fromSeqNo, long toSeqNo) throws IOException {