diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml index bb3c75f10aaea..beb45107bc313 100644 --- a/.buildkite/pipelines/intake.yml +++ b/.buildkite/pipelines/intake.yml @@ -62,7 +62,7 @@ steps: timeout_in_minutes: 300 matrix: setup: - BWC_VERSION: ["7.17.24", "8.15.1", "8.16.0"] + BWC_VERSION: ["7.17.24", "8.15.2", "8.16.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index 12729a9b6ebda..cd0bc8449f89e 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -594,8 +594,8 @@ steps: env: BWC_VERSION: 8.14.3 - - label: "{{matrix.image}} / 8.15.1 / packaging-tests-upgrade" - command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.1 + - label: "{{matrix.image}} / 8.15.2 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.2 timeout_in_minutes: 300 matrix: setup: @@ -609,7 +609,7 @@ steps: buildDirectory: /dev/shm/bk diskSizeGb: 250 env: - BWC_VERSION: 8.15.1 + BWC_VERSION: 8.15.2 - label: "{{matrix.image}} / 8.16.0 / packaging-tests-upgrade" command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.0 diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index 740fec13d1790..8f25a0fb11065 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -662,8 +662,8 @@ steps: - signal_reason: agent_stop limit: 3 - - label: 8.15.1 / bwc - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.1#bwcTest + - label: 8.15.2 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.2#bwcTest timeout_in_minutes: 300 agents: provider: gcp @@ -673,7 +673,7 @@ steps: preemptible: true diskSizeGb: 250 env: - BWC_VERSION: 8.15.1 + BWC_VERSION: 8.15.2 retry: automatic: - exit_status: "-1" @@ -771,7 +771,7 @@ steps: setup: ES_RUNTIME_JAVA: - openjdk17 - BWC_VERSION: ["7.17.24", "8.15.1", "8.16.0"] + BWC_VERSION: ["7.17.24", "8.15.2", "8.16.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 @@ -821,7 +821,7 @@ steps: - openjdk21 - openjdk22 - openjdk23 - BWC_VERSION: ["7.17.24", "8.15.1", "8.16.0"] + BWC_VERSION: ["7.17.24", "8.15.2", "8.16.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.buildkite/scripts/lucene-snapshot/update-branch.sh b/.buildkite/scripts/lucene-snapshot/update-branch.sh index d02123f3236e7..6a2d1e3df05f7 100755 --- a/.buildkite/scripts/lucene-snapshot/update-branch.sh +++ b/.buildkite/scripts/lucene-snapshot/update-branch.sh @@ -2,17 +2,17 @@ set -euo pipefail -if [[ "$BUILDKITE_BRANCH" != "lucene_snapshot" ]]; then - echo "Error: This script should only be run on the lucene_snapshot branch" +if [[ "$BUILDKITE_BRANCH" != "lucene_snapshot"* ]]; then + echo "Error: This script should only be run on lucene_snapshot branches" exit 1 fi -echo --- Updating lucene_snapshot branch with main +echo --- Updating "$BUILDKITE_BRANCH" branch with main git config --global user.name elasticsearchmachine git config --global user.email 'infra-root+elasticsearchmachine@elastic.co' -git checkout lucene_snapshot +git checkout "$BUILDKITE_BRANCH" git fetch origin main git merge --no-edit origin/main -git push origin lucene_snapshot +git push origin "$BUILDKITE_BRANCH" diff --git a/.buildkite/scripts/lucene-snapshot/update-es-snapshot.sh b/.buildkite/scripts/lucene-snapshot/update-es-snapshot.sh index 75f42a32cb590..7bec83d055139 100755 --- a/.buildkite/scripts/lucene-snapshot/update-es-snapshot.sh +++ b/.buildkite/scripts/lucene-snapshot/update-es-snapshot.sh @@ -2,8 +2,8 @@ set -euo pipefail -if [[ "$BUILDKITE_BRANCH" != "lucene_snapshot" ]]; then - echo "Error: This script should only be run on the lucene_snapshot branch" +if [[ "$BUILDKITE_BRANCH" != "lucene_snapshot"* ]]; then + echo "Error: This script should only be run on the lucene_snapshot branches" exit 1 fi diff --git a/.ci/bwcVersions b/.ci/bwcVersions index e43b3333dd755..b80309cdb3f8e 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -32,5 +32,5 @@ BWC_VERSION: - "8.12.2" - "8.13.4" - "8.14.3" - - "8.15.1" + - "8.15.2" - "8.16.0" diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions index 2eea118e57e2a..e41bbac68f1ec 100644 --- a/.ci/snapshotBwcVersions +++ b/.ci/snapshotBwcVersions @@ -1,4 +1,4 @@ BWC_VERSION: - "7.17.24" - - "8.15.1" + - "8.15.2" - "8.16.0" diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5b98444c044d2..f0d9068820029 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -70,3 +70,7 @@ server/src/main/java/org/elasticsearch/threadpool @elastic/es-core-infra # Security x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege @elastic/es-security x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java @elastic/es-security + +# Analytical engine +x-pack/plugin/esql @elastic/es-analytical-engine +x-pack/plugin/esql-core @elastic/es-analytical-engine diff --git a/.github/workflows/sync-main-to-jdk-branch.yml b/.github/workflows/sync-main-to-jdk-branch.yml new file mode 100644 index 0000000000000..eea3348529284 --- /dev/null +++ b/.github/workflows/sync-main-to-jdk-branch.yml @@ -0,0 +1,20 @@ +# Daily update of JDK update branch with changes from main +name: "Merge main to openjdk23-bundle branch" +on: + schedule: + - cron: '30 17 * * *' + workflow_dispatch: {} + +jobs: + merge-branch: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@master + + - name: merge + uses: devmasx/merge-branch@1.4.0 + with: + type: 'now' + target_branch: openjdk23-bundle + github_token: ${{ secrets.ELASTICSEARCHMACHINE_TOKEN }} diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle index 49e81a67e85f9..b16621aaaa471 100644 --- a/benchmarks/build.gradle +++ b/benchmarks/build.gradle @@ -1,4 +1,5 @@ import org.elasticsearch.gradle.internal.info.BuildParams +import org.elasticsearch.gradle.internal.test.TestUtil /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one @@ -29,6 +30,7 @@ tasks.named("javadoc").configure { enabled = false } configurations { expression painless + nativeLib } dependencies { @@ -37,6 +39,7 @@ dependencies { // us to invoke the JMH uberjar as usual. exclude group: 'net.sf.jopt-simple', module: 'jopt-simple' } + api(project(':libs:elasticsearch-h3')) api(project(':modules:aggregations')) api(project(':x-pack:plugin:esql-core')) api(project(':x-pack:plugin:esql')) @@ -44,6 +47,7 @@ dependencies { implementation project(path: ':libs:elasticsearch-simdvec') expression(project(path: ':modules:lang-expression', configuration: 'zip')) painless(project(path: ':modules:lang-painless', configuration: 'zip')) + nativeLib(project(':libs:elasticsearch-native')) api "org.openjdk.jmh:jmh-core:$versions.jmh" annotationProcessor "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh" // Dependencies of JMH @@ -75,17 +79,8 @@ tasks.register("copyPainless", Copy) { tasks.named("run").configure { executable = "${BuildParams.runtimeJavaHome}/bin/java" args << "-Dplugins.dir=${buildDir}/plugins" << "-Dtests.index=${buildDir}/index" - dependsOn "copyExpression", "copyPainless" - systemProperty 'java.library.path', file("../libs/native/libraries/build/platform/${platformName()}-${os.arch}") -} - -String platformName() { - String name = System.getProperty("os.name"); - if (name.startsWith("Mac")) { - return "darwin"; - } else { - return name.toLowerCase(Locale.ROOT); - } + dependsOn "copyExpression", "copyPainless", configurations.nativeLib + systemProperty 'es.nativelibs.path', TestUtil.getTestLibraryPath(file("../libs/native/libraries/build/platform/").toString()) } spotless { diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockBenchmark.java index 49603043e7bcc..59fdfff3025a1 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockBenchmark.java @@ -20,13 +20,10 @@ import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.BooleanBigArrayBlock; import org.elasticsearch.compute.data.BooleanBigArrayVector; -import org.elasticsearch.compute.data.BooleanBlock; import org.elasticsearch.compute.data.BooleanVector; -import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.BytesRefVector; import org.elasticsearch.compute.data.DoubleBigArrayBlock; import org.elasticsearch.compute.data.DoubleBigArrayVector; -import org.elasticsearch.compute.data.DoubleBlock; import org.elasticsearch.compute.data.DoubleVector; import org.elasticsearch.compute.data.IntBigArrayBlock; import org.elasticsearch.compute.data.IntBigArrayVector; @@ -34,39 +31,13 @@ import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.LongBigArrayBlock; import org.elasticsearch.compute.data.LongBigArrayVector; -import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.LongVector; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Level; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OperationsPerInvocation; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Param; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.TearDown; -import org.openjdk.jmh.annotations.Warmup; import java.util.ArrayList; import java.util.BitSet; -import java.util.Collections; -import java.util.List; import java.util.Random; -import java.util.concurrent.TimeUnit; -import java.util.stream.IntStream; - -@Warmup(iterations = 5) -@Measurement(iterations = 7) -@BenchmarkMode(Mode.AverageTime) -@OutputTimeUnit(TimeUnit.NANOSECONDS) -@State(Scope.Thread) -@Fork(1) -public class BlockBenchmark { +public class BlockBenchmark { /** * All data type/block kind combinations to be loaded before the benchmark. * It is important to be exhaustive here so that all implementers of {@link IntBlock#getInt(int)} are actually loaded when we benchmark @@ -114,35 +85,12 @@ public class BlockBenchmark { private static final int MAX_MV_ELEMENTS = 100; private static final int MAX_BYTES_REF_LENGTH = 255; - private static final Random random = new Random(); - - private static final BlockFactory blockFactory = BlockFactory.getInstance( - new NoopCircuitBreaker("noop"), - BigArrays.NON_RECYCLING_INSTANCE - ); - - static { - // Smoke test all the expected values and force loading subclasses more like prod - int totalPositions = 10; - long[] actualCheckSums = new long[NUM_BLOCKS_PER_ITERATION]; - - for (String paramString : RELEVANT_TYPE_BLOCK_COMBINATIONS) { - String[] params = paramString.split("/"); - String dataType = params[0]; - String blockKind = params[1]; - - BenchmarkBlocks data = buildBlocks(dataType, blockKind, totalPositions); - int[][] traversalOrders = createTraversalOrders(data.blocks, false); - run(dataType, data, traversalOrders, actualCheckSums); - assertCheckSums(data, actualCheckSums); - } - } + static final Random random = new Random(); - private record BenchmarkBlocks(Block[] blocks, long[] checkSums) {}; + static final BlockFactory blockFactory = BlockFactory.getInstance(new NoopCircuitBreaker("noop"), BigArrays.NON_RECYCLING_INSTANCE); - private static BenchmarkBlocks buildBlocks(String dataType, String blockKind, int totalPositions) { + static Block[] buildBlocks(String dataType, String blockKind, int totalPositions) { Block[] blocks = new Block[NUM_BLOCKS_PER_ITERATION]; - long[] checkSums = new long[NUM_BLOCKS_PER_ITERATION]; switch (dataType) { case "boolean" -> { @@ -237,11 +185,6 @@ private static BenchmarkBlocks buildBlocks(String dataType, String blockKind, in } } } - - for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { - BooleanBlock block = (BooleanBlock) blocks[blockIndex]; - checkSums[blockIndex] = computeBooleanCheckSum(block, IntStream.range(0, block.getPositionCount()).toArray()); - } } case "BytesRef" -> { for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { @@ -294,11 +237,6 @@ private static BenchmarkBlocks buildBlocks(String dataType, String blockKind, in } } } - - for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { - BytesRefBlock block = (BytesRefBlock) blocks[blockIndex]; - checkSums[blockIndex] = computeBytesRefCheckSum(block, IntStream.range(0, block.getPositionCount()).toArray()); - } } case "double" -> { for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { @@ -386,11 +324,6 @@ private static BenchmarkBlocks buildBlocks(String dataType, String blockKind, in } } } - - for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { - DoubleBlock block = (DoubleBlock) blocks[blockIndex]; - checkSums[blockIndex] = computeDoubleCheckSum(block, IntStream.range(0, block.getPositionCount()).toArray()); - } } case "int" -> { for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { @@ -478,11 +411,6 @@ private static BenchmarkBlocks buildBlocks(String dataType, String blockKind, in } } } - - for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { - IntBlock block = (IntBlock) blocks[blockIndex]; - checkSums[blockIndex] = computeIntCheckSum(block, IntStream.range(0, block.getPositionCount()).toArray()); - } } case "long" -> { for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { @@ -570,36 +498,12 @@ private static BenchmarkBlocks buildBlocks(String dataType, String blockKind, in } } } - - for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { - LongBlock block = (LongBlock) blocks[blockIndex]; - checkSums[blockIndex] = computeLongCheckSum(block, IntStream.range(0, block.getPositionCount()).toArray()); - } } default -> { throw new IllegalStateException("illegal data type [" + dataType + "]"); } } - - return new BenchmarkBlocks(blocks, checkSums); - } - - private static int[][] createTraversalOrders(Block[] blocks, boolean randomized) { - int[][] orders = new int[blocks.length][]; - - for (int i = 0; i < blocks.length; i++) { - IntStream positionsStream = IntStream.range(0, blocks[i].getPositionCount()); - - if (randomized) { - List positions = new java.util.ArrayList<>(positionsStream.boxed().toList()); - Collections.shuffle(positions, random); - orders[i] = positions.stream().mapToInt(x -> x).toArray(); - } else { - orders[i] = positionsStream.toArray(); - } - } - - return orders; + return blocks; } private static int[] randomFirstValueIndexes(int totalPositions) { @@ -631,220 +535,4 @@ private static BitSet randomNulls(int positionCount) { return nulls; } - - private static void run(String dataType, BenchmarkBlocks data, int[][] traversalOrders, long[] resultCheckSums) { - switch (dataType) { - case "boolean" -> { - for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { - BooleanBlock block = (BooleanBlock) data.blocks[blockIndex]; - - resultCheckSums[blockIndex] = computeBooleanCheckSum(block, traversalOrders[blockIndex]); - } - } - case "BytesRef" -> { - for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { - BytesRefBlock block = (BytesRefBlock) data.blocks[blockIndex]; - - resultCheckSums[blockIndex] = computeBytesRefCheckSum(block, traversalOrders[blockIndex]); - } - } - case "double" -> { - for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { - DoubleBlock block = (DoubleBlock) data.blocks[blockIndex]; - - resultCheckSums[blockIndex] = computeDoubleCheckSum(block, traversalOrders[blockIndex]); - } - } - case "int" -> { - for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { - IntBlock block = (IntBlock) data.blocks[blockIndex]; - - resultCheckSums[blockIndex] = computeIntCheckSum(block, traversalOrders[blockIndex]); - } - } - case "long" -> { - for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { - LongBlock block = (LongBlock) data.blocks[blockIndex]; - - resultCheckSums[blockIndex] = computeLongCheckSum(block, traversalOrders[blockIndex]); - } - } - default -> { - throw new IllegalStateException(); - } - } - } - - private static void assertCheckSums(BenchmarkBlocks data, long[] actualCheckSums) { - for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { - if (actualCheckSums[blockIndex] != data.checkSums[blockIndex]) { - throw new AssertionError("checksums do not match for block [" + blockIndex + "]"); - } - } - } - - private static long computeBooleanCheckSum(BooleanBlock block, int[] traversalOrder) { - long sum = 0; - - for (int position : traversalOrder) { - if (block.isNull(position)) { - continue; - } - int start = block.getFirstValueIndex(position); - int end = start + block.getValueCount(position); - for (int i = start; i < end; i++) { - sum += block.getBoolean(i) ? 1 : 0; - } - } - - return sum; - } - - private static long computeBytesRefCheckSum(BytesRefBlock block, int[] traversalOrder) { - long sum = 0; - BytesRef currentValue = new BytesRef(); - - for (int position : traversalOrder) { - if (block.isNull(position)) { - continue; - } - int start = block.getFirstValueIndex(position); - int end = start + block.getValueCount(position); - for (int i = start; i < end; i++) { - block.getBytesRef(i, currentValue); - sum += currentValue.length > 0 ? currentValue.bytes[0] : 0; - } - } - - return sum; - } - - private static long computeDoubleCheckSum(DoubleBlock block, int[] traversalOrder) { - long sum = 0; - - for (int position : traversalOrder) { - if (block.isNull(position)) { - continue; - } - int start = block.getFirstValueIndex(position); - int end = start + block.getValueCount(position); - for (int i = start; i < end; i++) { - // Use an operation that is not affected by rounding errors. Otherwise, the result may depend on the traversalOrder. - sum += (long) block.getDouble(i); - } - } - - return sum; - } - - private static long computeIntCheckSum(IntBlock block, int[] traversalOrder) { - int sum = 0; - - for (int position : traversalOrder) { - if (block.isNull(position)) { - continue; - } - int start = block.getFirstValueIndex(position); - int end = start + block.getValueCount(position); - for (int i = start; i < end; i++) { - sum += block.getInt(i); - } - } - - return sum; - } - - private static long computeLongCheckSum(LongBlock block, int[] traversalOrder) { - long sum = 0; - - for (int position : traversalOrder) { - if (block.isNull(position)) { - continue; - } - int start = block.getFirstValueIndex(position); - int end = start + block.getValueCount(position); - for (int i = start; i < end; i++) { - sum += block.getLong(i); - } - } - - return sum; - } - - private static boolean isRandom(String accessType) { - return accessType.equalsIgnoreCase("random"); - } - - /** - * Must be a subset of {@link BlockBenchmark#RELEVANT_TYPE_BLOCK_COMBINATIONS} - */ - @Param( - { - "boolean/array", - "boolean/array-multivalue-null", - "boolean/big-array", - "boolean/big-array-multivalue-null", - "boolean/vector", - "boolean/vector-big-array", - "boolean/vector-const", - "BytesRef/array", - "BytesRef/array-multivalue-null", - "BytesRef/vector", - "BytesRef/vector-const", - "double/array", - "double/array-multivalue-null", - "double/big-array", - "double/big-array-multivalue-null", - "double/vector", - "double/vector-big-array", - "double/vector-const", - "int/array", - "int/array-multivalue-null", - "int/big-array", - "int/big-array-multivalue-null", - "int/vector", - "int/vector-big-array", - "int/vector-const", - "long/array", - "long/array-multivalue-null", - "long/big-array", - "long/big-array-multivalue-null", - "long/vector", - "long/vector-big-array", - "long/vector-const" } - ) - public String dataTypeAndBlockKind; - - @Param({ "sequential", "random" }) - public String accessType; - - private BenchmarkBlocks data; - - private int[][] traversalOrders; - - private final long[] actualCheckSums = new long[NUM_BLOCKS_PER_ITERATION]; - - @Setup - public void setup() { - String[] params = dataTypeAndBlockKind.split("/"); - String dataType = params[0]; - String blockKind = params[1]; - - data = buildBlocks(dataType, blockKind, BLOCK_TOTAL_POSITIONS); - traversalOrders = createTraversalOrders(data.blocks, isRandom(accessType)); - } - - @Benchmark - @OperationsPerInvocation(NUM_BLOCKS_PER_ITERATION * BLOCK_TOTAL_POSITIONS) - public void run() { - String[] params = dataTypeAndBlockKind.split("/"); - String dataType = params[0]; - - run(dataType, data, traversalOrders, actualCheckSums); - } - - @TearDown(Level.Iteration) - public void assertCheckSums() { - assertCheckSums(data, actualCheckSums); - } } diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockKeepMaskBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockKeepMaskBenchmark.java new file mode 100644 index 0000000000000..23048ad188a37 --- /dev/null +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockKeepMaskBenchmark.java @@ -0,0 +1,295 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.benchmark.compute.operator; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.LongBlock; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OperationsPerInvocation; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; + +import java.util.concurrent.TimeUnit; + +@Warmup(iterations = 5) +@Measurement(iterations = 7) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Thread) +@Fork(1) +public class BlockKeepMaskBenchmark extends BlockBenchmark { + static { + // Smoke test all the expected values and force loading subclasses more like prod + int totalPositions = 10; + for (String paramString : RELEVANT_TYPE_BLOCK_COMBINATIONS) { + String[] params = paramString.split("/"); + String dataType = params[0]; + String blockKind = params[1]; + BooleanVector mask = buildMask(totalPositions); + + BenchmarkBlocks data = buildBenchmarkBlocks(dataType, blockKind, mask, totalPositions); + Block[] results = new Block[NUM_BLOCKS_PER_ITERATION]; + run(data, mask, results); + assertCheckSums(dataType, blockKind, data, results, totalPositions); + } + } + + record BenchmarkBlocks(Block[] blocks, long[] checkSums) {}; + + static BenchmarkBlocks buildBenchmarkBlocks(String dataType, String blockKind, BooleanVector mask, int totalPositions) { + Block[] blocks = BlockBenchmark.buildBlocks(dataType, blockKind, totalPositions); + return new BenchmarkBlocks(blocks, checksumsFor(dataType, blocks, mask)); + } + + static long[] checksumsFor(String dataType, Block[] blocks, BooleanVector mask) { + long[] checkSums = new long[NUM_BLOCKS_PER_ITERATION]; + switch (dataType) { + case "boolean" -> { + for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { + BooleanBlock block = (BooleanBlock) blocks[blockIndex]; + checkSums[blockIndex] = computeBooleanCheckSum(block, mask); + } + } + case "BytesRef" -> { + for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { + BytesRefBlock block = (BytesRefBlock) blocks[blockIndex]; + checkSums[blockIndex] = computeBytesRefCheckSum(block, mask); + } + } + case "double" -> { + for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { + DoubleBlock block = (DoubleBlock) blocks[blockIndex]; + checkSums[blockIndex] = computeDoubleCheckSum(block, mask); + } + } + case "int" -> { + for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { + IntBlock block = (IntBlock) blocks[blockIndex]; + checkSums[blockIndex] = computeIntCheckSum(block, mask); + } + } + case "long" -> { + for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { + LongBlock block = (LongBlock) blocks[blockIndex]; + checkSums[blockIndex] = computeLongCheckSum(block, mask); + } + } + // TODO float + default -> throw new IllegalStateException("illegal data type [" + dataType + "]"); + } + return checkSums; + } + + static BooleanVector buildMask(int totalPositions) { + try (BooleanVector.FixedBuilder builder = blockFactory.newBooleanVectorFixedBuilder(totalPositions)) { + for (int p = 0; p < totalPositions; p++) { + builder.appendBoolean(p % 2 == 0); + } + return builder.build(); + } + } + + private static void run(BenchmarkBlocks data, BooleanVector mask, Block[] results) { + for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { + results[blockIndex] = data.blocks[blockIndex].keepMask(mask); + } + } + + private static void assertCheckSums(String dataType, String blockKind, BenchmarkBlocks data, Block[] results, int positionCount) { + long[] checkSums = checksumsFor(dataType, results, blockFactory.newConstantBooleanVector(true, positionCount)); + for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { + if (checkSums[blockIndex] != data.checkSums[blockIndex]) { + throw new AssertionError( + "checksums do not match for block [" + + blockIndex + + "][" + + dataType + + "][" + + blockKind + + "]: " + + checkSums[blockIndex] + + " vs " + + data.checkSums[blockIndex] + ); + } + } + } + + private static long computeBooleanCheckSum(BooleanBlock block, BooleanVector mask) { + long sum = 0; + + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p) || mask.getBoolean(p) == false) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + sum += block.getBoolean(i) ? 1 : 0; + } + } + + return sum; + } + + private static long computeBytesRefCheckSum(BytesRefBlock block, BooleanVector mask) { + long sum = 0; + BytesRef scratch = new BytesRef(); + + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p) || mask.getBoolean(p) == false) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + BytesRef v = block.getBytesRef(i, scratch); + sum += v.length > 0 ? v.bytes[v.offset] : 0; + } + } + + return sum; + } + + private static long computeDoubleCheckSum(DoubleBlock block, BooleanVector mask) { + long sum = 0; + + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p) || mask.getBoolean(p) == false) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + sum += (long) block.getDouble(i); + } + } + + return sum; + } + + private static long computeIntCheckSum(IntBlock block, BooleanVector mask) { + int sum = 0; + + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p) || mask.getBoolean(p) == false) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + sum += block.getInt(i); + } + } + + return sum; + } + + private static long computeLongCheckSum(LongBlock block, BooleanVector mask) { + long sum = 0; + + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p) || mask.getBoolean(p) == false) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + sum += block.getLong(i); + } + } + + return sum; + } + + /** + * Must be a subset of {@link BlockBenchmark#RELEVANT_TYPE_BLOCK_COMBINATIONS} + */ + @Param( + { + "boolean/array", + "boolean/array-multivalue-null", + "boolean/big-array", + "boolean/big-array-multivalue-null", + "boolean/vector", + "boolean/vector-big-array", + "boolean/vector-const", + "BytesRef/array", + "BytesRef/array-multivalue-null", + "BytesRef/vector", + "BytesRef/vector-const", + "double/array", + "double/array-multivalue-null", + "double/big-array", + "double/big-array-multivalue-null", + "double/vector", + "double/vector-big-array", + "double/vector-const", + "int/array", + "int/array-multivalue-null", + "int/big-array", + "int/big-array-multivalue-null", + "int/vector", + "int/vector-big-array", + "int/vector-const", + "long/array", + "long/array-multivalue-null", + "long/big-array", + "long/big-array-multivalue-null", + "long/vector", + "long/vector-big-array", + "long/vector-const" } + ) + public String dataTypeAndBlockKind; + + private BenchmarkBlocks data; + + private final BooleanVector mask = buildMask(BLOCK_TOTAL_POSITIONS); + + private final Block[] results = new Block[NUM_BLOCKS_PER_ITERATION]; + + @Setup + public void setup() { + String[] params = dataTypeAndBlockKind.split("/"); + String dataType = params[0]; + String blockKind = params[1]; + + data = buildBenchmarkBlocks(dataType, blockKind, mask, BLOCK_TOTAL_POSITIONS); + } + + @Benchmark + @OperationsPerInvocation(NUM_BLOCKS_PER_ITERATION * BLOCK_TOTAL_POSITIONS) + public void run() { + run(data, mask, results); + } + + @TearDown(Level.Iteration) + public void assertCheckSums() { + String[] params = dataTypeAndBlockKind.split("/"); + String dataType = params[0]; + String blockKind = params[1]; + assertCheckSums(dataType, blockKind, data, results, BLOCK_TOTAL_POSITIONS); + } +} diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockReadBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockReadBenchmark.java new file mode 100644 index 0000000000000..327dcfcff3a28 --- /dev/null +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/BlockReadBenchmark.java @@ -0,0 +1,319 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.benchmark.compute.operator; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.util.*; +import org.elasticsearch.compute.data.*; +import org.openjdk.jmh.annotations.*; + +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.stream.IntStream; + +@Warmup(iterations = 5) +@Measurement(iterations = 7) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Thread) +@Fork(1) +public class BlockReadBenchmark extends BlockBenchmark { + static { + // Smoke test all the expected values and force loading subclasses more like prod + int totalPositions = 10; + long[] actualCheckSums = new long[NUM_BLOCKS_PER_ITERATION]; + + for (String paramString : RELEVANT_TYPE_BLOCK_COMBINATIONS) { + String[] params = paramString.split("/"); + String dataType = params[0]; + String blockKind = params[1]; + + BenchmarkBlocks data = buildBenchmarkBlocks(dataType, blockKind, totalPositions); + int[][] traversalOrders = createTraversalOrders(data.blocks(), false); + run(dataType, data, traversalOrders, actualCheckSums); + assertCheckSums(data, actualCheckSums); + } + } + + private static int[][] createTraversalOrders(Block[] blocks, boolean randomized) { + int[][] orders = new int[blocks.length][]; + + for (int i = 0; i < blocks.length; i++) { + IntStream positionsStream = IntStream.range(0, blocks[i].getPositionCount()); + + if (randomized) { + List positions = new ArrayList<>(positionsStream.boxed().toList()); + Collections.shuffle(positions, random); + orders[i] = positions.stream().mapToInt(x -> x).toArray(); + } else { + orders[i] = positionsStream.toArray(); + } + } + + return orders; + } + + record BenchmarkBlocks(Block[] blocks, long[] checkSums) {}; + + static BenchmarkBlocks buildBenchmarkBlocks(String dataType, String blockKind, int totalPositions) { + Block[] blocks = BlockBenchmark.buildBlocks(dataType, blockKind, totalPositions); + long[] checkSums = new long[NUM_BLOCKS_PER_ITERATION]; + switch (dataType) { + case "boolean" -> { + for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { + BooleanBlock block = (BooleanBlock) blocks[blockIndex]; + checkSums[blockIndex] = computeBooleanCheckSum(block, IntStream.range(0, block.getPositionCount()).toArray()); + } + } + case "BytesRef" -> { + for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { + BytesRefBlock block = (BytesRefBlock) blocks[blockIndex]; + checkSums[blockIndex] = computeBytesRefCheckSum(block, IntStream.range(0, block.getPositionCount()).toArray()); + } + } + case "double" -> { + for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { + DoubleBlock block = (DoubleBlock) blocks[blockIndex]; + checkSums[blockIndex] = computeDoubleCheckSum(block, IntStream.range(0, block.getPositionCount()).toArray()); + } + } + case "int" -> { + for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { + IntBlock block = (IntBlock) blocks[blockIndex]; + checkSums[blockIndex] = computeIntCheckSum(block, IntStream.range(0, block.getPositionCount()).toArray()); + } + } + case "long" -> { + for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { + LongBlock block = (LongBlock) blocks[blockIndex]; + checkSums[blockIndex] = computeLongCheckSum(block, IntStream.range(0, block.getPositionCount()).toArray()); + } + } + // TODO float + default -> throw new IllegalStateException("illegal data type [" + dataType + "]"); + } + return new BenchmarkBlocks(blocks, checkSums); + } + + private static void run(String dataType, BenchmarkBlocks data, int[][] traversalOrders, long[] resultCheckSums) { + switch (dataType) { + case "boolean" -> { + for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { + BooleanBlock block = (BooleanBlock) data.blocks[blockIndex]; + + resultCheckSums[blockIndex] = computeBooleanCheckSum(block, traversalOrders[blockIndex]); + } + } + case "BytesRef" -> { + for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { + BytesRefBlock block = (BytesRefBlock) data.blocks[blockIndex]; + + resultCheckSums[blockIndex] = computeBytesRefCheckSum(block, traversalOrders[blockIndex]); + } + } + case "double" -> { + for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { + DoubleBlock block = (DoubleBlock) data.blocks[blockIndex]; + + resultCheckSums[blockIndex] = computeDoubleCheckSum(block, traversalOrders[blockIndex]); + } + } + case "int" -> { + for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { + IntBlock block = (IntBlock) data.blocks[blockIndex]; + + resultCheckSums[blockIndex] = computeIntCheckSum(block, traversalOrders[blockIndex]); + } + } + case "long" -> { + for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { + LongBlock block = (LongBlock) data.blocks[blockIndex]; + + resultCheckSums[blockIndex] = computeLongCheckSum(block, traversalOrders[blockIndex]); + } + } + default -> { + throw new IllegalStateException(); + } + } + } + + private static void assertCheckSums(BenchmarkBlocks data, long[] actualCheckSums) { + for (int blockIndex = 0; blockIndex < NUM_BLOCKS_PER_ITERATION; blockIndex++) { + if (actualCheckSums[blockIndex] != data.checkSums[blockIndex]) { + throw new AssertionError("checksums do not match for block [" + blockIndex + "]"); + } + } + } + + private static long computeBooleanCheckSum(BooleanBlock block, int[] traversalOrder) { + long sum = 0; + + for (int position : traversalOrder) { + if (block.isNull(position)) { + continue; + } + int start = block.getFirstValueIndex(position); + int end = start + block.getValueCount(position); + for (int i = start; i < end; i++) { + sum += block.getBoolean(i) ? 1 : 0; + } + } + + return sum; + } + + private static long computeBytesRefCheckSum(BytesRefBlock block, int[] traversalOrder) { + long sum = 0; + BytesRef scratch = new BytesRef(); + + for (int position : traversalOrder) { + if (block.isNull(position)) { + continue; + } + int start = block.getFirstValueIndex(position); + int end = start + block.getValueCount(position); + for (int i = start; i < end; i++) { + BytesRef v = block.getBytesRef(i, scratch); + sum += v.length > 0 ? v.bytes[v.offset] : 0; + } + } + + return sum; + } + + private static long computeDoubleCheckSum(DoubleBlock block, int[] traversalOrder) { + long sum = 0; + + for (int position : traversalOrder) { + if (block.isNull(position)) { + continue; + } + int start = block.getFirstValueIndex(position); + int end = start + block.getValueCount(position); + for (int i = start; i < end; i++) { + // Use an operation that is not affected by rounding errors. Otherwise, the result may depend on the traversalOrder. + sum += (long) block.getDouble(i); + } + } + + return sum; + } + + private static long computeIntCheckSum(IntBlock block, int[] traversalOrder) { + int sum = 0; + + for (int position : traversalOrder) { + if (block.isNull(position)) { + continue; + } + int start = block.getFirstValueIndex(position); + int end = start + block.getValueCount(position); + for (int i = start; i < end; i++) { + sum += block.getInt(i); + } + } + + return sum; + } + + private static long computeLongCheckSum(LongBlock block, int[] traversalOrder) { + long sum = 0; + + for (int position : traversalOrder) { + if (block.isNull(position)) { + continue; + } + int start = block.getFirstValueIndex(position); + int end = start + block.getValueCount(position); + for (int i = start; i < end; i++) { + sum += block.getLong(i); + } + } + + return sum; + } + + private static boolean isRandom(String accessType) { + return accessType.equalsIgnoreCase("random"); + } + + /** + * Must be a subset of {@link BlockBenchmark#RELEVANT_TYPE_BLOCK_COMBINATIONS} + */ + @Param( + { + "boolean/array", + "boolean/array-multivalue-null", + "boolean/big-array", + "boolean/big-array-multivalue-null", + "boolean/vector", + "boolean/vector-big-array", + "boolean/vector-const", + "BytesRef/array", + "BytesRef/array-multivalue-null", + "BytesRef/vector", + "BytesRef/vector-const", + "double/array", + "double/array-multivalue-null", + "double/big-array", + "double/big-array-multivalue-null", + "double/vector", + "double/vector-big-array", + "double/vector-const", + "int/array", + "int/array-multivalue-null", + "int/big-array", + "int/big-array-multivalue-null", + "int/vector", + "int/vector-big-array", + "int/vector-const", + "long/array", + "long/array-multivalue-null", + "long/big-array", + "long/big-array-multivalue-null", + "long/vector", + "long/vector-big-array", + "long/vector-const" } + ) + public String dataTypeAndBlockKind; + + @Param({ "sequential", "random" }) + public String accessType; + + private BenchmarkBlocks data; + + private int[][] traversalOrders; + + private final long[] actualCheckSums = new long[NUM_BLOCKS_PER_ITERATION]; + + @Setup + public void setup() { + String[] params = dataTypeAndBlockKind.split("/"); + String dataType = params[0]; + String blockKind = params[1]; + + data = buildBenchmarkBlocks(dataType, blockKind, BLOCK_TOTAL_POSITIONS); + traversalOrders = createTraversalOrders(data.blocks(), isRandom(accessType)); + } + + @Benchmark + @OperationsPerInvocation(NUM_BLOCKS_PER_ITERATION * BLOCK_TOTAL_POSITIONS) + public void run() { + String[] params = dataTypeAndBlockKind.split("/"); + String dataType = params[0]; + + run(dataType, data, traversalOrders, actualCheckSums); + } + + @TearDown(Level.Iteration) + public void assertCheckSums() { + assertCheckSums(data, actualCheckSums); + } +} diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/h3/H3Benchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/h3/H3Benchmark.java new file mode 100644 index 0000000000000..2441acab7d405 --- /dev/null +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/h3/H3Benchmark.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.benchmark.h3; + +import org.elasticsearch.h3.H3; +import org.openjdk.jmh.Main; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.concurrent.TimeUnit; + +@OutputTimeUnit(TimeUnit.SECONDS) +@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 25, time = 1, timeUnit = TimeUnit.SECONDS) +@Fork(1) +public class H3Benchmark { + + @Benchmark + public void pointToH3(H3State state, Blackhole bh) { + for (int i = 0; i < state.points.length; i++) { + for (int res = 0; res <= 15; res++) { + bh.consume(H3.geoToH3(state.points[i][0], state.points[i][1], res)); + } + } + } + + @Benchmark + public void h3Boundary(H3State state, Blackhole bh) { + for (int i = 0; i < state.h3.length; i++) { + bh.consume(H3.h3ToGeoBoundary(state.h3[i])); + } + } + + public static void main(String[] args) throws Exception { + Main.main(args); + } +} diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/h3/H3State.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/h3/H3State.java new file mode 100644 index 0000000000000..5707e692a0750 --- /dev/null +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/h3/H3State.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.benchmark.h3; + +import org.elasticsearch.h3.H3; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; + +import java.io.IOException; +import java.util.Random; + +@State(Scope.Benchmark) +public class H3State { + + double[][] points = new double[1000][2]; + long[] h3 = new long[1000]; + + @Setup(Level.Trial) + public void setupTrial() throws IOException { + Random random = new Random(1234); + for (int i = 0; i < points.length; i++) { + points[i][0] = random.nextDouble() * 180 - 90; // lat + points[i][1] = random.nextDouble() * 360 - 180; // lon + int res = random.nextInt(16); // resolution + h3[i] = H3.geoToH3(points[i][0], points[i][1], res); + } + } +} diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java index 8c5de05a01648..d7a72615f4b93 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/ShardsAvailabilityHealthIndicatorBenchmark.java @@ -32,6 +32,7 @@ import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.telemetry.metric.MeterRegistry; +import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders; import org.elasticsearch.threadpool.ThreadPool; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; @@ -167,7 +168,7 @@ public void setUp() throws Exception { .build(); Settings settings = Settings.builder().put("node.name", ShardsAvailabilityHealthIndicatorBenchmark.class.getSimpleName()).build(); - ThreadPool threadPool = new ThreadPool(settings, MeterRegistry.NOOP); + ThreadPool threadPool = new ThreadPool(settings, MeterRegistry.NOOP, new DefaultBuiltInExecutorBuilders()); ClusterService clusterService = new ClusterService( Settings.EMPTY, diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java index 5a27abe8be2a4..fe221ec980dc3 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java @@ -186,6 +186,11 @@ public void setDocument(int docid) { public boolean needs_score() { return false; } + + @Override + public boolean needs_termStats() { + return false; + } }; }; } diff --git a/build-tools-internal/gradle/wrapper/gradle-wrapper.properties b/build-tools-internal/gradle/wrapper/gradle-wrapper.properties index efe2ff3449216..9036682bf0f0c 100644 --- a/build-tools-internal/gradle/wrapper/gradle-wrapper.properties +++ b/build-tools-internal/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=258e722ec21e955201e31447b0aed14201765a3bfbae296a46cf60b70e66db70 -distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-all.zip +distributionSha256Sum=682b4df7fe5accdca84a4d1ef6a3a6ab096b3efd5edf7de2bd8c758d95a93703 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-all.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy index b7c4908e39b62..737c448f23be6 100644 --- a/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy +++ b/build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/LegacyYamlRestCompatTestPluginFuncTest.groovy @@ -55,8 +55,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe def result = gradleRunner("yamlRestTestV${compatibleVersion}CompatTest", '--stacktrace').build() then: - // we set the task to be skipped if there are no matching tests in the compatibility test sourceSet - result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.SKIPPED + result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.NO_SOURCE result.task(':copyRestCompatApiTask').outcome == TaskOutcome.NO_SOURCE result.task(':copyRestCompatTestTask').outcome == TaskOutcome.NO_SOURCE result.task(transformTask).outcome == TaskOutcome.NO_SOURCE @@ -165,7 +164,7 @@ class LegacyYamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTe then: result.task(':check').outcome == TaskOutcome.UP_TO_DATE result.task(':checkRestCompat').outcome == TaskOutcome.UP_TO_DATE - result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.SKIPPED + result.task(":yamlRestTestV${compatibleVersion}CompatTest").outcome == TaskOutcome.NO_SOURCE result.task(':copyRestCompatApiTask').outcome == TaskOutcome.NO_SOURCE result.task(':copyRestCompatTestTask').outcome == TaskOutcome.NO_SOURCE result.task(transformTask).outcome == TaskOutcome.NO_SOURCE diff --git a/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle b/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle index 7cba4730e88da..d604973efcb4b 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle @@ -19,11 +19,14 @@ import java.time.LocalDateTime develocity { buildScan { - URL jenkinsUrl = System.getenv('JENKINS_URL') ? new URL(System.getenv('JENKINS_URL')) : null - String buildKiteUrl = System.getenv('BUILDKITE_BUILD_URL') ? System.getenv('BUILDKITE_BUILD_URL') : null + + def onCI = System.getenv('CI') ? Boolean.parseBoolean(System.getenv('CI')) : false + + // Disable async upload in CI to ensure scan upload completes before CI agent is terminated + uploadInBackground = onCI == false // Automatically publish scans from Elasticsearch CI - if (jenkinsUrl?.host?.endsWith('elastic.co') || jenkinsUrl?.host?.endsWith('elastic.dev') || System.getenv('BUILDKITE') == 'true') { + if (onCI) { publishing.onlyIf { true } server = 'https://gradle-enterprise.elastic.co' } else if( server.isPresent() == false) { @@ -39,72 +42,8 @@ develocity { tag 'FIPS' } - // Jenkins-specific build scan metadata - if (jenkinsUrl) { - // Disable async upload in CI to ensure scan upload completes before CI agent is terminated - uploadInBackground = false - - String buildNumber = System.getenv('BUILD_NUMBER') - String buildUrl = System.getenv('BUILD_URL') - String jobName = System.getenv('JOB_NAME') - String nodeName = System.getenv('NODE_NAME') - String jobBranch = System.getenv('ghprbTargetBranch') ?: System.getenv('JOB_BRANCH') - - // Link to Jenkins worker logs and system metrics - if (nodeName) { - link 'System logs', "https://ci-stats.elastic.co/app/infra#/logs?&logFilter=(expression:'host.name:${nodeName}',kind:kuery)" - buildFinished { - link 'System metrics', "https://ci-stats.elastic.co/app/metrics/detail/host/${nodeName}" - } - } - - // Parse job name in the case of matrix builds - // Matrix job names come in the form of "base-job-name/matrix_param1=value1,matrix_param2=value2" - def splitJobName = jobName.split('/') - if (splitJobName.length > 1 && splitJobName.last() ==~ /^([a-zA-Z0-9_\-]+=[a-zA-Z0-9_\-&\.]+,?)+$/) { - def baseJobName = splitJobName.dropRight(1).join('/') - tag baseJobName - tag splitJobName.last() - value 'Job Name', baseJobName - def matrixParams = splitJobName.last().split(',') - matrixParams.collect { it.split('=') }.each { param -> - value "MATRIX_${param[0].toUpperCase()}", param[1] - } - } else { - tag jobName - value 'Job Name', jobName - } - - tag 'CI' - link 'CI Build', buildUrl - link 'GCP Upload', - "https://console.cloud.google.com/storage/browser/_details/elasticsearch-ci-artifacts/jobs/${URLEncoder.encode(jobName, "UTF-8")}/build/${buildNumber}.tar.bz2" - value 'Job Number', buildNumber - if (jobBranch) { - tag jobBranch - value 'Git Branch', jobBranch - } - - System.getenv().getOrDefault('NODE_LABELS', '').split(' ').each { - value 'Jenkins Worker Label', it - } - - // Add SCM information - def isPrBuild = System.getenv('ROOT_BUILD_CAUSE_GHPRBCAUSE') != null - if (isPrBuild) { - value 'Git Commit ID', System.getenv('ghprbActualCommit') - tag "pr/${System.getenv('ghprbPullId')}" - tag 'pull-request' - link 'Source', "https://github.com/elastic/elasticsearch/tree/${System.getenv('ghprbActualCommit')}" - link 'Pull Request', System.getenv('ghprbPullLink') - } else { - value 'Git Commit ID', BuildParams.gitRevision - link 'Source', "https://github.com/elastic/elasticsearch/tree/${BuildParams.gitRevision}" - } - } else if (buildKiteUrl) { //Buildkite-specific build scan metadata - // Disable async upload in CI to ensure scan upload completes before CI agent is terminated - uploadInBackground = false - + if (onCI) { //Buildkite-specific build scan metadata + String buildKiteUrl = System.getenv('BUILDKITE_BUILD_URL') def branch = System.getenv('BUILDKITE_PULL_REQUEST_BASE_BRANCH') ?: System.getenv('BUILDKITE_BRANCH') def repoMatcher = System.getenv('BUILDKITE_REPO') =~ /(https:\/\/github\.com\/|git@github\.com:)(\S+)\.git/ def repository = repoMatcher.matches() ? repoMatcher.group(2) : "" diff --git a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle index 6cb22dad9bc79..dd8b582adb92f 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.ide.gradle @@ -167,9 +167,8 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') { vmParameters = [ '-ea', '-Djava.security.manager=allow', - '-Djava.locale.providers=SPI,COMPAT', - '-Djava.library.path=' + testLibraryPath, - '-Djna.library.path=' + testLibraryPath, + '-Djava.locale.providers=SPI,CLDR', + '-Des.nativelibs.path="' + testLibraryPath + '"', // TODO: only open these for mockito when it is modularized '--add-opens=java.base/java.security.cert=ALL-UNNAMED', '--add-opens=java.base/java.nio.channels=ALL-UNNAMED', diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java index 97172ec51e5b3..dd1d9d48252e1 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java @@ -12,27 +12,40 @@ * This class models the different Docker base images that are used to build Docker distributions of Elasticsearch. */ public enum DockerBase { - DEFAULT("ubuntu:20.04", ""), + DEFAULT("ubuntu:20.04", "", "apt-get"), // "latest" here is intentional, since the image name specifies "8" - UBI("docker.elastic.co/ubi8/ubi-minimal:latest", "-ubi8"), + UBI("docker.elastic.co/ubi8/ubi-minimal:latest", "-ubi8", "microdnf"), // The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build - IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank"), + IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank", "yum"), // Base image with extras for Cloud - CLOUD("ubuntu:20.04", "-cloud"), + CLOUD("ubuntu:20.04", "-cloud", "apt-get"), // Based on CLOUD above, with more extras. We don't set a base image because // we programmatically extend from the Cloud image. - CLOUD_ESS(null, "-cloud-ess"); + CLOUD_ESS(null, "-cloud-ess", "apt-get"), + + // Chainguard based wolfi image with latest jdk + WOLFI( + "docker.elastic.co/wolfi/chainguard-base:latest@sha256:c16d3ad6cebf387e8dd2ad769f54320c4819fbbaa21e729fad087c7ae223b4d0", + "wolfi", + "apk" + ); private final String image; private final String suffix; + private final String packageManager; DockerBase(String image, String suffix) { + this(image, suffix, "apt-get"); + } + + DockerBase(String image, String suffix, String packageManager) { this.image = image; this.suffix = suffix; + this.packageManager = packageManager; } public String getImage() { @@ -42,4 +55,8 @@ public String getImage() { public String getSuffix() { return suffix; } + + public String getPackageManager() { + return packageManager; + } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java index f95d9d72a473f..a3b1dd9731591 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java @@ -189,9 +189,7 @@ private static void configureNativeLibraryPath(Project project) { var libraryPath = (Supplier) () -> TestUtil.getTestLibraryPath(nativeConfigFiles.getAsPath()); test.dependsOn(nativeConfigFiles); - // we may use JNA or the JDK's foreign function api to load libraries, so we set both sysprops - systemProperties.systemProperty("java.library.path", libraryPath); - systemProperties.systemProperty("jna.library.path", libraryPath); + systemProperties.systemProperty("es.nativelibs.path", libraryPath); }); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java index 689c8ddecb057..2d6964c041fe2 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java @@ -92,7 +92,7 @@ public void execute(Task t) { mkdirs(test.getWorkingDir().toPath().resolve("temp").toFile()); // TODO remove once jvm.options are added to test system properties - test.systemProperty("java.locale.providers", "SPI,COMPAT"); + test.systemProperty("java.locale.providers", "SPI,CLDR"); } }); test.getJvmArgumentProviders().add(nonInputProperties); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java index f92789f701049..eeb4306ce6fb9 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java @@ -177,6 +177,9 @@ private static String distributionProjectName(ElasticsearchDistribution distribu if (distribution.getType() == InternalElasticsearchDistributionTypes.DOCKER_CLOUD_ESS) { return projectName + "cloud-ess-docker" + archString + "-export"; } + if (distribution.getType() == InternalElasticsearchDistributionTypes.DOCKER_WOLFI) { + return projectName + "wolfi-docker" + archString + "-export"; + } return projectName + distribution.getType().getName(); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerWolfiElasticsearchDistributionType.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerWolfiElasticsearchDistributionType.java new file mode 100644 index 0000000000000..d055337436a88 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/DockerWolfiElasticsearchDistributionType.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.gradle.internal.distribution; + +import org.elasticsearch.gradle.ElasticsearchDistributionType; + +public class DockerWolfiElasticsearchDistributionType implements ElasticsearchDistributionType { + + DockerWolfiElasticsearchDistributionType() {} + + @Override + public String getName() { + return "dockerWolfi"; + } + + @Override + public boolean isDocker() { + return true; + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java index 0b6ef212a63dd..5f8ef58e44a68 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/distribution/InternalElasticsearchDistributionTypes.java @@ -20,6 +20,7 @@ public class InternalElasticsearchDistributionTypes { public static ElasticsearchDistributionType DOCKER_IRONBANK = new DockerIronBankElasticsearchDistributionType(); public static ElasticsearchDistributionType DOCKER_CLOUD = new DockerCloudElasticsearchDistributionType(); public static ElasticsearchDistributionType DOCKER_CLOUD_ESS = new DockerCloudEssElasticsearchDistributionType(); + public static ElasticsearchDistributionType DOCKER_WOLFI = new DockerWolfiElasticsearchDistributionType(); public static List ALL_INTERNAL = List.of( DEB, @@ -28,6 +29,7 @@ public class InternalElasticsearchDistributionTypes { DOCKER_UBI, DOCKER_IRONBANK, DOCKER_CLOUD, - DOCKER_CLOUD_ESS + DOCKER_CLOUD_ESS, + DOCKER_WOLFI ); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java index 42d3a770dbbcc..5b1044bbb29a3 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java @@ -52,6 +52,7 @@ import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_CLOUD_ESS; import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_IRONBANK; import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_UBI; +import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.DOCKER_WOLFI; import static org.elasticsearch.gradle.internal.distribution.InternalElasticsearchDistributionTypes.RPM; /** @@ -93,6 +94,7 @@ public void apply(Project project) { for (ElasticsearchDistribution distribution : testDistributions) { String taskname = destructiveDistroTestTaskName(distribution); + ElasticsearchDistributionType type = distribution.getType(); TaskProvider destructiveTask = configureTestTask(project, taskname, distribution, t -> { t.onlyIf( "Docker is not available", @@ -106,12 +108,13 @@ public void apply(Project project) { if (distribution.getPlatform() == Platform.WINDOWS) { windowsTestTasks.add(destructiveTask); } else { - linuxTestTasks.computeIfAbsent(distribution.getType(), k -> new ArrayList<>()).add(destructiveTask); + linuxTestTasks.computeIfAbsent(type, k -> new ArrayList<>()).add(destructiveTask); } destructiveDistroTest.configure(t -> t.dependsOn(destructiveTask)); - lifecycleTasks.get(distribution.getType()).configure(t -> t.dependsOn(destructiveTask)); + TaskProvider lifecycleTask = lifecycleTasks.get(type); + lifecycleTask.configure(t -> t.dependsOn(destructiveTask)); - if ((distribution.getType() == DEB || distribution.getType() == RPM) && distribution.getBundledJdk()) { + if ((type == DEB || type == RPM) && distribution.getBundledJdk()) { for (Version version : BuildParams.getBwcVersions().getIndexCompatible()) { final ElasticsearchDistribution bwcDistro; if (version.equals(Version.fromString(distribution.getVersion()))) { @@ -121,7 +124,7 @@ public void apply(Project project) { bwcDistro = createDistro( allDistributions, distribution.getArchitecture(), - distribution.getType(), + type, distribution.getPlatform(), distribution.getBundledJdk(), version.toString() @@ -147,6 +150,7 @@ private static Map> lifecycleTask lifecyleTasks.put(DOCKER_IRONBANK, project.getTasks().register(taskPrefix + ".docker-ironbank")); lifecyleTasks.put(DOCKER_CLOUD, project.getTasks().register(taskPrefix + ".docker-cloud")); lifecyleTasks.put(DOCKER_CLOUD_ESS, project.getTasks().register(taskPrefix + ".docker-cloud-ess")); + lifecyleTasks.put(DOCKER_WOLFI, project.getTasks().register(taskPrefix + ".docker-wolfi")); lifecyleTasks.put(ARCHIVE, project.getTasks().register(taskPrefix + ".archives")); lifecyleTasks.put(DEB, project.getTasks().register(taskPrefix + ".packages")); lifecyleTasks.put(RPM, lifecyleTasks.get(DEB)); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestUtil.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestUtil.java index 96fde95d0dd17..965f3964c9a38 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestUtil.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestUtil.java @@ -11,7 +11,6 @@ import org.elasticsearch.gradle.Architecture; import org.elasticsearch.gradle.ElasticsearchDistribution; -import java.io.File; import java.util.Locale; public class TestUtil { @@ -19,8 +18,7 @@ public class TestUtil { public static String getTestLibraryPath(String nativeLibsDir) { String arch = Architecture.current().toString().toLowerCase(Locale.ROOT); String platform = String.format(Locale.ROOT, "%s-%s", ElasticsearchDistribution.CURRENT_PLATFORM, arch); - String existingLibraryPath = System.getProperty("java.library.path"); - return String.format(Locale.ROOT, "%s/%s%c%s", nativeLibsDir, platform, File.pathSeparatorChar, existingLibraryPath); + return String.format(Locale.ROOT, "%s/%s", nativeLibsDir, platform); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java index c6320394ef5b9..e0581ebf67081 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/AbstractYamlRestCompatTestPlugin.java @@ -35,6 +35,7 @@ import org.gradle.api.tasks.Sync; import org.gradle.api.tasks.TaskProvider; import org.gradle.api.tasks.testing.Test; +import org.gradle.language.jvm.tasks.ProcessResources; import java.io.File; import java.nio.file.Path; @@ -213,6 +214,17 @@ public void apply(Project project) { .named(RestResourcesPlugin.COPY_YAML_TESTS_TASK) .flatMap(CopyRestTestsTask::getOutputResourceDir); + // ensure we include other non rest spec related test resources + project.getTasks() + .withType(ProcessResources.class) + .named(yamlCompatTestSourceSet.getProcessResourcesTaskName()) + .configure(processResources -> { + processResources.from( + sourceSets.getByName(YamlRestTestPlugin.YAML_REST_TEST).getResources(), + spec -> { spec.exclude("rest-api-spec/**"); } + ); + }); + // setup the test task TaskProvider yamlRestCompatTestTask = registerTestTask(project, yamlCompatTestSourceSet); yamlRestCompatTestTask.configure(testTask -> { @@ -221,7 +233,7 @@ public void apply(Project project) { testTask.setTestClassesDirs( yamlTestSourceSet.getOutput().getClassesDirs().plus(yamlCompatTestSourceSet.getOutput().getClassesDirs()) ); - testTask.onlyIf("Compatibility tests are available", t -> yamlCompatTestSourceSet.getAllSource().isEmpty() == false); + testTask.onlyIf("Compatibility tests are available", t -> yamlCompatTestSourceSet.getOutput().isEmpty() == false); testTask.setClasspath( yamlCompatTestSourceSet.getRuntimeClasspath() // remove the "normal" api and tests diff --git a/build-tools-internal/src/main/resources/changelog-schema.json b/build-tools-internal/src/main/resources/changelog-schema.json index d8fc7d780ae58..593716954780b 100644 --- a/build-tools-internal/src/main/resources/changelog-schema.json +++ b/build-tools-internal/src/main/resources/changelog-schema.json @@ -28,6 +28,7 @@ "Autoscaling", "CAT APIs", "CCR", + "CCS", "CRUD", "Client", "Cluster Coordination", diff --git a/build-tools-internal/src/main/resources/minimumGradleVersion b/build-tools-internal/src/main/resources/minimumGradleVersion index f7b1c8ff61774..8d04a0f38fab0 100644 --- a/build-tools-internal/src/main/resources/minimumGradleVersion +++ b/build-tools-internal/src/main/resources/minimumGradleVersion @@ -1 +1 @@ -8.9 \ No newline at end of file +8.10 \ No newline at end of file diff --git a/catalog-info.yaml b/catalog-info.yaml index dfeeae51c1b3a..e57841c9de268 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -125,7 +125,7 @@ spec: ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true" SLACK_NOTIFICATIONS_CHANNEL: "#lucene" SLACK_NOTIFICATIONS_ALL_BRANCHES: "true" - branch_configuration: lucene_snapshot + branch_configuration: lucene_snapshot lucene_snapshot_10 default_branch: lucene_snapshot teams: elasticsearch-team: {} @@ -142,6 +142,10 @@ spec: branch: lucene_snapshot cronline: "0 2 * * * America/New_York" message: "Builds a new lucene snapshot 1x per day" + Periodically on lucene_snapshot_10: + branch: lucene_snapshot_10 + cronline: "0 2 * * * America/New_York" + message: "Builds a new lucene snapshot 1x per day" --- # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json apiVersion: backstage.io/v1alpha1 @@ -169,7 +173,7 @@ spec: ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true" SLACK_NOTIFICATIONS_CHANNEL: "#lucene" SLACK_NOTIFICATIONS_ALL_BRANCHES: "true" - branch_configuration: lucene_snapshot + branch_configuration: lucene_snapshot lucene_snapshot_10 default_branch: lucene_snapshot teams: elasticsearch-team: {} @@ -186,6 +190,10 @@ spec: branch: lucene_snapshot cronline: "0 6 * * * America/New_York" message: "Merges main into lucene_snapshot branch 1x per day" + Periodically on lucene_snapshot_10: + branch: lucene_snapshot_10 + cronline: "0 6 * * * America/New_York" + message: "Merges main into lucene_snapshot_10 branch 1x per day" --- # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json apiVersion: backstage.io/v1alpha1 @@ -213,7 +221,7 @@ spec: ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true" SLACK_NOTIFICATIONS_CHANNEL: "#lucene" SLACK_NOTIFICATIONS_ALL_BRANCHES: "true" - branch_configuration: lucene_snapshot + branch_configuration: lucene_snapshot lucene_snapshot_10 default_branch: lucene_snapshot teams: elasticsearch-team: {} @@ -230,6 +238,10 @@ spec: branch: lucene_snapshot cronline: "0 9,12,15,18 * * * America/New_York" message: "Runs tests against lucene_snapshot branch several times per day" + Periodically on lucene_snapshot_10: + branch: lucene_snapshot_10 + cronline: "0 9,12,15,18 * * * America/New_York" + message: "Runs tests against lucene_snapshot_10 branch several times per day" --- # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json apiVersion: backstage.io/v1alpha1 diff --git a/distribution/docker/README.md b/distribution/docker/README.md index 4c8052cfc26b3..eb0e7b296097d 100644 --- a/distribution/docker/README.md +++ b/distribution/docker/README.md @@ -6,11 +6,13 @@ the [DockerBase] enum. * Default - this is what most people use, and is based on Ubuntu * UBI - the same as the default image, but based upon [RedHat's UBI images][ubi], specifically their minimal flavour. + * Wolfi - the same as the default image, but based upon [Wolfi](https://github.com/wolfi-dev) * Iron Bank - this is the US Department of Defence's repository of digitally signed, binary container images including both Free and Open-Source software (FOSS) and Commercial off-the-shelf (COTS). In practice, this is another UBI build, this time on the regular UBI image, with extra hardening. See below for more details. + * Cloud - this is mostly the same as the default image, with some notable differences: * `filebeat` and `metricbeat` are included * `wget` is included diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 85e66ccba34b1..11c6010c22af0 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -21,8 +21,6 @@ apply plugin: 'elasticsearch.dra-artifacts' String buildId = providers.systemProperty('build.id').getOrNull() boolean useLocalArtifacts = buildId != null && buildId.isBlank() == false && useDra == false - - repositories { // Define a repository that allows Gradle to fetch a resource from GitHub. This // is only used to fetch the `tini` binary, when building the Iron Bank docker image @@ -131,7 +129,7 @@ ext.expansions = { Architecture architecture, DockerBase base -> 'config_dir' : base == DockerBase.IRON_BANK ? 'scripts' : 'config', 'git_revision' : BuildParams.gitRevision, 'license' : base == DockerBase.IRON_BANK ? 'Elastic License 2.0' : 'Elastic-License-2.0', - 'package_manager' : base == DockerBase.IRON_BANK ? 'yum' : (base == DockerBase.UBI ? 'microdnf' : 'apt-get'), + 'package_manager' : base.packageManager, 'docker_base' : base.name().toLowerCase(), 'version' : VersionProperties.elasticsearch, 'major_minor_version': "${major}.${minor}", @@ -182,21 +180,12 @@ ext.dockerBuildContext = { Architecture architecture, DockerBase base -> from projectDir.resolve("src/docker/config") } } - from(projectDir.resolve("src/docker/Dockerfile")) { expand(varExpansions) filter SquashNewlinesFilter } } } -// -//def createAndSetWritable(Object... locations) { -// locations.each { location -> -// File file = file(location) -// file.mkdirs() -// file.setWritable(true, false) -// } -//} tasks.register("copyNodeKeyMaterial", Sync) { def certsDir = file("build/certs") diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile index 32f35b05015b9..47f79749cbefa 100644 --- a/distribution/docker/src/docker/Dockerfile +++ b/distribution/docker/src/docker/Dockerfile @@ -22,7 +22,7 @@ <% if (docker_base == 'iron_bank') { %> ARG BASE_REGISTRY=registry1.dso.mil ARG BASE_IMAGE=ironbank/redhat/ubi/ubi9 -ARG BASE_TAG=9.3 +ARG BASE_TAG=9.4 <% } %> ################################################################################ @@ -43,29 +43,34 @@ RUN chmod 0555 /bin/tini # Install required packages to extract the Elasticsearch distribution <% if (docker_base == 'default' || docker_base == 'cloud') { %> RUN <%= retry.loop(package_manager, "${package_manager} update && DEBIAN_FRONTEND=noninteractive ${package_manager} install -y curl ") %> +<% } else if (docker_base == "wolfi") { %> +RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %> <% } else { %> RUN <%= retry.loop(package_manager, "${package_manager} install -y findutils tar gzip") %> <% } %> -# `tini` is a tiny but valid init for containers. This is used to cleanly -# control how ES and any child processes are shut down. -# -# The tini GitHub page gives instructions for verifying the binary using -# gpg, but the keyservers are slow to return the key and this can fail the -# build. Instead, we check the binary against the published checksum. -RUN set -eux ; \\ - tini_bin="" ; \\ - case "\$(arch)" in \\ - aarch64) tini_bin='tini-arm64' ;; \\ - x86_64) tini_bin='tini-amd64' ;; \\ - *) echo >&2 ; echo >&2 "Unsupported architecture \$(arch)" ; echo >&2 ; exit 1 ;; \\ - esac ; \\ - curl --retry 10 -S -L -O https://github.com/krallin/tini/releases/download/v0.19.0/\${tini_bin} ; \\ - curl --retry 10 -S -L -O https://github.com/krallin/tini/releases/download/v0.19.0/\${tini_bin}.sha256sum ; \\ - sha256sum -c \${tini_bin}.sha256sum ; \\ - rm \${tini_bin}.sha256sum ; \\ - mv \${tini_bin} /bin/tini ; \\ - chmod 0555 /bin/tini +<% if (docker_base != 'wolfi') { %> + # `tini` is a tiny but valid init for containers. This is used to cleanly + # control how ES and any child processes are shut down. + # For wolfi we pick it from the blessed wolfi package registry. + # + # The tini GitHub page gives instructions for verifying the binary using + # gpg, but the keyservers are slow to return the key and this can fail the + # build. Instead, we check the binary against the published checksum. + RUN set -eux ; \\ + tini_bin="" ; \\ + case "\$(arch)" in \\ + aarch64) tini_bin='tini-arm64' ;; \\ + x86_64) tini_bin='tini-amd64' ;; \\ + *) echo >&2 ; echo >&2 "Unsupported architecture \$(arch)" ; echo >&2 ; exit 1 ;; \\ + esac ; \\ + curl --retry 10 -S -L -O https://github.com/krallin/tini/releases/download/v0.19.0/\${tini_bin} ; \\ + curl --retry 10 -S -L -O https://github.com/krallin/tini/releases/download/v0.19.0/\${tini_bin}.sha256sum ; \\ + sha256sum -c \${tini_bin}.sha256sum ; \\ + rm \${tini_bin}.sha256sum ; \\ + mv \${tini_bin} /bin/tini ; \\ + chmod 0555 /bin/tini +<% } %> <% } %> @@ -152,6 +157,15 @@ RUN ${package_manager} update --setopt=tsflags=nodocs -y && \\ nc shadow-utils zip findutils unzip procps-ng && \\ ${package_manager} clean all +<% } else if (docker_base == "wolfi") { %> +RUN <%= retry.loop(package_manager, + "export DEBIAN_FRONTEND=noninteractive && \n" + + " ${package_manager} update && \n" + + " ${package_manager} upgrade && \n" + + " ${package_manager} add --no-cache \n" + + " bash ca-certificates curl libsystemd netcat-openbsd p11-kit p11-kit-trust shadow tini unzip zip zstd && \n" + + " rm -rf /var/cache/apk/* " + ) %> <% } else if (docker_base == "default" || docker_base == "cloud") { %> # Change default shell to bash, then install required packages with retries. @@ -185,6 +199,11 @@ RUN groupadd -g 1000 elasticsearch && \\ adduser --uid 1000 --gid 1000 --home /usr/share/elasticsearch elasticsearch && \\ adduser elasticsearch root && \\ chown -R 0:0 /usr/share/elasticsearch +<% } else if (docker_base == "wolfi") { %> +RUN groupadd -g 1000 elasticsearch && \ + adduser -G elasticsearch -u 1000 elasticsearch -D --home /usr/share/elasticsearch elasticsearch && \ + adduser elasticsearch root && \ + chown -R 0:0 /usr/share/elasticsearch <% } else { %> RUN groupadd -g 1000 elasticsearch && \\ adduser -u 1000 -g 1000 -G 0 -d /usr/share/elasticsearch elasticsearch && \\ @@ -196,7 +215,9 @@ ENV ELASTIC_CONTAINER true WORKDIR /usr/share/elasticsearch COPY --from=builder --chown=0:0 /usr/share/elasticsearch /usr/share/elasticsearch +<% if (docker_base != "wolfi") { %> COPY --from=builder --chown=0:0 /bin/tini /bin/tini +<% } %> <% if (docker_base == 'cloud') { %> COPY --from=builder --chown=0:0 /opt /opt @@ -280,7 +301,12 @@ CMD ["/app/elasticsearch.sh"] RUN mkdir /app && \\ echo -e '#!/bin/bash\\nexec /usr/local/bin/docker-entrypoint.sh eswrapper' > /app/elasticsearch.sh && \\ chmod 0555 /app/elasticsearch.sh - +<% } else if (docker_base == "wolfi") { %> +# Our actual entrypoint is `tini`, a minimal but functional init program. It +# calls the entrypoint we provide, while correctly forwarding signals. +ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/docker-entrypoint.sh"] +# Dummy overridable parameter parsed by entrypoint +CMD ["eswrapper"] <% } else { %> # Our actual entrypoint is `tini`, a minimal but functional init program. It # calls the entrypoint we provide, while correctly forwarding signals. diff --git a/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml b/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml index 38ce16a413af2..f4364c5008c09 100644 --- a/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml +++ b/distribution/docker/src/docker/iron_bank/hardening_manifest.yaml @@ -14,7 +14,7 @@ tags: # Build args passed to Dockerfile ARGs args: BASE_IMAGE: "redhat/ubi/ubi9" - BASE_TAG: "9.3" + BASE_TAG: "9.4" # Docker image labels labels: diff --git a/distribution/docker/wolfi-docker-aarch64-export/build.gradle b/distribution/docker/wolfi-docker-aarch64-export/build.gradle new file mode 100644 index 0000000000000..537b5a093683e --- /dev/null +++ b/distribution/docker/wolfi-docker-aarch64-export/build.gradle @@ -0,0 +1,2 @@ +// This file is intentionally blank. All configuration of the +// export is done in the parent project. diff --git a/distribution/docker/wolfi-docker-export/build.gradle b/distribution/docker/wolfi-docker-export/build.gradle new file mode 100644 index 0000000000000..537b5a093683e --- /dev/null +++ b/distribution/docker/wolfi-docker-export/build.gradle @@ -0,0 +1,2 @@ +// This file is intentionally blank. All configuration of the +// export is done in the parent project. diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java index 7b904d4cb5a89..bea7fbb7f63e8 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java @@ -32,6 +32,7 @@ import java.nio.file.Path; import java.util.Arrays; import java.util.Locale; +import java.util.concurrent.atomic.AtomicBoolean; /** * The main CLI for running Elasticsearch. @@ -44,6 +45,8 @@ class ServerCli extends EnvironmentAwareCommand { private final OptionSpecBuilder quietOption; private final OptionSpec enrollmentTokenOption; + // flag for indicating shutdown has begun. we use an AtomicBoolean to double as a synchronization object + private final AtomicBoolean shuttingDown = new AtomicBoolean(false); private volatile ServerProcess server; // visible for testing @@ -98,7 +101,14 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce syncPlugins(terminal, env, processInfo); ServerArgs args = createArgs(options, env, secrets, processInfo); - this.server = startServer(terminal, processInfo, args); + synchronized (shuttingDown) { + // if we are shutting down there is no reason to start the server + if (shuttingDown.get()) { + terminal.println("CLI is shutting down, skipping starting server process"); + return; + } + this.server = startServer(terminal, processInfo, args); + } } if (options.has(daemonizeOption)) { @@ -233,8 +243,11 @@ private ServerArgs createArgs(OptionSet options, Environment env, SecureSettings @Override public void close() throws IOException { - if (server != null) { - server.stop(); + synchronized (shuttingDown) { + shuttingDown.set(true); + if (server != null) { + server.stop(); + } } } diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java index 2a89f18209d11..2d707f150cc8b 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java @@ -10,11 +10,8 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; -import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.core.UpdateForV9; -import java.io.File; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -25,7 +22,6 @@ final class SystemJvmOptions { static List systemJvmOptions(Settings nodeSettings, final Map sysprops) { String distroType = sysprops.get("es.distribution.type"); boolean isHotspot = sysprops.getOrDefault("sun.management.compiler", "").contains("HotSpot"); - String libraryPath = findLibraryPath(sysprops); return Stream.concat( Stream.of( @@ -64,17 +60,11 @@ static List systemJvmOptions(Settings nodeSettings, final Map maybeWorkaroundG1Bug() { } return Stream.of(); } - - private static String findLibraryPath(Map sysprops) { - // working dir is ES installation, so we use relative path here - Path platformDir = Paths.get("lib", "platform"); - String existingPath = sysprops.get("java.library.path"); - assert existingPath != null; - - String osname = sysprops.get("os.name"); - String os; - if (osname.startsWith("Windows")) { - os = "windows"; - } else if (osname.startsWith("Linux")) { - os = "linux"; - } else if (osname.startsWith("Mac OS")) { - os = "darwin"; - } else { - os = "unsupported_os[" + osname + "]"; - } - String archname = sysprops.get("os.arch"); - String arch; - if (archname.equals("amd64") || archname.equals("x86_64")) { - arch = "x64"; - } else if (archname.equals("aarch64")) { - arch = archname; - } else { - arch = "unsupported_arch[" + archname + "]"; - } - return platformDir.resolve(os + "-" + arch).toAbsolutePath() + getPathSeparator() + existingPath; - } - - @SuppressForbidden(reason = "no way to get path separator with nio") - private static String getPathSeparator() { - return File.pathSeparator; - } } diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java index 87b7894a9135a..fc889f036a795 100644 --- a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java +++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/JvmOptionsParserTests.java @@ -17,7 +17,6 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.StringReader; -import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; @@ -30,12 +29,10 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; -import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.hasSize; @@ -44,14 +41,7 @@ @WithoutSecurityManager public class JvmOptionsParserTests extends ESTestCase { - private static final Map TEST_SYSPROPS = Map.of( - "os.name", - "Linux", - "os.arch", - "aarch64", - "java.library.path", - "/usr/lib" - ); + private static final Map TEST_SYSPROPS = Map.of("os.name", "Linux", "os.arch", "aarch64"); public void testSubstitution() { final List jvmOptions = JvmOptionsParser.substitutePlaceholders( @@ -390,40 +380,4 @@ public void testCommandLineDistributionType() { final List jvmOptions = SystemJvmOptions.systemJvmOptions(Settings.EMPTY, sysprops); assertThat(jvmOptions, hasItem("-Des.distribution.type=testdistro")); } - - public void testLibraryPath() { - assertLibraryPath("Mac OS", "aarch64", "darwin-aarch64"); - assertLibraryPath("Mac OS", "amd64", "darwin-x64"); - assertLibraryPath("Mac OS", "x86_64", "darwin-x64"); - assertLibraryPath("Linux", "aarch64", "linux-aarch64"); - assertLibraryPath("Linux", "amd64", "linux-x64"); - assertLibraryPath("Linux", "x86_64", "linux-x64"); - assertLibraryPath("Windows", "amd64", "windows-x64"); - assertLibraryPath("Windows", "x86_64", "windows-x64"); - assertLibraryPath("Unknown", "aarch64", "unsupported_os[Unknown]-aarch64"); - assertLibraryPath("Mac OS", "Unknown", "darwin-unsupported_arch[Unknown]"); - } - - private void assertLibraryPath(String os, String arch, String expected) { - String existingPath = "/usr/lib"; - var sysprops = Map.of("os.name", os, "os.arch", arch, "java.library.path", existingPath); - final List jvmOptions = SystemJvmOptions.systemJvmOptions(Settings.EMPTY, sysprops); - Map options = new HashMap<>(); - for (var jvmOption : jvmOptions) { - if (jvmOption.startsWith("-D")) { - String[] parts = jvmOption.substring(2).split("="); - assert parts.length == 2; - options.put(parts[0], parts[1]); - } - } - String separator = FileSystems.getDefault().getSeparator(); - assertThat( - options, - hasEntry(equalTo("java.library.path"), allOf(containsString("platform" + separator + expected), containsString(existingPath))) - ); - assertThat( - options, - hasEntry(equalTo("jna.library.path"), allOf(containsString("platform" + separator + expected), containsString(existingPath))) - ); - } } diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java index 38a64a778fc27..e603790051c0c 100644 --- a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java +++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/ServerCliTests.java @@ -36,6 +36,8 @@ import java.util.List; import java.util.Locale; import java.util.Optional; +import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.CyclicBarrier; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; @@ -50,6 +52,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.matchesRegex; import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.sameInstance; public class ServerCliTests extends CommandTestCase { @@ -383,6 +386,52 @@ public void testSecureSettingsLoaderWithNullPassword() throws Exception { assertEquals("", loader.password); } + public void testProcessCreationRace() throws Exception { + for (int i = 0; i < 10; ++i) { + CyclicBarrier raceStart = new CyclicBarrier(2); + TestServerCli cli = new TestServerCli() { + @Override + void syncPlugins(Terminal terminal, Environment env, ProcessInfo processInfo) throws Exception { + super.syncPlugins(terminal, env, processInfo); + raceStart.await(); + } + + @Override + public void close() throws IOException { + try { + raceStart.await(); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new AssertionError(ie); + } catch (BrokenBarrierException e) { + throw new AssertionError(e); + } + super.close(); + } + }; + Thread closeThread = new Thread(() -> { + try { + cli.close(); + } catch (IOException e) { + throw new AssertionError(e); + } + }); + closeThread.start(); + cli.main(new String[] {}, terminal, new ProcessInfo(sysprops, envVars, esHomeDir)); + closeThread.join(); + + if (cli.getServer() == null) { + // close won the race, so server should never have been started + assertThat(cli.startServerCalled, is(false)); + } else { + // creation won the race, so check we correctly waited on it and stopped + assertThat(cli.getServer(), sameInstance(mockServer)); + assertThat(mockServer.waitForCalled, is(true)); + assertThat(mockServer.stopCalled, is(true)); + } + } + } + private MockSecureSettingsLoader loadWithMockSecureSettingsLoader() throws Exception { var loader = new MockSecureSettingsLoader(); this.mockSecureSettingsLoader = loader; @@ -465,9 +514,9 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce } private class MockServerProcess extends ServerProcess { - boolean detachCalled = false; - boolean waitForCalled = false; - boolean stopCalled = false; + volatile boolean detachCalled = false; + volatile boolean waitForCalled = false; + volatile boolean stopCalled = false; MockServerProcess() { super(null, null); @@ -505,6 +554,8 @@ void reset() { } private class TestServerCli extends ServerCli { + boolean startServerCalled = false; + @Override protected Command loadTool(String toolname, String libs) { if (toolname.equals("auto-configure-node")) { @@ -551,20 +602,21 @@ protected SecureSettingsLoader secureSettingsLoader(Environment env) { return new KeystoreSecureSettingsLoader(); } + + @Override + protected ServerProcess startServer(Terminal terminal, ProcessInfo processInfo, ServerArgs args) throws Exception { + startServerCalled = true; + if (argsValidator != null) { + argsValidator.accept(args); + } + mockServer.reset(); + return mockServer; + } } @Override protected Command newCommand() { - return new TestServerCli() { - @Override - protected ServerProcess startServer(Terminal terminal, ProcessInfo processInfo, ServerArgs args) { - if (argsValidator != null) { - argsValidator.accept(args); - } - mockServer.reset(); - return mockServer; - } - }; + return new TestServerCli(); } static class MockSecureSettingsLoader implements SecureSettingsLoader { diff --git a/docs/changelog/109414.yaml b/docs/changelog/109414.yaml new file mode 100644 index 0000000000000..81b7541bde35b --- /dev/null +++ b/docs/changelog/109414.yaml @@ -0,0 +1,6 @@ +pr: 109414 +summary: Don't fail retention lease sync actions due to capacity constraints +area: CRUD +type: bug +issues: + - 105926 diff --git a/docs/changelog/110524.yaml b/docs/changelog/110524.yaml new file mode 100644 index 0000000000000..6274c99b09998 --- /dev/null +++ b/docs/changelog/110524.yaml @@ -0,0 +1,5 @@ +pr: 110524 +summary: Introduce mode `subobjects=auto` for objects +area: Mapping +type: enhancement +issues: [] diff --git a/docs/changelog/110633.yaml b/docs/changelog/110633.yaml new file mode 100644 index 0000000000000..d4d1dc68cdbcc --- /dev/null +++ b/docs/changelog/110633.yaml @@ -0,0 +1,5 @@ +pr: 110633 +summary: Add manage roles privilege +area: Authorization +type: enhancement +issues: [] diff --git a/docs/changelog/110847.yaml b/docs/changelog/110847.yaml new file mode 100644 index 0000000000000..214adc97ac7cb --- /dev/null +++ b/docs/changelog/110847.yaml @@ -0,0 +1,5 @@ +pr: 110847 +summary: SLM Interval based scheduling +area: ILM+SLM +type: feature +issues: [] diff --git a/docs/changelog/111161.yaml b/docs/changelog/111161.yaml new file mode 100644 index 0000000000000..c081d555ff1ee --- /dev/null +++ b/docs/changelog/111161.yaml @@ -0,0 +1,6 @@ +pr: 111161 +summary: Add support for templates when validating mappings in the simulate ingest + API +area: Ingest Node +type: enhancement +issues: [] diff --git a/docs/changelog/111181.yaml b/docs/changelog/111181.yaml new file mode 100644 index 0000000000000..7f9f5937b7652 --- /dev/null +++ b/docs/changelog/111181.yaml @@ -0,0 +1,5 @@ +pr: 111181 +summary: "[Inference API] Add Alibaba Cloud AI Search Model support to Inference API" +area: Machine Learning +type: enhancement +issues: [ ] diff --git a/docs/changelog/111226.yaml b/docs/changelog/111226.yaml new file mode 100644 index 0000000000000..1021a26fa789f --- /dev/null +++ b/docs/changelog/111226.yaml @@ -0,0 +1,5 @@ +pr: 111226 +summary: "ES|QL: add Telemetry API and track top functions" +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/111285.yaml b/docs/changelog/111285.yaml deleted file mode 100644 index e4856482b4d6e..0000000000000 --- a/docs/changelog/111285.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111285 -summary: "[Bugfix] Add `accessDeclaredMembers` permission to allow search application templates to parse floats" -area: Relevance -type: bug -issues: [] diff --git a/docs/changelog/111412.yaml b/docs/changelog/111412.yaml new file mode 100644 index 0000000000000..297fa77cd2664 --- /dev/null +++ b/docs/changelog/111412.yaml @@ -0,0 +1,6 @@ +pr: 111412 +summary: Make enrich cache based on memory usage +area: Ingest Node +type: enhancement +issues: + - 106081 diff --git a/docs/changelog/111413.yaml b/docs/changelog/111413.yaml new file mode 100644 index 0000000000000..0eae45b17d0c4 --- /dev/null +++ b/docs/changelog/111413.yaml @@ -0,0 +1,6 @@ +pr: 111413 +summary: "ESQL: Fix synthetic attribute pruning" +area: ES|QL +type: bug +issues: + - 105821 diff --git a/docs/changelog/111475.yaml b/docs/changelog/111475.yaml deleted file mode 100644 index 264c975444868..0000000000000 --- a/docs/changelog/111475.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111475 -summary: "ESQL: Fix for overzealous validation in case of invalid mapped fields" -area: ES|QL -type: bug -issues: - - 111452 diff --git a/docs/changelog/111516.yaml b/docs/changelog/111516.yaml new file mode 100644 index 0000000000000..96e8bd843f750 --- /dev/null +++ b/docs/changelog/111516.yaml @@ -0,0 +1,5 @@ +pr: 111516 +summary: Adding support for `allow_partial_search_results` in PIT +area: Search +type: enhancement +issues: [] diff --git a/docs/changelog/111523.yaml b/docs/changelog/111523.yaml new file mode 100644 index 0000000000000..202d16c5a426d --- /dev/null +++ b/docs/changelog/111523.yaml @@ -0,0 +1,5 @@ +pr: 111523 +summary: Search coordinator uses `event.ingested` in cluster state to do rewrites +area: Search +type: enhancement +issues: [] diff --git a/docs/changelog/111548.yaml b/docs/changelog/111548.yaml new file mode 100644 index 0000000000000..ca9e5ae622894 --- /dev/null +++ b/docs/changelog/111548.yaml @@ -0,0 +1,6 @@ +pr: 111548 +summary: Json parsing exceptions should not cause 500 errors +area: Infra/Core +type: bug +issues: + - 111542 diff --git a/docs/changelog/111673.yaml b/docs/changelog/111673.yaml deleted file mode 100644 index ebc211633fcab..0000000000000 --- a/docs/changelog/111673.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111673 -summary: Properly handle filters on `TextSimilarityRank` retriever -area: Ranking -type: bug -issues: [] diff --git a/docs/changelog/111690.yaml b/docs/changelog/111690.yaml new file mode 100644 index 0000000000000..36e715744ad88 --- /dev/null +++ b/docs/changelog/111690.yaml @@ -0,0 +1,5 @@ +pr: 111690 +summary: "ESQL: Support INLINESTATS grouped on expressions" +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/111729.yaml b/docs/changelog/111729.yaml deleted file mode 100644 index c75c14a997da9..0000000000000 --- a/docs/changelog/111729.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111729 -summary: Speed up dense/sparse vector stats -area: Vector Search -type: bug -issues: - - 111715 diff --git a/docs/changelog/111756.yaml b/docs/changelog/111756.yaml deleted file mode 100644 index e58345dbe696a..0000000000000 --- a/docs/changelog/111756.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111756 -summary: Fix `NullPointerException` when doing knn search on empty index without dims -area: Vector Search -type: bug -issues: - - 111733 diff --git a/docs/changelog/111758.yaml b/docs/changelog/111758.yaml deleted file mode 100644 index c95cdf48bc8a7..0000000000000 --- a/docs/changelog/111758.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111758 -summary: Revert "Avoid bucket copies in Aggs" -area: Aggregations -type: bug -issues: - - 111679 diff --git a/docs/changelog/111797.yaml b/docs/changelog/111797.yaml new file mode 100644 index 0000000000000..00b793a19d9c3 --- /dev/null +++ b/docs/changelog/111797.yaml @@ -0,0 +1,6 @@ +pr: 111797 +summary: "ESQL: fix for missing indices error message" +area: ES|QL +type: bug +issues: + - 111712 diff --git a/docs/changelog/111807.yaml b/docs/changelog/111807.yaml deleted file mode 100644 index 97c5e58461c34..0000000000000 --- a/docs/changelog/111807.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111807 -summary: Explain Function Score Query -area: Search -type: bug -issues: [] diff --git a/docs/changelog/111818.yaml b/docs/changelog/111818.yaml new file mode 100644 index 0000000000000..c3a632861aae6 --- /dev/null +++ b/docs/changelog/111818.yaml @@ -0,0 +1,5 @@ +pr: 111818 +summary: Add tier preference to security index settings allowlist +area: Security +type: enhancement +issues: [] diff --git a/docs/changelog/111843.yaml b/docs/changelog/111843.yaml deleted file mode 100644 index c8b20036520f3..0000000000000 --- a/docs/changelog/111843.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111843 -summary: Add maximum nested depth check to WKT parser -area: Geo -type: bug -issues: [] diff --git a/docs/changelog/111863.yaml b/docs/changelog/111863.yaml deleted file mode 100644 index 1724cd83f984b..0000000000000 --- a/docs/changelog/111863.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111863 -summary: Fixing incorrect bulk request took time -area: Ingest Node -type: bug -issues: - - 111854 diff --git a/docs/changelog/111866.yaml b/docs/changelog/111866.yaml deleted file mode 100644 index 34bf56da4dc9e..0000000000000 --- a/docs/changelog/111866.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111866 -summary: Fix windows memory locking -area: Infra/Core -type: bug -issues: - - 111847 diff --git a/docs/changelog/111915.yaml b/docs/changelog/111915.yaml new file mode 100644 index 0000000000000..f64c45b82d10c --- /dev/null +++ b/docs/changelog/111915.yaml @@ -0,0 +1,6 @@ +pr: 111915 +summary: Fix DLS & FLS sometimes being enforced when it is disabled +area: Authorization +type: bug +issues: + - 94709 diff --git a/docs/changelog/111917.yaml b/docs/changelog/111917.yaml new file mode 100644 index 0000000000000..0dc760d76a698 --- /dev/null +++ b/docs/changelog/111917.yaml @@ -0,0 +1,7 @@ +pr: 111917 +summary: "[ES|QL] Cast mixed numeric types to a common numeric type for Coalesce and\ + \ In at Analyzer" +area: ES|QL +type: enhancement +issues: + - 111486 diff --git a/docs/changelog/111932.yaml b/docs/changelog/111932.yaml new file mode 100644 index 0000000000000..ce840ecebcff0 --- /dev/null +++ b/docs/changelog/111932.yaml @@ -0,0 +1,6 @@ +pr: 111932 +summary: Fix union-types where one index is missing the field +area: ES|QL +type: bug +issues: + - 111912 diff --git a/docs/changelog/111943.yaml b/docs/changelog/111943.yaml deleted file mode 100644 index 6b9f03ccee31c..0000000000000 --- a/docs/changelog/111943.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111943 -summary: Fix synthetic source for empty nested objects -area: Mapping -type: bug -issues: - - 111811 diff --git a/docs/changelog/111950.yaml b/docs/changelog/111950.yaml new file mode 100644 index 0000000000000..3f23c17d8e652 --- /dev/null +++ b/docs/changelog/111950.yaml @@ -0,0 +1,6 @@ +pr: 111950 +summary: "[ES|QL] Name parameter with leading underscore" +area: ES|QL +type: enhancement +issues: + - 111821 diff --git a/docs/changelog/111966.yaml b/docs/changelog/111966.yaml deleted file mode 100644 index facf0a61c4d8a..0000000000000 --- a/docs/changelog/111966.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 111966 -summary: No error when `store_array_source` is used without synthetic source -area: Mapping -type: bug -issues: [] diff --git a/docs/changelog/111983.yaml b/docs/changelog/111983.yaml deleted file mode 100644 index d5043d0b44155..0000000000000 --- a/docs/changelog/111983.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111983 -summary: Avoid losing error message in failure collector -area: ES|QL -type: bug -issues: - - 111894 diff --git a/docs/changelog/111994.yaml b/docs/changelog/111994.yaml deleted file mode 100644 index ee62651c43987..0000000000000 --- a/docs/changelog/111994.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 111994 -summary: Merge multiple ignored source entires for the same field -area: Logs -type: bug -issues: - - 111694 diff --git a/docs/changelog/112005.yaml b/docs/changelog/112005.yaml deleted file mode 100644 index 2d84381e632b3..0000000000000 --- a/docs/changelog/112005.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 112005 -summary: Check for valid `parentDoc` before retrieving its previous -area: Mapping -type: bug -issues: - - 111990 diff --git a/docs/changelog/112019.yaml b/docs/changelog/112019.yaml new file mode 100644 index 0000000000000..7afb207864ed7 --- /dev/null +++ b/docs/changelog/112019.yaml @@ -0,0 +1,5 @@ +pr: 112019 +summary: Display effective retention in the relevant data stream APIs +area: Data streams +type: enhancement +issues: [] diff --git a/docs/changelog/112024.yaml b/docs/changelog/112024.yaml new file mode 100644 index 0000000000000..e426693fba964 --- /dev/null +++ b/docs/changelog/112024.yaml @@ -0,0 +1,5 @@ +pr: 112024 +summary: (API) Cluster Health report `unassigned_primary_shards` +area: Health +type: enhancement +issues: [] diff --git a/docs/changelog/112026.yaml b/docs/changelog/112026.yaml new file mode 100644 index 0000000000000..fedf001923ab4 --- /dev/null +++ b/docs/changelog/112026.yaml @@ -0,0 +1,5 @@ +pr: 112026 +summary: Create `StreamingHttpResultPublisher` +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/changelog/112046.yaml b/docs/changelog/112046.yaml deleted file mode 100644 index f3cda1ed7a7d2..0000000000000 --- a/docs/changelog/112046.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112046 -summary: Fix calculation of parent offset for ignored source in some cases -area: Mapping -type: bug -issues: [] diff --git a/docs/changelog/112055.yaml b/docs/changelog/112055.yaml new file mode 100644 index 0000000000000..cdf15b3b37468 --- /dev/null +++ b/docs/changelog/112055.yaml @@ -0,0 +1,6 @@ +pr: 112055 +summary: "ESQL: `mv_median_absolute_deviation` function" +area: ES|QL +type: feature +issues: + - 111590 diff --git a/docs/changelog/112066.yaml b/docs/changelog/112066.yaml new file mode 100644 index 0000000000000..5dd846766bc8e --- /dev/null +++ b/docs/changelog/112066.yaml @@ -0,0 +1,6 @@ +pr: 112066 +summary: Do not treat replica as unassigned if primary recently created and unassigned + time is below a threshold +area: Health +type: enhancement +issues: [] diff --git a/docs/changelog/112100.yaml b/docs/changelog/112100.yaml new file mode 100644 index 0000000000000..9135edecb4d77 --- /dev/null +++ b/docs/changelog/112100.yaml @@ -0,0 +1,5 @@ +pr: 112100 +summary: Exclude internal data streams from global retention +area: Data streams +type: bug +issues: [] diff --git a/docs/changelog/112123.yaml b/docs/changelog/112123.yaml new file mode 100644 index 0000000000000..0c0d7ac44cd17 --- /dev/null +++ b/docs/changelog/112123.yaml @@ -0,0 +1,5 @@ +pr: 112123 +summary: SLM interval schedule followup - add back `getFieldName` style getters +area: ILM+SLM +type: enhancement +issues: [] diff --git a/docs/changelog/112126.yaml b/docs/changelog/112126.yaml new file mode 100644 index 0000000000000..f6a7aeb893a5e --- /dev/null +++ b/docs/changelog/112126.yaml @@ -0,0 +1,5 @@ +pr: 112126 +summary: Add support for spatial relationships in point field mapper +area: Geo +type: enhancement +issues: [] diff --git a/docs/changelog/112133.yaml b/docs/changelog/112133.yaml new file mode 100644 index 0000000000000..11109402b7373 --- /dev/null +++ b/docs/changelog/112133.yaml @@ -0,0 +1,5 @@ +pr: 112133 +summary: Add telemetry for repository usage +area: Snapshot/Restore +type: enhancement +issues: [] diff --git a/docs/changelog/112151.yaml b/docs/changelog/112151.yaml new file mode 100644 index 0000000000000..f5cbfd8da07c2 --- /dev/null +++ b/docs/changelog/112151.yaml @@ -0,0 +1,5 @@ +pr: 112151 +summary: Store original source for keywords using a normalizer +area: Logs +type: enhancement +issues: [] diff --git a/docs/changelog/112199.yaml b/docs/changelog/112199.yaml new file mode 100644 index 0000000000000..eb22f215f9828 --- /dev/null +++ b/docs/changelog/112199.yaml @@ -0,0 +1,5 @@ +pr: 112199 +summary: Support docvalues only query in shape field +area: Geo +type: enhancement +issues: [] diff --git a/docs/changelog/112200.yaml b/docs/changelog/112200.yaml new file mode 100644 index 0000000000000..0c2c3d71e3ddf --- /dev/null +++ b/docs/changelog/112200.yaml @@ -0,0 +1,6 @@ +pr: 112200 +summary: "ES|QL: better validation of GROK patterns" +area: ES|QL +type: bug +issues: + - 112111 diff --git a/docs/changelog/112210.yaml b/docs/changelog/112210.yaml new file mode 100644 index 0000000000000..6483b8b01315c --- /dev/null +++ b/docs/changelog/112210.yaml @@ -0,0 +1,5 @@ +pr: 112210 +summary: Expose global retention settings via data stream lifecycle API +area: Data streams +type: enhancement +issues: [] diff --git a/docs/changelog/112214.yaml b/docs/changelog/112214.yaml new file mode 100644 index 0000000000000..430f95a72bb3f --- /dev/null +++ b/docs/changelog/112214.yaml @@ -0,0 +1,5 @@ +pr: 112214 +summary: '`ByteArrayStreamInput:` Return -1 when there are no more bytes to read' +area: Infra/Core +type: bug +issues: [] diff --git a/docs/changelog/112218.yaml b/docs/changelog/112218.yaml new file mode 100644 index 0000000000000..c426dd7ade4ed --- /dev/null +++ b/docs/changelog/112218.yaml @@ -0,0 +1,9 @@ +pr: 112218 +summary: "ESQL: Fix a bug in `MV_PERCENTILE`" +area: ES|QL +type: bug +issues: + - 112193 + - 112180 + - 112187 + - 112188 diff --git a/docs/changelog/112262.yaml b/docs/changelog/112262.yaml new file mode 100644 index 0000000000000..fe23c14c79c9e --- /dev/null +++ b/docs/changelog/112262.yaml @@ -0,0 +1,6 @@ +pr: 112262 +summary: Check for disabling own user in Put User API +area: Authentication +type: bug +issues: + - 90205 diff --git a/docs/changelog/112263.yaml b/docs/changelog/112263.yaml new file mode 100644 index 0000000000000..2d1321f327673 --- /dev/null +++ b/docs/changelog/112263.yaml @@ -0,0 +1,6 @@ +pr: 112263 +summary: Fix `TokenService` always appearing used in Feature Usage +area: License +type: bug +issues: + - 61956 diff --git a/docs/changelog/112270.yaml b/docs/changelog/112270.yaml new file mode 100644 index 0000000000000..1e6b9c7fc9290 --- /dev/null +++ b/docs/changelog/112270.yaml @@ -0,0 +1,5 @@ +pr: 112270 +summary: Support sparse embedding models in the elasticsearch inference service +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/changelog/112273.yaml b/docs/changelog/112273.yaml new file mode 100644 index 0000000000000..3182a1884a145 --- /dev/null +++ b/docs/changelog/112273.yaml @@ -0,0 +1,5 @@ +pr: 111181 +summary: "[Inference API] Add Docs for AlibabaCloud AI Search Support for the Inference API" +area: Machine Learning +type: enhancement +issues: [ ] diff --git a/docs/changelog/112277.yaml b/docs/changelog/112277.yaml new file mode 100644 index 0000000000000..eac474555999a --- /dev/null +++ b/docs/changelog/112277.yaml @@ -0,0 +1,5 @@ +pr: 112277 +summary: Upgrade `repository-azure` dependencies +area: Snapshot/Restore +type: upgrade +issues: [] diff --git a/docs/changelog/112282.yaml b/docs/changelog/112282.yaml new file mode 100644 index 0000000000000..beea119b06aef --- /dev/null +++ b/docs/changelog/112282.yaml @@ -0,0 +1,6 @@ +pr: 112282 +summary: Adds example plugin for custom ingest processor +area: Ingest Node +type: enhancement +issues: + - 111539 diff --git a/docs/changelog/112303.yaml b/docs/changelog/112303.yaml new file mode 100644 index 0000000000000..a363e621e4c48 --- /dev/null +++ b/docs/changelog/112303.yaml @@ -0,0 +1,5 @@ +pr: 112303 +summary: Add 'verbose' flag retrieving `maximum_timestamp` for get data stream API +area: Data streams +type: enhancement +issues: [] diff --git a/docs/changelog/112320.yaml b/docs/changelog/112320.yaml new file mode 100644 index 0000000000000..d35a08dfa4e91 --- /dev/null +++ b/docs/changelog/112320.yaml @@ -0,0 +1,5 @@ +pr: 112320 +summary: Upgrade xcontent to Jackson 2.17.2 +area: Infra/Core +type: upgrade +issues: [] diff --git a/docs/changelog/112330.yaml b/docs/changelog/112330.yaml new file mode 100644 index 0000000000000..498698f5175ba --- /dev/null +++ b/docs/changelog/112330.yaml @@ -0,0 +1,5 @@ +pr: 112330 +summary: Add links to network disconnect troubleshooting +area: Network +type: enhancement +issues: [] diff --git a/docs/changelog/112337.yaml b/docs/changelog/112337.yaml new file mode 100644 index 0000000000000..f7d667e23cfe9 --- /dev/null +++ b/docs/changelog/112337.yaml @@ -0,0 +1,5 @@ +pr: 112337 +summary: Add workaround for missing shard gen blob +area: Snapshot/Restore +type: enhancement +issues: [] diff --git a/docs/changelog/112341.yaml b/docs/changelog/112341.yaml new file mode 100644 index 0000000000000..8f44b53ad9998 --- /dev/null +++ b/docs/changelog/112341.yaml @@ -0,0 +1,5 @@ +pr: 112341 +summary: Fix DLS using runtime fields and synthetic source +area: Authorization +type: bug +issues: [] diff --git a/docs/changelog/112345.yaml b/docs/changelog/112345.yaml new file mode 100644 index 0000000000000..b922fe3754cbb --- /dev/null +++ b/docs/changelog/112345.yaml @@ -0,0 +1,8 @@ +pr: 112345 +summary: Allow dimension fields to have multiple values in standard and logsdb index + mode +area: Mapping +type: enhancement +issues: + - 112232 + - 112239 diff --git a/docs/changelog/112350.yaml b/docs/changelog/112350.yaml new file mode 100644 index 0000000000000..994cd3a65c633 --- /dev/null +++ b/docs/changelog/112350.yaml @@ -0,0 +1,5 @@ +pr: 112350 +summary: "[ESQL] Add `SPACE` function" +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/112369.yaml b/docs/changelog/112369.yaml new file mode 100644 index 0000000000000..fb1c4775f7a12 --- /dev/null +++ b/docs/changelog/112369.yaml @@ -0,0 +1,5 @@ +pr: 112369 +summary: Register Task while Streaming +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/changelog/112397.yaml b/docs/changelog/112397.yaml new file mode 100644 index 0000000000000..e67478ec69b1c --- /dev/null +++ b/docs/changelog/112397.yaml @@ -0,0 +1,5 @@ +pr: 112397 +summary: Control storing array source with index setting +area: Mapping +type: enhancement +issues: [] diff --git a/docs/changelog/112400.yaml b/docs/changelog/112400.yaml new file mode 100644 index 0000000000000..6d622e5fb5248 --- /dev/null +++ b/docs/changelog/112400.yaml @@ -0,0 +1,5 @@ +pr: 112400 +summary: Make sure file accesses in `DnRoleMapper` are done in stack frames with permissions +area: Infra/Core +type: bug +issues: [] diff --git a/docs/changelog/112409.yaml b/docs/changelog/112409.yaml new file mode 100644 index 0000000000000..bad94b9f5f2be --- /dev/null +++ b/docs/changelog/112409.yaml @@ -0,0 +1,6 @@ +pr: 112409 +summary: Include reason when no nodes are found +area: "Transform" +type: bug +issues: + - 112404 diff --git a/docs/changelog/112412.yaml b/docs/changelog/112412.yaml new file mode 100644 index 0000000000000..fda53ebd1ade0 --- /dev/null +++ b/docs/changelog/112412.yaml @@ -0,0 +1,5 @@ +pr: 112412 +summary: Expose `HexFormat` in Painless +area: Infra/Scripting +type: enhancement +issues: [] diff --git a/docs/changelog/112431.yaml b/docs/changelog/112431.yaml new file mode 100644 index 0000000000000..b8c1197bdc7ef --- /dev/null +++ b/docs/changelog/112431.yaml @@ -0,0 +1,6 @@ +pr: 112431 +summary: "Async search: Add ID and \"is running\" http headers" +area: Search +type: feature +issues: + - 109576 diff --git a/docs/changelog/112440.yaml b/docs/changelog/112440.yaml new file mode 100644 index 0000000000000..f208474fa2686 --- /dev/null +++ b/docs/changelog/112440.yaml @@ -0,0 +1,5 @@ +pr: 112440 +summary: "logs-apm.error-*: define log.level field as keyword" +area: Data streams +type: bug +issues: [] diff --git a/docs/changelog/112444.yaml b/docs/changelog/112444.yaml new file mode 100644 index 0000000000000..bfa4fd693f0e0 --- /dev/null +++ b/docs/changelog/112444.yaml @@ -0,0 +1,6 @@ +pr: 112444 +summary: Full coverage of ECS by ecs@mappings when `date_detection` is disabled +area: Mapping +type: bug +issues: + - 112398 diff --git a/docs/changelog/112489.yaml b/docs/changelog/112489.yaml new file mode 100644 index 0000000000000..ebc84927b0e76 --- /dev/null +++ b/docs/changelog/112489.yaml @@ -0,0 +1,6 @@ +pr: 112489 +summary: "ES|QL: better validation for RLIKE patterns" +area: ES|QL +type: bug +issues: + - 112485 diff --git a/docs/changelog/112508.yaml b/docs/changelog/112508.yaml new file mode 100644 index 0000000000000..3945ebd226ac4 --- /dev/null +++ b/docs/changelog/112508.yaml @@ -0,0 +1,5 @@ +pr: 112508 +summary: "[ML] Create Inference API will no longer return model_id and now only return inference_id" +area: Machine Learning +type: bug +issues: [] diff --git a/docs/changelog/112519.yaml b/docs/changelog/112519.yaml new file mode 100644 index 0000000000000..aa8a942ef0f58 --- /dev/null +++ b/docs/changelog/112519.yaml @@ -0,0 +1,5 @@ +pr: 112519 +summary: Lower the memory footprint when creating `DelayedBucket` +area: Aggregations +type: enhancement +issues: [] diff --git a/docs/changelog/112547.yaml b/docs/changelog/112547.yaml new file mode 100644 index 0000000000000..7f42f2a82976e --- /dev/null +++ b/docs/changelog/112547.yaml @@ -0,0 +1,5 @@ +pr: 112547 +summary: Remove reduce and `reduceContext` from `DelayedBucket` +area: Aggregations +type: enhancement +issues: [] diff --git a/docs/changelog/112581.yaml b/docs/changelog/112581.yaml new file mode 100644 index 0000000000000..489b4780c06fb --- /dev/null +++ b/docs/changelog/112581.yaml @@ -0,0 +1,5 @@ +pr: 112581 +summary: Fix missing header in `put_geoip_database` JSON spec +area: Ingest Node +type: bug +issues: [] diff --git a/docs/changelog/112612.yaml b/docs/changelog/112612.yaml new file mode 100644 index 0000000000000..d6037e34ff171 --- /dev/null +++ b/docs/changelog/112612.yaml @@ -0,0 +1,5 @@ +pr: 112612 +summary: Set `replica_unassigned_buffer_time` in constructor +area: Health +type: bug +issues: [] diff --git a/docs/painless/painless-contexts/painless-field-context.asciidoc b/docs/painless/painless-contexts/painless-field-context.asciidoc index 2f4e27dd11e6b..661af8e64d1e0 100644 --- a/docs/painless/painless-contexts/painless-field-context.asciidoc +++ b/docs/painless/painless-contexts/painless-field-context.asciidoc @@ -64,14 +64,14 @@ actors that appear in each play: ---- GET seats/_search { - "size": 2, + "size": 2, "query": { "match_all": {} }, "script_fields": { "day-of-week": { "script": { - "source": "doc['datetime'].value.getDayOfWeekEnum().getDisplayName(TextStyle.FULL, Locale.ROOT)" + "source": "doc['datetime'].value.getDayOfWeekEnum().getDisplayName(TextStyle.FULL, Locale.ENGLISH)" } }, "number-of-actors": { @@ -132,4 +132,4 @@ GET seats/_search } } ---- -// TESTRESPONSE[s/"took" : 68/"took" : "$body.took"/] \ No newline at end of file +// TESTRESPONSE[s/"took" : 68/"took" : "$body.took"/] diff --git a/docs/painless/painless-guide/painless-execute-script.asciidoc b/docs/painless/painless-guide/painless-execute-script.asciidoc index 4417daeb63efa..771a6818d45e8 100644 --- a/docs/painless/painless-guide/painless-execute-script.asciidoc +++ b/docs/painless/painless-guide/painless-execute-script.asciidoc @@ -749,7 +749,7 @@ POST /_scripts/painless/_execute { "script": { "source": """ - emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT)); + emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH)); """ }, "context": "keyword_field", diff --git a/docs/plugins/analysis-kuromoji.asciidoc b/docs/plugins/analysis-kuromoji.asciidoc index 1f114e9ad9ed6..b1d1d5a751057 100644 --- a/docs/plugins/analysis-kuromoji.asciidoc +++ b/docs/plugins/analysis-kuromoji.asciidoc @@ -624,3 +624,123 @@ Which results in: } ] } -------------------------------------------------- + +[[analysis-kuromoji-hiragana-uppercase]] +==== `hiragana_uppercase` token filter + +The `hiragana_uppercase` token filter normalizes small letters (捨て仮名) in hiragana into standard letters. +This filter is useful if you want to search against old style Japanese text such as +patents, legal documents, contract policies, etc. + +For example: + +[source,console] +-------------------------------------------------- +PUT kuromoji_sample +{ + "settings": { + "index": { + "analysis": { + "analyzer": { + "my_analyzer": { + "tokenizer": "kuromoji_tokenizer", + "filter": [ + "hiragana_uppercase" + ] + } + } + } + } + } +} + +GET kuromoji_sample/_analyze +{ + "analyzer": "my_analyzer", + "text": "ちょっとまって" +} +-------------------------------------------------- + +Which results in: + +[source,console-result] +-------------------------------------------------- +{ + "tokens": [ + { + "token": "ちよつと", + "start_offset": 0, + "end_offset": 4, + "type": "word", + "position": 0 + }, + { + "token": "まつ", + "start_offset": 4, + "end_offset": 6, + "type": "word", + "position": 1 + }, + { + "token": "て", + "start_offset": 6, + "end_offset": 7, + "type": "word", + "position": 2 + } + ] +} +-------------------------------------------------- + +[[analysis-kuromoji-katakana-uppercase]] +==== `katakana_uppercase` token filter + +The `katakana_uppercase` token filter normalizes small letters (捨て仮名) in katakana into standard letters. +This filter is useful if you want to search against old style Japanese text such as +patents, legal documents, contract policies, etc. + +For example: + +[source,console] +-------------------------------------------------- +PUT kuromoji_sample +{ + "settings": { + "index": { + "analysis": { + "analyzer": { + "my_analyzer": { + "tokenizer": "kuromoji_tokenizer", + "filter": [ + "katakana_uppercase" + ] + } + } + } + } + } +} + +GET kuromoji_sample/_analyze +{ + "analyzer": "my_analyzer", + "text": "ストップウォッチ" +} +-------------------------------------------------- + +Which results in: + +[source,console-result] +-------------------------------------------------- +{ + "tokens": [ + { + "token": "ストツプウオツチ", + "start_offset": 0, + "end_offset": 8, + "type": "word", + "position": 0 + } + ] +} +-------------------------------------------------- diff --git a/docs/plugins/development/creating-classic-plugins.asciidoc b/docs/plugins/development/creating-classic-plugins.asciidoc index f3f62a11f2993..cc03ad51275fa 100644 --- a/docs/plugins/development/creating-classic-plugins.asciidoc +++ b/docs/plugins/development/creating-classic-plugins.asciidoc @@ -32,12 +32,13 @@ for the plugin. If you need other resources, package them into a resources JAR. The {es} repository contains {es-repo}tree/main/plugins/examples[examples of plugins]. Some of these include: * a plugin with {es-repo}tree/main/plugins/examples/custom-settings[custom settings] +* a plugin with a {es-repo}tree/main/plugins/examples/custom-processor[custom ingest processor] * adding {es-repo}tree/main/plugins/examples/rest-handler[custom rest endpoints] * adding a {es-repo}tree/main/plugins/examples/rescore[custom rescorer] * a script {es-repo}tree/main/plugins/examples/script-expert-scoring[implemented in Java] These examples provide the bare bones needed to get started. For more -information about how to write a plugin, we recommend looking at the +information about how to write a plugin, we recommend looking at the {es-repo}tree/main/plugins/[source code of existing plugins] for inspiration. [discrete] @@ -88,4 +89,4 @@ for more information. [[plugin-descriptor-file-classic]] ==== The plugin descriptor file for classic plugins -include::plugin-descriptor-file.asciidoc[] \ No newline at end of file +include::plugin-descriptor-file.asciidoc[] diff --git a/docs/reference/aggregations/bucket/composite-aggregation.asciidoc b/docs/reference/aggregations/bucket/composite-aggregation.asciidoc index 807ec93132d37..ded01237c23c8 100644 --- a/docs/reference/aggregations/bucket/composite-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/composite-aggregation.asciidoc @@ -156,7 +156,7 @@ GET /_search "type": "keyword", "script": """ emit(doc['timestamp'].value.dayOfWeekEnum - .getDisplayName(TextStyle.FULL, Locale.ROOT)) + .getDisplayName(TextStyle.FULL, Locale.ENGLISH)) """ } }, diff --git a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc index 3511ec9e63b02..ef62f263a54a8 100644 --- a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc @@ -582,7 +582,7 @@ For example, the offset of `+19d` will result in buckets with names like `2022-0 Increasing the offset to `+20d`, each document will appear in a bucket for the previous month, with all bucket keys ending with the same day of the month, as normal. -However, further increasing to `+28d`, +However, further increasing to `+28d`, what used to be a February bucket has now become `"2022-03-01"`. [source,console,id=datehistogram-aggregation-offset-example-28d] @@ -819,7 +819,7 @@ POST /sales/_search?size=0 "runtime_mappings": { "date.day_of_week": { "type": "keyword", - "script": "emit(doc['date'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))" + "script": "emit(doc['date'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))" } }, "aggs": { diff --git a/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc b/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc index d7d837b2f8364..16879450c65d8 100644 --- a/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc @@ -6,6 +6,8 @@ A metric aggregation that executes using scripts to provide a metric output. +WARNING: `scripted_metric` is not available in {serverless-full}. + WARNING: Using scripts can result in slower search speeds. See <>. @@ -127,7 +129,7 @@ init_script:: Executed prior to any collection of documents. Allows the ag + In the above example, the `init_script` creates an array `transactions` in the `state` object. -map_script:: Executed once per document collected. This is a required script. +map_script:: Executed once per document collected. This is a required script. + In the above example, the `map_script` checks the value of the type field. If the value is 'sale' the value of the amount field is added to the transactions array. If the value of the type field is not 'sale' the negated value of the amount field is added @@ -282,4 +284,4 @@ params:: Optional. An object whose contents will be passed as variable If a parent bucket of the scripted metric aggregation does not collect any documents an empty aggregation response will be returned from the shard with a `null` value. In this case the `reduce_script`'s `states` variable will contain `null` as a response from that shard. -`reduce_script`'s should therefore expect and deal with `null` responses from shards. +`reduce_script`'s should therefore expect and deal with `null` responses from shards. diff --git a/docs/reference/cat/health.asciidoc b/docs/reference/cat/health.asciidoc index 04a11699d3ecf..ad39ace310807 100644 --- a/docs/reference/cat/health.asciidoc +++ b/docs/reference/cat/health.asciidoc @@ -6,8 +6,8 @@ [IMPORTANT] ==== -cat APIs are only intended for human consumption using the command line or {kib} -console. They are _not_ intended for use by applications. For application +cat APIs are only intended for human consumption using the command line or {kib} +console. They are _not_ intended for use by applications. For application consumption, use the <>. ==== @@ -87,8 +87,8 @@ The API returns the following response: [source,txt] -------------------------------------------------- -epoch timestamp cluster status node.total node.data shards pri relo init unassign pending_tasks max_task_wait_time active_shards_percent -1475871424 16:17:04 elasticsearch green 1 1 1 1 0 0 0 0 - 100.0% +epoch timestamp cluster status node.total node.data shards pri relo init unassign unassign.pri pending_tasks max_task_wait_time active_shards_percent +1475871424 16:17:04 elasticsearch green 1 1 1 1 0 0 0 0 0 - 100.0% -------------------------------------------------- // TESTRESPONSE[s/1475871424 16:17:04/\\d+ \\d+:\\d+:\\d+/] // TESTRESPONSE[s/elasticsearch/[^ ]+/ s/0 -/\\d+ (-|\\d+(\\.\\d+)?[ms]+)/ non_json] @@ -107,11 +107,13 @@ The API returns the following response: [source,txt] -------------------------------------------------- -cluster status node.total node.data shards pri relo init unassign pending_tasks max_task_wait_time active_shards_percent -elasticsearch green 1 1 1 1 0 0 0 0 - 100.0% +cluster status node.total node.data shards pri relo init unassign unassign.pri pending_tasks max_task_wait_time active_shards_percent +elasticsearch green 1 1 1 1 0 0 0 0 0 - 100.0% -------------------------------------------------- // TESTRESPONSE[s/elasticsearch/[^ ]+/ s/0 -/\\d+ (-|\\d+(\\.\\d+)?[ms]+)/ non_json] +**Note**: The reported number of unassigned primary shards may be lower than the true value if your cluster contains nodes running a version below 8.16. For a more accurate count in this scenario, please use the <>. + [[cat-health-api-example-across-nodes]] ===== Example across nodes You can use the cat health API to verify the health of a cluster across nodes. @@ -121,11 +123,11 @@ For example: -------------------------------------------------- % pssh -i -h list.of.cluster.hosts curl -s localhost:9200/_cat/health [1] 20:20:52 [SUCCESS] es3.vm -1384309218 18:20:18 foo green 3 3 3 3 0 0 0 0 +1384309218 18:20:18 foo green 3 3 3 3 0 0 0 0 0 [2] 20:20:52 [SUCCESS] es1.vm -1384309218 18:20:18 foo green 3 3 3 3 0 0 0 0 +1384309218 18:20:18 foo green 3 3 3 3 0 0 0 0 0 [3] 20:20:52 [SUCCESS] es2.vm -1384309218 18:20:18 foo green 3 3 3 3 0 0 0 0 +1384309218 18:20:18 foo green 3 3 3 3 0 0 0 0 0 -------------------------------------------------- // NOTCONSOLE @@ -138,10 +140,10 @@ in a delayed loop. For example: [source,sh] -------------------------------------------------- % while true; do curl localhost:9200/_cat/health; sleep 120; done -1384309446 18:24:06 foo red 3 3 20 20 0 0 1812 0 -1384309566 18:26:06 foo yellow 3 3 950 916 0 12 870 0 -1384309686 18:28:06 foo yellow 3 3 1328 916 0 12 492 0 -1384309806 18:30:06 foo green 3 3 1832 916 4 0 0 +1384309446 18:24:06 foo red 3 3 20 20 0 0 1812 1121 0 +1384309566 18:26:06 foo yellow 3 3 950 916 0 12 870 421 0 +1384309686 18:28:06 foo yellow 3 3 1328 916 0 12 492 301 0 +1384309806 18:30:06 foo green 3 3 1832 916 4 0 0 0 ^C -------------------------------------------------- // NOTCONSOLE @@ -149,4 +151,4 @@ in a delayed loop. For example: In this example, the recovery took roughly six minutes, from `18:24:06` to `18:30:06`. If this recovery took hours, you could continue to monitor the number of `UNASSIGNED` shards, which should drop. If the number of `UNASSIGNED` -shards remains static, it would indicate an issue with the cluster recovery. \ No newline at end of file +shards remains static, it would indicate an issue with the cluster recovery. diff --git a/docs/reference/cluster/allocation-explain.asciidoc b/docs/reference/cluster/allocation-explain.asciidoc index 809c9d74f1450..7547dd74c5ecd 100644 --- a/docs/reference/cluster/allocation-explain.asciidoc +++ b/docs/reference/cluster/allocation-explain.asciidoc @@ -81,6 +81,7 @@ you might expect otherwise. ===== Unassigned primary shard +====== Conflicting settings The following request gets an allocation explanation for an unassigned primary shard. @@ -158,6 +159,56 @@ node. <5> The decider which led to the `no` decision for the node. <6> An explanation as to why the decider returned a `no` decision, with a helpful hint pointing to the setting that led to the decision. In this example, a newly created index has <> that requires that it only be allocated to a node named `nonexistent_node`, which does not exist, so the index is unable to allocate. +====== Maximum number of retries exceeded + +The following response contains an allocation explanation for an unassigned +primary shard that has reached the maximum number of allocation retry attempts. + +[source,js] +---- +{ + "index" : "my-index-000001", + "shard" : 0, + "primary" : true, + "current_state" : "unassigned", + "unassigned_info" : { + "at" : "2017-01-04T18:03:28.464Z", + "failed shard on node [mEKjwwzLT1yJVb8UxT6anw]: failed recovery, failure RecoveryFailedException", + "reason": "ALLOCATION_FAILED", + "failed_allocation_attempts": 5, + "last_allocation_status": "no", + }, + "can_allocate": "no", + "allocate_explanation": "cannot allocate because allocation is not permitted to any of the nodes", + "node_allocation_decisions" : [ + { + "node_id" : "3sULLVJrRneSg0EfBB-2Ew", + "node_name" : "node_t0", + "transport_address" : "127.0.0.1:9400", + "roles" : ["data_content", "data_hot"], + "node_decision" : "no", + "store" : { + "matching_size" : "4.2kb", + "matching_size_in_bytes" : 4325 + }, + "deciders" : [ + { + "decider": "max_retry", + "decision" : "NO", + "explanation": "shard has exceeded the maximum number of retries [5] on failed allocation attempts - manually call [/_cluster/reroute?retry_failed=true] to retry, [unassigned_info[[reason=ALLOCATION_FAILED], at[2024-07-30T21:04:12.166Z], failed_attempts[5], failed_nodes[[mEKjwwzLT1yJVb8UxT6anw]], delayed=false, details[failed shard on node [mEKjwwzLT1yJVb8UxT6anw]: failed recovery, failure RecoveryFailedException], allocation_status[deciders_no]]]" + } + ] + } + ] +} +---- +// NOTCONSOLE + +If decider message indicates a transient allocation issue, use +<> to retry allocation. + +====== No valid shard copy + The following response contains an allocation explanation for an unassigned primary shard that was previously allocated. @@ -184,6 +235,8 @@ TIP: If a shard is unassigned with an allocation status of `no_valid_shard_copy` ===== Unassigned replica shard +====== Allocation delayed + The following response contains an allocation explanation for a replica that's unassigned due to <>. @@ -241,8 +294,52 @@ unassigned due to <>. <2> The remaining delay before allocating the replica shard. <3> Information about the shard data found on a node. +====== Allocation throttled + +The following response contains an allocation explanation for a replica that's +queued to allocate but currently waiting on other queued shards. + +[source,js] +---- +{ + "index" : "my-index-000001", + "shard" : 0, + "primary" : false, + "current_state" : "unassigned", + "unassigned_info" : { + "reason" : "NODE_LEFT", + "at" : "2017-01-04T18:53:59.498Z", + "details" : "node_left[G92ZwuuaRY-9n8_tc-IzEg]", + "last_allocation_status" : "no_attempt" + }, + "can_allocate": "throttled", + "allocate_explanation": "Elasticsearch is currently busy with other activities. It expects to be able to allocate this shard when those activities finish. Please wait.", + "node_allocation_decisions" : [ + { + "node_id" : "3sULLVJrRneSg0EfBB-2Ew", + "node_name" : "node_t0", + "transport_address" : "127.0.0.1:9400", + "roles" : ["data_content", "data_hot"], + "node_decision" : "no", + "deciders" : [ + { + "decider": "throttling", + "decision": "THROTTLE", + "explanation": "reached the limit of incoming shard recoveries [2], cluster setting [cluster.routing.allocation.node_concurrent_incoming_recoveries=2] (can also be set via [cluster.routing.allocation.node_concurrent_recoveries])" + } + ] + } + ] +} +---- +// NOTCONSOLE + +This is a transient message that might appear when a large amount of shards are allocating. + ===== Assigned shard +====== Cannot remain on current node + The following response contains an allocation explanation for an assigned shard. The response indicates the shard is not allowed to remain on its current node and must be reallocated. @@ -295,6 +392,8 @@ and must be reallocated. <2> The deciders that factored into the decision of why the shard is not allowed to remain on its current node. <3> Whether the shard is allowed to be allocated to another node. +====== Must remain on current node + The following response contains an allocation explanation for a shard that must remain on its current node. Moving the shard to another node would not improve cluster balance. @@ -338,7 +437,7 @@ cluster balance. ===== No arguments If you call the API with no arguments, {es} retrieves an allocation explanation -for an arbitrary unassigned primary or replica shard. +for an arbitrary unassigned primary or replica shard, returning any unassigned primary shards first. [source,console] ---- diff --git a/docs/reference/cluster/health.asciidoc b/docs/reference/cluster/health.asciidoc index 3a4058a55ce16..94eb80a03d12e 100644 --- a/docs/reference/cluster/health.asciidoc +++ b/docs/reference/cluster/health.asciidoc @@ -20,22 +20,22 @@ Returns the health status of a cluster. [[cluster-health-api-desc]] ==== {api-description-title} -The cluster health API returns a simple status on the health of the +The cluster health API returns a simple status on the health of the cluster. You can also use the API to get the health status of only specified data streams and indices. For data streams, the API retrieves the health status of the stream's backing indices. -The cluster health status is: `green`, `yellow` or `red`. On the shard level, a -`red` status indicates that the specific shard is not allocated in the cluster, -`yellow` means that the primary shard is allocated but replicas are not, and -`green` means that all shards are allocated. The index level status is -controlled by the worst shard status. The cluster status is controlled by the +The cluster health status is: `green`, `yellow` or `red`. On the shard level, a +`red` status indicates that the specific shard is not allocated in the cluster, +`yellow` means that the primary shard is allocated but replicas are not, and +`green` means that all shards are allocated. The index level status is +controlled by the worst shard status. The cluster status is controlled by the worst index status. -One of the main benefits of the API is the ability to wait until the cluster -reaches a certain high water-mark health level. For example, the following will -wait for 50 seconds for the cluster to reach the `yellow` level (if it reaches -the `green` or `yellow` status before 50 seconds elapse, it will return at that +One of the main benefits of the API is the ability to wait until the cluster +reaches a certain high water-mark health level. For example, the following will +wait for 50 seconds for the cluster to reach the `yellow` level (if it reaches +the `green` or `yellow` status before 50 seconds elapse, it will return at that point): [source,console] @@ -58,31 +58,31 @@ To target all data streams and indices in a cluster, omit this parameter or use ==== {api-query-parms-title} `level`:: - (Optional, string) Can be one of `cluster`, `indices` or `shards`. Controls + (Optional, string) Can be one of `cluster`, `indices` or `shards`. Controls the details level of the health information returned. Defaults to `cluster`. - + include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local] - + include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] `wait_for_active_shards`:: - (Optional, string) A number controlling to how many active shards to wait - for, `all` to wait for all shards in the cluster to be active, or `0` to not + (Optional, string) A number controlling to how many active shards to wait + for, `all` to wait for all shards in the cluster to be active, or `0` to not wait. Defaults to `0`. - + `wait_for_events`:: - (Optional, string) Can be one of `immediate`, `urgent`, `high`, `normal`, - `low`, `languid`. Wait until all currently queued events with the given + (Optional, string) Can be one of `immediate`, `urgent`, `high`, `normal`, + `low`, `languid`. Wait until all currently queued events with the given priority are processed. `wait_for_no_initializing_shards`:: - (Optional, Boolean) A boolean value which controls whether to wait (until - the timeout provided) for the cluster to have no shard initializations. + (Optional, Boolean) A boolean value which controls whether to wait (until + the timeout provided) for the cluster to have no shard initializations. Defaults to false, which means it will not wait for initializing shards. `wait_for_no_relocating_shards`:: - (Optional, Boolean) A boolean value which controls whether to wait (until - the timeout provided) for the cluster to have no shard relocations. Defaults + (Optional, Boolean) A boolean value which controls whether to wait (until + the timeout provided) for the cluster to have no shard relocations. Defaults to false, which means it will not wait for relocating shards. `wait_for_nodes`:: @@ -92,7 +92,7 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] `lt(N)` notation. `wait_for_status`:: - (Optional, string) One of `green`, `yellow` or `red`. Will wait (until the + (Optional, string) One of `green`, `yellow` or `red`. Will wait (until the timeout provided) until the status of the cluster changes to the one provided or better, i.e. `green` > `yellow` > `red`. By default, will not wait for any status. @@ -107,7 +107,7 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cluster-health-status] `timed_out`:: - (Boolean) If `false` the response returned within the period of + (Boolean) If `false` the response returned within the period of time that is specified by the `timeout` parameter (`30s` by default). `number_of_nodes`:: @@ -131,23 +131,26 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cluster-health-status] `unassigned_shards`:: (integer) The number of shards that are not allocated. +`unassigned_primary_shards`:: + (integer) The number of shards that are primary but not allocated. **Note**: This number may be lower than the true value if your cluster contains nodes running a version below 8.16. For a more accurate count in this scenario, please use the <>. + `delayed_unassigned_shards`:: - (integer) The number of shards whose allocation has been delayed by the + (integer) The number of shards whose allocation has been delayed by the timeout settings. `number_of_pending_tasks`:: - (integer) The number of cluster-level changes that have not yet been + (integer) The number of cluster-level changes that have not yet been executed. `number_of_in_flight_fetch`:: (integer) The number of unfinished fetches. `task_max_waiting_in_queue_millis`:: - (integer) The time expressed in milliseconds since the earliest initiated task + (integer) The time expressed in milliseconds since the earliest initiated task is waiting for being performed. `active_shards_percent_as_number`:: - (float) The ratio of active shards in the cluster expressed as a percentage. + (float) The ratio of active shards in the cluster expressed as a percentage. [[cluster-health-api-example]] ==== {api-examples-title} @@ -158,7 +161,7 @@ GET _cluster/health -------------------------------------------------- // TEST[s/^/PUT test1\n/] -The API returns the following response in case of a quiet single node cluster +The API returns the following response in case of a quiet single node cluster with a single index with one shard and one replica: [source,console-result] @@ -174,6 +177,7 @@ with a single index with one shard and one replica: "relocating_shards" : 0, "initializing_shards" : 0, "unassigned_shards" : 1, + "unassigned_primary_shards" : 0, "delayed_unassigned_shards": 0, "number_of_pending_tasks" : 0, "number_of_in_flight_fetch": 0, diff --git a/docs/reference/cluster/stats.asciidoc b/docs/reference/cluster/stats.asciidoc index 3b429ef427071..c39bc0dcd2878 100644 --- a/docs/reference/cluster/stats.asciidoc +++ b/docs/reference/cluster/stats.asciidoc @@ -1282,6 +1282,31 @@ They are included here for expert users, but should otherwise be ignored. ===== +==== + +`repositories`:: +(object) Contains statistics about the <> repositories defined in the cluster, broken down +by repository type. ++ +.Properties of `repositories` +[%collapsible%open] +===== + +`count`::: +(integer) The number of repositories of this type in the cluster. + +`read_only`::: +(integer) The number of repositories of this type in the cluster which are registered read-only. + +`read_write`::: +(integer) The number of repositories of this type in the cluster which are not registered as read-only. + +Each repository type may also include other statistics about the repositories of that type here. + +===== + +==== + [[cluster-stats-api-example]] ==== {api-examples-title} @@ -1579,6 +1604,9 @@ The API returns the following response: }, "snapshots": { ... + }, + "repositories": { + ... } } -------------------------------------------------- @@ -1589,6 +1617,7 @@ The API returns the following response: // TESTRESPONSE[s/"count": \{[^\}]*\}/"count": $body.$_path/] // TESTRESPONSE[s/"packaging_types": \[[^\]]*\]/"packaging_types": $body.$_path/] // TESTRESPONSE[s/"snapshots": \{[^\}]*\}/"snapshots": $body.$_path/] +// TESTRESPONSE[s/"repositories": \{[^\}]*\}/"repositories": $body.$_path/] // TESTRESPONSE[s/"field_types": \[[^\]]*\]/"field_types": $body.$_path/] // TESTRESPONSE[s/"runtime_field_types": \[[^\]]*\]/"runtime_field_types": $body.$_path/] // TESTRESPONSE[s/"search": \{[^\}]*\}/"search": $body.$_path/] @@ -1600,7 +1629,7 @@ The API returns the following response: // the plugins that will be in it. And because we figure folks don't need to // see an exhaustive list anyway. // 2. Similarly, ignore the contents of `network_types`, `discovery_types`, -// `packaging_types` and `snapshots`. +// `packaging_types`, `snapshots` and `repositories`. // 3. Ignore the contents of the (nodes) count object, as what's shown here // depends on the license. Voting-only nodes are e.g. only shown when this // test runs with a basic license. diff --git a/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc index c83572a4e0795..6323fac1eac2f 100644 --- a/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc +++ b/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc @@ -67,8 +67,18 @@ Name of the data stream. ===== `data_retention`:: (Optional, string) +If defined, it represents the retention requested by the data stream owner for this data stream. + +`effective_retention`:: +(Optional, string) If defined, every document added to this data stream will be stored at least for this time frame. Any time after this -duration the document could be deleted. When undefined, every document in this data stream will be stored indefinitely. +duration the document could be deleted. When empty, every document in this data stream will be stored indefinitely. +duration the document could be deleted. When empty, every document in this data stream will be stored indefinitely. The +effective retention is calculated as described in the <>. + +`retention_determined_by`:: +(Optional, string) +The source of the retention, it can be one of three values, `data_stream_configuration`, `default_retention` or `max_retention`. `rollover`:: (Optional, object) @@ -78,6 +88,21 @@ when the query param `include_defaults` is set to `true`. The contents of this f ===== ==== +`global_retention`:: +(object) +Contains the global max and default retention. When no global retention is configured, this will be an empty object. ++ +.Properties of `global_retention` +[%collapsible%open] +==== +`max_retention`:: +(Optional, string) +The effective retention of data streams managed by the data stream lifecycle cannot exceed this value. +`default_retention`:: +(Optional, string) +This will be the effective retention of data streams managed by the data stream lifecycle that do not specify `data_retention`. +==== + [[data-streams-get-lifecycle-example]] ==== {api-examples-title} @@ -128,16 +153,21 @@ The response will look like the following: "name": "my-data-stream-1", "lifecycle": { "enabled": true, - "data_retention": "7d" + "data_retention": "7d", + "effective_retention": "7d", + "retention_determined_by": "data_stream_configuration" } }, { "name": "my-data-stream-2", "lifecycle": { "enabled": true, - "data_retention": "7d" + "data_retention": "7d", + "effective_retention": "7d", + "retention_determined_by": "data_stream_configuration" } } - ] + ], + "global_retention": {} } -------------------------------------------------- diff --git a/docs/reference/data-streams/lifecycle/index.asciidoc b/docs/reference/data-streams/lifecycle/index.asciidoc index 16ccf2ef82391..e4d5acfb704d3 100644 --- a/docs/reference/data-streams/lifecycle/index.asciidoc +++ b/docs/reference/data-streams/lifecycle/index.asciidoc @@ -14,10 +14,11 @@ To achieve that, it supports: * Automatic <>, which chunks your incoming data in smaller pieces to facilitate better performance and backwards incompatible mapping changes. * Configurable retention, which allows you to configure the time period for which your data is guaranteed to be stored. -{es} is allowed at a later time to delete data older than this time period. +{es} is allowed at a later time to delete data older than this time period. Retention can be configured on the data stream level +or on a global level. Read more about the different options in this <>. A data stream lifecycle also supports downsampling the data stream backing indices. -See <> for +See <> for more details. [discrete] @@ -33,16 +34,17 @@ each data stream and performs the following steps: 3. After an index is not the write index anymore (i.e. the data stream has been rolled over), automatically tail merges the index. Data stream lifecycle executes a merge operation that only targets the long tail of small segments instead of the whole shard. As the segments are organised -into tiers of exponential sizes, merging the long tail of small segments is only a +into tiers of exponential sizes, merging the long tail of small segments is only a fraction of the cost of force merging to a single segment. The small segments would usually hold the most recent data so tail merging will focus the merging resources on the higher-value data that is most likely to keep being queried. -4. If <> is configured it will execute +4. If <> is configured it will execute all the configured downsampling rounds. 5. Applies retention to the remaining backing indices. This means deleting the backing indices whose -`generation_time` is longer than the configured retention period. The `generation_time` is only applicable to rolled over backing -indices and it is either the time since the backing index got rolled over, or the time optionally configured in the -<> setting. +`generation_time` is longer than the effective retention period (read more about the +<>). The `generation_time` is only applicable to rolled +over backing indices and it is either the time since the backing index got rolled over, or the time optionally configured +in the <> setting. IMPORTANT: We use the `generation_time` instead of the creation time because this ensures that all data in the backing index have passed the retention period. As a result, the retention period is not the exact time data gets deleted, but @@ -75,4 +77,6 @@ include::tutorial-manage-new-data-stream.asciidoc[] include::tutorial-manage-existing-data-stream.asciidoc[] +include::tutorial-manage-data-stream-retention.asciidoc[] + include::tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc[] diff --git a/docs/reference/data-streams/lifecycle/tutorial-manage-data-stream-retention.asciidoc b/docs/reference/data-streams/lifecycle/tutorial-manage-data-stream-retention.asciidoc new file mode 100644 index 0000000000000..a7f0379a45167 --- /dev/null +++ b/docs/reference/data-streams/lifecycle/tutorial-manage-data-stream-retention.asciidoc @@ -0,0 +1,228 @@ +[role="xpack"] +[[tutorial-manage-data-stream-retention]] +=== Tutorial: Data stream retention + +In this tutorial, we are going to go over the data stream lifecycle retention; we will define it, go over how it can be configured +and how it can gets applied. Keep in mind, the following options apply only to data streams that are managed by the data stream lifecycle. + +. <> +. <> +. <> +. <> + +You can verify if a data steam is managed by the data stream lifecycle via the <>: + +//// +[source,console] +---- +PUT /_index_template/template +{ + "index_patterns": ["my-data-stream*"], + "template": { + "lifecycle": {} + }, + "data_stream": { } +} + +PUT /_data_stream/my-data-stream +---- +// TESTSETUP +//// + +//// +[source,console] +---- +DELETE /_data_stream/my-data-stream* +DELETE /_index_template/template +PUT /_cluster/settings +{ + "persistent" : { + "data_streams.lifecycle.retention.*" : null + } +} +---- +// TEARDOWN +//// + +[source,console] +-------------------------------------------------- +GET _data_stream/my-data-stream/_lifecycle +-------------------------------------------------- + +The result should look like this: + +[source,console-result] +-------------------------------------------------- +{ + "data_streams": [ + { + "name": "my-data-stream", <1> + "lifecycle": { + "enabled": true <2> + } + } + ] +} +-------------------------------------------------- +// TESTRESPONSE[skip:the result is for illustrating purposes only] +<1> The name of your data stream. +<2> Ensure that the lifecycle is enabled, meaning this should be `true`. + +[discrete] +[[what-is-retention]] +==== What is data stream retention? + +We define retention as the least amount of time the data of a data stream are going to be kept in {es}. After this time period +has passed, {es} is allowed to remove these data to free up space and/or manage costs. + +NOTE: Retention does not define the period that the data will be removed, but the minimum time period they will be kept. + +We define 4 different types of retention: + +* The data stream retention, or `data_retention`, which is the retention configured on the data stream level. It can be +set via an <> for future data streams or via the <> for an existing data stream. When the data stream retention is not set, it implies that the data +need to be kept forever. +* The global default retention, let's call it `default_retention`, which is a retention configured via the cluster setting +<> and will be +applied to all data streams managed by data stream lifecycle that do not have `data_retention` configured. Effectively, +it ensures that there will be no data streams keeping their data forever. This can be set via the +<>. +* The global max retention, let's call it `max_retention`, which is a retention configured via the cluster setting +<> and will be applied to +all data streams managed by data stream lifecycle. Effectively, it ensures that there will be no data streams whose retention +will exceed this time period. This can be set via the <>. +* The effective retention, or `effective_retention`, which is the retention applied at a data stream on a given moment. +Effective retention cannot be set, it is derived by taking into account all the configured retention listed above and is +calculated as it is described <>. + +NOTE: Global default and max retention do not apply to data streams internal to elastic. Internal data streams are recognised + either by having the `system` flag set to `true` or if their name is prefixed with a dot (`.`). + +[discrete] +[[retention-configuration]] +==== How to configure retention? + +- By setting the `data_retention` on the data stream level. This retention can be configured in two ways: ++ +-- For new data streams, it can be defined in the index template that would be applied during the data stream's creation. +You can use the <>, for example: ++ +[source,console] +-------------------------------------------------- +PUT _index_template/template +{ + "index_patterns": ["my-data-stream*"], + "data_stream": { }, + "priority": 500, + "template": { + "lifecycle": { + "data_retention": "7d" + } + }, + "_meta": { + "description": "Template with data stream lifecycle" + } +} +-------------------------------------------------- +-- For an existing data stream, it can be set via the <>. ++ +[source,console] +---- +PUT _data_stream/my-data-stream/_lifecycle +{ + "data_retention": "30d" <1> +} +---- +// TEST[continued] +<1> The retention period of this data stream is set to 30 days. + +- By setting the global retention via the `data_streams.lifecycle.retention.default` and/or `data_streams.lifecycle.retention.max` +that are set on a cluster level. You can be set via the <>. For example: ++ +[source,console] +-------------------------------------------------- +PUT /_cluster/settings +{ + "persistent" : { + "data_streams.lifecycle.retention.default" : "7d", + "data_streams.lifecycle.retention.max" : "90d" + } +} +-------------------------------------------------- +// TEST[continued] + +[discrete] +[[effective-retention-calculation]] +==== How is the effective retention calculated? +The effective is calculated in the following way: + +- The `effective_retention` is the `default_retention`, when `default_retention` is defined and the data stream does not +have `data_retention`. +- The `effective_retention` is the `data_retention`, when `data_retention` is defined and if `max_retention` is defined, +it is less than the `max_retention`. +- The `effective_retention` is the `max_retention`, when `max_retention` is defined, and the data stream has either no +`data_retention` or its `data_retention` is greater than the `max_retention`. + +The above is demonstrated in the examples below: + +|=== +|`default_retention` |`max_retention` |`data_retention` |`effective_retention` |Retention determined by + +|Not set |Not set |Not set |Infinite |N/A +|Not relevant |12 months |**30 days** |30 days |`data_retention` +|Not relevant |Not set |**30 days** |30 days |`data_retention` +|**30 days** |12 months |Not set |30 days |`default_retention` +|**30 days** |30 days |Not set |30 days |`default_retention` +|Not relevant |**30 days** |12 months |30 days |`max_retention` +|Not set |**30 days** |Not set |30 days |`max_retention` +|=== + +Considering our example, if we retrieve the lifecycle of `my-data-stream`: +[source,console] +---- +GET _data_stream/my-data-stream/_lifecycle +---- +// TEST[continued] + +We see that it will remain the same with what the user configured: +[source,console-result] +---- +{ + "global_retention" : { + "max_retention" : "90d", <1> + "default_retention" : "7d" <2> + }, + "data_streams": [ + { + "name": "my-data-stream", + "lifecycle": { + "enabled": true, + "data_retention": "30d", <3> + "effective_retention": "30d", <4> + "retention_determined_by": "data_stream_configuration" <5> + } + } + ] +} +---- +<1> The maximum retention configured in the cluster. +<2> The default retention configured in the cluster. +<3> The requested retention for this data stream. +<4> The retention that is applied by the data stream lifecycle on this data stream. +<5> The configuration that determined the effective retention. In this case it's the `data_configuration` because +it is less than the `max_retention`. + +[discrete] +[[effective-retention-application]] +==== How is the effective retention applied? + +Retention is applied to the remaining backing indices of a data stream as the last step of +<>. Data stream lifecycle will retrieve the backing indices +whose `generation_time` is longer than the effective retention period and delete them. The `generation_time` is only +applicable to rolled over backing indices and it is either the time since the backing index got rolled over, or the time +optionally configured in the <> setting. + +IMPORTANT: We use the `generation_time` instead of the creation time because this ensures that all data in the backing +index have passed the retention period. As a result, the retention period is not the exact time data get deleted, but +the minimum time data will be stored. diff --git a/docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc b/docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc index c34340a096046..173b7a75dd28e 100644 --- a/docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc +++ b/docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc @@ -91,19 +91,23 @@ The result will look like this: { "data_streams": [ { - "name": "my-data-stream",<1> + "name": "my-data-stream", <1> "lifecycle": { - "enabled": true, <2> - "data_retention": "7d" <3> + "enabled": true, <2> + "data_retention": "7d", <3> + "effective_retention": "7d", <4> + "retention_determined_by": "data_stream_configuration" } } - ] + ], + "global_retention": {} } -------------------------------------------------- <1> The name of your data stream. <2> Shows if the data stream lifecycle is enabled for this data stream. -<3> The retention period of the data indexed in this data stream, this means that the data in this data stream will -be kept at least for 7 days. After that {es} can delete it at its own discretion. +<3> The retention period of the data indexed in this data stream, as configured by the user. +<4> The retention period that will be applied by the data stream lifecycle. This means that the data in this data stream will + be kept at least for 7 days. After that {es} can delete it at its own discretion. If you want to see more information about how the data stream lifecycle is applied on individual backing indices use the <>: diff --git a/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc b/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc index 8d959d8f4ad84..a2c12466b7f2b 100644 --- a/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc +++ b/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc @@ -1,14 +1,14 @@ [role="xpack"] [[tutorial-migrate-data-stream-from-ilm-to-dsl]] -=== Tutorial: Migrate ILM managed data stream to data stream lifecycle +=== Tutorial: Migrate ILM managed data stream to data stream lifecycle In this tutorial we'll look at migrating an existing data stream from <> to -<>. The existing {ilm-init} managed backing indices will continue +<>. The existing {ilm-init} managed backing indices will continue to be managed by {ilm-init} until they age out and get deleted by {ilm-init}; however, -the new backing indices will be managed by data stream lifecycle. -This way, a data stream is gradually migrated away from being managed by {ilm-init} to +the new backing indices will be managed by data stream lifecycle. +This way, a data stream is gradually migrated away from being managed by {ilm-init} to being managed by data stream lifecycle. As we'll see, {ilm-init} and data stream lifecycle -can co-manage a data stream; however, an index can only be managed by one system at +can co-manage a data stream; however, an index can only be managed by one system at a time. [discrete] @@ -17,7 +17,7 @@ a time. To migrate a data stream from {ilm-init} to data stream lifecycle we'll have to execute two steps: -1. Update the index template that's backing the data stream to set <> +1. Update the index template that's backing the data stream to set <> to `false`, and to configure data stream lifecycle. 2. Configure the data stream lifecycle for the _existing_ data stream using the <>. @@ -174,8 +174,8 @@ in the index template). To migrate the `dsl-data-stream` to data stream lifecycle we'll have to execute two steps: -1. Update the index template that's backing the data stream to set <> -to `false`, and to configure data stream lifecycle. +1. Update the index template that's backing the data stream to set <> +to `false`, and to configure data stream lifecycle. 2. Configure the data stream lifecycle for the _existing_ `dsl-data-stream` using the <>. @@ -209,9 +209,9 @@ PUT _index_template/dsl-data-stream-template // TEST[continued] <1> The `prefer_ilm` setting will now be configured on the **new** backing indices -(created by rolling over the data stream) such that {ilm-init} does _not_ take +(created by rolling over the data stream) such that {ilm-init} does _not_ take precedence over data stream lifecycle. -<2> We're configuring the data stream lifecycle so _new_ data streams will be +<2> We're configuring the data stream lifecycle so _new_ data streams will be managed by data stream lifecycle. We've now made sure that new data streams will be managed by data stream lifecycle. @@ -227,7 +227,7 @@ PUT _data_stream/dsl-data-stream/_lifecycle ---- // TEST[continued] -We can inspect the data stream to check that the next generation will indeed be +We can inspect the data stream to check that the next generation will indeed be managed by data stream lifecycle: [source,console] @@ -266,7 +266,9 @@ GET _data_stream/dsl-data-stream "template": "dsl-data-stream-template", "lifecycle": { "enabled": true, - "data_retention": "7d" + "data_retention": "7d", + "effective_retention": "7d", + "retention_determined_by": "data_stream_configuration" }, "ilm_policy": "pre-dsl-ilm-policy", "next_generation_managed_by": "Data stream lifecycle", <3> @@ -292,7 +294,7 @@ GET _data_stream/dsl-data-stream <4> The `prefer_ilm` setting value we configured in the index template is reflected and will be configured accordingly for new backing indices. -We'll now rollover the data stream to see the new generation index being managed by +We'll now rollover the data stream to see the new generation index being managed by data stream lifecycle: [source,console] @@ -344,7 +346,9 @@ GET _data_stream/dsl-data-stream "template": "dsl-data-stream-template", "lifecycle": { "enabled": true, - "data_retention": "7d" + "data_retention": "7d", + "effective_retention": "7d", + "retention_determined_by": "data_stream_configuration" }, "ilm_policy": "pre-dsl-ilm-policy", "next_generation_managed_by": "Data stream lifecycle", @@ -375,9 +379,9 @@ in the index template [discrete] [[migrate-from-dsl-to-ilm]] ==== Migrate data stream back to ILM -We can easily change this data stream to be managed by {ilm-init} because we didn't remove -the {ilm-init} policy when we <>. +We can easily change this data stream to be managed by {ilm-init} because we didn't remove +the {ilm-init} policy when we <>. We can achieve this in two ways: diff --git a/docs/reference/esql/functions/description/cosh.asciidoc b/docs/reference/esql/functions/description/cosh.asciidoc index bfe51f9152875..ddace7da54343 100644 --- a/docs/reference/esql/functions/description/cosh.asciidoc +++ b/docs/reference/esql/functions/description/cosh.asciidoc @@ -2,4 +2,4 @@ *Description* -Returns the {wikipedia}/Hyperbolic_functions[hyperbolic cosine] of an angle. +Returns the {wikipedia}/Hyperbolic_functions[hyperbolic cosine] of a number. diff --git a/docs/reference/esql/functions/description/locate.asciidoc b/docs/reference/esql/functions/description/locate.asciidoc index e5a6fba512432..b3f9d2a1ad78e 100644 --- a/docs/reference/esql/functions/description/locate.asciidoc +++ b/docs/reference/esql/functions/description/locate.asciidoc @@ -2,4 +2,4 @@ *Description* -Returns an integer that indicates the position of a keyword substring within another string. +Returns an integer that indicates the position of a keyword substring within another string. Returns `0` if the substring cannot be found. Note that string positions start from `1`. diff --git a/docs/reference/esql/functions/description/mv_median_absolute_deviation.asciidoc b/docs/reference/esql/functions/description/mv_median_absolute_deviation.asciidoc new file mode 100644 index 0000000000000..765c4d322c3dc --- /dev/null +++ b/docs/reference/esql/functions/description/mv_median_absolute_deviation.asciidoc @@ -0,0 +1,7 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Converts a multivalued field into a single valued field containing the median absolute deviation. It is calculated as the median of each data point's deviation from the median of the entire sample. That is, for a random variable `X`, the median absolute deviation is `median(|median(X) - X|)`. + +NOTE: If the field has an even number of values, the medians will be calculated as the average of the middle two values. If the value is not a floating point number, the averages are rounded towards 0. diff --git a/docs/reference/esql/functions/description/sin.asciidoc b/docs/reference/esql/functions/description/sin.asciidoc index ba12ba88ca37a..40f5a46d1863d 100644 --- a/docs/reference/esql/functions/description/sin.asciidoc +++ b/docs/reference/esql/functions/description/sin.asciidoc @@ -2,4 +2,4 @@ *Description* -Returns ths {wikipedia}/Sine_and_cosine[Sine] trigonometric function of an angle. +Returns the {wikipedia}/Sine_and_cosine[sine] of an angle. diff --git a/docs/reference/esql/functions/description/sinh.asciidoc b/docs/reference/esql/functions/description/sinh.asciidoc index bb7761e2a0254..be7aee68f5932 100644 --- a/docs/reference/esql/functions/description/sinh.asciidoc +++ b/docs/reference/esql/functions/description/sinh.asciidoc @@ -2,4 +2,4 @@ *Description* -Returns the {wikipedia}/Hyperbolic_functions[hyperbolic sine] of an angle. +Returns the {wikipedia}/Hyperbolic_functions[hyperbolic sine] of a number. diff --git a/docs/reference/esql/functions/description/space.asciidoc b/docs/reference/esql/functions/description/space.asciidoc new file mode 100644 index 0000000000000..ee01da64f590f --- /dev/null +++ b/docs/reference/esql/functions/description/space.asciidoc @@ -0,0 +1,5 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Description* + +Returns a string made of `number` spaces. diff --git a/docs/reference/esql/functions/description/tan.asciidoc b/docs/reference/esql/functions/description/tan.asciidoc index 925bebf044a7b..dae37126f0ad3 100644 --- a/docs/reference/esql/functions/description/tan.asciidoc +++ b/docs/reference/esql/functions/description/tan.asciidoc @@ -2,4 +2,4 @@ *Description* -Returns the {wikipedia}/Sine_and_cosine[Tangent] trigonometric function of an angle. +Returns the {wikipedia}/Sine_and_cosine[tangent] of an angle. diff --git a/docs/reference/esql/functions/description/tanh.asciidoc b/docs/reference/esql/functions/description/tanh.asciidoc index 7ee5e457dfe48..42c73a7536dc3 100644 --- a/docs/reference/esql/functions/description/tanh.asciidoc +++ b/docs/reference/esql/functions/description/tanh.asciidoc @@ -2,4 +2,4 @@ *Description* -Returns the {wikipedia}/Hyperbolic_functions[Tangent] hyperbolic function of an angle. +Returns the {wikipedia}/Hyperbolic_functions[hyperbolic tangent] of a number. diff --git a/docs/reference/esql/functions/examples/median_absolute_deviation.asciidoc b/docs/reference/esql/functions/examples/median_absolute_deviation.asciidoc index 20891126c20fb..9084c008e890a 100644 --- a/docs/reference/esql/functions/examples/median_absolute_deviation.asciidoc +++ b/docs/reference/esql/functions/examples/median_absolute_deviation.asciidoc @@ -4,19 +4,19 @@ [source.merge.styled,esql] ---- -include::{esql-specs}/stats_percentile.csv-spec[tag=median-absolute-deviation] +include::{esql-specs}/median_absolute_deviation.csv-spec[tag=median-absolute-deviation] ---- [%header.monospaced.styled,format=dsv,separator=|] |=== -include::{esql-specs}/stats_percentile.csv-spec[tag=median-absolute-deviation-result] +include::{esql-specs}/median_absolute_deviation.csv-spec[tag=median-absolute-deviation-result] |=== The expression can use inline functions. For example, to calculate the the median absolute deviation of the maximum values of a multivalued column, first use `MV_MAX` to get the maximum value per row, and use the result with the `MEDIAN_ABSOLUTE_DEVIATION` function [source.merge.styled,esql] ---- -include::{esql-specs}/stats_percentile.csv-spec[tag=docsStatsMADNestedExpression] +include::{esql-specs}/median_absolute_deviation.csv-spec[tag=docsStatsMADNestedExpression] ---- [%header.monospaced.styled,format=dsv,separator=|] |=== -include::{esql-specs}/stats_percentile.csv-spec[tag=docsStatsMADNestedExpression-result] +include::{esql-specs}/median_absolute_deviation.csv-spec[tag=docsStatsMADNestedExpression-result] |=== diff --git a/docs/reference/esql/functions/examples/mv_median_absolute_deviation.asciidoc b/docs/reference/esql/functions/examples/mv_median_absolute_deviation.asciidoc new file mode 100644 index 0000000000000..b36bc18a80174 --- /dev/null +++ b/docs/reference/esql/functions/examples/mv_median_absolute_deviation.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/mv_median_absolute_deviation.csv-spec[tag=example] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/mv_median_absolute_deviation.csv-spec[tag=example-result] +|=== + diff --git a/docs/reference/esql/functions/examples/space.asciidoc b/docs/reference/esql/functions/examples/space.asciidoc new file mode 100644 index 0000000000000..cef3cd6139021 --- /dev/null +++ b/docs/reference/esql/functions/examples/space.asciidoc @@ -0,0 +1,13 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Example* + +[source.merge.styled,esql] +---- +include::{esql-specs}/string.csv-spec[tag=space] +---- +[%header.monospaced.styled,format=dsv,separator=|] +|=== +include::{esql-specs}/string.csv-spec[tag=space-result] +|=== + diff --git a/docs/reference/esql/functions/kibana/definition/cosh.json b/docs/reference/esql/functions/kibana/definition/cosh.json index a34eee15be37e..dca261d971c40 100644 --- a/docs/reference/esql/functions/kibana/definition/cosh.json +++ b/docs/reference/esql/functions/kibana/definition/cosh.json @@ -2,15 +2,15 @@ "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", "type" : "eval", "name" : "cosh", - "description" : "Returns the hyperbolic cosine of an angle.", + "description" : "Returns the hyperbolic cosine of a number.", "signatures" : [ { "params" : [ { - "name" : "angle", + "name" : "number", "type" : "double", "optional" : false, - "description" : "An angle, in radians. If `null`, the function returns `null`." + "description" : "Numeric expression. If `null`, the function returns `null`." } ], "variadic" : false, @@ -19,10 +19,10 @@ { "params" : [ { - "name" : "angle", + "name" : "number", "type" : "integer", "optional" : false, - "description" : "An angle, in radians. If `null`, the function returns `null`." + "description" : "Numeric expression. If `null`, the function returns `null`." } ], "variadic" : false, @@ -31,10 +31,10 @@ { "params" : [ { - "name" : "angle", + "name" : "number", "type" : "long", "optional" : false, - "description" : "An angle, in radians. If `null`, the function returns `null`." + "description" : "Numeric expression. If `null`, the function returns `null`." } ], "variadic" : false, @@ -43,10 +43,10 @@ { "params" : [ { - "name" : "angle", + "name" : "number", "type" : "unsigned_long", "optional" : false, - "description" : "An angle, in radians. If `null`, the function returns `null`." + "description" : "Numeric expression. If `null`, the function returns `null`." } ], "variadic" : false, diff --git a/docs/reference/esql/functions/kibana/definition/locate.json b/docs/reference/esql/functions/kibana/definition/locate.json index 2097c90b41958..a9ddc8c52368a 100644 --- a/docs/reference/esql/functions/kibana/definition/locate.json +++ b/docs/reference/esql/functions/kibana/definition/locate.json @@ -2,7 +2,7 @@ "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", "type" : "eval", "name" : "locate", - "description" : "Returns an integer that indicates the position of a keyword substring within another string.", + "description" : "Returns an integer that indicates the position of a keyword substring within another string.\nReturns `0` if the substring cannot be found.\nNote that string positions start from `1`.", "signatures" : [ { "params" : [ diff --git a/docs/reference/esql/functions/kibana/definition/mv_median_absolute_deviation.json b/docs/reference/esql/functions/kibana/definition/mv_median_absolute_deviation.json new file mode 100644 index 0000000000000..d6f1174a4e259 --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/mv_median_absolute_deviation.json @@ -0,0 +1,60 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "mv_median_absolute_deviation", + "description" : "Converts a multivalued field into a single valued field containing the median absolute deviation.\n\nIt is calculated as the median of each data point's deviation from the median of the entire sample. That is, for a random variable `X`, the median absolute deviation is `median(|median(X) - X|)`.", + "note" : "If the field has an even number of values, the medians will be calculated as the average of the middle two values. If the value is not a floating point number, the averages are rounded towards 0.", + "signatures" : [ + { + "params" : [ + { + "name" : "number", + "type" : "double", + "optional" : false, + "description" : "Multivalue expression." + } + ], + "variadic" : false, + "returnType" : "double" + }, + { + "params" : [ + { + "name" : "number", + "type" : "integer", + "optional" : false, + "description" : "Multivalue expression." + } + ], + "variadic" : false, + "returnType" : "integer" + }, + { + "params" : [ + { + "name" : "number", + "type" : "long", + "optional" : false, + "description" : "Multivalue expression." + } + ], + "variadic" : false, + "returnType" : "long" + }, + { + "params" : [ + { + "name" : "number", + "type" : "unsigned_long", + "optional" : false, + "description" : "Multivalue expression." + } + ], + "variadic" : false, + "returnType" : "unsigned_long" + } + ], + "examples" : [ + "ROW values = [0, 2, 5, 6]\n| EVAL median_absolute_deviation = MV_MEDIAN_ABSOLUTE_DEVIATION(values), median = MV_MEDIAN(values)" + ] +} diff --git a/docs/reference/esql/functions/kibana/definition/sin.json b/docs/reference/esql/functions/kibana/definition/sin.json index 8d092bd0c15a3..ce46fa66a2acc 100644 --- a/docs/reference/esql/functions/kibana/definition/sin.json +++ b/docs/reference/esql/functions/kibana/definition/sin.json @@ -2,7 +2,7 @@ "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", "type" : "eval", "name" : "sin", - "description" : "Returns ths Sine trigonometric function of an angle.", + "description" : "Returns the sine of an angle.", "signatures" : [ { "params" : [ diff --git a/docs/reference/esql/functions/kibana/definition/sinh.json b/docs/reference/esql/functions/kibana/definition/sinh.json index 2261b18134f6c..e773e95e5e9e1 100644 --- a/docs/reference/esql/functions/kibana/definition/sinh.json +++ b/docs/reference/esql/functions/kibana/definition/sinh.json @@ -2,15 +2,15 @@ "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", "type" : "eval", "name" : "sinh", - "description" : "Returns the hyperbolic sine of an angle.", + "description" : "Returns the hyperbolic sine of a number.", "signatures" : [ { "params" : [ { - "name" : "angle", + "name" : "number", "type" : "double", "optional" : false, - "description" : "An angle, in radians. If `null`, the function returns `null`." + "description" : "Numeric expression. If `null`, the function returns `null`." } ], "variadic" : false, @@ -19,10 +19,10 @@ { "params" : [ { - "name" : "angle", + "name" : "number", "type" : "integer", "optional" : false, - "description" : "An angle, in radians. If `null`, the function returns `null`." + "description" : "Numeric expression. If `null`, the function returns `null`." } ], "variadic" : false, @@ -31,10 +31,10 @@ { "params" : [ { - "name" : "angle", + "name" : "number", "type" : "long", "optional" : false, - "description" : "An angle, in radians. If `null`, the function returns `null`." + "description" : "Numeric expression. If `null`, the function returns `null`." } ], "variadic" : false, @@ -43,10 +43,10 @@ { "params" : [ { - "name" : "angle", + "name" : "number", "type" : "unsigned_long", "optional" : false, - "description" : "An angle, in radians. If `null`, the function returns `null`." + "description" : "Numeric expression. If `null`, the function returns `null`." } ], "variadic" : false, diff --git a/docs/reference/esql/functions/kibana/definition/space.json b/docs/reference/esql/functions/kibana/definition/space.json new file mode 100644 index 0000000000000..acf7466284d3b --- /dev/null +++ b/docs/reference/esql/functions/kibana/definition/space.json @@ -0,0 +1,23 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", + "type" : "eval", + "name" : "space", + "description" : "Returns a string made of `number` spaces.", + "signatures" : [ + { + "params" : [ + { + "name" : "number", + "type" : "integer", + "optional" : false, + "description" : "Number of spaces in result." + } + ], + "variadic" : false, + "returnType" : "keyword" + } + ], + "examples" : [ + "ROW message = CONCAT(\"Hello\", SPACE(1), \"World!\");" + ] +} diff --git a/docs/reference/esql/functions/kibana/definition/tan.json b/docs/reference/esql/functions/kibana/definition/tan.json index 7498964dc1a2c..f5452f310a995 100644 --- a/docs/reference/esql/functions/kibana/definition/tan.json +++ b/docs/reference/esql/functions/kibana/definition/tan.json @@ -2,7 +2,7 @@ "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", "type" : "eval", "name" : "tan", - "description" : "Returns the Tangent trigonometric function of an angle.", + "description" : "Returns the tangent of an angle.", "signatures" : [ { "params" : [ diff --git a/docs/reference/esql/functions/kibana/definition/tanh.json b/docs/reference/esql/functions/kibana/definition/tanh.json index 507f62d394be3..081d606b64217 100644 --- a/docs/reference/esql/functions/kibana/definition/tanh.json +++ b/docs/reference/esql/functions/kibana/definition/tanh.json @@ -2,15 +2,15 @@ "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", "type" : "eval", "name" : "tanh", - "description" : "Returns the Tangent hyperbolic function of an angle.", + "description" : "Returns the hyperbolic tangent of a number.", "signatures" : [ { "params" : [ { - "name" : "angle", + "name" : "number", "type" : "double", "optional" : false, - "description" : "An angle, in radians. If `null`, the function returns `null`." + "description" : "Numeric expression. If `null`, the function returns `null`." } ], "variadic" : false, @@ -19,10 +19,10 @@ { "params" : [ { - "name" : "angle", + "name" : "number", "type" : "integer", "optional" : false, - "description" : "An angle, in radians. If `null`, the function returns `null`." + "description" : "Numeric expression. If `null`, the function returns `null`." } ], "variadic" : false, @@ -31,10 +31,10 @@ { "params" : [ { - "name" : "angle", + "name" : "number", "type" : "long", "optional" : false, - "description" : "An angle, in radians. If `null`, the function returns `null`." + "description" : "Numeric expression. If `null`, the function returns `null`." } ], "variadic" : false, @@ -43,10 +43,10 @@ { "params" : [ { - "name" : "angle", + "name" : "number", "type" : "unsigned_long", "optional" : false, - "description" : "An angle, in radians. If `null`, the function returns `null`." + "description" : "Numeric expression. If `null`, the function returns `null`." } ], "variadic" : false, diff --git a/docs/reference/esql/functions/kibana/docs/cosh.md b/docs/reference/esql/functions/kibana/docs/cosh.md index d5cc126650e44..0338429521781 100644 --- a/docs/reference/esql/functions/kibana/docs/cosh.md +++ b/docs/reference/esql/functions/kibana/docs/cosh.md @@ -3,7 +3,7 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ --> ### COSH -Returns the hyperbolic cosine of an angle. +Returns the hyperbolic cosine of a number. ``` ROW a=1.8 diff --git a/docs/reference/esql/functions/kibana/docs/locate.md b/docs/reference/esql/functions/kibana/docs/locate.md index 75275068d3096..412832e9b1587 100644 --- a/docs/reference/esql/functions/kibana/docs/locate.md +++ b/docs/reference/esql/functions/kibana/docs/locate.md @@ -4,6 +4,8 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ ### LOCATE Returns an integer that indicates the position of a keyword substring within another string. +Returns `0` if the substring cannot be found. +Note that string positions start from `1`. ``` row a = "hello" diff --git a/docs/reference/esql/functions/kibana/docs/mv_median_absolute_deviation.md b/docs/reference/esql/functions/kibana/docs/mv_median_absolute_deviation.md new file mode 100644 index 0000000000000..191ce3ce60ae1 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/mv_median_absolute_deviation.md @@ -0,0 +1,14 @@ + + +### MV_MEDIAN_ABSOLUTE_DEVIATION +Converts a multivalued field into a single valued field containing the median absolute deviation. + +It is calculated as the median of each data point's deviation from the median of the entire sample. That is, for a random variable `X`, the median absolute deviation is `median(|median(X) - X|)`. + +``` +ROW values = [0, 2, 5, 6] +| EVAL median_absolute_deviation = MV_MEDIAN_ABSOLUTE_DEVIATION(values), median = MV_MEDIAN(values) +``` +Note: If the field has an even number of values, the medians will be calculated as the average of the middle two values. If the value is not a floating point number, the averages are rounded towards 0. diff --git a/docs/reference/esql/functions/kibana/docs/sin.md b/docs/reference/esql/functions/kibana/docs/sin.md index 1e1fc5ee9c938..d1128350d12fb 100644 --- a/docs/reference/esql/functions/kibana/docs/sin.md +++ b/docs/reference/esql/functions/kibana/docs/sin.md @@ -3,7 +3,7 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ --> ### SIN -Returns ths Sine trigonometric function of an angle. +Returns the sine of an angle. ``` ROW a=1.8 diff --git a/docs/reference/esql/functions/kibana/docs/sinh.md b/docs/reference/esql/functions/kibana/docs/sinh.md index 886b3b95b09f8..249c9bb0906c3 100644 --- a/docs/reference/esql/functions/kibana/docs/sinh.md +++ b/docs/reference/esql/functions/kibana/docs/sinh.md @@ -3,7 +3,7 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ --> ### SINH -Returns the hyperbolic sine of an angle. +Returns the hyperbolic sine of a number. ``` ROW a=1.8 diff --git a/docs/reference/esql/functions/kibana/docs/space.md b/docs/reference/esql/functions/kibana/docs/space.md new file mode 100644 index 0000000000000..3112bf953dd65 --- /dev/null +++ b/docs/reference/esql/functions/kibana/docs/space.md @@ -0,0 +1,10 @@ + + +### SPACE +Returns a string made of `number` spaces. + +``` +ROW message = CONCAT("Hello", SPACE(1), "World!"); +``` diff --git a/docs/reference/esql/functions/kibana/docs/tan.md b/docs/reference/esql/functions/kibana/docs/tan.md index f1594f4de7476..41bd3c814b171 100644 --- a/docs/reference/esql/functions/kibana/docs/tan.md +++ b/docs/reference/esql/functions/kibana/docs/tan.md @@ -3,7 +3,7 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ --> ### TAN -Returns the Tangent trigonometric function of an angle. +Returns the tangent of an angle. ``` ROW a=1.8 diff --git a/docs/reference/esql/functions/kibana/docs/tanh.md b/docs/reference/esql/functions/kibana/docs/tanh.md index c4a70dec00ba8..365add190de82 100644 --- a/docs/reference/esql/functions/kibana/docs/tanh.md +++ b/docs/reference/esql/functions/kibana/docs/tanh.md @@ -3,7 +3,7 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ --> ### TANH -Returns the Tangent hyperbolic function of an angle. +Returns the hyperbolic tangent of a number. ``` ROW a=1.8 diff --git a/docs/reference/esql/functions/layout/mv_median_absolute_deviation.asciidoc b/docs/reference/esql/functions/layout/mv_median_absolute_deviation.asciidoc new file mode 100644 index 0000000000000..b594d589e6108 --- /dev/null +++ b/docs/reference/esql/functions/layout/mv_median_absolute_deviation.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-mv_median_absolute_deviation]] +=== `MV_MEDIAN_ABSOLUTE_DEVIATION` + +*Syntax* + +[.text-center] +image::esql/functions/signature/mv_median_absolute_deviation.svg[Embedded,opts=inline] + +include::../parameters/mv_median_absolute_deviation.asciidoc[] +include::../description/mv_median_absolute_deviation.asciidoc[] +include::../types/mv_median_absolute_deviation.asciidoc[] +include::../examples/mv_median_absolute_deviation.asciidoc[] diff --git a/docs/reference/esql/functions/layout/space.asciidoc b/docs/reference/esql/functions/layout/space.asciidoc new file mode 100644 index 0000000000000..22355d1e24978 --- /dev/null +++ b/docs/reference/esql/functions/layout/space.asciidoc @@ -0,0 +1,15 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +[discrete] +[[esql-space]] +=== `SPACE` + +*Syntax* + +[.text-center] +image::esql/functions/signature/space.svg[Embedded,opts=inline] + +include::../parameters/space.asciidoc[] +include::../description/space.asciidoc[] +include::../types/space.asciidoc[] +include::../examples/space.asciidoc[] diff --git a/docs/reference/esql/functions/mv-functions.asciidoc b/docs/reference/esql/functions/mv-functions.asciidoc index bd5f14cdd3557..4093e44c16911 100644 --- a/docs/reference/esql/functions/mv-functions.asciidoc +++ b/docs/reference/esql/functions/mv-functions.asciidoc @@ -17,6 +17,7 @@ * <> * <> * <> +* <> * <> * <> * <> @@ -34,6 +35,7 @@ include::layout/mv_first.asciidoc[] include::layout/mv_last.asciidoc[] include::layout/mv_max.asciidoc[] include::layout/mv_median.asciidoc[] +include::layout/mv_median_absolute_deviation.asciidoc[] include::layout/mv_min.asciidoc[] include::layout/mv_pseries_weighted_sum.asciidoc[] include::layout/mv_slice.asciidoc[] diff --git a/docs/reference/esql/functions/parameters/cosh.asciidoc b/docs/reference/esql/functions/parameters/cosh.asciidoc index a1c3f7edf30ce..65013f4c21265 100644 --- a/docs/reference/esql/functions/parameters/cosh.asciidoc +++ b/docs/reference/esql/functions/parameters/cosh.asciidoc @@ -2,5 +2,5 @@ *Parameters* -`angle`:: -An angle, in radians. If `null`, the function returns `null`. +`number`:: +Numeric expression. If `null`, the function returns `null`. diff --git a/docs/reference/esql/functions/parameters/mv_median_absolute_deviation.asciidoc b/docs/reference/esql/functions/parameters/mv_median_absolute_deviation.asciidoc new file mode 100644 index 0000000000000..47859c7e2b320 --- /dev/null +++ b/docs/reference/esql/functions/parameters/mv_median_absolute_deviation.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`number`:: +Multivalue expression. diff --git a/docs/reference/esql/functions/parameters/sinh.asciidoc b/docs/reference/esql/functions/parameters/sinh.asciidoc index a1c3f7edf30ce..65013f4c21265 100644 --- a/docs/reference/esql/functions/parameters/sinh.asciidoc +++ b/docs/reference/esql/functions/parameters/sinh.asciidoc @@ -2,5 +2,5 @@ *Parameters* -`angle`:: -An angle, in radians. If `null`, the function returns `null`. +`number`:: +Numeric expression. If `null`, the function returns `null`. diff --git a/docs/reference/esql/functions/parameters/space.asciidoc b/docs/reference/esql/functions/parameters/space.asciidoc new file mode 100644 index 0000000000000..de4efd34c0ba4 --- /dev/null +++ b/docs/reference/esql/functions/parameters/space.asciidoc @@ -0,0 +1,6 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Parameters* + +`number`:: +Number of spaces in result. diff --git a/docs/reference/esql/functions/parameters/tanh.asciidoc b/docs/reference/esql/functions/parameters/tanh.asciidoc index a1c3f7edf30ce..65013f4c21265 100644 --- a/docs/reference/esql/functions/parameters/tanh.asciidoc +++ b/docs/reference/esql/functions/parameters/tanh.asciidoc @@ -2,5 +2,5 @@ *Parameters* -`angle`:: -An angle, in radians. If `null`, the function returns `null`. +`number`:: +Numeric expression. If `null`, the function returns `null`. diff --git a/docs/reference/esql/functions/signature/cosh.svg b/docs/reference/esql/functions/signature/cosh.svg index 11b14d922929a..9b9eddd3cb808 100644 --- a/docs/reference/esql/functions/signature/cosh.svg +++ b/docs/reference/esql/functions/signature/cosh.svg @@ -1 +1 @@ -COSH(angle) \ No newline at end of file +COSH(number) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/mv_median_absolute_deviation.svg b/docs/reference/esql/functions/signature/mv_median_absolute_deviation.svg new file mode 100644 index 0000000000000..7d8a131a91015 --- /dev/null +++ b/docs/reference/esql/functions/signature/mv_median_absolute_deviation.svg @@ -0,0 +1 @@ +MV_MEDIAN_ABSOLUTE_DEVIATION(number) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/sinh.svg b/docs/reference/esql/functions/signature/sinh.svg index 0bb4ac31dee30..16e7ddb6b6534 100644 --- a/docs/reference/esql/functions/signature/sinh.svg +++ b/docs/reference/esql/functions/signature/sinh.svg @@ -1 +1 @@ -SINH(angle) \ No newline at end of file +SINH(number) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/space.svg b/docs/reference/esql/functions/signature/space.svg new file mode 100644 index 0000000000000..c506c25dfcb16 --- /dev/null +++ b/docs/reference/esql/functions/signature/space.svg @@ -0,0 +1 @@ +SPACE(number) \ No newline at end of file diff --git a/docs/reference/esql/functions/signature/tanh.svg b/docs/reference/esql/functions/signature/tanh.svg index f7b968f8b30c4..c2edfe2d6942f 100644 --- a/docs/reference/esql/functions/signature/tanh.svg +++ b/docs/reference/esql/functions/signature/tanh.svg @@ -1 +1 @@ -TANH(angle) \ No newline at end of file +TANH(number) \ No newline at end of file diff --git a/docs/reference/esql/functions/string-functions.asciidoc b/docs/reference/esql/functions/string-functions.asciidoc index d4b120ad1c45b..ed97769b900e7 100644 --- a/docs/reference/esql/functions/string-functions.asciidoc +++ b/docs/reference/esql/functions/string-functions.asciidoc @@ -19,6 +19,7 @@ * <> * <> * <> +* <> * <> * <> * <> @@ -39,6 +40,7 @@ include::layout/repeat.asciidoc[] include::layout/replace.asciidoc[] include::layout/right.asciidoc[] include::layout/rtrim.asciidoc[] +include::layout/space.asciidoc[] include::layout/split.asciidoc[] include::layout/starts_with.asciidoc[] include::layout/substring.asciidoc[] diff --git a/docs/reference/esql/functions/types/cosh.asciidoc b/docs/reference/esql/functions/types/cosh.asciidoc index d96a34b678531..7cda278abdb56 100644 --- a/docs/reference/esql/functions/types/cosh.asciidoc +++ b/docs/reference/esql/functions/types/cosh.asciidoc @@ -4,7 +4,7 @@ [%header.monospaced.styled,format=dsv,separator=|] |=== -angle | result +number | result double | double integer | double long | double diff --git a/docs/reference/esql/functions/types/mv_median_absolute_deviation.asciidoc b/docs/reference/esql/functions/types/mv_median_absolute_deviation.asciidoc new file mode 100644 index 0000000000000..d81bbf36ae3fe --- /dev/null +++ b/docs/reference/esql/functions/types/mv_median_absolute_deviation.asciidoc @@ -0,0 +1,12 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +number | result +double | double +integer | integer +long | long +unsigned_long | unsigned_long +|=== diff --git a/docs/reference/esql/functions/types/sinh.asciidoc b/docs/reference/esql/functions/types/sinh.asciidoc index d96a34b678531..7cda278abdb56 100644 --- a/docs/reference/esql/functions/types/sinh.asciidoc +++ b/docs/reference/esql/functions/types/sinh.asciidoc @@ -4,7 +4,7 @@ [%header.monospaced.styled,format=dsv,separator=|] |=== -angle | result +number | result double | double integer | double long | double diff --git a/docs/reference/esql/functions/types/space.asciidoc b/docs/reference/esql/functions/types/space.asciidoc new file mode 100644 index 0000000000000..3f2e89f80d3e5 --- /dev/null +++ b/docs/reference/esql/functions/types/space.asciidoc @@ -0,0 +1,9 @@ +// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it. + +*Supported types* + +[%header.monospaced.styled,format=dsv,separator=|] +|=== +number | result +integer | keyword +|=== diff --git a/docs/reference/esql/functions/types/tanh.asciidoc b/docs/reference/esql/functions/types/tanh.asciidoc index d96a34b678531..7cda278abdb56 100644 --- a/docs/reference/esql/functions/types/tanh.asciidoc +++ b/docs/reference/esql/functions/types/tanh.asciidoc @@ -4,7 +4,7 @@ [%header.monospaced.styled,format=dsv,separator=|] |=== -angle | result +number | result double | double integer | double long | double diff --git a/docs/reference/esql/processing-commands/stats.asciidoc b/docs/reference/esql/processing-commands/stats.asciidoc index 7377522a93201..0c479c1f62b76 100644 --- a/docs/reference/esql/processing-commands/stats.asciidoc +++ b/docs/reference/esql/processing-commands/stats.asciidoc @@ -3,7 +3,7 @@ === `STATS ... BY` The `STATS ... BY` processing command groups rows according to a common value -and calculate one or more aggregated values over the grouped rows. +and calculates one or more aggregated values over the grouped rows. **Syntax** @@ -41,6 +41,10 @@ The following <> are supported: include::../functions/aggregation-functions.asciidoc[tag=agg_list] +The following <> are supported: + +include::../functions/grouping-functions.asciidoc[tag=group_list] + NOTE: `STATS` without any groups is much much faster than adding a group. NOTE: Grouping on a single expression is currently much more optimized than grouping diff --git a/docs/reference/index-modules/similarity.asciidoc b/docs/reference/index-modules/similarity.asciidoc index afc3b6556c67d..51a01b5b7c7e8 100644 --- a/docs/reference/index-modules/similarity.asciidoc +++ b/docs/reference/index-modules/similarity.asciidoc @@ -5,6 +5,8 @@ A similarity (scoring / ranking model) defines how matching documents are scored. Similarity is per field, meaning that via the mapping one can define a different similarity per field. +Similarity is only applicable for text type and keyword type fields. + Configuring a custom similarity is considered an expert feature and the builtin similarities are most likely sufficient as is described in <>. diff --git a/docs/reference/indices/get-data-stream.asciidoc b/docs/reference/indices/get-data-stream.asciidoc index b88a1a1be2a7e..6bf150897acab 100644 --- a/docs/reference/indices/get-data-stream.asciidoc +++ b/docs/reference/indices/get-data-stream.asciidoc @@ -105,6 +105,10 @@ Defaults to `open`. (Optional, Boolean) Functionality in preview:[]. If `true`, return all default settings in the response. Defaults to `false`. +`verbose`:: +(Optional, Boolean). If `true`, Returns the `maximum_timestamp` corresponding to the `@timestamp` field for documents in the data stream. +Defaults to `false`. + [role="child_attributes"] [[get-data-stream-api-response-body]] ==== {api-response-body-title} diff --git a/docs/reference/indices/index-templates.asciidoc b/docs/reference/indices/index-templates.asciidoc index 538fb5b97860a..66911716ffee2 100644 --- a/docs/reference/indices/index-templates.asciidoc +++ b/docs/reference/indices/index-templates.asciidoc @@ -102,7 +102,7 @@ PUT _component_template/runtime_component_template "day_of_week": { "type": "keyword", "script": { - "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))" + "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))" } } } diff --git a/docs/reference/inference/inference-apis.asciidoc b/docs/reference/inference/inference-apis.asciidoc index 33db148755d8e..8fdf8aecc2ae5 100644 --- a/docs/reference/inference/inference-apis.asciidoc +++ b/docs/reference/inference/inference-apis.asciidoc @@ -39,6 +39,7 @@ include::delete-inference.asciidoc[] include::get-inference.asciidoc[] include::post-inference.asciidoc[] include::put-inference.asciidoc[] +include::service-alibabacloud-ai-search.asciidoc[] include::service-amazon-bedrock.asciidoc[] include::service-anthropic.asciidoc[] include::service-azure-ai-studio.asciidoc[] diff --git a/docs/reference/inference/put-inference.asciidoc b/docs/reference/inference/put-inference.asciidoc index 57485e0720cca..ba26a563541fc 100644 --- a/docs/reference/inference/put-inference.asciidoc +++ b/docs/reference/inference/put-inference.asciidoc @@ -39,6 +39,7 @@ The create {infer} API enables you to create an {infer} endpoint and configure a The following services are available through the {infer} API, click the links to review the configuration details of the services: +* <> * <> * <> * <> diff --git a/docs/reference/inference/service-alibabacloud-ai-search.asciidoc b/docs/reference/inference/service-alibabacloud-ai-search.asciidoc new file mode 100644 index 0000000000000..23a3d532635ac --- /dev/null +++ b/docs/reference/inference/service-alibabacloud-ai-search.asciidoc @@ -0,0 +1,184 @@ +[[infer-service-alibabacloud-ai-search]] +=== AlibabaCloud AI Search {infer} service + +Creates an {infer} endpoint to perform an {infer} task with the `alibabacloud-ai-search` service. + +[discrete] +[[infer-service-alibabacloud-ai-search-api-request]] +==== {api-request-title} + +`PUT /_inference//` + +[discrete] +[[infer-service-alibabacloud-ai-search-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) +include::inference-shared.asciidoc[tag=inference-id] + +``:: +(Required, string) +include::inference-shared.asciidoc[tag=task-type] ++ +-- +Available task types: + +* `text_embedding`, +* `sparse_embedding`, +* `rerank`. +-- + +[discrete] +[[infer-service-alibabacloud-ai-search-api-request-body]] +==== {api-request-body-title} + +`service`:: +(Required, string) The type of service supported for the specified task type. +In this case, +`alibabacloud-ai-search`. + +`service_settings`:: +(Required, object) +include::inference-shared.asciidoc[tag=service-settings] ++ +-- +These settings are specific to the `alibabacloud-ai-search` service. +-- + +`api_key`::: +(Required, string) +A valid API key for the AlibabaCloud AI Search API. + +`service_id`::: +(Required, string) +The name of the model service to use for the {infer} task. ++ +-- +Available service_ids for the `text_embedding` task: + +* `ops-text-embedding-001` +* `ops-text-embedding-zh-001` +* `ops-text-embedding-en-001` +* `ops-text-embedding-002` + +For the supported `text_embedding` service_ids, refer to the https://help.aliyun.com/zh/open-search/search-platform/developer-reference/text-embedding-api-details[documentation]. + +Available service_id for the `sparse_embedding` task: + +* `ops-text-sparse-embedding-001` + +For the supported `sparse_embedding` service_id, refer to the https://help.aliyun.com/zh/open-search/search-platform/developer-reference/text-sparse-embedding-api-details[documentation]. + +Available service_id for the `rerank` task is: + +* `ops-bge-reranker-larger` + +For the supported `rerank` service_id, refer to the https://help.aliyun.com/zh/open-search/search-platform/developer-reference/ranker-api-details[documentation]. +-- + +`host`::: +(Required, string) +The name of the host address used for the {infer} task. You can find the host address at https://opensearch.console.aliyun.com/cn-shanghai/rag/api-key[ the API keys section] of the documentation. + +`workspace`::: +(Required, string) +The name of the workspace used for the {infer} task. + +`rate_limit`::: +(Optional, object) +By default, the `alibabacloud-ai-search` service sets the number of requests allowed per minute to `1000`. +This helps to minimize the number of rate limit errors returned from AlibabaCloud AI Search. +To modify this, set the `requests_per_minute` setting of this object in your service settings: ++ +-- +include::inference-shared.asciidoc[tag=request-per-minute-example] +-- + + +`task_settings`:: +(Optional, object) +include::inference-shared.asciidoc[tag=task-settings] ++ +.`task_settings` for the `text_embedding` task type +[%collapsible%closed] +===== +`input_type`::: +(Optional, string) +Specifies the type of input passed to the model. +Valid values are: +* `ingest`: for storing document embeddings in a vector database. +* `search`: for storing embeddings of search queries run against a vector database to find relevant documents. +===== ++ +.`task_settings` for the `sparse_embedding` task type +[%collapsible%closed] +===== +`input_type`::: +(Optional, string) +Specifies the type of input passed to the model. +Valid values are: +* `ingest`: for storing document embeddings in a vector database. +* `search`: for storing embeddings of search queries run against a vector database to find relevant documents. + +`return_token`::: +(Optional, boolean) +If `true`, the token name will be returned in the response. Defaults to `false` which means only the token ID will be returned in the response. +===== + +[discrete] +[[inference-example-alibabacloud-ai-search]] +==== AlibabaCloud AI Search service examples + +The following example shows how to create an {infer} endpoint called `alibabacloud_ai_search_embeddings` to perform a `text_embedding` task type. + +[source,console] +------------------------------------------------------------ +PUT _inference/text_embedding/alibabacloud_ai_search_embeddings +{ + "service": "alibabacloud-ai-search", + "service_settings": { + "api_key": "", + "service_id": "ops-text-embedding-001", + "host": "default-j01.platform-cn-shanghai.opensearch.aliyuncs.com", + "workspace": "default" + } +} +------------------------------------------------------------ +// TEST[skip:TBD] + +The following example shows how to create an {infer} endpoint called +`alibabacloud_ai_search_sparse` to perform a `sparse_embedding` task type. + +[source,console] +------------------------------------------------------------ +PUT _inference/sparse_embedding/alibabacloud_ai_search_sparse +{ + "service": "alibabacloud-ai-search", + "service_settings": { + "api_key": "", + "service_id": "ops-text-sparse-embedding-001", + "host": "default-j01.platform-cn-shanghai.opensearch.aliyuncs.com", + "workspace": "default" + } +} +------------------------------------------------------------ +// TEST[skip:TBD] + +The next example shows how to create an {infer} endpoint called +`alibabacloud_ai_search_rerank` to perform a `rerank` task type. + +[source,console] +------------------------------------------------------------ +PUT _inference/rerank/alibabacloud_ai_search_rerank +{ + "service": "alibabacloud-ai-search", + "service_settings": { + "api_key": "", + "service_id": "ops-bge-reranker-larger", + "host": "default-j01.platform-cn-shanghai.opensearch.aliyuncs.com", + "workspace": "default" + } +} +------------------------------------------------------------ +// TEST[skip:TBD] diff --git a/docs/reference/inference/service-amazon-bedrock.asciidoc b/docs/reference/inference/service-amazon-bedrock.asciidoc index 4ffa368613a0e..dbffd5c26fbcc 100644 --- a/docs/reference/inference/service-amazon-bedrock.asciidoc +++ b/docs/reference/inference/service-amazon-bedrock.asciidoc @@ -122,14 +122,6 @@ Only available for `anthropic`, `cohere`, and `mistral` providers. Alternative to `temperature`. Limits samples to the top-K most likely words, balancing coherence and variability. Should not be used if `temperature` is specified. -===== -+ -.`task_settings` for the `text_embedding` task type -[%collapsible%closed] -===== - -There are no `task_settings` available for the `text_embedding` task type. - ===== [discrete] diff --git a/docs/reference/inference/service-elasticsearch.asciidoc b/docs/reference/inference/service-elasticsearch.asciidoc index 99fd41ee2db65..572cad591fba6 100644 --- a/docs/reference/inference/service-elasticsearch.asciidoc +++ b/docs/reference/inference/service-elasticsearch.asciidoc @@ -31,6 +31,7 @@ include::inference-shared.asciidoc[tag=task-type] Available task types: * `rerank`, +* `sparse_embedding`, * `text_embedding`. -- @@ -182,4 +183,4 @@ PUT _inference/text_embedding/my-e5-model } } ------------------------------------------------------------ -// TEST[skip:TBD] \ No newline at end of file +// TEST[skip:TBD] diff --git a/docs/reference/ingest/apis/simulate-ingest.asciidoc b/docs/reference/ingest/apis/simulate-ingest.asciidoc index 36f1f089ce90e..ee84a39ee6f65 100644 --- a/docs/reference/ingest/apis/simulate-ingest.asciidoc +++ b/docs/reference/ingest/apis/simulate-ingest.asciidoc @@ -119,7 +119,11 @@ as well the same way that a non-simulated ingest would. No data is indexed into {es}. Instead, the transformed document is returned, along with the list of pipelines that have been executed and the name of the index where the document would have been indexed if this were -not a simulation. This differs from the +not a simulation. The transformed document is validated against the +mappings that would apply to this index, and any validation error is +reported in the result. + +This API differs from the <> in that you specify a single pipeline for that API, and it only runs that one pipeline. The simulate pipeline API is more useful for developing a single pipeline, diff --git a/docs/reference/ingest/enrich.asciidoc b/docs/reference/ingest/enrich.asciidoc index 6642cdc2a74ce..4bd50641149c0 100644 --- a/docs/reference/ingest/enrich.asciidoc +++ b/docs/reference/ingest/enrich.asciidoc @@ -230,12 +230,12 @@ Instead, you can: [[ingest-enrich-components]] ==== Enrich components -The enrich coordinator is a component that manages and performs the searches +The enrich coordinator is a component that manages and performs the searches required to enrich documents on each ingest node. It combines searches from all enrich processors in all pipelines into bulk <>. -The enrich policy executor is a component that manages the executions of all -enrich policies. When an enrich policy is executed, this component creates +The enrich policy executor is a component that manages the executions of all +enrich policies. When an enrich policy is executed, this component creates a new enrich index and removes the previous enrich index. The enrich policy executions are managed from the elected master node. The execution of these policies occurs on a different node. @@ -249,9 +249,15 @@ enrich policy executor. The enrich coordinator supports the following node settings: `enrich.cache_size`:: -Maximum number of searches to cache for enriching documents. Defaults to `1000`. -There is a single cache for all enrich processors in the cluster. This setting -determines the size of that cache. +Maximum size of the cache that caches searches for enriching documents. +The size can be specified in three units: the raw number of +cached searches (e.g. `1000`), an absolute size in bytes (e.g. `100Mb`), +or a percentage of the max heap space of the node (e.g. `1%`). +Both for the absolute byte size and the percentage of heap space, +{es} does not guarantee that the enrich cache size will adhere exactly to that maximum, +as {es} uses the byte size of the serialized search response +which is is a good representation of the used space on the heap, but not an exact match. +Defaults to `1%`. There is a single cache for all enrich processors in the cluster. `enrich.coordinator_proxy.max_concurrent_requests`:: Maximum number of concurrent <> to @@ -280,4 +286,4 @@ Maximum number of enrich policies to execute concurrently. Defaults to `50`. include::geo-match-enrich-policy-type-ex.asciidoc[] include::match-enrich-policy-type-ex.asciidoc[] -include::range-enrich-policy-type-ex.asciidoc[] \ No newline at end of file +include::range-enrich-policy-type-ex.asciidoc[] diff --git a/docs/reference/ingest/processors/community-id.asciidoc b/docs/reference/ingest/processors/community-id.asciidoc index 03e65ac04a209..2d86bd21fa1e9 100644 --- a/docs/reference/ingest/processors/community-id.asciidoc +++ b/docs/reference/ingest/processors/community-id.asciidoc @@ -23,11 +23,12 @@ configuration is required. | `source_port` | no | `source.port` | Field containing the source port. | `destination_ip` | no | `destination.ip` | Field containing the destination IP address. | `destination_port` | no | `destination.port` | Field containing the destination port. -| `iana_number` | no | `network.iana_number` | Field containing the IANA number. The following protocol numbers are currently supported: `1` ICMP, `2` IGMP, `6` TCP, `17` UDP, `47` GRE, `58` ICMP IPv6, `88` EIGRP, `89` OSPF, `103` PIM, and `132` SCTP. +| `iana_number` | no | `network.iana_number` | Field containing the IANA number. | `icmp_type` | no | `icmp.type` | Field containing the ICMP type. | `icmp_code` | no | `icmp.code` | Field containing the ICMP code. -| `transport` | no | `network.transport` | Field containing the transport protocol. -Used only when the `iana_number` field is not present. +| `transport` | no | `network.transport` | Field containing the transport protocol name or number. +Used only when the `iana_number` field is not present. The following protocol names are currently supported: +`ICMP`, `IGMP`, `TCP`, `UDP`, `GRE`, `ICMP IPv6`, `EIGRP`, `OSPF`, `PIM`, and `SCTP`. | `target_field` | no | `network.community_id` | Output field for the community ID. | `seed` | no | `0` | Seed for the community ID hash. Must be between 0 and 65535 (inclusive). The seed can prevent hash collisions between network domains, such as diff --git a/docs/reference/intro.asciidoc b/docs/reference/intro.asciidoc index 3fc23b44994a7..3ad5a9bd71c08 100644 --- a/docs/reference/intro.asciidoc +++ b/docs/reference/intro.asciidoc @@ -1,95 +1,163 @@ [[elasticsearch-intro]] == What is {es}? -_**You know, for search (and analysis)**_ - -{es} is the distributed search and analytics engine at the heart of -the {stack}. {ls} and {beats} facilitate collecting, aggregating, and -enriching your data and storing it in {es}. {kib} enables you to -interactively explore, visualize, and share insights into your data and manage -and monitor the stack. {es} is where the indexing, search, and analysis -magic happens. - -{es} provides near real-time search and analytics for all types of data. Whether you -have structured or unstructured text, numerical data, or geospatial data, -{es} can efficiently store and index it in a way that supports fast searches. -You can go far beyond simple data retrieval and aggregate information to discover -trends and patterns in your data. And as your data and query volume grows, the -distributed nature of {es} enables your deployment to grow seamlessly right -along with it. - -While not _every_ problem is a search problem, {es} offers speed and flexibility -to handle data in a wide variety of use cases: - -* Add a search box to an app or website -* Store and analyze logs, metrics, and security event data -* Use machine learning to automatically model the behavior of your data in real - time -* Use {es} as a vector database to create, store, and search vector embeddings -* Automate business workflows using {es} as a storage engine -* Manage, integrate, and analyze spatial information using {es} as a geographic - information system (GIS) -* Store and process genetic data using {es} as a bioinformatics research tool - -We’re continually amazed by the novel ways people use search. But whether -your use case is similar to one of these, or you're using {es} to tackle a new -problem, the way you work with your data, documents, and indices in {es} is -the same. +{es-repo}[{es}] is a distributed search and analytics engine, scalable data store, and vector database built on Apache Lucene. +It's optimized for speed and relevance on production-scale workloads. +Use {es} to search, index, store, and analyze data of all shapes and sizes in near real time. + +[TIP] +==== +{es} has a lot of features. Explore the full list on the https://www.elastic.co/elasticsearch/features[product webpage^]. +==== + +{es} is the heart of the {estc-welcome-current}/stack-components.html[Elastic Stack] and powers the Elastic https://www.elastic.co/enterprise-search[Search], https://www.elastic.co/observability[Observability] and https://www.elastic.co/security[Security] solutions. + +{es} is used for a wide and growing range of use cases. Here are a few examples: + +* *Monitor log and event data*. Store logs, metrics, and event data for observability and security information and event management (SIEM). +* *Build search applications*. Add search capabilities to apps or websites, or build enterprise search engines over your organization's internal data sources. +* *Vector database*. Store and search vectorized data, and create vector embeddings with built-in and third-party natural language processing (NLP) models. +* *Retrieval augmented generation (RAG)*. Use {es} as a retrieval engine to augment Generative AI models. +* *Application and security monitoring*. Monitor and analyze application performance and security data effectively. +* *Machine learning*. Use {ml} to automatically model the behavior of your data in real-time. + +This is just a sample of search, observability, and security use cases enabled by {es}. +Refer to our https://www.elastic.co/customers/success-stories[customer success stories] for concrete examples across a range of industries. +// Link to demos, search labs chatbots + +[discrete] +[[elasticsearch-intro-elastic-stack]] +.What is the Elastic Stack? +******************************* +{es} is the core component of the Elastic Stack, a suite of products for collecting, storing, searching, and visualizing data. +https://www.elastic.co/guide/en/starting-with-the-elasticsearch-platform-and-its-solutions/current/stack-components.html[Learn more about the Elastic Stack]. +******************************* +// TODO: Remove once we've moved Stack Overview to a subpage? + +[discrete] +[[elasticsearch-intro-deploy]] +=== Deployment options + +To use {es}, you need a running instance of the {es} service. +You can deploy {es} in various ways: + +* <>. Get started quickly with a minimal local Docker setup. +* {cloud}/ec-getting-started-trial.html[*Elastic Cloud*]. {es} is available as part of our hosted Elastic Stack offering, deployed in the cloud with your provider of choice. Sign up for a https://cloud.elastic.co/registration[14 day free trial]. +* {serverless-docs}/general/sign-up-trial[*Elastic Cloud Serverless* (technical preview)]. Create serverless projects for autoscaled and fully managed {es} deployments. Sign up for a https://cloud.elastic.co/serverless-registration[14 day free trial]. + +**Advanced deployment options** + +* <>. Install, configure, and run {es} on your own premises. +* {ece-ref}/Elastic-Cloud-Enterprise-overview.html[*Elastic Cloud Enterprise*]. Deploy Elastic Cloud on public or private clouds, virtual machines, or your own premises. +* {eck-ref}/k8s-overview.html[*Elastic Cloud on Kubernetes*]. Deploy Elastic Cloud on Kubernetes. + +[discrete] +[[elasticsearch-next-steps]] +=== Learn more + +Some resources to help you get started: + +* <>. A beginner's guide to deploying your first {es} instance, indexing data, and running queries. +* https://elastic.co/webinars/getting-started-elasticsearch[Webinar: Introduction to {es}]. Register for our live webinars to learn directly from {es} experts. +* https://www.elastic.co/search-labs[Elastic Search Labs]. Tutorials and blogs that explore AI-powered search using the latest {es} features. +** Follow our tutorial https://www.elastic.co/search-labs/tutorials/search-tutorial/welcome[to build a hybrid search solution in Python]. +** Check out the https://github.com/elastic/elasticsearch-labs?tab=readme-ov-file#elasticsearch-examples--apps[`elasticsearch-labs` repository] for a range of Python notebooks and apps for various use cases. + +// new html page [[documents-indices]] -=== Data in: documents and indices - -{es} is a distributed document store. Instead of storing information as rows of -columnar data, {es} stores complex data structures that have been serialized -as JSON documents. When you have multiple {es} nodes in a cluster, stored -documents are distributed across the cluster and can be accessed immediately -from any node. - -When a document is stored, it is indexed and fully searchable in <>--within 1 second. {es} uses a data structure called an -inverted index that supports very fast full-text searches. An inverted index -lists every unique word that appears in any document and identifies all of the -documents each word occurs in. - -An index can be thought of as an optimized collection of documents and each -document is a collection of fields, which are the key-value pairs that contain -your data. By default, {es} indexes all data in every field and each indexed -field has a dedicated, optimized data structure. For example, text fields are -stored in inverted indices, and numeric and geo fields are stored in BKD trees. -The ability to use the per-field data structures to assemble and return search -results is what makes {es} so fast. - -{es} also has the ability to be schema-less, which means that documents can be -indexed without explicitly specifying how to handle each of the different fields -that might occur in a document. When dynamic mapping is enabled, {es} -automatically detects and adds new fields to the index. This default -behavior makes it easy to index and explore your data--just start -indexing documents and {es} will detect and map booleans, floating point and -integer values, dates, and strings to the appropriate {es} data types. - -Ultimately, however, you know more about your data and how you want to use it -than {es} can. You can define rules to control dynamic mapping and explicitly -define mappings to take full control of how fields are stored and indexed. - -Defining your own mappings enables you to: - -* Distinguish between full-text string fields and exact value string fields -* Perform language-specific text analysis -* Optimize fields for partial matching -* Use custom date formats -* Use data types such as `geo_point` and `geo_shape` that cannot be automatically -detected - -It’s often useful to index the same field in different ways for different -purposes. For example, you might want to index a string field as both a text -field for full-text search and as a keyword field for sorting or aggregating -your data. Or, you might choose to use more than one language analyzer to -process the contents of a string field that contains user input. - -The analysis chain that is applied to a full-text field during indexing is also -used at search time. When you query a full-text field, the query text undergoes -the same analysis before the terms are looked up in the index. +=== Indices, documents, and fields +++++ +Indices and documents +++++ + +The index is the fundamental unit of storage in {es}, a logical namespace for storing data that share similar characteristics. +After you have {es} <>, you'll get started by creating an index to store your data. + +[TIP] +==== +A closely related concept is a <>. +This index abstraction is optimized for append-only time-series data, and is made up of hidden, auto-generated backing indices. +If you're working with time-series data, we recommend the {observability-guide}[Elastic Observability] solution. +==== +Some key facts about indices: + +* An index is a collection of documents +* An index has a unique name +* An index can also be referred to by an alias +* An index has a mapping that defines the schema of its documents + +[discrete] +[[elasticsearch-intro-documents-fields]] +==== Documents and fields + +{es} serializes and stores data in the form of JSON documents. +A document is a set of fields, which are key-value pairs that contain your data. +Each document has a unique ID, which you can create or have {es} auto-generate. + +A simple {es} document might look like this: + +[source,js] +---- +{ + "_index": "my-first-elasticsearch-index", + "_id": "DyFpo5EBxE8fzbb95DOa", + "_version": 1, + "_seq_no": 0, + "_primary_term": 1, + "found": true, + "_source": { + "email": "john@smith.com", + "first_name": "John", + "last_name": "Smith", + "info": { + "bio": "Eco-warrior and defender of the weak", + "age": 25, + "interests": [ + "dolphins", + "whales" + ] + }, + "join_date": "2024/05/01" + } +} +---- +// NOTCONSOLE + +[discrete] +[[elasticsearch-intro-documents-fields-data-metadata]] +==== Data and metadata + +An indexed document contains data and metadata. +In {es}, metadata fields are prefixed with an underscore. + +The most important metadata fields are: + +* `_source`. Contains the original JSON document. +* `_index`. The name of the index where the document is stored. +* `_id`. The document's ID. IDs must be unique per index. + +[discrete] +[[elasticsearch-intro-documents-fields-mappings]] +==== Mappings and data types + +Each index has a <> or schema for how the fields in your documents are indexed. +A mapping defines the <> for each field, how the field should be indexed, +and how it should be stored. +When adding documents to {es}, you have two options for mappings: + +* <>. Let {es} automatically detect the data types and create the mappings for you. This is great for getting started quickly. +* <>. Define the mappings up front by specifying data types for each field. Recommended for production use cases. + +[TIP] +==== +You can use a combination of dynamic and explicit mapping on the same index. +This is useful when you have a mix of known and unknown fields in your data. +==== + +// New html page [[search-analyze]] -=== Information out: search and analyze +=== Search and analyze While you can use {es} as a document store and retrieve documents and their metadata, the real power comes from being able to easily access the full suite @@ -160,27 +228,8 @@ size 70 needles, you’re displaying a count of the size 70 needles that match your users' search criteria--for example, all size 70 _non-stick embroidery_ needles. -[discrete] -[[more-features]] -===== But wait, there’s more - -Want to automate the analysis of your time series data? You can use -{ml-docs}/ml-ad-overview.html[machine learning] features to create accurate -baselines of normal behavior in your data and identify anomalous patterns. With -machine learning, you can detect: - -* Anomalies related to temporal deviations in values, counts, or frequencies -* Statistical rarity -* Unusual behaviors for a member of a population - -And the best part? You can do this without having to specify algorithms, models, -or other data science-related configurations. - [[scalability]] -=== Scalability and resilience: clusters, nodes, and shards -++++ -Scalability and resilience -++++ +=== Scalability and resilience {es} is built to be always available and to scale with your needs. It does this by being distributed by nature. You can add servers (nodes) to a cluster to @@ -209,7 +258,7 @@ interrupting indexing or query operations. [discrete] [[it-depends]] -==== It depends... +==== Shard size and number of shards There are a number of performance considerations and trade offs with respect to shard size and the number of primary shards configured for an index. The more @@ -237,7 +286,7 @@ testing with your own data and queries]. [discrete] [[disaster-ccr]] -==== In case of disaster +==== Disaster recovery A cluster's nodes need good, reliable connections to each other. To provide better connections, you typically co-locate the nodes in the same data center or @@ -257,7 +306,7 @@ secondary clusters are read-only followers. [discrete] [[admin]] -==== Care and feeding +==== Security, management, and monitoring As with any enterprise system, you need tools to secure, manage, and monitor your {es} clusters. Security, monitoring, and administrative features @@ -265,3 +314,5 @@ that are integrated into {es} enable you to use {kibana-ref}/introduction.html[{ as a control center for managing a cluster. Features like <> and <> help you intelligently manage your data over time. + +Refer to <> for more information. \ No newline at end of file diff --git a/docs/reference/mapping/runtime.asciidoc b/docs/reference/mapping/runtime.asciidoc index dc21fcfb9261e..190081fa801b4 100644 --- a/docs/reference/mapping/runtime.asciidoc +++ b/docs/reference/mapping/runtime.asciidoc @@ -135,7 +135,7 @@ PUT my-index-000001/ "day_of_week": { "type": "keyword", "script": { - "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))" + "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))" } } }, @@ -291,7 +291,7 @@ GET my-index-000001/_search "day_of_week": { "type": "keyword", "script": { - "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))" + "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))" } } }, @@ -667,7 +667,7 @@ PUT my-index-000001/ "day_of_week": { "type": "keyword", "script": { - "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))" + "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))" } } }, diff --git a/docs/reference/mapping/types/geo-shape.asciidoc b/docs/reference/mapping/types/geo-shape.asciidoc index 20f79df8950af..e50c7d73b1b76 100644 --- a/docs/reference/mapping/types/geo-shape.asciidoc +++ b/docs/reference/mapping/types/geo-shape.asciidoc @@ -18,9 +18,8 @@ Documents using this type can be used: ** a <> (for example, intersecting polygons). * to aggregate documents by geographic grids: ** either <> -** or <>. - -Grid aggregations over `geo_hex` grids are not supported for `geo_shape` fields. +** or <> +** or <> [[geo-shape-mapping-options]] [discrete] diff --git a/docs/reference/mapping/types/semantic-text.asciidoc b/docs/reference/mapping/types/semantic-text.asciidoc index 522a0c54c8aad..a006f288dc66d 100644 --- a/docs/reference/mapping/types/semantic-text.asciidoc +++ b/docs/reference/mapping/types/semantic-text.asciidoc @@ -7,8 +7,8 @@ beta[] -The `semantic_text` field type automatically generates embeddings for text -content using an inference endpoint. +The `semantic_text` field type automatically generates embeddings for text content using an inference endpoint. +Long passages are <> to smaller sections to enable the processing of larger corpuses of text. The `semantic_text` field type specifies an inference endpoint identifier that will be used to generate embeddings. You can create the inference endpoint by using the <>. diff --git a/docs/reference/mapping/types/sparse-vector.asciidoc b/docs/reference/mapping/types/sparse-vector.asciidoc index d0c2c83b8a8fa..b24f65fcf97ca 100644 --- a/docs/reference/mapping/types/sparse-vector.asciidoc +++ b/docs/reference/mapping/types/sparse-vector.asciidoc @@ -91,7 +91,7 @@ NOTE: `sparse_vector` fields can not be included in indices that were *created* NOTE: `sparse_vector` fields only support strictly positive values. Negative values will be rejected. -NOTE: `sparse_vector` fields do not support querying, sorting or aggregating. +NOTE: `sparse_vector` fields do not support <>, querying, sorting or aggregating. They may only be used within specialized queries. The recommended query to use on these fields are <> queries. They may also be used within legacy <> queries. diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc index 44c2012f502e1..97122141d7558 100644 --- a/docs/reference/ml/ml-shared.asciidoc +++ b/docs/reference/ml/ml-shared.asciidoc @@ -951,7 +951,7 @@ For example: "day_of_week": { "type": "keyword", "script": { - "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))" + "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))" } } } diff --git a/docs/reference/ml/trained-models/apis/put-trained-models.asciidoc b/docs/reference/ml/trained-models/apis/put-trained-models.asciidoc index eef90630eb35b..e29bc8823ab29 100644 --- a/docs/reference/ml/trained-models/apis/put-trained-models.asciidoc +++ b/docs/reference/ml/trained-models/apis/put-trained-models.asciidoc @@ -588,7 +588,7 @@ Refer to <> to review the properties of the `tokenization` object. ===== -`text_similarity`:::: +`text_similarity`::: (Object, optional) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-similarity] + diff --git a/docs/reference/modules/cluster/remote-clusters-settings.asciidoc b/docs/reference/modules/cluster/remote-clusters-settings.asciidoc index 2308ec259da48..537783ef6ff01 100644 --- a/docs/reference/modules/cluster/remote-clusters-settings.asciidoc +++ b/docs/reference/modules/cluster/remote-clusters-settings.asciidoc @@ -6,7 +6,10 @@ mode are described separately. `cluster.remote..mode`:: The mode used for a remote cluster connection. The only supported modes are - `sniff` and `proxy`. + `sniff` and `proxy`. The default is `sniff`. See <> for + further information about these modes, and <> + and <> for further information about their + settings. `cluster.remote.initial_connect_timeout`:: @@ -97,6 +100,11 @@ you configure the remotes. [[remote-cluster-sniff-settings]] ==== Sniff mode remote cluster settings +To use <> to connect to a remote cluster, set +`cluster.remote..mode: sniff` and then configure the following +settings. You may also leave `cluster.remote..mode` unset since +`sniff` is the default mode. + `cluster.remote..seeds`:: The list of seed nodes used to sniff the remote cluster state. @@ -117,6 +125,10 @@ you configure the remotes. [[remote-cluster-proxy-settings]] ==== Proxy mode remote cluster settings +To use <> to connect to a remote cluster, set +`cluster.remote..mode: proxy` and then configure the following +settings. + `cluster.remote..proxy_address`:: The address used for all remote connections. diff --git a/docs/reference/modules/discovery/fault-detection.asciidoc b/docs/reference/modules/discovery/fault-detection.asciidoc index 383e4c6044c67..21f4ae2317e6a 100644 --- a/docs/reference/modules/discovery/fault-detection.asciidoc +++ b/docs/reference/modules/discovery/fault-detection.asciidoc @@ -35,269 +35,30 @@ starting from the beginning of the cluster state update. Refer to [[cluster-fault-detection-troubleshooting]] ==== Troubleshooting an unstable cluster -//tag::troubleshooting[] -Normally, a node will only leave a cluster if deliberately shut down. If a node -leaves the cluster unexpectedly, it's important to address the cause. A cluster -in which nodes leave unexpectedly is unstable and can create several issues. -For instance: -* The cluster health may be yellow or red. - -* Some shards will be initializing and other shards may be failing. - -* Search, indexing, and monitoring operations may fail and report exceptions in -logs. - -* The `.security` index may be unavailable, blocking access to the cluster. - -* The master may appear busy due to frequent cluster state updates. - -To troubleshoot a cluster in this state, first ensure the cluster has a -<>. Next, focus on the nodes -unexpectedly leaving the cluster ahead of all other issues. It will not be -possible to solve other issues until the cluster has a stable master node and -stable node membership. - -Diagnostics and statistics are usually not useful in an unstable cluster. These -tools only offer a view of the state of the cluster at a single point in time. -Instead, look at the cluster logs to see the pattern of behaviour over time. -Focus particularly on logs from the elected master. When a node leaves the -cluster, logs for the elected master include a message like this (with line -breaks added to make it easier to read): - -[source,text] ----- -[2022-03-21T11:02:35,513][INFO ][o.e.c.c.NodeLeftExecutor] [instance-0000000000] - node-left: [{instance-0000000004}{bfcMDTiDRkietFb9v_di7w}{aNlyORLASam1ammv2DzYXA}{172.27.47.21}{172.27.47.21:19054}{m}] - with reason [disconnected] ----- - -This message says that the `NodeLeftExecutor` on the elected master -(`instance-0000000000`) processed a `node-left` task, identifying the node that -was removed and the reason for its removal. When the node joins the cluster -again, logs for the elected master will include a message like this (with line -breaks added to make it easier to read): - -[source,text] ----- -[2022-03-21T11:02:59,892][INFO ][o.e.c.c.NodeJoinExecutor] [instance-0000000000] - node-join: [{instance-0000000004}{bfcMDTiDRkietFb9v_di7w}{UNw_RuazQCSBskWZV8ID_w}{172.27.47.21}{172.27.47.21:19054}{m}] - with reason [joining after restart, removed [24s] ago with reason [disconnected]] ----- - -This message says that the `NodeJoinExecutor` on the elected master -(`instance-0000000000`) processed a `node-join` task, identifying the node that -was added to the cluster and the reason for the task. - -Other nodes may log similar messages, but report fewer details: - -[source,text] ----- -[2020-01-29T11:02:36,985][INFO ][o.e.c.s.ClusterApplierService] - [instance-0000000001] removed { - {instance-0000000004}{bfcMDTiDRkietFb9v_di7w}{aNlyORLASam1ammv2DzYXA}{172.27.47.21}{172.27.47.21:19054}{m} - {tiebreaker-0000000003}{UNw_RuazQCSBskWZV8ID_w}{bltyVOQ-RNu20OQfTHSLtA}{172.27.161.154}{172.27.161.154:19251}{mv} - }, term: 14, version: 1653415, reason: Publication{term=14, version=1653415} ----- - -These messages are not especially useful for troubleshooting, so focus on the -ones from the `NodeLeftExecutor` and `NodeJoinExecutor` which are only emitted -on the elected master and which contain more details. If you don't see the -messages from the `NodeLeftExecutor` and `NodeJoinExecutor`, check that: - -* You're looking at the logs for the elected master node. - -* The logs cover the correct time period. - -* Logging is enabled at `INFO` level. - -Nodes will also log a message containing `master node changed` whenever they -start or stop following the elected master. You can use these messages to -determine each node's view of the state of the master over time. - -If a node restarts, it will leave the cluster and then join the cluster again. -When it rejoins, the `NodeJoinExecutor` will log that it processed a -`node-join` task indicating that the node is `joining after restart`. If a node -is unexpectedly restarting, look at the node's logs to see why it is shutting -down. - -The <> API on the affected node will also provide some useful -information about the situation. - -If the node did not restart then you should look at the reason for its -departure more closely. Each reason has different troubleshooting steps, -described below. There are three possible reasons: - -* `disconnected`: The connection from the master node to the removed node was -closed. - -* `lagging`: The master published a cluster state update, but the removed node -did not apply it within the permitted timeout. By default, this timeout is 2 -minutes. Refer to <> for information about the -settings which control this mechanism. - -* `followers check retry count exceeded`: The master sent a number of -consecutive health checks to the removed node. These checks were rejected or -timed out. By default, each health check times out after 10 seconds and {es} -removes the node removed after three consecutively failed health checks. Refer -to <> for information about the settings which -control this mechanism. +See <>. [discrete] ===== Diagnosing `disconnected` nodes -Nodes typically leave the cluster with reason `disconnected` when they shut -down, but if they rejoin the cluster without restarting then there is some -other problem. - -{es} is designed to run on a fairly reliable network. It opens a number of TCP -connections between nodes and expects these connections to remain open forever. -If a connection is closed then {es} will try and reconnect, so the occasional -blip should have limited impact on the cluster even if the affected node -briefly leaves the cluster. In contrast, repeatedly-dropped connections will -severely affect its operation. - -The connections from the elected master node to every other node in the cluster -are particularly important. The elected master never spontaneously closes its -outbound connections to other nodes. Similarly, once a connection is fully -established, a node never spontaneously close its inbound connections unless -the node is shutting down. - -If you see a node unexpectedly leave the cluster with the `disconnected` -reason, something other than {es} likely caused the connection to close. A -common cause is a misconfigured firewall with an improper timeout or another -policy that's <>. It could also -be caused by general connectivity issues, such as packet loss due to faulty -hardware or network congestion. If you're an advanced user, configure the -following loggers to get more detailed information about network exceptions: - -[source,yaml] ----- -logger.org.elasticsearch.transport.TcpTransport: DEBUG -logger.org.elasticsearch.xpack.core.security.transport.netty4.SecurityNetty4Transport: DEBUG ----- - -If these logs do not show enough information to diagnose the problem, obtain a -packet capture simultaneously from the nodes at both ends of an unstable -connection and analyse it alongside the {es} logs from those nodes to determine -if traffic between the nodes is being disrupted by another device on the -network. +See <>. [discrete] ===== Diagnosing `lagging` nodes -{es} needs every node to process cluster state updates reasonably quickly. If a -node takes too long to process a cluster state update, it can be harmful to the -cluster. The master will remove these nodes with the `lagging` reason. Refer to -<> for information about the settings which control -this mechanism. - -Lagging is typically caused by performance issues on the removed node. However, -a node may also lag due to severe network delays. To rule out network delays, -ensure that `net.ipv4.tcp_retries2` is <>. Log messages that contain `warn threshold` may provide more -information about the root cause. - -If you're an advanced user, you can get more detailed information about what -the node was doing when it was removed by configuring the following logger: - -[source,yaml] ----- -logger.org.elasticsearch.cluster.coordination.LagDetector: DEBUG ----- - -When this logger is enabled, {es} will attempt to run the -<> API on the faulty node and report the results in -the logs on the elected master. The results are compressed, encoded, and split -into chunks to avoid truncation: - -[source,text] ----- -[DEBUG][o.e.c.c.LagDetector ] [master] hot threads from node [{node}{g3cCUaMDQJmQ2ZLtjr-3dg}{10.0.0.1:9300}] lagging at version [183619] despite commit of cluster state version [183620] [part 1]: H4sIAAAAAAAA/x... -[DEBUG][o.e.c.c.LagDetector ] [master] hot threads from node [{node}{g3cCUaMDQJmQ2ZLtjr-3dg}{10.0.0.1:9300}] lagging at version [183619] despite commit of cluster state version [183620] [part 2]: p7x3w1hmOQVtuV... -[DEBUG][o.e.c.c.LagDetector ] [master] hot threads from node [{node}{g3cCUaMDQJmQ2ZLtjr-3dg}{10.0.0.1:9300}] lagging at version [183619] despite commit of cluster state version [183620] [part 3]: v7uTboMGDbyOy+... -[DEBUG][o.e.c.c.LagDetector ] [master] hot threads from node [{node}{g3cCUaMDQJmQ2ZLtjr-3dg}{10.0.0.1:9300}] lagging at version [183619] despite commit of cluster state version [183620] [part 4]: 4tse0RnPnLeDNN... -[DEBUG][o.e.c.c.LagDetector ] [master] hot threads from node [{node}{g3cCUaMDQJmQ2ZLtjr-3dg}{10.0.0.1:9300}] lagging at version [183619] despite commit of cluster state version [183620] (gzip compressed, base64-encoded, and split into 4 parts on preceding log lines) ----- - -To reconstruct the output, base64-decode the data and decompress it using -`gzip`. For instance, on Unix-like systems: - -[source,sh] ----- -cat lagdetector.log | sed -e 's/.*://' | base64 --decode | gzip --decompress ----- +See <>. [discrete] ===== Diagnosing `follower check retry count exceeded` nodes -Nodes sometimes leave the cluster with reason `follower check retry count -exceeded` when they shut down, but if they rejoin the cluster without -restarting then there is some other problem. - -{es} needs every node to respond to network messages successfully and -reasonably quickly. If a node rejects requests or does not respond at all then -it can be harmful to the cluster. If enough consecutive checks fail then the -master will remove the node with reason `follower check retry count exceeded` -and will indicate in the `node-left` message how many of the consecutive -unsuccessful checks failed and how many of them timed out. Refer to -<> for information about the settings which control -this mechanism. - -Timeouts and failures may be due to network delays or performance problems on -the affected nodes. Ensure that `net.ipv4.tcp_retries2` is -<> to eliminate network delays as -a possible cause for this kind of instability. Log messages containing -`warn threshold` may give further clues about the cause of the instability. - -If the last check failed with an exception then the exception is reported, and -typically indicates the problem that needs to be addressed. If any of the -checks timed out then narrow down the problem as follows. - -include::../../troubleshooting/network-timeouts.asciidoc[tag=troubleshooting-network-timeouts-gc-vm] - -include::../../troubleshooting/network-timeouts.asciidoc[tag=troubleshooting-network-timeouts-packet-capture-fault-detection] - -include::../../troubleshooting/network-timeouts.asciidoc[tag=troubleshooting-network-timeouts-threads] - -By default the follower checks will time out after 30s, so if node departures -are unpredictable then capture stack dumps every 15s to be sure that at least -one stack dump was taken at the right time. +See <>. [discrete] ===== Diagnosing `ShardLockObtainFailedException` failures -If a node leaves and rejoins the cluster then {es} will usually shut down and -re-initialize its shards. If the shards do not shut down quickly enough then -{es} may fail to re-initialize them due to a `ShardLockObtainFailedException`. +See <>. -To gather more information about the reason for shards shutting down slowly, -configure the following logger: - -[source,yaml] ----- -logger.org.elasticsearch.env.NodeEnvironment: DEBUG ----- - -When this logger is enabled, {es} will attempt to run the -<> API whenever it encounters a -`ShardLockObtainFailedException`. The results are compressed, encoded, and -split into chunks to avoid truncation: - -[source,text] ----- -[DEBUG][o.e.e.NodeEnvironment ] [master] hot threads while failing to obtain shard lock for [index][0] [part 1]: H4sIAAAAAAAA/x... -[DEBUG][o.e.e.NodeEnvironment ] [master] hot threads while failing to obtain shard lock for [index][0] [part 2]: p7x3w1hmOQVtuV... -[DEBUG][o.e.e.NodeEnvironment ] [master] hot threads while failing to obtain shard lock for [index][0] [part 3]: v7uTboMGDbyOy+... -[DEBUG][o.e.e.NodeEnvironment ] [master] hot threads while failing to obtain shard lock for [index][0] [part 4]: 4tse0RnPnLeDNN... -[DEBUG][o.e.e.NodeEnvironment ] [master] hot threads while failing to obtain shard lock for [index][0] (gzip compressed, base64-encoded, and split into 4 parts on preceding log lines) ----- - -To reconstruct the output, base64-decode the data and decompress it using -`gzip`. For instance, on Unix-like systems: +[discrete] +===== Diagnosing other network disconnections -[source,sh] ----- -cat shardlock.log | sed -e 's/.*://' | base64 --decode | gzip --decompress ----- -//end::troubleshooting[] +See <>. diff --git a/docs/reference/modules/network.asciidoc b/docs/reference/modules/network.asciidoc index 593aa79ded4d9..8fdc9f2e4f9cb 100644 --- a/docs/reference/modules/network.asciidoc +++ b/docs/reference/modules/network.asciidoc @@ -5,7 +5,9 @@ Each {es} node has two different network interfaces. Clients send requests to {es}'s REST APIs using its <>, but nodes communicate with other nodes using the <>. The transport interface is also used for communication with -<>. +<>. The transport interface uses a custom +binary protocol sent over <> TCP channels. +Both interfaces can be configured to use <>. You can configure both of these interfaces at the same time using the `network.*` settings. If you have a more complicated network, you might need to diff --git a/docs/reference/modules/remote-clusters.asciidoc b/docs/reference/modules/remote-clusters.asciidoc index 25217302b7631..ca1c507aa4ed9 100644 --- a/docs/reference/modules/remote-clusters.asciidoc +++ b/docs/reference/modules/remote-clusters.asciidoc @@ -1,7 +1,7 @@ [[remote-clusters]] == Remote clusters You can connect a local cluster to other {es} clusters, known as _remote -clusters_. Remote clusters can be located in different datacenters or +clusters_. Remote clusters can be located in different datacenters or geographic regions, and contain indices or data streams that can be replicated with {ccr} or searched by a local cluster using {ccs}. @@ -30,9 +30,9 @@ capabilities, the local and remote cluster must be on the same [discrete] === Add remote clusters -NOTE: The instructions that follow describe how to create a remote connection from a -self-managed cluster. You can also set up {ccs} and {ccr} from an -link:https://www.elastic.co/guide/en/cloud/current/ec-enable-ccs.html[{ess} deployment] +NOTE: The instructions that follow describe how to create a remote connection from a +self-managed cluster. You can also set up {ccs} and {ccr} from an +link:https://www.elastic.co/guide/en/cloud/current/ec-enable-ccs.html[{ess} deployment] or from an link:https://www.elastic.co/guide/en/cloud-enterprise/current/ece-enable-ccs.html[{ece} deployment]. To add remote clusters, you can choose between @@ -52,7 +52,7 @@ controls. <>. Certificate based security model:: Uses mutual TLS authentication for cross-cluster operations. User authentication -is performed on the local cluster and a user's role names are passed to the +is performed on the local cluster and a user's role names are passed to the remote cluster. In this model, a superuser on the local cluster gains total read access to the remote cluster, so it is only suitable for clusters that are in the same security domain. <>. @@ -63,13 +63,17 @@ the same security domain. <>. [[sniff-mode]] Sniff mode:: -In sniff mode, a cluster is created using a name and a list of seed nodes. When -a remote cluster is registered, its cluster state is retrieved from one of the -seed nodes and up to three _gateway nodes_ are selected as part of remote -cluster requests. This mode requires that the gateway node's publish addresses -are accessible by the local cluster. +In sniff mode, a cluster alias is registered with a name of your choosing and a +list of addresses of _seed_ nodes specified with the +`cluster.remote..seeds` setting. When you register a remote +cluster using sniff mode, {es} retrieves from one of the seed nodes the +addresses of up to three _gateway nodes_. Each `remote_cluster_client` node in +the local {es} cluster then opens several TCP connections to the publish +addresses of the gateway nodes. This mode therefore requires that the gateway +nodes' publish addresses are accessible to nodes in the local cluster. + -Sniff mode is the default connection mode. +Sniff mode is the default connection mode. See <> +for more information about configuring sniff mode. + [[gateway-nodes-selection]] The _gateway nodes_ selection depends on the following criteria: @@ -84,13 +88,21 @@ However, such nodes still have to satisfy the two above requirements. [[proxy-mode]] Proxy mode:: -In proxy mode, a cluster is created using a name and a single proxy address. -When you register a remote cluster, a configurable number of socket connections -are opened to the proxy address. The proxy is required to route those -connections to the remote cluster. Proxy mode does not require remote cluster -nodes to have accessible publish addresses. +In proxy mode, a cluster alias is registered with a name of your choosing and +the address of a TCP (layer 4) reverse proxy specified with the +`cluster.remote..proxy_address` setting. You must configure this +proxy to route connections to one or more nodes of the remote cluster. When you +register a remote cluster using proxy mode, {es} opens several TCP connections +to the proxy address and uses these connections to communicate with the remote +cluster. In proxy mode {es} disregards the publish addresses of the remote +cluster nodes which means that the publish addresses of the remote cluster +nodes need not be accessible to the local cluster. ++ +Proxy mode is not the default connection mode, so you must set +`cluster.remote..mode: proxy` to use it. See +<> for more information about configuring proxy +mode. + -The proxy mode is not the default connection mode and must be configured. Proxy mode has the same <> as sniff mode. diff --git a/docs/reference/modules/transport.asciidoc b/docs/reference/modules/transport.asciidoc index d08da2cfc1d2f..fc7b6831ca848 100644 --- a/docs/reference/modules/transport.asciidoc +++ b/docs/reference/modules/transport.asciidoc @@ -185,16 +185,18 @@ configured, and defaults otherwise to `transport.tcp.reuse_address`. A transport connection between two nodes is made up of a number of long-lived TCP connections, some of which may be idle for an extended period of time. -Nonetheless, Elasticsearch requires these connections to remain open, and it -can disrupt the operation of your cluster if any inter-node connections are -closed by an external influence such as a firewall. It is important to -configure your network to preserve long-lived idle connections between -Elasticsearch nodes, for instance by leaving `*.tcp.keep_alive` enabled and -ensuring that the keepalive interval is shorter than any timeout that might -cause idle connections to be closed, or by setting `transport.ping_schedule` if -keepalives cannot be configured. Devices which drop connections when they reach -a certain age are a common source of problems to Elasticsearch clusters, and -must not be used. +Nonetheless, {es} requires these connections to remain open, and it can disrupt +the operation of your cluster if any inter-node connections are closed by an +external influence such as a firewall. It is important to configure your network +to preserve long-lived idle connections between {es} nodes, for instance by +leaving `*.tcp.keep_alive` enabled and ensuring that the keepalive interval is +shorter than any timeout that might cause idle connections to be closed, or by +setting `transport.ping_schedule` if keepalives cannot be configured. Devices +which drop connections when they reach a certain age are a common source of +problems to {es} clusters, and must not be used. + +For information about troubleshooting unexpected network disconnections, see +<>. [[request-compression]] ===== Request compression diff --git a/docs/reference/monitoring/production.asciidoc b/docs/reference/monitoring/production.asciidoc index 381f67e254041..86ffa99fa7f59 100644 --- a/docs/reference/monitoring/production.asciidoc +++ b/docs/reference/monitoring/production.asciidoc @@ -73,7 +73,9 @@ credentials must be valid on both the {kib} server and the monitoring cluster. *** If you plan to use {agent}, create a user that has the `remote_monitoring_collector` -<>. +<> and that the +monitoring related {fleet-guide}/install-uninstall-integration-assets.html#install-integration-assets[integration assets have been installed] +on the remote monitoring cluster. *** If you plan to use {metricbeat}, create a user that has the `remote_monitoring_collector` built-in role and a diff --git a/docs/reference/quickstart/run-elasticsearch-locally.asciidoc b/docs/reference/quickstart/run-elasticsearch-locally.asciidoc index 0db395ba34b0a..8c75510ae860f 100644 --- a/docs/reference/quickstart/run-elasticsearch-locally.asciidoc +++ b/docs/reference/quickstart/run-elasticsearch-locally.asciidoc @@ -125,6 +125,10 @@ The service is started with a trial license. The trial license enables all featu To connect to the {es} cluster from a language client, you can use basic authentication with the `elastic` username and the password you set in the environment variable. +.*Expand* for details +[%collapsible] +============== + You'll use the following connection details: * **{es} endpoint**: `http://localhost:9200` @@ -160,6 +164,8 @@ curl -u elastic:$ELASTIC_PASSWORD \ ---- // NOTCONSOLE +============== + [discrete] [[local-dev-next-steps]] === Next steps diff --git a/docs/reference/release-notes/8.15.0.asciidoc b/docs/reference/release-notes/8.15.0.asciidoc index 2069c1bd96ff0..bed1912fc1b84 100644 --- a/docs/reference/release-notes/8.15.0.asciidoc +++ b/docs/reference/release-notes/8.15.0.asciidoc @@ -26,6 +26,10 @@ memory lock feature (issue: {es-issue}111847[#111847]) <> assume that this value will be within a particular range (e.g. that it fits into a 32-bit signed integer) may encounter errors (issue: {es-issue}111854[#111854]) +* Elasticsearch will not start if custom role mappings are configured using the +`xpack.security.authc.realms.*.files.role_mapping` configuration option. As a workaround, custom role mappings +can be configured using the https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-put-role-mapping.html[REST API] (issue: {es-issue}112503[#112503]) + [[breaking-8.15.0]] [float] === Breaking changes diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc index e5ab10b7d71ba..fabd495cdc525 100644 --- a/docs/reference/rest-api/common-parms.asciidoc +++ b/docs/reference/rest-api/common-parms.asciidoc @@ -1327,13 +1327,21 @@ that lower ranked documents have more influence. This value must be greater than equal to `1`. Defaults to `60`. end::rrf-rank-constant[] -tag::rrf-window-size[] -`window_size`:: +tag::rrf-rank-window-size[] +`rank_window_size`:: (Optional, integer) + This value determines the size of the individual result sets per query. A higher value will improve result relevance at the cost of performance. The final ranked result set is pruned down to the search request's <>. -`window_size` must be greater than or equal to `size` and greater than or equal to `1`. +`rank_window_size` must be greater than or equal to `size` and greater than or equal to `1`. Defaults to the `size` parameter. -end::rrf-window-size[] +end::rrf-rank-window-size[] + +tag::rrf-filter[] +`filter`:: +(Optional, <>) ++ +Applies the specified <> to all of the specified sub-retrievers, +according to each retriever's specifications. +end::rrf-filter[] diff --git a/docs/reference/search/point-in-time-api.asciidoc b/docs/reference/search/point-in-time-api.asciidoc index 2e32324cb44d9..9cd91626c7600 100644 --- a/docs/reference/search/point-in-time-api.asciidoc +++ b/docs/reference/search/point-in-time-api.asciidoc @@ -78,6 +78,44 @@ IMPORTANT: The open point in time request and each subsequent search request can return different `id`; thus always use the most recently received `id` for the next search request. +In addition to the `keep_alive` parameter, the `allow_partial_search_results` parameter +can also be defined. +This parameter determines whether the <> +should tolerate unavailable shards or <> when +initially creating the PIT. +If set to true, the PIT will be created with the available shards, along with a +reference to any missing ones. +If set to false, the operation will fail if any shard is unavailable. +The default value is false. + +The PIT response includes a summary of the total number of shards, as well as the number +of successful shards when creating the PIT. + +[source,console] +-------------------------------------------------- +POST /my-index-000001/_pit?keep_alive=1m&allow_partial_search_results=true +-------------------------------------------------- +// TEST[setup:my_index] + +[source,js] +-------------------------------------------------- +{ + "id": "46ToAwMDaWR5BXV1aWQyKwZub2RlXzMAAAAAAAAAACoBYwADaWR4BXV1aWQxAgZub2RlXzEAAAAAAAAAAAEBYQADaWR5BXV1aWQyKgZub2RlXzIAAAAAAAAAAAwBYgACBXV1aWQyAAAFdXVpZDEAAQltYXRjaF9hbGw_gAAAAA=", + "_shards": { + "total": 10, + "successful": 10, + "skipped": 0, + "failed": 0 + } +} +-------------------------------------------------- +// NOTCONSOLE + +When a PIT that contains shard failures is used in a search request, the missing are +always reported in the search response as a NoShardAvailableActionException exception. +To get rid of these exceptions, a new PIT needs to be created so that shards missing +from the previous PIT can be handled, assuming they become available in the meantime. + [[point-in-time-keep-alive]] ==== Keeping point in time alive The `keep_alive` parameter, which is passed to a open point in time request and diff --git a/docs/reference/search/retriever.asciidoc b/docs/reference/search/retriever.asciidoc index b52b296220029..58cc8ce9ef459 100644 --- a/docs/reference/search/retriever.asciidoc +++ b/docs/reference/search/retriever.asciidoc @@ -198,7 +198,7 @@ GET my-embeddings/_search An <> retriever returns top documents based on the RRF formula, equally weighting two or more child retrievers. -Reciprocal rank fusion (RRF) is a method for combining multiple result +Reciprocal rank fusion (RRF) is a method for combining multiple result sets with different relevance indicators into a single result set. ===== Parameters @@ -207,7 +207,9 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=rrf-retrievers] include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=rrf-rank-constant] -include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=rrf-window-size] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=rrf-rank-window-size] + +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=rrf-filter] ===== Restrictions @@ -225,7 +227,7 @@ A simple hybrid search example (lexical search + dense vector search) combining ---- GET /restaurants/_search { - "retriever": { + "retriever": { "rrf": { <1> "retrievers": [ <2> { @@ -340,6 +342,10 @@ Currently you can: ** Refer to the <> on this page for a step-by-step guide. ===== Parameters +`retriever`:: +(Required, <>) ++ +The child retriever that generates the initial set of top documents to be re-ranked. `field`:: (Required, `string`) @@ -366,6 +372,13 @@ The number of top documents to consider in the re-ranking process. Defaults to ` + Sets a minimum threshold score for including documents in the re-ranked results. Documents with similarity scores below this threshold will be excluded. Note that score calculations vary depending on the model used. +`filter`:: +(Optional, <>) ++ +Applies the specified <> to the child <>. +If the child retriever already specifies any filters, then this top-level filter is applied in conjuction +with the filter defined in the child retriever. + ===== Restrictions A text similarity re-ranker retriever is a compound retriever. Child retrievers may not use elements that are restricted by having a compound retriever as part of the retriever tree. @@ -441,13 +454,13 @@ eland_import_hub_model \ + [source,js] ---- -PUT _inference/rerank/my-msmarco-minilm-model +PUT _inference/rerank/my-msmarco-minilm-model { "service": "elasticsearch", "service_settings": { "num_allocations": 1, "num_threads": 1, - "model_id": "cross-encoder__ms-marco-minilm-l-6-v2" + "model_id": "cross-encoder__ms-marco-minilm-l-6-v2" } } ---- diff --git a/docs/reference/search/rrf.asciidoc b/docs/reference/search/rrf.asciidoc index fb474fe6bf4e6..2525dfff23b94 100644 --- a/docs/reference/search/rrf.asciidoc +++ b/docs/reference/search/rrf.asciidoc @@ -1,9 +1,7 @@ [[rrf]] === Reciprocal rank fusion -preview::["This functionality is in technical preview and may be changed or removed in a future release. -The syntax will likely change before GA. -Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] +preview::["This functionality is in technical preview and may be changed or removed in a future release. The syntax will likely change before GA. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf[Reciprocal rank fusion (RRF)] is a method for combining multiple result sets with different relevance indicators into a single result set. @@ -43,7 +41,7 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=rrf-retrievers] include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=rrf-rank-constant] -include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=rrf-window-size] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=rrf-rank-window-size] An example request using RRF: diff --git a/docs/reference/search/search-your-data/near-real-time.asciidoc b/docs/reference/search/search-your-data/near-real-time.asciidoc index 46a996c237c38..47618ecd9fd7a 100644 --- a/docs/reference/search/search-your-data/near-real-time.asciidoc +++ b/docs/reference/search/search-your-data/near-real-time.asciidoc @@ -2,7 +2,7 @@ [[near-real-time]] === Near real-time search -The overview of <> indicates that when a document is stored in {es}, it is indexed and fully searchable in _near real-time_--within 1 second. What defines near real-time search? +When a document is stored in {es}, it is indexed and fully searchable in _near real-time_--within 1 second. What defines near real-time search? Lucene, the Java libraries on which {es} is based, introduced the concept of per-segment search. A _segment_ is similar to an inverted index, but the word _index_ in Lucene means "a collection of segments plus a commit point". After a commit, a new segment is added to the commit point and the buffer is cleared. diff --git a/docs/reference/search/search-your-data/paginate-search-results.asciidoc b/docs/reference/search/search-your-data/paginate-search-results.asciidoc index edd1546dd0854..f69fd60be0484 100644 --- a/docs/reference/search/search-your-data/paginate-search-results.asciidoc +++ b/docs/reference/search/search-your-data/paginate-search-results.asciidoc @@ -106,9 +106,9 @@ The search response includes an array of `sort` values for each hit: "_id" : "654322", "_score" : null, "_source" : ..., - "sort" : [ + "sort" : [ 1463538855, - "654322" + "654322" ] }, { @@ -118,7 +118,7 @@ The search response includes an array of `sort` values for each hit: "_source" : ..., "sort" : [ <1> 1463538857, - "654323" + "654323" ] } ] @@ -150,7 +150,7 @@ GET twitter/_search -------------------------------------------------- //TEST[continued] -Repeat this process by updating the `search_after` array every time you retrieve a +Repeat this process by updating the `search_after` array every time you retrieve a new page of results. If a <> occurs between these requests, the order of your results may change, causing inconsistent results across pages. To prevent this, you can create a <> to @@ -167,10 +167,12 @@ The API returns a PIT ID. [source,console-result] ---- { - "id": "46ToAwMDaWR5BXV1aWQyKwZub2RlXzMAAAAAAAAAACoBYwADaWR4BXV1aWQxAgZub2RlXzEAAAAAAAAAAAEBYQADaWR5BXV1aWQyKgZub2RlXzIAAAAAAAAAAAwBYgACBXV1aWQyAAAFdXVpZDEAAQltYXRjaF9hbGw_gAAAAA==" + "id": "46ToAwMDaWR5BXV1aWQyKwZub2RlXzMAAAAAAAAAACoBYwADaWR4BXV1aWQxAgZub2RlXzEAAAAAAAAAAAEBYQADaWR5BXV1aWQyKgZub2RlXzIAAAAAAAAAAAwBYgACBXV1aWQyAAAFdXVpZDEAAQltYXRjaF9hbGw_gAAAAA==", + "_shards": ... } ---- // TESTRESPONSE[s/"id": "46ToAwMDaWR5BXV1aWQyKwZub2RlXzMAAAAAAAAAACoBYwADaWR4BXV1aWQxAgZub2RlXzEAAAAAAAAAAAEBYQADaWR5BXV1aWQyKgZub2RlXzIAAAAAAAAAAAwBYgACBXV1aWQyAAAFdXVpZDEAAQltYXRjaF9hbGw_gAAAAA=="/"id": $body.id/] +// TESTRESPONSE[s/"_shards": \.\.\./"_shards": "$body._shards"/] To get the first page of results, submit a search request with a `sort` argument. If using a PIT, specify the PIT ID in the `pit.id` parameter and omit diff --git a/docs/reference/search/search-your-data/retrievers-reranking/retrievers-overview.asciidoc b/docs/reference/search/search-your-data/retrievers-reranking/retrievers-overview.asciidoc index 99659ae76e092..c0fe7471946f3 100644 --- a/docs/reference/search/search-your-data/retrievers-reranking/retrievers-overview.asciidoc +++ b/docs/reference/search/search-your-data/retrievers-reranking/retrievers-overview.asciidoc @@ -13,23 +13,23 @@ For implementation details, including notable restrictions, check out the [discrete] [[retrievers-overview-types]] -==== Retriever types +==== Retriever types Retrievers come in various types, each tailored for different search operations. The following retrievers are currently available: -* <>. Returns top documents from a -traditional https://www.elastic.co/guide/en/elasticsearch/reference/master/query-dsl.html[query]. -Mimics a traditional query but in the context of a retriever framework. This -ensures backward compatibility as existing `_search` requests remain supported. -That way you can transition to the new abstraction at your own pace without +* <>. Returns top documents from a +traditional https://www.elastic.co/guide/en/elasticsearch/reference/master/query-dsl.html[query]. +Mimics a traditional query but in the context of a retriever framework. This +ensures backward compatibility as existing `_search` requests remain supported. +That way you can transition to the new abstraction at your own pace without mixing syntaxes. -* <>. Returns top documents from a <>, +* <>. Returns top documents from a <>, in the context of a retriever framework. * <>. Combines and ranks multiple first-stage retrievers using -the reciprocal rank fusion (RRF) algorithm. Allows you to combine multiple result sets +the reciprocal rank fusion (RRF) algorithm. Allows you to combine multiple result sets with different relevance indicators into a single result set. -An RRF retriever is a *compound retriever*, where its `filter` element is +An RRF retriever is a *compound retriever*, where its `filter` element is propagated to its sub retrievers. + Sub retrievers may not use elements that are restricted by having a compound retriever as part of the retriever tree. @@ -38,7 +38,7 @@ See the <> for detaile Requires first creating a `rerank` task using the <>. [discrete] -==== What makes retrievers useful? +==== What makes retrievers useful? Here's an overview of what makes retrievers useful and how they differ from regular queries. @@ -140,7 +140,7 @@ GET example-index/_search ], "rank":{ "rrf":{ - "window_size":50, + "rank_window_size":50, "rank_constant":20 } } @@ -155,14 +155,14 @@ GET example-index/_search Here are some important terms: -* *Retrieval Pipeline*. Defines the entire retrieval and ranking logic to +* *Retrieval Pipeline*. Defines the entire retrieval and ranking logic to produce top hits. * *Retriever Tree*. A hierarchical structure that defines how retrievers interact. * *First-stage Retriever*. Returns an initial set of candidate documents. -* *Compound Retriever*. Builds on one or more retrievers, +* *Compound Retriever*. Builds on one or more retrievers, enhancing document retrieval and ranking logic. -* *Combiners*. Compound retrievers that merge top hits -from multiple sub-retrievers. +* *Combiners*. Compound retrievers that merge top hits +from multiple sub-retrievers. * *Rerankers*. Special compound retrievers that reorder hits and may adjust the number of hits, with distinctions between first-stage and second-stage rerankers. [discrete] @@ -180,4 +180,4 @@ Refer to the {kibana-ref}/playground.html[Playground documentation] for more inf [[retrievers-overview-api-reference]] ==== API reference -For implementation details, including notable restrictions, check out the <> in the Search API docs. \ No newline at end of file +For implementation details, including notable restrictions, check out the <> in the Search API docs. diff --git a/docs/reference/search/search-your-data/search-api.asciidoc b/docs/reference/search/search-your-data/search-api.asciidoc index 496812a0cedb4..98c5a48b7559b 100644 --- a/docs/reference/search/search-your-data/search-api.asciidoc +++ b/docs/reference/search/search-your-data/search-api.asciidoc @@ -173,7 +173,7 @@ GET /my-index-000001/_search "script": { "source": """emit(doc['@timestamp'].value.dayOfWeekEnum - .getDisplayName(TextStyle.FULL, Locale.ROOT))""" + .getDisplayName(TextStyle.FULL, Locale.ENGLISH))""" } } }, diff --git a/docs/reference/search/search-your-data/semantic-search-inference.asciidoc b/docs/reference/search/search-your-data/semantic-search-inference.asciidoc index f74bc65e31bf0..dee91a6aa4ec4 100644 --- a/docs/reference/search/search-your-data/semantic-search-inference.asciidoc +++ b/docs/reference/search/search-your-data/semantic-search-inference.asciidoc @@ -9,15 +9,20 @@ The instructions in this tutorial shows you how to use the {infer} API workflow IMPORTANT: For the easiest way to perform semantic search in the {stack}, refer to the <> end-to-end tutorial. -The following examples use Cohere's `embed-english-v3.0` model, the `all-mpnet-base-v2` model from HuggingFace, and OpenAI's `text-embedding-ada-002` second generation embedding model. +The following examples use the: + +* `embed-english-v3.0` model for https://docs.cohere.com/docs/cohere-embed[Cohere] +* `all-mpnet-base-v2` model from https://huggingface.co/sentence-transformers/all-mpnet-base-v2[HuggingFace] +* `text-embedding-ada-002` second generation embedding model for OpenAI +* models available through https://ai.azure.com/explore/models?selectedTask=embeddings[Azure AI Studio] or https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models[Azure OpenAI] +* `text-embedding-004` model for https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api[Google Vertex AI] +* `mistral-embed` model for https://docs.mistral.ai/getting-started/models/[Mistral] +* `amazon.titan-embed-text-v1` model for https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids.html[Amazon Bedrock] +* `ops-text-embedding-zh-001` model for https://help.aliyun.com/zh/open-search/search-platform/developer-reference/text-embedding-api-details[AlibabaCloud AI] + You can use any Cohere and OpenAI models, they are all supported by the {infer} API. For a list of recommended models available on HuggingFace, refer to <>. -Azure based examples use models available through https://ai.azure.com/explore/models?selectedTask=embeddings[Azure AI Studio] -or https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models[Azure OpenAI]. -Mistral examples use the `mistral-embed` model from https://docs.mistral.ai/getting-started/models/[the Mistral API]. -Amazon Bedrock examples use the `amazon.titan-embed-text-v1` model from https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids.html[the Amazon Bedrock base models]. - Click the name of the service you want to use on any of the widgets below to review the corresponding instructions. [discrete] @@ -73,8 +78,8 @@ Once the upload is complete, you can see an index named `test-data` with 182469 [[reindexing-data-infer]] ==== Ingest the data through the {infer} ingest pipeline -Create the embeddings from the text by reindexing the data through the {infer} -pipeline that uses the chosen model as the inference model. +Create embeddings from the text by reindexing the data through the {infer} pipeline that uses your chosen model. +This step uses the {ref}/docs-reindex.html[reindex API] to simulate data ingestion through a pipeline. include::{es-ref-dir}/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc[] @@ -113,5 +118,6 @@ include::{es-ref-dir}/tab-widgets/inference-api/infer-api-search-widget.asciidoc You can also find tutorials in an interactive Colab notebook format using the {es} Python client: + * https://colab.research.google.com/github/elastic/elasticsearch-labs/blob/main/notebooks/integrations/cohere/inference-cohere.ipynb[Cohere {infer} tutorial notebook] * https://colab.research.google.com/github/elastic/elasticsearch-labs/blob/main/notebooks/search/07-inference.ipynb[OpenAI {infer} tutorial notebook] diff --git a/docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc b/docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc index 5d838eb86dcf3..b47bc2370ab10 100644 --- a/docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc +++ b/docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc @@ -24,7 +24,10 @@ For more information, see <>. ==== {api-description-title} This API mounts a snapshot as a searchable snapshot index. -Note that manually mounting {ilm-init}-managed snapshots can <> with <>. + +Don't use this API for snapshots managed by {ilm-init}. Manually mounting +{ilm-init}-managed snapshots can <> with +<>. [[searchable-snapshots-api-mount-path-params]] ==== {api-path-parms-title} diff --git a/docs/reference/searchable-snapshots/index.asciidoc b/docs/reference/searchable-snapshots/index.asciidoc index a8a9ef36dc9a6..a38971a0bae6a 100644 --- a/docs/reference/searchable-snapshots/index.asciidoc +++ b/docs/reference/searchable-snapshots/index.asciidoc @@ -176,9 +176,12 @@ nodes that have a shared cache. ==== Manually mounting snapshots captured by an Index Lifecycle Management ({ilm-init}) policy can interfere with {ilm-init}'s automatic management. This may lead to issues such as data loss -or complications with snapshot handling. For optimal results, allow {ilm-init} to manage -snapshots automatically. If manual mounting is necessary, be aware of its potential -impact on {ilm-init} processes. For more information, learn about <>. +or complications with snapshot handling. + +For optimal results, allow {ilm-init} to manage +snapshots automatically. + +<>. ==== [[searchable-snapshots-shared-cache]] diff --git a/docs/reference/security/authorization/field-and-document-access-control.asciidoc b/docs/reference/security/authorization/field-and-document-access-control.asciidoc index f4d4fcd49a35f..7c7ea75ece161 100644 --- a/docs/reference/security/authorization/field-and-document-access-control.asciidoc +++ b/docs/reference/security/authorization/field-and-document-access-control.asciidoc @@ -54,8 +54,11 @@ specify any field restrictions. If you assign a user both roles, `role_a` gives the user access to all documents and `role_b` gives the user access to all fields. +[IMPORTANT] +=========== If you need to restrict access to both documents and fields, consider splitting documents by index instead. +=========== include::role-templates.asciidoc[] include::set-security-user.asciidoc[] diff --git a/docs/reference/security/authorization/privileges.asciidoc b/docs/reference/security/authorization/privileges.asciidoc index f15654bef2d1f..747b1eef40441 100644 --- a/docs/reference/security/authorization/privileges.asciidoc +++ b/docs/reference/security/authorization/privileges.asciidoc @@ -101,6 +101,9 @@ deprecated[7.5] Use `manage_transform` instead. + This privilege is not available in {serverless-full}. +`manage_data_stream_global_retention`:: +This privilege has no effect.deprecated[8.16] + `manage_enrich`:: All operations related to managing and executing enrich policies. @@ -223,6 +226,9 @@ security roles of the user who created or updated them. All cluster read-only operations, like cluster health and state, hot threads, node info, node and cluster stats, and pending cluster tasks. +`monitor_data_stream_global_retention`:: +This privilege has no effect.deprecated[8.16] + `monitor_enrich`:: All read-only operations related to managing and executing enrich policies. diff --git a/docs/reference/security/fips-140-compliance.asciidoc b/docs/reference/security/fips-140-compliance.asciidoc index bf880213c2073..5bf73d43541d6 100644 --- a/docs/reference/security/fips-140-compliance.asciidoc +++ b/docs/reference/security/fips-140-compliance.asciidoc @@ -55,7 +55,8 @@ so that the JVM uses FIPS validated implementations of NIST recommended cryptogr Elasticsearch has been tested with Bouncy Castle's https://repo1.maven.org/maven2/org/bouncycastle/bc-fips/1.0.2.4/bc-fips-1.0.2.4.jar[bc-fips 1.0.2.4] and https://repo1.maven.org/maven2/org/bouncycastle/bctls-fips/1.0.17/bctls-fips-1.0.17.jar[bctls-fips 1.0.17]. -Please refer to the [Support Matrix] for details on which combinations of JVM and security provider are supported in FIPS mode. Elasticsearch does not ship with a FIPS certified provider. It is the responsibility of the user +Please refer to the {es} +https://www.elastic.co/support/matrix#matrix_jvm[JVM support matrix] for details on which combinations of JVM and security provider are supported in FIPS mode. Elasticsearch does not ship with a FIPS certified provider. It is the responsibility of the user to install and configure the security provider to ensure compliance with FIPS 140-2. Using a FIPS certified provider will ensure that only approved cryptographic algorithms are used. diff --git a/docs/reference/setup/install/docker/docker-compose.yml b/docs/reference/setup/install/docker/docker-compose.yml index 4b4ecf401b7d4..15d8c11e2f12f 100644 --- a/docs/reference/setup/install/docker/docker-compose.yml +++ b/docs/reference/setup/install/docker/docker-compose.yml @@ -117,6 +117,7 @@ services: - cluster.name=${CLUSTER_NAME} - cluster.initial_master_nodes=es01,es02,es03 - discovery.seed_hosts=es01,es03 + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} - bootstrap.memory_lock=true - xpack.security.enabled=true - xpack.security.http.ssl.enabled=true @@ -156,6 +157,7 @@ services: - cluster.name=${CLUSTER_NAME} - cluster.initial_master_nodes=es01,es02,es03 - discovery.seed_hosts=es01,es02 + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} - bootstrap.memory_lock=true - xpack.security.enabled=true - xpack.security.http.ssl.enabled=true diff --git a/docs/reference/slm/apis/slm-put.asciidoc b/docs/reference/slm/apis/slm-put.asciidoc index be265554deef5..51ad571ee12e7 100644 --- a/docs/reference/slm/apis/slm-put.asciidoc +++ b/docs/reference/slm/apis/slm-put.asciidoc @@ -100,13 +100,19 @@ Minimum number of snapshots to retain, even if the snapshots have expired. ==== `schedule`:: -(Required, <>) +(Required, <> or <>) Periodic or absolute schedule at which the policy creates snapshots. {slm-init} applies `schedule` changes immediately. +Schedule may be either a Cron schedule or a time unit describing the interval between snapshots. +When using a time unit interval, the first snapshot is scheduled one interval after the policy modification time, and then again every interval after. + [[slm-api-put-example]] ==== {api-examples-title} + +[[slm-api-put-daily-policy]] +===== Create a policy Create a `daily-snapshots` lifecycle policy: [source,console] @@ -138,4 +144,25 @@ PUT /_slm/policy/daily-snapshots <6> Optional retention configuration <7> Keep snapshots for 30 days <8> Always keep at least 5 successful snapshots, even if they're more than 30 days old -<9> Keep no more than 50 successful snapshots, even if they're less than 30 days old \ No newline at end of file +<9> Keep no more than 50 successful snapshots, even if they're less than 30 days old + + +[[slm-api-put-hourly-policy]] +===== Use Interval Scheduling +Create an `hourly-snapshots` lifecycle policy using interval scheduling: + +[source,console] +-------------------------------------------------- +PUT /_slm/policy/hourly-snapshots +{ + "schedule": "1h", + "name": "", + "repository": "my_repository", + "config": { + "indices": ["data-*", "important"] + } +} +-------------------------------------------------- +// TEST[setup:setup-repository] +Creates a snapshot once every hour. The first snapshot will be created one hour after the policy is modified, +with subsequent snapshots being created every hour afterward. diff --git a/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc index 997dbbe8a20e6..d8d1cfaa2a2c7 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc @@ -37,6 +37,12 @@ id="infer-api-ingest-azure-ai-studio"> Azure AI Studio + +
+
+
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline.asciidoc index 6adf3d2ebbf46..f1652c1d8aff8 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline.asciidoc @@ -165,6 +165,32 @@ and the `output_field` that will contain the {infer} results. // end::azure-ai-studio[] +// tag::google-vertex-ai[] + +[source,console] +-------------------------------------------------- +PUT _ingest/pipeline/google_vertex_ai_embeddings +{ + "processors": [ + { + "inference": { + "model_id": "google_vertex_ai_embeddings", <1> + "input_output": { <2> + "input_field": "content", + "output_field": "content_embedding" + } + } + } + ] +} +-------------------------------------------------- +<1> The name of the inference endpoint you created by using the +<>, it's referred to as `inference_id` in that step. +<2> Configuration object that defines the `input_field` for the {infer} process +and the `output_field` that will contain the {infer} results. + +// end::google-vertex-ai[] + // tag::mistral[] [source,console] @@ -216,3 +242,29 @@ PUT _ingest/pipeline/amazon_bedrock_embeddings and the `output_field` that will contain the {infer} results. // end::amazon-bedrock[] + +// tag::alibabacloud-ai-search[] + +[source,console] +-------------------------------------------------- +PUT _ingest/pipeline/alibabacloud_ai_search_embeddings +{ + "processors": [ + { + "inference": { + "model_id": "alibabacloud_ai_search_embeddings", <1> + "input_output": { <2> + "input_field": "content", + "output_field": "content_embedding" + } + } + } + ] +} +-------------------------------------------------- +<1> The name of the inference endpoint you created by using the +<>, it's referred to as `inference_id` in that step. +<2> Configuration object that defines the `input_field` for the {infer} process +and the `output_field` that will contain the {infer} results. + +// end::alibabacloud-ai-search[] diff --git a/docs/reference/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc index 4e3a453a7bbea..ac09e9e5c7cdc 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc @@ -37,6 +37,12 @@ id="infer-api-mapping-azure-ai-studio"> Azure AI Studio + +
+
+
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-mapping.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-mapping.asciidoc index abeeb87f03e75..2b18e644e097b 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-mapping.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-mapping.asciidoc @@ -202,6 +202,39 @@ the {infer} pipeline configuration in the next step. // end::azure-ai-studio[] +// tag::google-vertex-ai[] + +[source,console] +-------------------------------------------------- +PUT google-vertex-ai-embeddings +{ + "mappings": { + "properties": { + "content_embedding": { <1> + "type": "dense_vector", <2> + "dims": 768, <3> + "element_type": "float", + "similarity": "dot_product" <4> + }, + "content": { <5> + "type": "text" <6> + } + } + } +} +-------------------------------------------------- +<1> The name of the field to contain the generated embeddings. It must be referenced in the {infer} pipeline configuration in the next step. +<2> The field to contain the embeddings is a `dense_vector` field. +<3> The output dimensions of the model. This value may be found on the https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api[Google Vertex AI model reference]. +The {infer} API attempts to calculate the output dimensions automatically if `dims` are not specified. +<4> For Google Vertex AI embeddings, the `dot_product` function should be used to calculate similarity. +<5> The name of the field from which to create the dense vector representation. +In this example, the name of the field is `content`. It must be referenced in +the {infer} pipeline configuration in the next step. +<6> The field type which is `text` in this example. + +// end::google-vertex-ai[] + // tag::mistral[] [source,console] @@ -270,3 +303,35 @@ the {infer} pipeline configuration in the next step. <6> The field type which is text in this example. // end::amazon-bedrock[] + +// tag::alibabacloud-ai-search[] + +[source,console] +-------------------------------------------------- +PUT alibabacloud-ai-search-embeddings +{ + "mappings": { + "properties": { + "content_embedding": { <1> + "type": "dense_vector", <2> + "dims": 1024, <3> + "element_type": "float" + }, + "content": { <4> + "type": "text" <5> + } + } + } +} +-------------------------------------------------- +<1> The name of the field to contain the generated tokens. It must be referenced +in the {infer} pipeline configuration in the next step. +<2> The field to contain the tokens is a `dense_vector` field. +<3> The output dimensions of the model. This value may be different depending on the underlying model used. +See the https://help.aliyun.com/zh/open-search/search-platform/developer-reference/text-embedding-api-details[AlibabaCloud AI Search embedding model] documentation. +<4> The name of the field from which to create the dense vector representation. +In this example, the name of the field is `content`. It must be referenced in +the {infer} pipeline configuration in the next step. +<5> The field type which is text in this example. + +// end::alibabacloud-ai-search[] diff --git a/docs/reference/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc index 45cb9fc51b9f1..d24c751cdbfc6 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc @@ -37,6 +37,12 @@ id="infer-api-reindex-azure-ai-studio"> Azure AI Studio + +
+
+
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-reindex.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-reindex.asciidoc index d961ec8bd39bd..0b0ab9a9cfe0e 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-reindex.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-reindex.asciidoc @@ -155,6 +155,28 @@ might affect the throughput of the reindexing process. If this happens, change // end::azure-ai-studio[] +// tag::google-vertex-ai[] + +[source,console] +---- +POST _reindex?wait_for_completion=false +{ + "source": { + "index": "test-data", + "size": 50 <1> + }, + "dest": { + "index": "google-vertex-ai-embeddings", + "pipeline": "google_vertex_ai_embeddings" + } +} +---- +// TEST[skip:TBD] +<1> The default batch size for reindexing is 1000. Reducing `size` will make updates to the reindexing process faster. This enables you to +follow the progress closely and detect errors early. + +// end::google-vertex-ai[] + // tag::mistral[] [source,console] @@ -200,3 +222,26 @@ number makes the update of the reindexing process quicker which enables you to follow the progress closely and detect errors early. // end::amazon-bedrock[] + +// tag::alibabacloud-ai-search[] + +[source,console] +---- +POST _reindex?wait_for_completion=false +{ + "source": { + "index": "test-data", + "size": 50 <1> + }, + "dest": { + "index": "alibabacloud-ai-search-embeddings", + "pipeline": "alibabacloud_ai_search_embeddings" + } +} +---- +// TEST[skip:TBD] +<1> The default batch size for reindexing is 1000. Reducing `size` to a smaller +number makes the update of the reindexing process quicker which enables you to +follow the progress closely and detect errors early. + +// end::alibabacloud-ai-search[] diff --git a/docs/reference/tab-widgets/inference-api/infer-api-requirements-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-requirements-widget.asciidoc index c867b39b88e3b..430129f2f2edb 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-requirements-widget.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-requirements-widget.asciidoc @@ -35,7 +35,13 @@ aria-selected="false" aria-controls="infer-api-requirements-azure-ai-studio-tab" id="infer-api-requirements-azure-ai-studio"> - Azure AI Studio + Azure AI Studio + + +
+
+
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc index 603cd85a8f93d..eeecb4718658a 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc @@ -41,6 +41,15 @@ You can apply for access to Azure OpenAI by completing the form at https://aka.m // end::azure-ai-studio[] +// tag::google-vertex-ai[] +* A https://console.cloud.google.com/[Google Cloud account] +* A project in Google Cloud +* The Vertex AI API enabled in your project +* A valid service account for the Google Vertex AI API +* The service account must have the Vertex AI User role and the `aiplatform.endpoints.predict` permission. + +// end::google-vertex-ai[] + // tag::mistral[] * A Mistral Account on https://console.mistral.ai/[La Plateforme] * An API key generated for your account @@ -52,3 +61,9 @@ You can apply for access to Azure OpenAI by completing the form at https://aka.m * A pair of access and secret keys used to access Amazon Bedrock // end::amazon-bedrock[] + +// tag::alibabacloud-ai-search[] +* An AlibabaCloud Account with https://console.aliyun.com[AlibabaCloud] access +* An API key generated for your account from the https://opensearch.console.aliyun.com/cn-shanghai/rag/api-key[API keys section] + +// end::alibabacloud-ai-search[] diff --git a/docs/reference/tab-widgets/inference-api/infer-api-search-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-search-widget.asciidoc index fa4a11c59a158..b4f7f56a19b94 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-search-widget.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-search-widget.asciidoc @@ -36,6 +36,12 @@ aria-controls="infer-api-search-azure-ai-studio-tab" id="infer-api-search-azure-ai-studio"> Azure AI Studio + + +
+
+
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-search.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-search.asciidoc index f23ed1dfef05d..706db42669210 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-search.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-search.asciidoc @@ -402,6 +402,71 @@ query from the `azure-ai-studio-embeddings` index sorted by their proximity to t // end::azure-ai-studio[] +// tag::google-vertex-ai[] + +[source,console] +-------------------------------------------------- +GET google-vertex-ai-embeddings/_search +{ + "knn": { + "field": "content_embedding", + "query_vector_builder": { + "text_embedding": { + "model_id": "google_vertex_ai_embeddings", + "model_text": "Calculate fuel cost" + } + }, + "k": 10, + "num_candidates": 100 + }, + "_source": [ + "id", + "content" + ] +} +-------------------------------------------------- +// TEST[skip:TBD] + +As a result, you receive the top 10 documents that are closest in meaning to the +query from the `mistral-embeddings` index sorted by their proximity to the query: + +[source,console-result] +-------------------------------------------------- +"hits": [ + { + "_index": "google-vertex-ai-embeddings", + "_id": "Ryv0nZEBBFPLbFsdCbGn", + "_score": 0.86815524, + "_source": { + "id": 3041038, + "content": "For example, the cost of the fuel could be 96.9, the amount could be 10 pounds, and the distance covered could be 80 miles. To convert between Litres per 100KM and Miles Per Gallon, please provide a value and click on the required button.o calculate how much fuel you'll need for a given journey, please provide the distance in miles you will be covering on your journey, and the estimated MPG of your vehicle. To work out what MPG you are really getting, please provide the cost of the fuel, how much you spent on the fuel, and how far it took you." + } + }, + { + "_index": "google-vertex-ai-embeddings", + "_id": "w4j0nZEBZ1nFq1oiHQvK", + "_score": 0.8676357, + "_source": { + "id": 1541469, + "content": "This driving cost calculator takes into consideration the fuel economy of the vehicle that you are travelling in as well as the fuel cost. This road trip gas calculator will give you an idea of how much would it cost to drive before you actually travel.his driving cost calculator takes into consideration the fuel economy of the vehicle that you are travelling in as well as the fuel cost. This road trip gas calculator will give you an idea of how much would it cost to drive before you actually travel." + } + }, + { + "_index": "google-vertex-ai-embeddings", + "_id": "Hoj0nZEBZ1nFq1oiHQjJ", + "_score": 0.80510974, + "_source": { + "id": 7982559, + "content": "What's that light cost you? 1 Select your electric rate (or click to enter your own). 2 You can calculate results for up to four types of lights. 3 Select the type of lamp (i.e. 4 Select the lamp wattage (lamp lumens). 5 Enter the number of lights in use. 6 Select how long the lamps are in use (or click to enter your own; enter hours on per year). 7 Finally, ..." + } + }, + (...) + ] +-------------------------------------------------- +// NOTCONSOLE + +// end::google-vertex-ai[] + // tag::mistral[] [source,console] @@ -531,3 +596,68 @@ query from the `amazon-bedrock-embeddings` index sorted by their proximity to th // NOTCONSOLE // end::amazon-bedrock[] + +// tag::alibabacloud-ai-search[] + +[source,console] +-------------------------------------------------- +GET alibabacloud-ai-search-embeddings/_search +{ + "knn": { + "field": "content_embedding", + "query_vector_builder": { + "text_embedding": { + "model_id": "alibabacloud_ai_search_embeddings", + "model_text": "Calculate fuel cost" + } + }, + "k": 10, + "num_candidates": 100 + }, + "_source": [ + "id", + "content" + ] +} +-------------------------------------------------- +// TEST[skip:TBD] + +As a result, you receive the top 10 documents that are closest in meaning to the +query from the `alibabacloud-ai-search-embeddings` index sorted by their proximity to the query: + +[source,consol-result] +-------------------------------------------------- +"hits": [ + { + "_index": "alibabacloud-ai-search-embeddings", + "_id": "DDd5OowBHxQKHyc3TDSC", + "_score": 0.83704096, + "_source": { + "id": 862114, + "body": "How to calculate fuel cost for a road trip. By Tara Baukus Mello • Bankrate.com. Dear Driving for Dollars, My family is considering taking a long road trip to finish off the end of the summer, but I'm a little worried about gas prices and our overall fuel cost.It doesn't seem easy to calculate since we'll be traveling through many states and we are considering several routes.y family is considering taking a long road trip to finish off the end of the summer, but I'm a little worried about gas prices and our overall fuel cost. It doesn't seem easy to calculate since we'll be traveling through many states and we are considering several routes." + } + }, + { + "_index": "alibabacloud-ai-search-embeddings", + "_id": "ajd5OowBHxQKHyc3TDSC", + "_score": 0.8345704, + "_source": { + "id": 820622, + "body": "Home Heating Calculator. Typically, approximately 50% of the energy consumed in a home annually is for space heating. When deciding on a heating system, many factors will come into play: cost of fuel, installation cost, convenience and life style are all important.This calculator can help you estimate the cost of fuel for different heating appliances.hen deciding on a heating system, many factors will come into play: cost of fuel, installation cost, convenience and life style are all important. This calculator can help you estimate the cost of fuel for different heating appliances." + } + }, + { + "_index": "alibabacloud-ai-search-embeddings", + "_id": "Djd5OowBHxQKHyc3TDSC", + "_score": 0.8327426, + "_source": { + "id": 8202683, + "body": "Fuel is another important cost. This cost will depend on your boat, how far you travel, and how fast you travel. A 33-foot sailboat traveling at 7 knots should be able to travel 300 miles on 50 gallons of diesel fuel.If you are paying $4 per gallon, the trip would cost you $200.Most boats have much larger gas tanks than cars.uel is another important cost. This cost will depend on your boat, how far you travel, and how fast you travel. A 33-foot sailboat traveling at 7 knots should be able to travel 300 miles on 50 gallons of diesel fuel." + } + }, + (...) + ] +-------------------------------------------------- +// NOTCONSOLE + +// end::alibabacloud-ai-search[] diff --git a/docs/reference/tab-widgets/inference-api/infer-api-task-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-task-widget.asciidoc index f12be341d866d..97d471af0d2fb 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-task-widget.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-task-widget.asciidoc @@ -37,6 +37,12 @@ id="infer-api-task-azure-ai-studio"> Azure AI Studio + +
+
+
diff --git a/docs/reference/tab-widgets/inference-api/infer-api-task.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-task.asciidoc index b186b2c58ccc5..6a2bdbb5e79a4 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-task.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-task.asciidoc @@ -178,6 +178,30 @@ Also, when using this model the recommended similarity measure to use in the // end::azure-ai-studio[] +// tag::google-vertex-ai[] + +[source,console] +------------------------------------------------------------ +PUT _inference/text_embedding/google_vertex_ai_embeddings <1> +{ + "service": "googlevertexai", + "service_settings": { + "service_account_json": "", <2> + "model_id": "text-embedding-004", <3> + "location": "", <4> + "project_id": "" <5> + } +} +------------------------------------------------------------ +// TEST[skip:TBD] +<1> The task type is `text_embedding` per the path. `google_vertex_ai_embeddings` is the unique identifier of the {infer} endpoint (its `inference_id`). +<2> A valid service account in JSON format for the Google Vertex AI API. +<3> For the list of the available models, refer to the https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api[Text embeddings API] page. +<4> The name of the location to use for the {infer} task. Refer to https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations[Generative AI on Vertex AI locations] for available locations. +<5> The name of the project to use for the {infer} task. + +// end::google-vertex-ai[] + // tag::mistral[] [source,console] @@ -223,3 +247,32 @@ PUT _inference/text_embedding/amazon_bedrock_embeddings <1> <6> The model ID or ARN of the model to use. // end::amazon-bedrock[] + +// tag::alibabacloud-ai-search[] + +[source,console] +------------------------------------------------------------ +PUT _inference/text_embedding/alibabacloud_ai_search_embeddings <1> +{ + "service": "alibabacloud-ai-search", + "service_settings": { + "api_key": "", <2> + "service_id": "", <3> + "host": "", <4> + "workspace": "" <5> + } +} +------------------------------------------------------------ +// TEST[skip:TBD] +<1> The task type is `text_embedding` in the path and the `inference_id` which is the unique identifier of the {infer} endpoint is `alibabacloud_ai_search_embeddings`. +<2> The API key for accessing the AlibabaCloud AI Search API. You can find your API keys in +your AlibabaCloud account under the +https://opensearch.console.aliyun.com/cn-shanghai/rag/api-key[API keys section]. You need to provide +your API key only once. The <> does not return your API +key. +<3> The AlibabaCloud AI Search embeddings model name, for example `ops-text-embedding-zh-001`. +<4> The name our your AlibabaCloud AI Search host address. +<5> The name our your AlibabaCloud AI Search workspace. + +// end::alibabacloud-ai-search[] + diff --git a/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc b/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc index 387ebcdcd43c0..cbb35f7731034 100644 --- a/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc +++ b/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc @@ -1,4 +1,316 @@ [[troubleshooting-unstable-cluster]] == Troubleshooting an unstable cluster -include::../modules/discovery/fault-detection.asciidoc[tag=troubleshooting,leveloffset=-2] \ No newline at end of file +Normally, a node will only leave a cluster if deliberately shut down. If a node +leaves the cluster unexpectedly, it's important to address the cause. A cluster +in which nodes leave unexpectedly is unstable and can create several issues. +For instance: + +* The cluster health may be yellow or red. + +* Some shards will be initializing and other shards may be failing. + +* Search, indexing, and monitoring operations may fail and report exceptions in +logs. + +* The `.security` index may be unavailable, blocking access to the cluster. + +* The master may appear busy due to frequent cluster state updates. + +To troubleshoot a cluster in this state, first ensure the cluster has a +<>. Next, focus on the nodes +unexpectedly leaving the cluster ahead of all other issues. It will not be +possible to solve other issues until the cluster has a stable master node and +stable node membership. + +Diagnostics and statistics are usually not useful in an unstable cluster. These +tools only offer a view of the state of the cluster at a single point in time. +Instead, look at the cluster logs to see the pattern of behaviour over time. +Focus particularly on logs from the elected master. When a node leaves the +cluster, logs for the elected master include a message like this (with line +breaks added to make it easier to read): + +[source,text] +---- +[2022-03-21T11:02:35,513][INFO ][o.e.c.c.NodeLeftExecutor] [instance-0000000000] + node-left: [{instance-0000000004}{bfcMDTiDRkietFb9v_di7w}{aNlyORLASam1ammv2DzYXA}{172.27.47.21}{172.27.47.21:19054}{m}] + with reason [disconnected] +---- + +This message says that the `NodeLeftExecutor` on the elected master +(`instance-0000000000`) processed a `node-left` task, identifying the node that +was removed and the reason for its removal. When the node joins the cluster +again, logs for the elected master will include a message like this (with line +breaks added to make it easier to read): + +[source,text] +---- +[2022-03-21T11:02:59,892][INFO ][o.e.c.c.NodeJoinExecutor] [instance-0000000000] + node-join: [{instance-0000000004}{bfcMDTiDRkietFb9v_di7w}{UNw_RuazQCSBskWZV8ID_w}{172.27.47.21}{172.27.47.21:19054}{m}] + with reason [joining after restart, removed [24s] ago with reason [disconnected]] +---- + +This message says that the `NodeJoinExecutor` on the elected master +(`instance-0000000000`) processed a `node-join` task, identifying the node that +was added to the cluster and the reason for the task. + +Other nodes may log similar messages, but report fewer details: + +[source,text] +---- +[2020-01-29T11:02:36,985][INFO ][o.e.c.s.ClusterApplierService] + [instance-0000000001] removed { + {instance-0000000004}{bfcMDTiDRkietFb9v_di7w}{aNlyORLASam1ammv2DzYXA}{172.27.47.21}{172.27.47.21:19054}{m} + {tiebreaker-0000000003}{UNw_RuazQCSBskWZV8ID_w}{bltyVOQ-RNu20OQfTHSLtA}{172.27.161.154}{172.27.161.154:19251}{mv} + }, term: 14, version: 1653415, reason: Publication{term=14, version=1653415} +---- + +These messages are not especially useful for troubleshooting, so focus on the +ones from the `NodeLeftExecutor` and `NodeJoinExecutor` which are only emitted +on the elected master and which contain more details. If you don't see the +messages from the `NodeLeftExecutor` and `NodeJoinExecutor`, check that: + +* You're looking at the logs for the elected master node. + +* The logs cover the correct time period. + +* Logging is enabled at `INFO` level. + +Nodes will also log a message containing `master node changed` whenever they +start or stop following the elected master. You can use these messages to +determine each node's view of the state of the master over time. + +If a node restarts, it will leave the cluster and then join the cluster again. +When it rejoins, the `NodeJoinExecutor` will log that it processed a +`node-join` task indicating that the node is `joining after restart`. If a node +is unexpectedly restarting, look at the node's logs to see why it is shutting +down. + +The <> API on the affected node will also provide some useful +information about the situation. + +If the node did not restart then you should look at the reason for its +departure more closely. Each reason has different troubleshooting steps, +described below. There are three possible reasons: + +* `disconnected`: The connection from the master node to the removed node was +closed. + +* `lagging`: The master published a cluster state update, but the removed node +did not apply it within the permitted timeout. By default, this timeout is 2 +minutes. Refer to <> for information about the +settings which control this mechanism. + +* `followers check retry count exceeded`: The master sent a number of +consecutive health checks to the removed node. These checks were rejected or +timed out. By default, each health check times out after 10 seconds and {es} +removes the node removed after three consecutively failed health checks. Refer +to <> for information about the settings which +control this mechanism. + +[discrete] +[[troubleshooting-unstable-cluster-disconnected]] +=== Diagnosing `disconnected` nodes + +Nodes typically leave the cluster with reason `disconnected` when they shut +down, but if they rejoin the cluster without restarting then there is some +other problem. + +{es} is designed to run on a fairly reliable network. It opens a number of TCP +connections between nodes and expects these connections to remain open +<>. If a connection is closed then {es} will +try and reconnect, so the occasional blip may fail some in-flight operations +but should otherwise have limited impact on the cluster. In contrast, +repeatedly-dropped connections will severely affect its operation. + +The connections from the elected master node to every other node in the cluster +are particularly important. The elected master never spontaneously closes its +outbound connections to other nodes. Similarly, once an inbound connection is +fully established, a node never spontaneously it unless the node is shutting +down. + +If you see a node unexpectedly leave the cluster with the `disconnected` +reason, something other than {es} likely caused the connection to close. A +common cause is a misconfigured firewall with an improper timeout or another +policy that's <>. It could also +be caused by general connectivity issues, such as packet loss due to faulty +hardware or network congestion. If you're an advanced user, configure the +following loggers to get more detailed information about network exceptions: + +[source,yaml] +---- +logger.org.elasticsearch.transport.TcpTransport: DEBUG +logger.org.elasticsearch.xpack.core.security.transport.netty4.SecurityNetty4Transport: DEBUG +---- + +If these logs do not show enough information to diagnose the problem, obtain a +packet capture simultaneously from the nodes at both ends of an unstable +connection and analyse it alongside the {es} logs from those nodes to determine +if traffic between the nodes is being disrupted by another device on the +network. + +[discrete] +[[troubleshooting-unstable-cluster-lagging]] +=== Diagnosing `lagging` nodes + +{es} needs every node to process cluster state updates reasonably quickly. If a +node takes too long to process a cluster state update, it can be harmful to the +cluster. The master will remove these nodes with the `lagging` reason. Refer to +<> for information about the settings which control +this mechanism. + +Lagging is typically caused by performance issues on the removed node. However, +a node may also lag due to severe network delays. To rule out network delays, +ensure that `net.ipv4.tcp_retries2` is <>. Log messages that contain `warn threshold` may provide more +information about the root cause. + +If you're an advanced user, you can get more detailed information about what +the node was doing when it was removed by configuring the following logger: + +[source,yaml] +---- +logger.org.elasticsearch.cluster.coordination.LagDetector: DEBUG +---- + +When this logger is enabled, {es} will attempt to run the +<> API on the faulty node and report the results in +the logs on the elected master. The results are compressed, encoded, and split +into chunks to avoid truncation: + +[source,text] +---- +[DEBUG][o.e.c.c.LagDetector ] [master] hot threads from node [{node}{g3cCUaMDQJmQ2ZLtjr-3dg}{10.0.0.1:9300}] lagging at version [183619] despite commit of cluster state version [183620] [part 1]: H4sIAAAAAAAA/x... +[DEBUG][o.e.c.c.LagDetector ] [master] hot threads from node [{node}{g3cCUaMDQJmQ2ZLtjr-3dg}{10.0.0.1:9300}] lagging at version [183619] despite commit of cluster state version [183620] [part 2]: p7x3w1hmOQVtuV... +[DEBUG][o.e.c.c.LagDetector ] [master] hot threads from node [{node}{g3cCUaMDQJmQ2ZLtjr-3dg}{10.0.0.1:9300}] lagging at version [183619] despite commit of cluster state version [183620] [part 3]: v7uTboMGDbyOy+... +[DEBUG][o.e.c.c.LagDetector ] [master] hot threads from node [{node}{g3cCUaMDQJmQ2ZLtjr-3dg}{10.0.0.1:9300}] lagging at version [183619] despite commit of cluster state version [183620] [part 4]: 4tse0RnPnLeDNN... +[DEBUG][o.e.c.c.LagDetector ] [master] hot threads from node [{node}{g3cCUaMDQJmQ2ZLtjr-3dg}{10.0.0.1:9300}] lagging at version [183619] despite commit of cluster state version [183620] (gzip compressed, base64-encoded, and split into 4 parts on preceding log lines) +---- + +To reconstruct the output, base64-decode the data and decompress it using +`gzip`. For instance, on Unix-like systems: + +[source,sh] +---- +cat lagdetector.log | sed -e 's/.*://' | base64 --decode | gzip --decompress +---- + +[discrete] +[[troubleshooting-unstable-cluster-follower-check]] +=== Diagnosing `follower check retry count exceeded` nodes + +Nodes sometimes leave the cluster with reason `follower check retry count +exceeded` when they shut down, but if they rejoin the cluster without +restarting then there is some other problem. + +{es} needs every node to respond to network messages successfully and +reasonably quickly. If a node rejects requests or does not respond at all then +it can be harmful to the cluster. If enough consecutive checks fail then the +master will remove the node with reason `follower check retry count exceeded` +and will indicate in the `node-left` message how many of the consecutive +unsuccessful checks failed and how many of them timed out. Refer to +<> for information about the settings which control +this mechanism. + +Timeouts and failures may be due to network delays or performance problems on +the affected nodes. Ensure that `net.ipv4.tcp_retries2` is +<> to eliminate network delays as +a possible cause for this kind of instability. Log messages containing +`warn threshold` may give further clues about the cause of the instability. + +If the last check failed with an exception then the exception is reported, and +typically indicates the problem that needs to be addressed. If any of the +checks timed out then narrow down the problem as follows. + +include::network-timeouts.asciidoc[tag=troubleshooting-network-timeouts-gc-vm] + +include::network-timeouts.asciidoc[tag=troubleshooting-network-timeouts-packet-capture-fault-detection] + +include::network-timeouts.asciidoc[tag=troubleshooting-network-timeouts-threads] + +By default the follower checks will time out after 30s, so if node departures +are unpredictable then capture stack dumps every 15s to be sure that at least +one stack dump was taken at the right time. + +[discrete] +[[troubleshooting-unstable-cluster-shardlockobtainfailedexception]] +=== Diagnosing `ShardLockObtainFailedException` failures + +If a node leaves and rejoins the cluster then {es} will usually shut down and +re-initialize its shards. If the shards do not shut down quickly enough then +{es} may fail to re-initialize them due to a `ShardLockObtainFailedException`. + +To gather more information about the reason for shards shutting down slowly, +configure the following logger: + +[source,yaml] +---- +logger.org.elasticsearch.env.NodeEnvironment: DEBUG +---- + +When this logger is enabled, {es} will attempt to run the +<> API whenever it encounters a +`ShardLockObtainFailedException`. The results are compressed, encoded, and +split into chunks to avoid truncation: + +[source,text] +---- +[DEBUG][o.e.e.NodeEnvironment ] [master] hot threads while failing to obtain shard lock for [index][0] [part 1]: H4sIAAAAAAAA/x... +[DEBUG][o.e.e.NodeEnvironment ] [master] hot threads while failing to obtain shard lock for [index][0] [part 2]: p7x3w1hmOQVtuV... +[DEBUG][o.e.e.NodeEnvironment ] [master] hot threads while failing to obtain shard lock for [index][0] [part 3]: v7uTboMGDbyOy+... +[DEBUG][o.e.e.NodeEnvironment ] [master] hot threads while failing to obtain shard lock for [index][0] [part 4]: 4tse0RnPnLeDNN... +[DEBUG][o.e.e.NodeEnvironment ] [master] hot threads while failing to obtain shard lock for [index][0] (gzip compressed, base64-encoded, and split into 4 parts on preceding log lines) +---- + +To reconstruct the output, base64-decode the data and decompress it using +`gzip`. For instance, on Unix-like systems: + +[source,sh] +---- +cat shardlock.log | sed -e 's/.*://' | base64 --decode | gzip --decompress +---- + +[discrete] +[[troubleshooting-unstable-cluster-network]] +=== Diagnosing other network disconnections + +{es} is designed to run on a fairly reliable network. It opens a number of TCP +connections between nodes and expects these connections to remain open +<>. If a connection is closed then {es} will +try and reconnect, so the occasional blip may fail some in-flight operations +but should otherwise have limited impact on the cluster. In contrast, +repeatedly-dropped connections will severely affect its operation. + +{es} nodes will only actively close an outbound connection to another node if +the other node leaves the cluster. See +<> for further information about +identifying and troubleshooting this situation. If an outbound connection +closes for some other reason, nodes will log a message such as the following: + +[source,text] +---- +[INFO ][o.e.t.ClusterConnectionManager] [node-1] transport connection to [{node-2}{g3cCUaMDQJmQ2ZLtjr-3dg}{10.0.0.1:9300}] closed by remote +---- + +Similarly, once an inbound connection is fully established, a node never +spontaneously closes it unless the node is shutting down. + +Therefore if you see a node report that a connection to another node closed +unexpectedly, something other than {es} likely caused the connection to close. +A common cause is a misconfigured firewall with an improper timeout or another +policy that's <>. It could also +be caused by general connectivity issues, such as packet loss due to faulty +hardware or network congestion. If you're an advanced user, configure the +following loggers to get more detailed information about network exceptions: + +[source,yaml] +---- +logger.org.elasticsearch.transport.TcpTransport: DEBUG +logger.org.elasticsearch.xpack.core.security.transport.netty4.SecurityNetty4Transport: DEBUG +---- + +If these logs do not show enough information to diagnose the problem, obtain a +packet capture simultaneously from the nodes at both ends of an unstable +connection and analyse it alongside the {es} logs from those nodes to determine +if traffic between the nodes is being disrupted by another device on the +network. diff --git a/gradle/build.versions.toml b/gradle/build.versions.toml index 792330fd3613b..35c26ef10f9ec 100644 --- a/gradle/build.versions.toml +++ b/gradle/build.versions.toml @@ -11,7 +11,7 @@ apache-compress = "org.apache.commons:commons-compress:1.26.1" apache-rat = "org.apache.rat:apache-rat:0.11" asm = { group = "org.ow2.asm", name="asm", version.ref="asm" } asm-tree = { group = "org.ow2.asm", name="asm-tree", version.ref="asm" } -bytebuddy = "net.bytebuddy:byte-buddy:1.12.10" +bytebuddy = "net.bytebuddy:byte-buddy:1.14.12" checkstyle = "com.puppycrawl.tools:checkstyle:10.3" commons-codec = "commons-codec:commons-codec:1.11" commmons-io = "commons-io:commons-io:2.2" @@ -44,6 +44,6 @@ snakeyaml = { group = "org.yaml", name = "snakeyaml", version = { strictly = "2. spock-core = { group = "org.spockframework", name="spock-core", version.ref="spock" } spock-junit4 = { group = "org.spockframework", name="spock-junit4", version.ref="spock" } spock-platform = { group = "org.spockframework", name="spock-bom", version.ref="spock" } -spotless-plugin = "com.diffplug.spotless:spotless-plugin-gradle:6.22.0" +spotless-plugin = "com.diffplug.spotless:spotless-plugin-gradle:6.25.0" wiremock = "com.github.tomakehurst:wiremock-jre8-standalone:2.23.2" xmlunit-core = "org.xmlunit:xmlunit-core:2.8.2" diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 1001ab2b709dd..472a65f9c6f24 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -119,44 +119,44 @@ - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + @@ -204,19 +204,14 @@ - - - - - - - - + + + @@ -229,36 +224,19 @@ - - - + + + - - - - - - - - - - - - - - - - - - - - + + + - - - + + + @@ -311,6 +289,11 @@ + + + + + @@ -346,6 +329,11 @@ + + + + + @@ -361,6 +349,11 @@ + + + + + @@ -381,6 +374,11 @@ + + + + + @@ -411,9 +409,9 @@ - - - + + + @@ -581,11 +579,6 @@ - - - - - @@ -601,6 +594,11 @@ + + + + + @@ -611,14 +609,9 @@ - - - - - - - - + + + @@ -636,11 +629,6 @@ - - - - - @@ -666,6 +654,11 @@ + + + + + @@ -901,9 +894,9 @@ - - - + + + @@ -1018,14 +1011,9 @@ - - - - - - - - + + + @@ -1038,14 +1026,9 @@ - - - - - - - - + + + @@ -1318,14 +1301,14 @@ - - - + + + - - - + + + @@ -1681,11 +1664,6 @@ - - - - - @@ -3264,11 +3242,6 @@ - - - - - @@ -3289,16 +3262,16 @@ + + + + + - - - - - @@ -3394,6 +3367,11 @@ + + + + + @@ -3409,14 +3387,14 @@ - - - + + + - - - + + + @@ -3534,69 +3512,49 @@ - - - - - - - - - - - - - - - - - - - - - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + @@ -3604,19 +3562,19 @@ - - - + + + - - - + + + - - - + + + @@ -3624,9 +3582,14 @@ - - - + + + + + + + + @@ -3714,54 +3677,24 @@ - - - - - - - - - - - - - - - - - - + + + - - - + + + - - - + + + - - - - - - - - - - - - - - - - - - + + + @@ -4204,6 +4137,11 @@ + + + + + diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 2c3521197d7c4..a4b76b9530d66 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index efe2ff3449216..9036682bf0f0c 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=258e722ec21e955201e31447b0aed14201765a3bfbae296a46cf60b70e66db70 -distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-all.zip +distributionSha256Sum=682b4df7fe5accdca84a4d1ef6a3a6ab096b3efd5edf7de2bd8c758d95a93703 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-all.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/libs/core/src/main/java/org/elasticsearch/core/Releasables.java b/libs/core/src/main/java/org/elasticsearch/core/Releasables.java index 6e595436c9f54..45d98b7761110 100644 --- a/libs/core/src/main/java/org/elasticsearch/core/Releasables.java +++ b/libs/core/src/main/java/org/elasticsearch/core/Releasables.java @@ -149,8 +149,9 @@ public static Releasable assertOnce(final Releasable delegate) { private final AtomicReference firstCompletion = new AtomicReference<>(); private void assertFirstRun() { - var previousRun = firstCompletion.compareAndExchange(null, new Exception(delegate.toString())); - assert previousRun == null : previousRun; // reports the stack traces of both completions + var previousRun = firstCompletion.compareAndExchange(null, new Exception("already executed")); + // reports the stack traces of both completions + assert previousRun == null : new AssertionError(delegate.toString(), previousRun); } @Override diff --git a/libs/core/src/main/java/org/elasticsearch/core/UpdateForV10.java b/libs/core/src/main/java/org/elasticsearch/core/UpdateForV10.java new file mode 100644 index 0000000000000..0fe816bd3721d --- /dev/null +++ b/libs/core/src/main/java/org/elasticsearch/core/UpdateForV10.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.core; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Annotation to identify a block of code (a whole class, a method, or a field) that needs to be reviewed (for cleanup, remove or change) + * before releasing 10.0 + */ +@Retention(RetentionPolicy.SOURCE) +@Target({ ElementType.LOCAL_VARIABLE, ElementType.CONSTRUCTOR, ElementType.FIELD, ElementType.METHOD, ElementType.TYPE }) +public @interface UpdateForV10 { +} diff --git a/libs/core/src/test/java/org/elasticsearch/core/ReleasablesTests.java b/libs/core/src/test/java/org/elasticsearch/core/ReleasablesTests.java index d54c9b8104e8b..602437c27bd49 100644 --- a/libs/core/src/test/java/org/elasticsearch/core/ReleasablesTests.java +++ b/libs/core/src/test/java/org/elasticsearch/core/ReleasablesTests.java @@ -69,7 +69,7 @@ public String toString() { .anyMatch(ste -> ste.toString().contains("CloserWithIdentifiableMethodNames.closeMethod2")) ); assertTrue( - Arrays.stream(assertionError.getCause().getStackTrace()) + Arrays.stream(assertionError.getCause().getCause().getStackTrace()) .anyMatch(ste -> ste.toString().contains("CloserWithIdentifiableMethodNames.closeMethod1")) ); } diff --git a/libs/grok/src/main/java/org/elasticsearch/grok/PatternBank.java b/libs/grok/src/main/java/org/elasticsearch/grok/PatternBank.java index bcf9253866931..3b10d58815169 100644 --- a/libs/grok/src/main/java/org/elasticsearch/grok/PatternBank.java +++ b/libs/grok/src/main/java/org/elasticsearch/grok/PatternBank.java @@ -8,12 +8,17 @@ package org.elasticsearch.grok; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collections; +import java.util.Deque; +import java.util.HashSet; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; public class PatternBank { @@ -57,52 +62,102 @@ public PatternBank extendWith(Map extraPatterns) { } /** - * Checks whether patterns reference each other in a circular manner and if so fail with an exception. + * Checks whether patterns reference each other in a circular manner and if so fail with an IllegalArgumentException. It will also + * fail if any pattern value contains a pattern name that does not exist in the bank. *

* In a pattern, anything between %{ and } or : is considered * a reference to another named pattern. This method will navigate to all these named patterns and * check for a circular reference. */ static void forbidCircularReferences(Map bank) { - // first ensure that the pattern bank contains no simple circular references (i.e., any pattern - // containing an immediate reference to itself) as those can cause the remainder of this algorithm - // to recurse infinitely - for (Map.Entry entry : bank.entrySet()) { - if (patternReferencesItself(entry.getValue(), entry.getKey())) { - throw new IllegalArgumentException("circular reference in pattern [" + entry.getKey() + "][" + entry.getValue() + "]"); + Set allVisitedNodes = new HashSet<>(); + Set nodesVisitedMoreThanOnceInAPath = new HashSet<>(); + // Walk the full path starting at each node in the graph: + for (String traversalStartNode : bank.keySet()) { + if (nodesVisitedMoreThanOnceInAPath.contains(traversalStartNode) == false && allVisitedNodes.contains(traversalStartNode)) { + // If we have seen this node before in a path, and it only appeared once in that path, there is no need to check it again + continue; } - } - - // next, recursively check any other pattern names referenced in each pattern - for (Map.Entry entry : bank.entrySet()) { - String name = entry.getKey(); - String pattern = entry.getValue(); - innerForbidCircularReferences(bank, name, new ArrayList<>(), pattern); + Set visitedFromThisStartNode = new LinkedHashSet<>(); + /* + * This stack records where we are in the graph. Each String[] in the stack represents a collection of neighbors to the first + * non-null node in the layer below it. Null means that the path from that location has been fully traversed. Once all nodes + * at a layer have been set to null, the layer is popped. So for example say we have the graph + * ( 1 -> (2 -> (4, 5, 8), 3 -> (6, 7))) then when we are at 6 via 1 -> 3 -> 6, the stack looks like this: + * [6, 7] + * [null, 3] + * [1] + */ + Deque stack = new ArrayDeque<>(); + stack.push(new String[] { traversalStartNode }); + // This is used so that we know that we're unwinding the stack and know not to get the current node's neighbors again. + boolean unwinding = false; + while (stack.isEmpty() == false) { + String[] currentLevel = stack.peek(); + int firstNonNullIndex = findFirstNonNull(currentLevel); + String node = currentLevel[firstNonNullIndex]; + boolean endOfThisPath = false; + if (unwinding) { + // We have completed all of this node's neighbors and have popped back to the node + endOfThisPath = true; + } else if (traversalStartNode.equals(node) && stack.size() > 1) { + Deque reversedPath = new ArrayDeque<>(); + for (String[] level : stack) { + reversedPath.push(level[findFirstNonNull(level)]); + } + throw new IllegalArgumentException("circular reference detected: " + String.join("->", reversedPath)); + } else if (visitedFromThisStartNode.contains(node)) { + /* + * We are only looking for a cycle starting and ending at traversalStartNode right now. But this node has been + * visited more than once in the path rooted at traversalStartNode. This could be because it is a cycle, or could be + * because two nodes in the path both point to it. We add it to nodesVisitedMoreThanOnceInAPath so that we make sure + * to check the path rooted at this node later. + */ + nodesVisitedMoreThanOnceInAPath.add(node); + endOfThisPath = true; + } else { + visitedFromThisStartNode.add(node); + String[] neighbors = getPatternNamesForPattern(bank, node); + if (neighbors.length == 0) { + endOfThisPath = true; + } else { + stack.push(neighbors); + } + } + if (endOfThisPath) { + if (firstNonNullIndex == currentLevel.length - 1) { + // We have handled all the neighbors at this level -- there are no more non-null ones + stack.pop(); + unwinding = true; + } else { + currentLevel[firstNonNullIndex] = null; + unwinding = false; + } + } else { + unwinding = false; + } + } + allVisitedNodes.addAll(visitedFromThisStartNode); } } - private static void innerForbidCircularReferences(Map bank, String patternName, List path, String pattern) { - if (patternReferencesItself(pattern, patternName)) { - String message; - if (path.isEmpty()) { - message = "circular reference in pattern [" + patternName + "][" + pattern + "]"; - } else { - message = "circular reference in pattern [" - + path.remove(path.size() - 1) - + "][" - + pattern - + "] back to pattern [" - + patternName - + "]"; - // add rest of the path: - if (path.isEmpty() == false) { - message += " via patterns [" + String.join("=>", path) + "]"; - } + private static int findFirstNonNull(String[] level) { + for (int i = 0; i < level.length; i++) { + if (level[i] != null) { + return i; } - throw new IllegalArgumentException(message); } + return -1; + } - // next check any other pattern names found in the pattern + /** + * This method returns the array of pattern names (if any) found in the bank for the pattern named patternName. If no pattern names + * are found, an empty array is returned. If any of the list of pattern names to be returned does not exist in the bank, an exception + * is thrown. + */ + private static String[] getPatternNamesForPattern(Map bank, String patternName) { + String pattern = bank.get(patternName); + List patternReferences = new ArrayList<>(); for (int i = pattern.indexOf("%{"); i != -1; i = pattern.indexOf("%{", i + 1)) { int begin = i + 2; int bracketIndex = pattern.indexOf('}', begin); @@ -112,25 +167,22 @@ private static void innerForbidCircularReferences(Map bank, Stri end = bracketIndex; } else if (columnIndex != -1 && bracketIndex == -1) { end = columnIndex; - } else if (bracketIndex != -1 && columnIndex != -1) { + } else if (bracketIndex != -1) { end = Math.min(bracketIndex, columnIndex); } else { throw new IllegalArgumentException("pattern [" + pattern + "] has an invalid syntax"); } String otherPatternName = pattern.substring(begin, end); - path.add(otherPatternName); - String otherPattern = bank.get(otherPatternName); - if (otherPattern == null) { - throw new IllegalArgumentException( - "pattern [" + patternName + "] is referencing a non-existent pattern [" + otherPatternName + "]" - ); + if (patternReferences.contains(otherPatternName) == false) { + patternReferences.add(otherPatternName); + String otherPattern = bank.get(otherPatternName); + if (otherPattern == null) { + throw new IllegalArgumentException( + "pattern [" + patternName + "] is referencing a non-existent pattern [" + otherPatternName + "]" + ); + } } - - innerForbidCircularReferences(bank, patternName, path, otherPattern); } - } - - private static boolean patternReferencesItself(String pattern, String patternName) { - return pattern.contains("%{" + patternName + "}") || pattern.contains("%{" + patternName + ":"); + return patternReferences.toArray(new String[0]); } } diff --git a/libs/grok/src/test/java/org/elasticsearch/grok/MatcherWatchdogTests.java b/libs/grok/src/test/java/org/elasticsearch/grok/MatcherWatchdogTests.java index b66778743aec0..5ed1a7d13b80a 100644 --- a/libs/grok/src/test/java/org/elasticsearch/grok/MatcherWatchdogTests.java +++ b/libs/grok/src/test/java/org/elasticsearch/grok/MatcherWatchdogTests.java @@ -7,12 +7,12 @@ */ package org.elasticsearch.grok; +import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.test.ESTestCase; import org.joni.Matcher; import org.mockito.Mockito; import java.util.Map; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -77,16 +77,17 @@ public void testIdleIfNothingRegistered() throws Exception { ); // Periodic action is not scheduled because no thread is registered verifyNoMoreInteractions(threadPool); - CompletableFuture commandFuture = new CompletableFuture<>(); + + PlainActionFuture commandFuture = new PlainActionFuture<>(); // Periodic action is scheduled because a thread is registered doAnswer(invocationOnMock -> { - commandFuture.complete((Runnable) invocationOnMock.getArguments()[0]); + commandFuture.onResponse(invocationOnMock.getArgument(0)); return null; }).when(threadPool).schedule(any(Runnable.class), eq(interval), eq(TimeUnit.MILLISECONDS)); Matcher matcher = mock(Matcher.class); watchdog.register(matcher); // Registering the first thread should have caused the command to get scheduled again - Runnable command = commandFuture.get(1L, TimeUnit.MILLISECONDS); + Runnable command = safeGet(commandFuture); Mockito.reset(threadPool); watchdog.unregister(matcher); command.run(); diff --git a/libs/grok/src/test/java/org/elasticsearch/grok/PatternBankTests.java b/libs/grok/src/test/java/org/elasticsearch/grok/PatternBankTests.java index dcc7ab431611a..08a4965cdb371 100644 --- a/libs/grok/src/test/java/org/elasticsearch/grok/PatternBankTests.java +++ b/libs/grok/src/test/java/org/elasticsearch/grok/PatternBankTests.java @@ -11,8 +11,13 @@ import org.elasticsearch.test.ESTestCase; import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.Map; -import java.util.TreeMap; +import java.util.Set; + +import static org.elasticsearch.test.ESTestCase.randomBoolean; +import static org.hamcrest.Matchers.containsString; public class PatternBankTests extends ESTestCase { @@ -32,7 +37,7 @@ public void testBankCannotBeNull() { public void testConstructorValidatesCircularReferences() { var e = expectThrows(IllegalArgumentException.class, () -> new PatternBank(Map.of("NAME", "!!!%{NAME}!!!"))); - assertEquals("circular reference in pattern [NAME][!!!%{NAME}!!!]", e.getMessage()); + assertEquals("circular reference detected: NAME->NAME", e.getMessage()); } public void testExtendWith() { @@ -48,36 +53,36 @@ public void testExtendWith() { public void testCircularReference() { var e = expectThrows(IllegalArgumentException.class, () -> PatternBank.forbidCircularReferences(Map.of("NAME", "!!!%{NAME}!!!"))); - assertEquals("circular reference in pattern [NAME][!!!%{NAME}!!!]", e.getMessage()); + assertEquals("circular reference detected: NAME->NAME", e.getMessage()); e = expectThrows(IllegalArgumentException.class, () -> PatternBank.forbidCircularReferences(Map.of("NAME", "!!!%{NAME:name}!!!"))); - assertEquals("circular reference in pattern [NAME][!!!%{NAME:name}!!!]", e.getMessage()); + assertEquals("circular reference detected: NAME->NAME", e.getMessage()); e = expectThrows( IllegalArgumentException.class, () -> { PatternBank.forbidCircularReferences(Map.of("NAME", "!!!%{NAME:name:int}!!!")); } ); - assertEquals("circular reference in pattern [NAME][!!!%{NAME:name:int}!!!]", e.getMessage()); + assertEquals("circular reference detected: NAME->NAME", e.getMessage()); e = expectThrows(IllegalArgumentException.class, () -> { - Map bank = new TreeMap<>(); + Map bank = new LinkedHashMap<>(); bank.put("NAME1", "!!!%{NAME2}!!!"); bank.put("NAME2", "!!!%{NAME1}!!!"); PatternBank.forbidCircularReferences(bank); }); - assertEquals("circular reference in pattern [NAME2][!!!%{NAME1}!!!] back to pattern [NAME1]", e.getMessage()); + assertEquals("circular reference detected: NAME1->NAME2->NAME1", e.getMessage()); e = expectThrows(IllegalArgumentException.class, () -> { - Map bank = new TreeMap<>(); + Map bank = new LinkedHashMap<>(); bank.put("NAME1", "!!!%{NAME2}!!!"); bank.put("NAME2", "!!!%{NAME3}!!!"); bank.put("NAME3", "!!!%{NAME1}!!!"); PatternBank.forbidCircularReferences(bank); }); - assertEquals("circular reference in pattern [NAME3][!!!%{NAME1}!!!] back to pattern [NAME1] via patterns [NAME2]", e.getMessage()); + assertEquals("circular reference detected: NAME1->NAME2->NAME3->NAME1", e.getMessage()); e = expectThrows(IllegalArgumentException.class, () -> { - Map bank = new TreeMap<>(); + Map bank = new LinkedHashMap<>(); bank.put("NAME1", "!!!%{NAME2}!!!"); bank.put("NAME2", "!!!%{NAME3}!!!"); bank.put("NAME3", "!!!%{NAME4}!!!"); @@ -85,10 +90,78 @@ public void testCircularReference() { bank.put("NAME5", "!!!%{NAME1}!!!"); PatternBank.forbidCircularReferences(bank); }); - assertEquals( - "circular reference in pattern [NAME5][!!!%{NAME1}!!!] back to pattern [NAME1] via patterns [NAME2=>NAME3=>NAME4]", - e.getMessage() - ); + assertEquals("circular reference detected: NAME1->NAME2->NAME3->NAME4->NAME5->NAME1", e.getMessage()); + + e = expectThrows(IllegalArgumentException.class, () -> { + Map bank = new LinkedHashMap<>(); + bank.put("NAME1", "!!!%{NAME2}!!!"); + bank.put("NAME2", "!!!%{NAME3}!!!"); + bank.put("NAME3", "!!!%{NAME2}!!!"); + PatternBank.forbidCircularReferences(bank); + }); + assertEquals("circular reference detected: NAME2->NAME3->NAME2", e.getMessage()); + + e = expectThrows(IllegalArgumentException.class, () -> { + Map bank = new LinkedHashMap<>(); + bank.put("NAME1", "!!!%{NAME2}!!!"); + bank.put("NAME2", "!!!%{NAME2}!!%{NAME3}!"); + bank.put("NAME3", "!!!%{NAME1}!!!"); + PatternBank.forbidCircularReferences(bank); + }); + assertEquals("circular reference detected: NAME1->NAME2->NAME3->NAME1", e.getMessage()); + + { + Map bank = new HashMap<>(); + bank.put("NAME1", "!!!%{NAME2}!!!%{NAME3}%{NAME4}"); + bank.put("NAME2", "!!!%{NAME3}!!!"); + bank.put("NAME3", "!!!!!!"); + bank.put("NAME4", "!!!%{NAME5}!!!"); + bank.put("NAME5", "!!!!!!"); + PatternBank.forbidCircularReferences(bank); + } + + e = expectThrows(IllegalArgumentException.class, () -> { + Map bank = new LinkedHashMap<>(); + bank.put("NAME1", "!!!%{NAME2}!!!%{NAME3}%{NAME4}"); + bank.put("NAME2", "!!!%{NAME3}!!!"); + bank.put("NAME3", "!!!!!!"); + bank.put("NAME4", "!!!%{NAME5}!!!"); + bank.put("NAME5", "!!!%{NAME1}!!!"); + PatternBank.forbidCircularReferences(bank); + }); + assertEquals("circular reference detected: NAME1->NAME4->NAME5->NAME1", e.getMessage()); + + { + Map bank = new HashMap<>(); + bank.put("NAME1", "!!!%{NAME2}!!!"); + bank.put("NAME2", "!!!%{NAME3}!!!"); + bank.put("NAME3", "!!!!!!"); + bank.put("NAME4", "!!!%{NAME5}!!!"); + bank.put("NAME5", "!!!%{NAME1}!!!"); + PatternBank.forbidCircularReferences(bank); + } + + e = expectThrows(IllegalArgumentException.class, () -> { + Map bank = new LinkedHashMap<>(); + bank.put("NAME1", "!!!%{NAME2} %{NAME3}!!!"); + bank.put("NAME2", "!!!%{NAME4} %{NAME5}!!!"); + bank.put("NAME3", "!!!!!!"); + bank.put("NAME4", "!!!!!!"); + bank.put("NAME5", "!!!%{NAME1}!!!"); + PatternBank.forbidCircularReferences(bank); + }); + assertEquals("circular reference detected: NAME1->NAME2->NAME5->NAME1", e.getMessage()); + + e = expectThrows(IllegalArgumentException.class, () -> { + Map bank = new LinkedHashMap<>(); + bank.put("NAME1", "!!!%{NAME2} %{NAME3}!!!"); + bank.put("NAME2", "!!!%{NAME4} %{NAME5}!!!"); + bank.put("NAME3", "!!!%{NAME1}!!!"); + bank.put("NAME4", "!!!!!!"); + bank.put("NAME5", "!!!!!!"); + PatternBank.forbidCircularReferences(bank); + }); + assertEquals("circular reference detected: NAME1->NAME3->NAME1", e.getMessage()); } public void testCircularSelfReference() { @@ -96,7 +169,7 @@ public void testCircularSelfReference() { IllegalArgumentException.class, () -> PatternBank.forbidCircularReferences(Map.of("ANOTHER", "%{INT}", "INT", "%{INT}")) ); - assertEquals("circular reference in pattern [INT][%{INT}]", e.getMessage()); + assertEquals("circular reference detected: INT->INT", e.getMessage()); } public void testInvalidPatternReferences() { @@ -112,4 +185,80 @@ public void testInvalidPatternReferences() { ); assertEquals("pattern [%{VALID] has an invalid syntax", e.getMessage()); } + + public void testDeepGraphOfPatterns() { + Map patternBankMap = randomBoolean() ? new HashMap<>() : new LinkedHashMap<>(); + final int nodeCount = 20_000; + for (int i = 0; i < nodeCount - 1; i++) { + patternBankMap.put("FOO" + i, "%{FOO" + (i + 1) + "}"); + } + patternBankMap.put("FOO" + (nodeCount - 1), "foo"); + new PatternBank(patternBankMap); + } + + public void testRandomBanksWithoutCycles() { + /* + * This creates a large number of pattens, each of which refers to a large number of patterns. But there are no cycles in any of + * these since each pattern only references patterns with a higher ID. We don't expect any exceptions here. + */ + Map patternBankMap = randomBoolean() ? new HashMap<>() : new LinkedHashMap<>(); + final int nodeCount = 500; + for (int i = 0; i < nodeCount - 1; i++) { + StringBuilder patternBuilder = new StringBuilder(); + for (int j = 0; j < randomIntBetween(0, 20); j++) { + patternBuilder.append("%{FOO-" + randomIntBetween(i + 1, nodeCount - 1) + "}"); + } + patternBankMap.put("FOO-" + i, patternBuilder.toString()); + } + patternBankMap.put("FOO-" + (nodeCount - 1), "foo"); + new PatternBank(patternBankMap); + } + + public void testRandomBanksWithCycles() { + /* + * This creates a large number of pattens, each of which refers to a large number of patterns. We have at least one cycle because + * we pick a node at random, and make sure that a node that it links (or one of its descendants) to links back. If no descendant + * links back to it, we create an artificial cycle at the end. + */ + Map patternBankMap = new LinkedHashMap<>(); + final int nodeCount = 500; + int nodeToHaveCycle = randomIntBetween(0, nodeCount); + int nodeToPotentiallyCreateCycle = -1; + boolean haveCreatedCycle = false; + for (int i = 0; i < nodeCount - 1; i++) { + StringBuilder patternBuilder = new StringBuilder(); + int numberOfLinkedPatterns = randomIntBetween(1, 20); + int nodeToLinkBackIndex = randomIntBetween(0, numberOfLinkedPatterns); + Set childNodes = new HashSet<>(); + for (int j = 0; j < numberOfLinkedPatterns; j++) { + int childNode = randomIntBetween(i + 1, nodeCount - 1); + childNodes.add(childNode); + patternBuilder.append("%{FOO-" + childNode + "}"); + if (i == nodeToHaveCycle) { + if (nodeToLinkBackIndex == j) { + nodeToPotentiallyCreateCycle = childNode; + } + } + } + if (i == nodeToPotentiallyCreateCycle) { + // We either create the cycle here, or randomly pick a child node to maybe create the cycle + if (randomBoolean()) { + patternBuilder.append("%{FOO-" + nodeToHaveCycle + "}"); + haveCreatedCycle = true; + } else { + nodeToPotentiallyCreateCycle = randomFrom(childNodes); + } + } + patternBankMap.put("FOO-" + i, patternBuilder.toString()); + } + if (haveCreatedCycle) { + patternBankMap.put("FOO-" + (nodeCount - 1), "foo"); + } else { + // We didn't randomly create a cycle, so just force one in this last pattern + nodeToHaveCycle = nodeCount - 1; + patternBankMap.put("FOO-" + nodeToHaveCycle, "%{FOO-" + nodeToHaveCycle + "}"); + } + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new PatternBank(patternBankMap)); + assertThat(e.getMessage(), containsString("FOO-" + nodeToHaveCycle)); + } } diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/threadpool/ThreadPoolBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/threadpool/ThreadPoolBridge.java index 13218a9b206a5..30801b4f0b078 100644 --- a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/threadpool/ThreadPoolBridge.java +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/threadpool/ThreadPoolBridge.java @@ -10,6 +10,7 @@ import org.elasticsearch.logstashbridge.StableBridgeAPI; import org.elasticsearch.logstashbridge.common.SettingsBridge; import org.elasticsearch.telemetry.metric.MeterRegistry; +import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders; import org.elasticsearch.threadpool.ThreadPool; import java.util.concurrent.TimeUnit; @@ -17,7 +18,7 @@ public class ThreadPoolBridge extends StableBridgeAPI.Proxy { public ThreadPoolBridge(final SettingsBridge settingsBridge) { - this(new ThreadPool(settingsBridge.unwrap(), MeterRegistry.NOOP)); + this(new ThreadPool(settingsBridge.unwrap(), MeterRegistry.NOOP, new DefaultBuiltInExecutorBuilders())); } public ThreadPoolBridge(final ThreadPool delegate) { diff --git a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaNativeLibraryProvider.java b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaNativeLibraryProvider.java index 454581ae70b51..e0233187425ea 100644 --- a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaNativeLibraryProvider.java +++ b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaNativeLibraryProvider.java @@ -8,14 +8,15 @@ package org.elasticsearch.nativeaccess.jna; +import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.nativeaccess.lib.JavaLibrary; import org.elasticsearch.nativeaccess.lib.Kernel32Library; import org.elasticsearch.nativeaccess.lib.LinuxCLibrary; +import org.elasticsearch.nativeaccess.lib.LoaderHelper; import org.elasticsearch.nativeaccess.lib.MacCLibrary; import org.elasticsearch.nativeaccess.lib.NativeLibrary; import org.elasticsearch.nativeaccess.lib.NativeLibraryProvider; import org.elasticsearch.nativeaccess.lib.PosixCLibrary; -import org.elasticsearch.nativeaccess.lib.SystemdLibrary; import org.elasticsearch.nativeaccess.lib.VectorLibrary; import org.elasticsearch.nativeaccess.lib.ZstdLibrary; @@ -24,6 +25,10 @@ public class JnaNativeLibraryProvider extends NativeLibraryProvider { + static { + setJnaLibraryPath(); + } + public JnaNativeLibraryProvider() { super( "jna", @@ -38,8 +43,6 @@ public JnaNativeLibraryProvider() { JnaMacCLibrary::new, Kernel32Library.class, JnaKernel32Library::new, - SystemdLibrary.class, - JnaSystemdLibrary::new, ZstdLibrary.class, JnaZstdLibrary::new, VectorLibrary.class, @@ -48,6 +51,11 @@ public JnaNativeLibraryProvider() { ); } + @SuppressForbidden(reason = "jna library path must be set for load library to work with our own libs") + private static void setJnaLibraryPath() { + System.setProperty("jna.library.path", LoaderHelper.platformLibDir.toString()); + } + private static Supplier notImplemented() { return () -> { throw new AssertionError(); }; } diff --git a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaSystemdLibrary.java b/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaSystemdLibrary.java deleted file mode 100644 index f06361e8807c5..0000000000000 --- a/libs/native/jna/src/main/java/org/elasticsearch/nativeaccess/jna/JnaSystemdLibrary.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.nativeaccess.jna; - -import com.sun.jna.Library; -import com.sun.jna.Native; - -import org.elasticsearch.nativeaccess.lib.SystemdLibrary; - -class JnaSystemdLibrary implements SystemdLibrary { - private interface NativeFunctions extends Library { - int sd_notify(int unset_environment, String state); - } - - private final NativeFunctions functions; - - JnaSystemdLibrary() { - this.functions = Native.load("libsystemd.so.0", NativeFunctions.class); - } - - @Override - public int sd_notify(int unset_environment, String state) { - return functions.sd_notify(unset_environment, state); - } -} diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/LoaderHelper.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/LoaderHelper.java new file mode 100644 index 0000000000000..42ca60b81a027 --- /dev/null +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/LoaderHelper.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.nativeaccess.lib; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * A utility for loading libraries from Elasticsearch's platform specific lib dir. + */ +public class LoaderHelper { + public static final Path platformLibDir = findPlatformLibDir(); + + private static Path findPlatformLibDir() { + // tests don't have an ES install, so the platform dir must be passed in explicitly + String path = System.getProperty("es.nativelibs.path"); + if (path != null) { + return Paths.get(path); + } + + Path platformDir = Paths.get("lib", "platform"); + + String osname = System.getProperty("os.name"); + String os; + if (osname.startsWith("Windows")) { + os = "windows"; + } else if (osname.startsWith("Linux")) { + os = "linux"; + } else if (osname.startsWith("Mac OS")) { + os = "darwin"; + } else { + os = "unsupported_os[" + osname + "]"; + } + String archname = System.getProperty("os.arch"); + String arch; + if (archname.equals("amd64") || archname.equals("x86_64")) { + arch = "x64"; + } else if (archname.equals("aarch64")) { + arch = archname; + } else { + arch = "unsupported_arch[" + archname + "]"; + } + return platformDir.resolve(os + "-" + arch); + } + + public static void loadLibrary(String libname) { + Path libpath = platformLibDir.resolve(System.mapLibraryName(libname)); + if (Files.exists(libpath) == false) { + throw new UnsatisfiedLinkError("Native library [" + libpath + "] does not exist"); + } + System.load(libpath.toAbsolutePath().toString()); + } + + private LoaderHelper() {} // no construction +} diff --git a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibrary.java b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibrary.java index faa0e861dc63f..cdd0a56c52a90 100644 --- a/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibrary.java +++ b/libs/native/src/main/java/org/elasticsearch/nativeaccess/lib/NativeLibrary.java @@ -9,5 +9,5 @@ package org.elasticsearch.nativeaccess.lib; /** A marker interface for libraries that can be loaded by {@link org.elasticsearch.nativeaccess.lib.NativeLibraryProvider} */ -public sealed interface NativeLibrary permits JavaLibrary, PosixCLibrary, LinuxCLibrary, MacCLibrary, Kernel32Library, SystemdLibrary, - VectorLibrary, ZstdLibrary {} +public sealed interface NativeLibrary permits JavaLibrary, PosixCLibrary, LinuxCLibrary, MacCLibrary, Kernel32Library, VectorLibrary, + ZstdLibrary {} diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkNativeLibraryProvider.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkNativeLibraryProvider.java index cbd43a394379b..1ac7d6c6f897d 100644 --- a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkNativeLibraryProvider.java +++ b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkNativeLibraryProvider.java @@ -14,7 +14,6 @@ import org.elasticsearch.nativeaccess.lib.MacCLibrary; import org.elasticsearch.nativeaccess.lib.NativeLibraryProvider; import org.elasticsearch.nativeaccess.lib.PosixCLibrary; -import org.elasticsearch.nativeaccess.lib.SystemdLibrary; import org.elasticsearch.nativeaccess.lib.VectorLibrary; import org.elasticsearch.nativeaccess.lib.ZstdLibrary; @@ -36,8 +35,6 @@ public JdkNativeLibraryProvider() { JdkMacCLibrary::new, Kernel32Library.class, JdkKernel32Library::new, - SystemdLibrary.class, - JdkSystemdLibrary::new, ZstdLibrary.class, JdkZstdLibrary::new, VectorLibrary.class, diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkSystemdLibrary.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkSystemdLibrary.java deleted file mode 100644 index c34c8c070edc5..0000000000000 --- a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkSystemdLibrary.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.nativeaccess.jdk; - -import org.elasticsearch.nativeaccess.lib.SystemdLibrary; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.lang.foreign.Arena; -import java.lang.foreign.FunctionDescriptor; -import java.lang.foreign.MemorySegment; -import java.lang.invoke.MethodHandle; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import static java.lang.foreign.ValueLayout.ADDRESS; -import static java.lang.foreign.ValueLayout.JAVA_INT; -import static org.elasticsearch.nativeaccess.jdk.LinkerHelper.downcallHandle; - -class JdkSystemdLibrary implements SystemdLibrary { - - static { - // Find and load libsystemd. We attempt all instances of - // libsystemd in case of multiarch systems, and stop when - // one is successfully loaded. If none can be loaded, - // UnsatisfiedLinkError will be thrown. - List paths = findLibSystemd(); - if (paths.isEmpty()) { - String libpath = System.getProperty("java.library.path"); - throw new UnsatisfiedLinkError("Could not find libsystemd in java.library.path: " + libpath); - } - UnsatisfiedLinkError last = null; - for (String path : paths) { - try { - System.load(path); - last = null; - break; - } catch (UnsatisfiedLinkError e) { - last = e; - } - } - if (last != null) { - throw last; - } - } - - // findLibSystemd returns a list of paths to instances of libsystemd - // found within java.library.path. - static List findLibSystemd() { - // Note: on some systems libsystemd does not have a non-versioned symlink. - // System.loadLibrary only knows how to find non-versioned library files, - // so we must manually check the library path to find what we need. - final Path libsystemd = Paths.get("libsystemd.so.0"); - final String libpath = System.getProperty("java.library.path"); - final List foundPaths = new ArrayList<>(); - Arrays.stream(libpath.split(":")).map(Paths::get).filter(Files::exists).forEach(rootPath -> { - try { - Files.walkFileTree(rootPath, new SimpleFileVisitor<>() { - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) { - if (Files.isReadable(dir)) { - return FileVisitResult.CONTINUE; - } - return FileVisitResult.SKIP_SUBTREE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { - if (file.getFileName().equals(libsystemd)) { - foundPaths.add(file.toAbsolutePath().toString()); - } - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFileFailed(Path file, IOException exc) { - return FileVisitResult.CONTINUE; - } - }); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }); - return foundPaths; - } - - private static final MethodHandle sd_notify$mh = downcallHandle("sd_notify", FunctionDescriptor.of(JAVA_INT, JAVA_INT, ADDRESS)); - - @Override - public int sd_notify(int unset_environment, String state) { - try (Arena arena = Arena.ofConfined()) { - MemorySegment nativeState = MemorySegmentUtil.allocateString(arena, state); - return (int) sd_notify$mh.invokeExact(unset_environment, nativeState); - } catch (Throwable t) { - throw new AssertionError(t); - } - } -} diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkVectorLibrary.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkVectorLibrary.java index c92ad654c9b9a..a1032f1381d94 100644 --- a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkVectorLibrary.java +++ b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkVectorLibrary.java @@ -9,6 +9,7 @@ package org.elasticsearch.nativeaccess.jdk; import org.elasticsearch.nativeaccess.VectorSimilarityFunctions; +import org.elasticsearch.nativeaccess.lib.LoaderHelper; import org.elasticsearch.nativeaccess.lib.VectorLibrary; import java.lang.foreign.FunctionDescriptor; @@ -29,7 +30,7 @@ public final class JdkVectorLibrary implements VectorLibrary { static final VectorSimilarityFunctions INSTANCE; static { - System.loadLibrary("vec"); + LoaderHelper.loadLibrary("vec"); final MethodHandle vecCaps$mh = downcallHandle("vec_caps", FunctionDescriptor.of(JAVA_INT)); try { diff --git a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkZstdLibrary.java b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkZstdLibrary.java index e3e972bc19d72..284ac134d2036 100644 --- a/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkZstdLibrary.java +++ b/libs/native/src/main21/java/org/elasticsearch/nativeaccess/jdk/JdkZstdLibrary.java @@ -9,6 +9,7 @@ package org.elasticsearch.nativeaccess.jdk; import org.elasticsearch.nativeaccess.CloseableByteBuffer; +import org.elasticsearch.nativeaccess.lib.LoaderHelper; import org.elasticsearch.nativeaccess.lib.ZstdLibrary; import java.lang.foreign.FunctionDescriptor; @@ -24,7 +25,7 @@ class JdkZstdLibrary implements ZstdLibrary { static { - System.loadLibrary("zstd"); + LoaderHelper.loadLibrary("zstd"); } private static final MethodHandle compressBound$mh = downcallHandle("ZSTD_compressBound", FunctionDescriptor.of(JAVA_LONG, JAVA_INT)); diff --git a/libs/native/src/test/java/org/elasticsearch/nativeaccess/VectorSystemPropertyTests.java b/libs/native/src/test/java/org/elasticsearch/nativeaccess/VectorSystemPropertyTests.java index 9875878d8658a..cda4fc8c55444 100644 --- a/libs/native/src/test/java/org/elasticsearch/nativeaccess/VectorSystemPropertyTests.java +++ b/libs/native/src/test/java/org/elasticsearch/nativeaccess/VectorSystemPropertyTests.java @@ -49,7 +49,7 @@ public void testSystemPropertyDisabled() throws Exception { "-Xms4m", "-cp", jarPath + File.pathSeparator + System.getProperty("java.class.path"), - "-Djava.library.path=" + System.getProperty("java.library.path"), + "-Des.nativelibs.path=" + System.getProperty("es.nativelibs.path"), "p.Test" ).start(); String output = new String(process.getInputStream().readAllBytes(), UTF_8); diff --git a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java index 172b0f24dfd99..fc22bda52e104 100644 --- a/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java +++ b/libs/tdigest/src/main/java/org/elasticsearch/tdigest/MergingDigest.java @@ -92,7 +92,7 @@ public class MergingDigest extends AbstractTDigest { private final int[] order; // if true, alternate upward and downward merge passes - public boolean useAlternatingSort = false; + public boolean useAlternatingSort = true; // if true, use higher working value of compression during construction, then reduce on presentation public boolean useTwoLevelCompression = true; diff --git a/libs/x-content/impl/build.gradle b/libs/x-content/impl/build.gradle index 829b75524baeb..6cf278e826d4c 100644 --- a/libs/x-content/impl/build.gradle +++ b/libs/x-content/impl/build.gradle @@ -12,7 +12,7 @@ base { archivesName = "x-content-impl" } -String jacksonVersion = "2.17.0" +String jacksonVersion = "2.17.2" dependencies { compileOnly project(':libs:elasticsearch-core') diff --git a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentParser.java b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentParser.java index c8e429d4c1490..c59f003d9cb04 100644 --- a/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentParser.java +++ b/libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/JsonXContentParser.java @@ -57,7 +57,8 @@ public Token nextToken() throws IOException { try { return convertToken(parser.nextToken()); } catch (JsonEOFException e) { - throw new XContentEOFException(e); + JsonLocation location = e.getLocation(); + throw new XContentEOFException(new XContentLocation(location.getLineNr(), location.getColumnNr()), "Unexpected end of file", e); } catch (JsonParseException e) { throw newXContentParseException(e); } diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentEOFException.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentEOFException.java index de9ea6fb04f26..01a2407598159 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentEOFException.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentEOFException.java @@ -8,11 +8,9 @@ package org.elasticsearch.xcontent; -import java.io.IOException; +public class XContentEOFException extends XContentParseException { -public class XContentEOFException extends IOException { - - public XContentEOFException(IOException cause) { - super(cause); + public XContentEOFException(XContentLocation location, String message, Exception cause) { + super(location, message, cause); } } diff --git a/modules/aggregations/build.gradle b/modules/aggregations/build.gradle index a773c751eeaf5..91f3303d9d4a8 100644 --- a/modules/aggregations/build.gradle +++ b/modules/aggregations/build.gradle @@ -54,6 +54,9 @@ tasks.named("yamlRestTestV7CompatTransform").configure { task -> task.skipTest("search.aggregation/180_percentiles_tdigest_metric/Filtered test", "Hybrid t-digest produces different results.") task.skipTest("search.aggregation/420_percentile_ranks_tdigest_metric/filtered", "Hybrid t-digest produces different results.") + // Something has changed with response codes + task.skipTest("search.aggregation/20_terms/IP test", "Hybrid t-digest produces different results.") + task.addAllowedWarningRegex("\\[types removal\\].*") } diff --git a/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/30_tokenizers.yml b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/30_tokenizers.yml index 802e599b89f12..71c26372dac59 100644 --- a/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/30_tokenizers.yml +++ b/modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/30_tokenizers.yml @@ -317,22 +317,24 @@ body: text: "a/b/c" explain: true - tokenizer: - type: PathHierarchy + tokenizer: path_hierarchy - length: { detail.tokenizer.tokens: 3 } - - match: { detail.tokenizer.name: __anonymous__PathHierarchy } + - match: { detail.tokenizer.name: path_hierarchy } - match: { detail.tokenizer.tokens.0.token: a } - match: { detail.tokenizer.tokens.1.token: a/b } - match: { detail.tokenizer.tokens.2.token: a/b/c } +--- +"PathHierarchy": - do: indices.analyze: body: text: "a/b/c" explain: true - tokenizer: path_hierarchy + tokenizer: + type: PathHierarchy - length: { detail.tokenizer.tokens: 3 } - - match: { detail.tokenizer.name: path_hierarchy } + - match: { detail.tokenizer.name: __anonymous__PathHierarchy } - match: { detail.tokenizer.tokens.0.token: a } - match: { detail.tokenizer.tokens.1.token: a/b } - match: { detail.tokenizer.tokens.2.token: a/b/c } diff --git a/modules/data-streams/build.gradle b/modules/data-streams/build.gradle index a0375c61d7c29..daf0c188cc83e 100644 --- a/modules/data-streams/build.gradle +++ b/modules/data-streams/build.gradle @@ -1,4 +1,5 @@ import org.elasticsearch.gradle.internal.info.BuildParams +import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask apply plugin: 'elasticsearch.test-with-dependencies' apply plugin: 'elasticsearch.internal-cluster-test' @@ -23,11 +24,7 @@ dependencies { internalClusterTestImplementation project(":modules:mapper-extras") } -tasks.named('yamlRestTest') { - usesDefaultDistribution() -} - -tasks.named('javaRestTest') { +tasks.withType(StandaloneRestIntegTestTask).configureEach { usesDefaultDistribution() } diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java new file mode 100644 index 0000000000000..a52016e8c7f0b --- /dev/null +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java @@ -0,0 +1,428 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.datastreams; + +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; +import org.elasticsearch.action.admin.indices.alias.TransportIndicesAliasesAction; +import org.elasticsearch.action.admin.indices.readonly.AddIndexBlockRequest; +import org.elasticsearch.action.admin.indices.readonly.TransportAddIndexBlockAction; +import org.elasticsearch.action.admin.indices.rollover.RolloverAction; +import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; +import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.FailureStoreMetrics; +import org.elasticsearch.action.datastreams.CreateDataStreamAction; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.ingest.PutPipelineRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.core.Strings; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin; +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.IngestTestPlugin; +import org.elasticsearch.ingest.Processor; +import org.elasticsearch.ingest.TestProcessor; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.telemetry.Measurement; +import org.elasticsearch.telemetry.TestTelemetryPlugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xcontent.XContentType; +import org.junit.Before; + +import java.io.IOException; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.function.Consumer; + +import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; + +/** + * An integration test that verifies how different paths/scenarios affect the APM metrics for failure stores. + */ +@ESIntegTestCase.ClusterScope(numDataNodes = 0, numClientNodes = 0, scope = ESIntegTestCase.Scope.SUITE) +public class IngestFailureStoreMetricsIT extends ESIntegTestCase { + + private static final List METRICS = List.of( + FailureStoreMetrics.METRIC_TOTAL, + FailureStoreMetrics.METRIC_FAILURE_STORE, + FailureStoreMetrics.METRIC_REJECTED + ); + + private String template; + private String dataStream; + private String pipeline; + + @Before + public void initializeRandomNames() { + template = "template-" + randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + dataStream = "data-stream-" + randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + pipeline = "pipeline-" + randomAlphaOfLength(10).toLowerCase(Locale.ROOT); + logger.info( + "--> running [{}] with generated names data stream [{}], template [{}] and pipeline [{}]", + getTestName(), + dataStream, + template, + pipeline + ); + } + + @Override + protected Collection> nodePlugins() { + return List.of(DataStreamsPlugin.class, CustomIngestTestPlugin.class, TestTelemetryPlugin.class, MapperExtrasPlugin.class); + } + + public void testNoPipelineNoFailures() throws IOException { + putComposableIndexTemplate(true); + createDataStream(); + + int nrOfDocs = randomIntBetween(5, 10); + indexDocs(dataStream, nrOfDocs, null); + + var measurements = collectTelemetry(); + assertMeasurements(measurements.get(FailureStoreMetrics.METRIC_TOTAL), nrOfDocs, dataStream); + assertEquals(0, measurements.get(FailureStoreMetrics.METRIC_FAILURE_STORE).size()); + assertEquals(0, measurements.get(FailureStoreMetrics.METRIC_REJECTED).size()); + } + + public void testFailingPipelineNoFailureStore() throws IOException { + putComposableIndexTemplate(false); + createDataStream(); + createBasicPipeline("fail"); + + int nrOfSuccessfulDocs = randomIntBetween(5, 10); + indexDocs(dataStream, nrOfSuccessfulDocs, null); + int nrOfFailingDocs = randomIntBetween(5, 10); + indexDocs(dataStream, nrOfFailingDocs, pipeline); + + var measurements = collectTelemetry(); + assertMeasurements(measurements.get(FailureStoreMetrics.METRIC_TOTAL), nrOfSuccessfulDocs + nrOfFailingDocs, dataStream); + assertEquals(0, measurements.get(FailureStoreMetrics.METRIC_FAILURE_STORE).size()); + assertMeasurements( + measurements.get(FailureStoreMetrics.METRIC_REJECTED), + nrOfFailingDocs, + dataStream, + FailureStoreMetrics.ErrorLocation.PIPELINE, + false + ); + } + + public void testFailingPipelineWithFailureStore() throws IOException { + putComposableIndexTemplate(true); + createDataStream(); + createBasicPipeline("fail"); + + int nrOfSuccessfulDocs = randomIntBetween(5, 10); + indexDocs(dataStream, nrOfSuccessfulDocs, null); + int nrOfFailingDocs = randomIntBetween(5, 10); + indexDocs(dataStream, nrOfFailingDocs, pipeline); + + var measurements = collectTelemetry(); + assertMeasurements(measurements.get(FailureStoreMetrics.METRIC_TOTAL), nrOfSuccessfulDocs + nrOfFailingDocs, dataStream); + assertMeasurements( + measurements.get(FailureStoreMetrics.METRIC_FAILURE_STORE), + nrOfFailingDocs, + dataStream, + FailureStoreMetrics.ErrorLocation.PIPELINE + ); + assertEquals(0, measurements.get(FailureStoreMetrics.METRIC_REJECTED).size()); + } + + public void testShardFailureNoFailureStore() throws IOException { + putComposableIndexTemplate(false); + createDataStream(); + + int nrOfSuccessfulDocs = randomIntBetween(5, 10); + indexDocs(dataStream, nrOfSuccessfulDocs, null); + int nrOfFailingDocs = randomIntBetween(5, 10); + indexDocs(dataStream, nrOfFailingDocs, "\"foo\"", null); + + var measurements = collectTelemetry(); + assertMeasurements(measurements.get(FailureStoreMetrics.METRIC_TOTAL), nrOfSuccessfulDocs + nrOfFailingDocs, dataStream); + assertEquals(0, measurements.get(FailureStoreMetrics.METRIC_FAILURE_STORE).size()); + assertMeasurements( + measurements.get(FailureStoreMetrics.METRIC_REJECTED), + nrOfFailingDocs, + dataStream, + FailureStoreMetrics.ErrorLocation.SHARD, + false + ); + } + + public void testShardFailureWithFailureStore() throws IOException { + putComposableIndexTemplate(true); + createDataStream(); + + int nrOfSuccessfulDocs = randomIntBetween(5, 10); + indexDocs(dataStream, nrOfSuccessfulDocs, null); + int nrOfFailingDocs = randomIntBetween(5, 10); + indexDocs(dataStream, nrOfFailingDocs, "\"foo\"", null); + + var measurements = collectTelemetry(); + assertMeasurements(measurements.get(FailureStoreMetrics.METRIC_TOTAL), nrOfSuccessfulDocs + nrOfFailingDocs, dataStream); + assertMeasurements( + measurements.get(FailureStoreMetrics.METRIC_FAILURE_STORE), + nrOfFailingDocs, + dataStream, + FailureStoreMetrics.ErrorLocation.SHARD + ); + assertEquals(0, measurements.get(FailureStoreMetrics.METRIC_REJECTED).size()); + } + + /** + * Make sure the rejected counter gets incremented when there were shard-level failures while trying to redirect a document to the + * failure store. + */ + public void testRejectionFromFailureStore() throws IOException { + putComposableIndexTemplate(true); + createDataStream(); + + // Initialize failure store. + var rolloverRequest = new RolloverRequest(dataStream, null); + rolloverRequest.setIndicesOptions( + IndicesOptions.builder(rolloverRequest.indicesOptions()) + .failureStoreOptions(opts -> opts.includeFailureIndices(true).includeRegularIndices(false)) + .build() + ); + var rolloverResponse = client().execute(RolloverAction.INSTANCE, rolloverRequest).actionGet(); + var failureStoreIndex = rolloverResponse.getNewIndex(); + // Add a write block to the failure store index, which causes shard-level "failures". + var addIndexBlockRequest = new AddIndexBlockRequest(IndexMetadata.APIBlock.WRITE, failureStoreIndex); + client().execute(TransportAddIndexBlockAction.TYPE, addIndexBlockRequest).actionGet(); + + int nrOfSuccessfulDocs = randomIntBetween(5, 10); + indexDocs(dataStream, nrOfSuccessfulDocs, null); + int nrOfFailingDocs = randomIntBetween(5, 10); + indexDocs(dataStream, nrOfFailingDocs, "\"foo\"", null); + + var measurements = collectTelemetry(); + assertMeasurements(measurements.get(FailureStoreMetrics.METRIC_TOTAL), nrOfSuccessfulDocs + nrOfFailingDocs, dataStream); + assertMeasurements( + measurements.get(FailureStoreMetrics.METRIC_FAILURE_STORE), + nrOfFailingDocs, + dataStream, + FailureStoreMetrics.ErrorLocation.SHARD + ); + assertMeasurements( + measurements.get(FailureStoreMetrics.METRIC_REJECTED), + nrOfFailingDocs, + dataStream, + FailureStoreMetrics.ErrorLocation.SHARD, + true + ); + } + + /** + * Make sure metrics get the correct data_stream attribute after a reroute. + */ + public void testRerouteSuccessfulCorrectName() throws IOException { + putComposableIndexTemplate(false); + createDataStream(); + + String destination = dataStream + "-destination"; + final var createDataStreamRequest = new CreateDataStreamAction.Request(destination); + assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).actionGet()); + createReroutePipeline(destination); + + int nrOfDocs = randomIntBetween(5, 10); + indexDocs(dataStream, nrOfDocs, pipeline); + + var measurements = collectTelemetry(); + assertMeasurements(measurements.get(FailureStoreMetrics.METRIC_TOTAL), nrOfDocs, destination); + assertEquals(0, measurements.get(FailureStoreMetrics.METRIC_FAILURE_STORE).size()); + assertEquals(0, measurements.get(FailureStoreMetrics.METRIC_REJECTED).size()); + } + + public void testDropping() throws IOException { + putComposableIndexTemplate(true); + createDataStream(); + createBasicPipeline("drop"); + + int nrOfDocs = randomIntBetween(5, 10); + indexDocs(dataStream, nrOfDocs, pipeline); + + var measurements = collectTelemetry(); + assertMeasurements(measurements.get(FailureStoreMetrics.METRIC_TOTAL), nrOfDocs, dataStream); + assertEquals(0, measurements.get(FailureStoreMetrics.METRIC_FAILURE_STORE).size()); + assertEquals(0, measurements.get(FailureStoreMetrics.METRIC_REJECTED).size()); + } + + public void testDataStreamAlias() throws IOException { + putComposableIndexTemplate(false); + createDataStream(); + var indicesAliasesRequest = new IndicesAliasesRequest(); + indicesAliasesRequest.addAliasAction( + IndicesAliasesRequest.AliasActions.add().alias("some-alias").index(dataStream).writeIndex(true) + ); + client().execute(TransportIndicesAliasesAction.TYPE, indicesAliasesRequest).actionGet(); + + int nrOfDocs = randomIntBetween(5, 10); + indexDocs("some-alias", nrOfDocs, null); + + var measurements = collectTelemetry(); + assertMeasurements(measurements.get(FailureStoreMetrics.METRIC_TOTAL), nrOfDocs, dataStream); + assertEquals(0, measurements.get(FailureStoreMetrics.METRIC_FAILURE_STORE).size()); + assertEquals(0, measurements.get(FailureStoreMetrics.METRIC_REJECTED).size()); + } + + private void putComposableIndexTemplate(boolean failureStore) throws IOException { + TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request(template); + request.indexTemplate( + ComposableIndexTemplate.builder() + .indexPatterns(List.of(dataStream + "*")) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false, failureStore)) + .template(new Template(null, new CompressedXContent(""" + { + "dynamic": false, + "properties": { + "@timestamp": { + "type": "date" + }, + "count": { + "type": "long" + } + } + }"""), null)) + .build() + ); + client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); + } + + private void createDataStream() { + final var createDataStreamRequest = new CreateDataStreamAction.Request(dataStream); + assertAcked(client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).actionGet()); + } + + private void createBasicPipeline(String processorType) { + createPipeline(Strings.format("\"%s\": {}", processorType)); + } + + private void createReroutePipeline(String destination) { + createPipeline(Strings.format("\"reroute\": {\"destination\": \"%s\"}", destination)); + } + + private void createPipeline(String processor) { + String pipelineDefinition = Strings.format("{\"processors\": [{%s}]}", processor); + BytesReference bytes = new BytesArray(pipelineDefinition); + clusterAdmin().putPipeline(new PutPipelineRequest(pipeline, bytes, XContentType.JSON)).actionGet(); + } + + private void indexDocs(String dataStream, int numDocs, String pipeline) { + indexDocs(dataStream, numDocs, "1", pipeline); + } + + private void indexDocs(String dataStream, int numDocs, String value, String pipeline) { + BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + for (int i = 0; i < numDocs; i++) { + String time = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(System.currentTimeMillis()); + bulkRequest.add( + new IndexRequest(dataStream).opType(DocWriteRequest.OpType.CREATE) + .source(Strings.format("{\"%s\":\"%s\", \"count\": %s}", DEFAULT_TIMESTAMP_FIELD, time, value), XContentType.JSON) + .setPipeline(pipeline) + ); + } + client().bulk(bulkRequest).actionGet(); + } + + private static Map> collectTelemetry() { + Map> measurements = new HashMap<>(); + for (PluginsService pluginsService : internalCluster().getInstances(PluginsService.class)) { + final TestTelemetryPlugin telemetryPlugin = pluginsService.filterPlugins(TestTelemetryPlugin.class).findFirst().orElseThrow(); + + telemetryPlugin.collect(); + + for (String metricName : METRICS) { + measurements.put(metricName, telemetryPlugin.getLongCounterMeasurement(metricName)); + } + } + return measurements; + } + + private void assertMeasurements(List measurements, int expectedSize, String expectedDataStream) { + assertMeasurements(measurements, expectedSize, expectedDataStream, (Consumer) null); + } + + private void assertMeasurements( + List measurements, + int expectedSize, + String expectedDataStream, + FailureStoreMetrics.ErrorLocation location + ) { + assertMeasurements( + measurements, + expectedSize, + expectedDataStream, + measurement -> assertEquals(location.name(), measurement.attributes().get("error_location")) + ); + } + + private void assertMeasurements( + List measurements, + int expectedSize, + String expectedDataStream, + FailureStoreMetrics.ErrorLocation location, + boolean failureStore + ) { + assertMeasurements(measurements, expectedSize, expectedDataStream, measurement -> { + assertEquals(location.name(), measurement.attributes().get("error_location")); + assertEquals(failureStore, measurement.attributes().get("failure_store")); + }); + } + + private void assertMeasurements( + List measurements, + int expectedSize, + String expectedDataStream, + Consumer customAssertion + ) { + assertEquals(expectedSize, measurements.size()); + for (Measurement measurement : measurements) { + assertEquals(expectedDataStream, measurement.attributes().get("data_stream")); + if (customAssertion != null) { + customAssertion.accept(measurement); + } + } + } + + public static class CustomIngestTestPlugin extends IngestTestPlugin { + @Override + public Map getProcessors(Processor.Parameters parameters) { + Map processors = new HashMap<>(); + processors.put( + "drop", + (factories, tag, description, config) -> new TestProcessor(tag, "drop", description, ingestDocument -> null) + ); + processors.put("reroute", (factories, tag, description, config) -> { + String destination = (String) config.remove("destination"); + return new TestProcessor( + tag, + "reroute", + description, + (Consumer) ingestDocument -> ingestDocument.reroute(destination) + ); + }); + processors.put( + "fail", + (processorFactories, tag, description, config) -> new TestProcessor(tag, "fail", description, new RuntimeException()) + ); + return processors; + } + } +} diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java index 24c373df72144..a0a0681dbd245 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.Template; @@ -35,6 +36,7 @@ import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.indices.InvalidIndexTemplateException; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; @@ -457,6 +459,16 @@ public void testTrimId() throws Exception { indexName = bulkResponse.getItems()[0].getIndex(); } client().admin().indices().refresh(new RefreshRequest(dataStreamName)).actionGet(); + + // In rare cases we can end up with a single segment shard, which means we can't trim away the _id later. + // So update an existing doc to create a new segment without adding a new document after force merging: + var indexRequest = new IndexRequest(indexName).setIfPrimaryTerm(1L) + .setIfSeqNo((numBulkRequests * numDocsPerBulk) - 1) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + indexRequest.source(DOC.replace("$time", formatInstant(time.minusMillis(1))), XContentType.JSON); + var res = client().index(indexRequest).actionGet(); + assertThat(res.status(), equalTo(RestStatus.OK)); + assertThat(res.getVersion(), equalTo(2L)); } // Check whether there are multiple segments: @@ -494,7 +506,7 @@ public void testTrimId() throws Exception { assertThat(retentionLeasesStats.retentionLeases().leases(), hasSize(1)); assertThat( retentionLeasesStats.retentionLeases().leases().iterator().next().retainingSequenceNumber(), - equalTo((long) numBulkRequests * numDocsPerBulk) + equalTo((long) numBulkRequests * numDocsPerBulk + 1) ); }); diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamGlobalRetentionIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamGlobalRetentionIT.java new file mode 100644 index 0000000000000..514eb6d8742ea --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamGlobalRetentionIT.java @@ -0,0 +1,190 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.datastreams.lifecycle; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.WarningFailureException; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.datastreams.DisabledSecurityDataStreamTestCase; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; + +public class DataStreamGlobalRetentionIT extends DisabledSecurityDataStreamTestCase { + + @Before + public void setup() throws IOException { + updateClusterSettings( + Settings.builder() + .put("data_streams.lifecycle.poll_interval", "1s") + .put("cluster.lifecycle.default.rollover", "min_docs=1,max_docs=1") + .build() + ); + // Create a template with the default lifecycle + Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/1"); + putComposableIndexTemplateRequest.setJsonEntity(""" + { + "index_patterns": ["my-data-stream*"], + "data_stream": {}, + "template": { + "lifecycle": {} + } + } + """); + assertOK(client().performRequest(putComposableIndexTemplateRequest)); + + // Create a data streams with one doc + Request createDocRequest = new Request("POST", "/my-data-stream/_doc?refresh=true"); + createDocRequest.setJsonEntity("{ \"@timestamp\": \"2022-12-12\"}"); + assertOK(client().performRequest(createDocRequest)); + } + + @After + public void cleanUp() throws IOException { + adminClient().performRequest(new Request("DELETE", "_data_stream/*")); + updateClusterSettings( + Settings.builder().putNull("data_streams.lifecycle.retention.default").putNull("data_streams.lifecycle.retention.max").build() + ); + } + + @SuppressWarnings("unchecked") + public void testDataStreamRetention() throws Exception { + // Set global retention and add retention to the data stream + { + updateClusterSettings( + Settings.builder() + .put("data_streams.lifecycle.retention.default", "7d") + .put("data_streams.lifecycle.retention.default", "90d") + .build() + ); + Request request = new Request("PUT", "_data_stream/my-data-stream/_lifecycle"); + request.setJsonEntity(""" + { + "data_retention": "10s" + }"""); + assertAcknowledged(client().performRequest(request)); + } + + // Verify that the effective retention matches the default retention + { + Request request = new Request("GET", "/_data_stream/my-data-stream"); + Response response = client().performRequest(request); + List dataStreams = (List) entityAsMap(response).get("data_streams"); + assertThat(dataStreams.size(), is(1)); + Map dataStream = (Map) dataStreams.get(0); + assertThat(dataStream.get("name"), is("my-data-stream")); + Map lifecycle = (Map) dataStream.get("lifecycle"); + assertThat(lifecycle.get("effective_retention"), is("10s")); + assertThat(lifecycle.get("retention_determined_by"), is("data_stream_configuration")); + assertThat(lifecycle.get("data_retention"), is("10s")); + } + + // Verify that the first generation index was removed + assertBusy(() -> { + Response response = client().performRequest(new Request("GET", "/_data_stream/my-data-stream")); + Map dataStream = ((List>) entityAsMap(response).get("data_streams")).get(0); + assertThat(dataStream.get("name"), is("my-data-stream")); + List backingIndices = (List) dataStream.get("indices"); + assertThat(backingIndices.size(), is(1)); + // 2 backing indices created + 1 for the deleted index + assertThat(dataStream.get("generation"), is(3)); + }, 20, TimeUnit.SECONDS); + } + + @SuppressWarnings("unchecked") + public void testDefaultRetention() throws Exception { + // Set default global retention + updateClusterSettings(Settings.builder().put("data_streams.lifecycle.retention.default", "10s").build()); + + // Verify that the effective retention matches the default retention + { + Request request = new Request("GET", "/_data_stream/my-data-stream"); + Response response = client().performRequest(request); + List dataStreams = (List) entityAsMap(response).get("data_streams"); + assertThat(dataStreams.size(), is(1)); + Map dataStream = (Map) dataStreams.get(0); + assertThat(dataStream.get("name"), is("my-data-stream")); + Map lifecycle = (Map) dataStream.get("lifecycle"); + assertThat(lifecycle.get("effective_retention"), is("10s")); + assertThat(lifecycle.get("retention_determined_by"), is("default_global_retention")); + assertThat(lifecycle.get("data_retention"), nullValue()); + } + + // Verify that the first generation index was removed + assertBusy(() -> { + Response response = client().performRequest(new Request("GET", "/_data_stream/my-data-stream")); + Map dataStream = ((List>) entityAsMap(response).get("data_streams")).get(0); + assertThat(dataStream.get("name"), is("my-data-stream")); + List backingIndices = (List) dataStream.get("indices"); + assertThat(backingIndices.size(), is(1)); + // 2 backing indices created + 1 for the deleted index + assertThat(dataStream.get("generation"), is(3)); + }, 20, TimeUnit.SECONDS); + } + + @SuppressWarnings("unchecked") + public void testMaxRetention() throws Exception { + // Set default global retention + updateClusterSettings(Settings.builder().put("data_streams.lifecycle.retention.max", "10s").build()); + boolean withDataStreamLevelRetention = randomBoolean(); + if (withDataStreamLevelRetention) { + try { + Request request = new Request("PUT", "_data_stream/my-data-stream/_lifecycle"); + request.setJsonEntity(""" + { + "data_retention": "30d" + }"""); + assertAcknowledged(client().performRequest(request)); + fail("Should have returned a warning about data retention exceeding the max retention"); + } catch (WarningFailureException warningFailureException) { + assertThat( + warningFailureException.getMessage(), + containsString("The retention provided [30d] is exceeding the max allowed data retention of this project [10s]") + ); + } + } + + // Verify that the effective retention matches the max retention + { + Request request = new Request("GET", "/_data_stream/my-data-stream"); + Response response = client().performRequest(request); + List dataStreams = (List) entityAsMap(response).get("data_streams"); + assertThat(dataStreams.size(), is(1)); + Map dataStream = (Map) dataStreams.get(0); + assertThat(dataStream.get("name"), is("my-data-stream")); + Map lifecycle = (Map) dataStream.get("lifecycle"); + assertThat(lifecycle.get("effective_retention"), is("10s")); + assertThat(lifecycle.get("retention_determined_by"), is("max_global_retention")); + if (withDataStreamLevelRetention) { + assertThat(lifecycle.get("data_retention"), is("30d")); + } else { + assertThat(lifecycle.get("data_retention"), nullValue()); + } + } + + // Verify that the first generation index was removed + assertBusy(() -> { + Response response = client().performRequest(new Request("GET", "/_data_stream/my-data-stream")); + Map dataStream = ((List>) entityAsMap(response).get("data_streams")).get(0); + assertThat(dataStream.get("name"), is("my-data-stream")); + List backingIndices = (List) dataStream.get("indices"); + assertThat(backingIndices.size(), is(1)); + // 2 backing indices created + 1 for the deleted index + assertThat(dataStream.get("generation"), is(3)); + }, 20, TimeUnit.SECONDS); + } +} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/AbstractChallengeRestTest.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/AbstractChallengeRestTest.java index 6292b06d44c9a..1d36a04657e9c 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/AbstractChallengeRestTest.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/AbstractChallengeRestTest.java @@ -233,9 +233,11 @@ private Response indexDocuments( final CheckedSupplier, IOException> documentsSupplier ) throws IOException { final StringBuilder sb = new StringBuilder(); + int id = 0; for (var document : documentsSupplier.get()) { - sb.append("{ \"create\": {} }").append("\n"); + sb.append(Strings.format("{ \"create\": { \"_id\" : \"%d\" } }", id)).append("\n"); sb.append(Strings.toString(document)).append("\n"); + id++; } var request = new Request("POST", "/" + dataStreamName + "/_bulk"); request.setJsonEntity(sb.toString()); diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java index 5824f8fa764f4..9bf1c394f9105 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java @@ -34,8 +34,11 @@ import java.io.IOException; import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; import java.time.temporal.ChronoUnit; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.Map; @@ -178,11 +181,8 @@ protected static void waitForLogs(RestClient client) throws Exception { } public void testMatchAllQuery() throws IOException { - final List documents = new ArrayList<>(); int numberOfDocuments = ESTestCase.randomIntBetween(100, 200); - for (int i = 0; i < numberOfDocuments; i++) { - documents.add(generateDocument(Instant.now().plus(i, ChronoUnit.SECONDS))); - } + final List documents = generateDocuments(numberOfDocuments); assertDocumentIndexing(documents); @@ -199,11 +199,8 @@ public void testMatchAllQuery() throws IOException { } public void testTermsQuery() throws IOException { - final List documents = new ArrayList<>(); - int numberOfDocuments = randomIntBetween(100, 200); - for (int i = 0; i < numberOfDocuments; i++) { - documents.add(generateDocument(Instant.now().plus(i, ChronoUnit.SECONDS))); - } + int numberOfDocuments = ESTestCase.randomIntBetween(100, 200); + final List documents = generateDocuments(numberOfDocuments); assertDocumentIndexing(documents); @@ -220,11 +217,8 @@ public void testTermsQuery() throws IOException { } public void testHistogramAggregation() throws IOException { - final List documents = new ArrayList<>(); - int numberOfDocuments = randomIntBetween(100, 200); - for (int i = 0; i < numberOfDocuments; i++) { - documents.add(generateDocument(Instant.now().plus(i, ChronoUnit.SECONDS))); - } + int numberOfDocuments = ESTestCase.randomIntBetween(100, 200); + final List documents = generateDocuments(numberOfDocuments); assertDocumentIndexing(documents); @@ -241,11 +235,8 @@ public void testHistogramAggregation() throws IOException { } public void testTermsAggregation() throws IOException { - final List documents = new ArrayList<>(); - int numberOfDocuments = randomIntBetween(100, 200); - for (int i = 0; i < numberOfDocuments; i++) { - documents.add(generateDocument(Instant.now().plus(i, ChronoUnit.SECONDS))); - } + int numberOfDocuments = ESTestCase.randomIntBetween(100, 200); + final List documents = generateDocuments(numberOfDocuments); assertDocumentIndexing(documents); @@ -262,11 +253,8 @@ public void testTermsAggregation() throws IOException { } public void testDateHistogramAggregation() throws IOException { - final List documents = new ArrayList<>(); - int numberOfDocuments = randomIntBetween(100, 200); - for (int i = 0; i < numberOfDocuments; i++) { - documents.add(generateDocument(Instant.now().plus(i, ChronoUnit.SECONDS))); - } + int numberOfDocuments = ESTestCase.randomIntBetween(100, 200); + final List documents = generateDocuments(numberOfDocuments); assertDocumentIndexing(documents); @@ -282,6 +270,17 @@ public void testDateHistogramAggregation() throws IOException { assertTrue(matchResult.getMessage(), matchResult.isMatch()); } + private List generateDocuments(int numberOfDocuments) throws IOException { + final List documents = new ArrayList<>(); + // This is static in order to be able to identify documents between test runs. + var startingPoint = ZonedDateTime.of(2024, 1, 1, 10, 0, 0, 0, ZoneId.of("UTC")).toInstant(); + for (int i = 0; i < numberOfDocuments; i++) { + documents.add(generateDocument(startingPoint.plus(i, ChronoUnit.SECONDS))); + } + + return documents; + } + protected XContentBuilder generateDocument(final Instant timestamp) throws IOException { return XContentFactory.jsonBuilder() .startObject() @@ -301,7 +300,10 @@ private static List> getQueryHits(final Response response) t final List> hitsList = (List>) hitsMap.get("hits"); assertThat(hitsList.size(), greaterThan(0)); - return hitsList.stream().map(hit -> (Map) hit.get("_source")).toList(); + return hitsList.stream() + .sorted(Comparator.comparingInt((Map hit) -> Integer.parseInt((String) hit.get("_id")))) + .map(hit -> (Map) hit.get("_source")) + .toList(); } @SuppressWarnings("unchecked") diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java index 4e123c1630457..ad4302cb04b44 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java @@ -8,10 +8,15 @@ package org.elasticsearch.datastreams.logsdb.qa; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.FormatNames; +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.ObjectMapper; import org.elasticsearch.logsdb.datageneration.DataGenerator; import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification; +import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceHandler; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; @@ -25,42 +30,31 @@ import java.time.Instant; import java.util.HashMap; import java.util.List; +import java.util.Map; /** * Challenge test (see {@link StandardVersusLogsIndexModeChallengeRestIT}) that uses randomly generated * mapping and documents in order to cover more code paths and permutations. */ public class StandardVersusLogsIndexModeRandomDataChallengeRestIT extends StandardVersusLogsIndexModeChallengeRestIT { - private final boolean fullyDynamicMapping; - private final boolean subobjectsDisabled; + private final ObjectMapper.Subobjects subobjects; + private final boolean keepArraySource; private final DataGenerator dataGenerator; public StandardVersusLogsIndexModeRandomDataChallengeRestIT() { super(); - this.fullyDynamicMapping = randomBoolean(); - this.subobjectsDisabled = randomBoolean(); + this.subobjects = randomFrom(ObjectMapper.Subobjects.values()); + this.keepArraySource = randomBoolean(); - var specificationBuilder = DataGeneratorSpecification.builder(); - if (subobjectsDisabled) { + var specificationBuilder = DataGeneratorSpecification.builder().withFullyDynamicMapping(randomBoolean()); + if (subobjects != ObjectMapper.Subobjects.ENABLED) { specificationBuilder = specificationBuilder.withNestedFieldsLimit(0); } this.dataGenerator = new DataGenerator(specificationBuilder.withDataSourceHandlers(List.of(new DataSourceHandler() { @Override - public DataSourceResponse.FieldTypeGenerator handle(DataSourceRequest.FieldTypeGenerator request) { - // Unsigned long is not used with dynamic mapping - // since it can initially look like long - // but later fail to parse once big values arrive. - // Double is not used since it maps to float with dynamic mapping - // resulting in precision loss compared to original source. - var excluded = fullyDynamicMapping ? List.of(FieldType.DOUBLE, FieldType.SCALED_FLOAT, FieldType.UNSIGNED_LONG) : List.of(); - return new DataSourceResponse.FieldTypeGenerator( - () -> randomValueOtherThanMany(excluded::contains, () -> randomFrom(FieldType.values())) - ); - } - public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequest.ObjectMappingParametersGenerator request) { - if (subobjectsDisabled == false) { + if (subobjects == ObjectMapper.Subobjects.ENABLED) { // Use default behavior return null; } @@ -71,52 +65,69 @@ public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequ // "dynamic: false/strict/runtime" is not compatible with subobjects: false return new DataSourceResponse.ObjectMappingParametersGenerator(() -> { var parameters = new HashMap(); + parameters.put("subobjects", subobjects.toString()); if (ESTestCase.randomBoolean()) { parameters.put("dynamic", "true"); } if (ESTestCase.randomBoolean()) { parameters.put("enabled", "true"); } - return parameters; }); } - })).withPredefinedFields(List.of(new PredefinedField("host.name", FieldType.KEYWORD))).build()); + })) + .withPredefinedFields( + List.of( + new PredefinedField.WithType("host.name", FieldType.KEYWORD), + // Needed for terms query + new PredefinedField.WithGenerator("method", new FieldDataGenerator() { + @Override + public CheckedConsumer mappingWriter() { + return b -> b.startObject().field("type", "keyword").endObject(); + } + + @Override + public CheckedConsumer fieldValueGenerator() { + return b -> b.value(randomFrom("put", "post", "get")); + } + }), + + // Needed for histogram aggregation + new PredefinedField.WithGenerator("memory_usage_bytes", new FieldDataGenerator() { + @Override + public CheckedConsumer mappingWriter() { + return b -> b.startObject().field("type", "long").endObject(); + } + + @Override + public CheckedConsumer fieldValueGenerator() { + // We can generate this using standard long field but we would get "too many buckets" + return b -> b.value(randomLongBetween(1000, 2000)); + } + }) + ) + ) + .build()); } @Override public void baselineMappings(XContentBuilder builder) throws IOException { - if (fullyDynamicMapping == false) { - dataGenerator.writeMapping(builder); - } else { - // We want dynamic mapping, but we need host.name to be a keyword instead of text to support aggregations. - builder.startObject() - .startObject("properties") - - .startObject("host.name") - .field("type", "keyword") - .field("ignore_above", randomIntBetween(1000, 1200)) - .endObject() - - .endObject() - .endObject(); - } + dataGenerator.writeMapping(builder); } @Override public void contenderMappings(XContentBuilder builder) throws IOException { - if (fullyDynamicMapping == false) { - if (subobjectsDisabled) { - dataGenerator.writeMapping(builder, b -> builder.field("subobjects", false)); - } else { - dataGenerator.writeMapping(builder); - } + if (subobjects != ObjectMapper.Subobjects.ENABLED) { + dataGenerator.writeMapping(builder, Map.of("subobjects", subobjects.toString())); } else { - builder.startObject(); - if (subobjectsDisabled) { - builder.field("subobjects", false); - } - builder.endObject(); + dataGenerator.writeMapping(builder); + } + } + + @Override + public void contenderSettings(Settings.Builder builder) { + if (keepArraySource) { + builder.put(Mapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING.getKey(), "arrays"); } } @@ -125,10 +136,6 @@ protected XContentBuilder generateDocument(final Instant timestamp) throws IOExc var document = XContentFactory.jsonBuilder(); dataGenerator.generateDocument(document, doc -> { doc.field("@timestamp", DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(timestamp)); - // Needed for terms query - doc.field("method", randomFrom("put", "post", "get")); - // We can generate this but we would get "too many buckets" - doc.field("memory_usage_bytes", randomLongBetween(1000, 2000)); }); return document; diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ListEqualMatcher.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ListEqualMatcher.java index bb5751b8873f2..ae18129a77111 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ListEqualMatcher.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/ListEqualMatcher.java @@ -43,7 +43,7 @@ private MatchResult matchListEquals(final List actualList, final List match(List actual, List expected) { + if (expected == null) { + return Optional.empty(); + } + + // Floating point values are always mapped as float with dynamic mapping. + var isDouble = expected.stream().filter(Objects::nonNull).findFirst().map(o -> o instanceof Double).orElse(false); + if (isDouble) { + assert expected.stream().allMatch(o -> o == null || o instanceof Double); + + var normalizedActual = normalizeDoubles(actual); + var normalizedExpected = normalizeDoubles(expected); + + var matchResult = normalizedActual.equals(normalizedExpected) + ? MatchResult.match() + : MatchResult.noMatch( + formatErrorMessage( + actualMappings, + actualSettings, + expectedMappings, + expectedSettings, + "Values of dynamically mapped field containing double values don't match after normalization, normalized " + + prettyPrintCollections(normalizedActual, normalizedExpected) + ) + ); + return Optional.of(matchResult); + } + + return Optional.empty(); + } + + private static Set normalizeDoubles(List values) { + if (values == null) { + return Set.of(); + } + + Function toFloat = (o) -> o instanceof Number n ? n.floatValue() : Float.parseFloat((String) o); + return values.stream().filter(Objects::nonNull).map(toFloat).collect(Collectors.toSet()); + } +} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/FieldSpecificMatcher.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/FieldSpecificMatcher.java index 10b1922e1e217..253fb4b0e9688 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/FieldSpecificMatcher.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/FieldSpecificMatcher.java @@ -198,7 +198,7 @@ public MatchResult match( actualSettings, expectedMappings, expectedSettings, - "Values of type [scaled_float] don't match after normalization, normalized " + "Values of type [unsigned_long] don't match after normalization, normalized " + prettyPrintCollections(actualNormalized, expectedNormalized) ) ); diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/MappingTransforms.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/MappingTransforms.java index 4ca3142310b44..eade6f10e48fe 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/MappingTransforms.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/MappingTransforms.java @@ -8,10 +8,21 @@ package org.elasticsearch.datastreams.logsdb.qa.matchers.source; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; class MappingTransforms { + /** + * Container for mapping of a field. Contains field mapping parameters and mapping parameters of parent fields (if present) + * in order of increasing distance (direct parent first). + * This is needed because some parent mapping parameters influence how source of the field is stored (e.g. `enabled: false`). + * @param mappingParameters + * @param parentMappingParameters + */ + record FieldMapping(Map mappingParameters, List> parentMappingParameters) {} + /** * Normalize mapping to have the same structure as normalized source and enable field mapping lookup. * Similar to {@link SourceTransforms#normalize(Map)} but needs to get rid of intermediate nodes @@ -20,8 +31,8 @@ class MappingTransforms { * @param map raw mapping document converted to map * @return map from normalized field name (like a.b.c) to a map of mapping parameters (like type) */ - public static Map> normalizeMapping(Map map) { - var flattened = new HashMap>(); + public static Map normalizeMapping(Map map) { + var flattened = new HashMap(); descend(null, map, flattened); @@ -29,21 +40,36 @@ public static Map> normalizeMapping(Map currentLevel, Map> flattened) { + private static void descend(String pathFromRoot, Map currentLevel, Map flattened) { for (var entry : currentLevel.entrySet()) { if (entry.getKey().equals("_doc") || entry.getKey().equals("properties")) { descend(pathFromRoot, (Map) entry.getValue(), flattened); } else { if (entry.getValue() instanceof Map map) { var pathToField = pathFromRoot == null ? entry.getKey() : pathFromRoot + "." + entry.getKey(); - descend(pathToField, (Map) map, flattened); - } else { - if (pathFromRoot == null) { - // Ignore top level mapping parameters for now - continue; + + // Descending to subobject, we need to remember parent mapping + if (pathFromRoot != null) { + var parentMapping = flattened.computeIfAbsent( + pathFromRoot, + k -> new FieldMapping(new HashMap<>(), new ArrayList<>()) + ); + var childMapping = flattened.computeIfAbsent( + pathToField, + k -> new FieldMapping(new HashMap<>(), new ArrayList<>()) + ); + childMapping.parentMappingParameters.add(parentMapping.mappingParameters); + childMapping.parentMappingParameters.addAll(parentMapping.parentMappingParameters); } - flattened.computeIfAbsent(pathFromRoot, k -> new HashMap<>()).put(entry.getKey(), entry.getValue()); + descend(pathToField, (Map) map, flattened); + } else { + var pathToField = pathFromRoot == null ? "_doc" : pathFromRoot; + // We are either at the lowest level of mapping or it's a leaf field of top level object + flattened.computeIfAbsent(pathToField, k -> new FieldMapping(new HashMap<>(), new ArrayList<>())).mappingParameters.put( + entry.getKey(), + entry.getValue() + ); } } } diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/SourceMatcher.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/SourceMatcher.java index f0e188a17631f..5eb93cee67d74 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/SourceMatcher.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/matchers/source/SourceMatcher.java @@ -10,15 +10,12 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.time.DateFormatter; -import org.elasticsearch.common.time.FormatNames; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.datastreams.logsdb.qa.matchers.GenericEqualsMatcher; import org.elasticsearch.datastreams.logsdb.qa.matchers.ListEqualMatcher; import org.elasticsearch.datastreams.logsdb.qa.matchers.MatchResult; import org.elasticsearch.xcontent.XContentBuilder; -import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Objects; @@ -28,10 +25,11 @@ import static org.elasticsearch.datastreams.logsdb.qa.matchers.Messages.prettyPrintCollections; public class SourceMatcher extends GenericEqualsMatcher>> { - private final Map> actualNormalizedMapping; - private final Map> expectedNormalizedMapping; + private final Map actualNormalizedMapping; + private final Map expectedNormalizedMapping; private final Map fieldSpecificMatchers; + private final DynamicFieldMatcher dynamicFieldMatcher; public SourceMatcher( final XContentBuilder actualMappings, @@ -60,6 +58,7 @@ public SourceMatcher( "unsigned_long", new FieldSpecificMatcher.UnsignedLongMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings) ); + this.dynamicFieldMatcher = new DynamicFieldMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings); } @Override @@ -76,14 +75,8 @@ public MatchResult match() { ); } - var sortedAndFlattenedActual = actual.stream() - .sorted(Comparator.comparing((Map m) -> parseTimestampToEpochMillis(m.get("@timestamp")))) - .map(SourceTransforms::normalize) - .toList(); - var sortedAndFlattenedExpected = expected.stream() - .sorted(Comparator.comparing((Map m) -> parseTimestampToEpochMillis(m.get("@timestamp")))) - .map(SourceTransforms::normalize) - .toList(); + var sortedAndFlattenedActual = actual.stream().map(SourceTransforms::normalize).toList(); + var sortedAndFlattenedExpected = expected.stream().map(SourceTransforms::normalize).toList(); for (int i = 0; i < sortedAndFlattenedActual.size(); i++) { var actual = sortedAndFlattenedActual.get(i); @@ -91,7 +84,8 @@ public MatchResult match() { var result = compareSource(actual, expected); if (result.isMatch() == false) { - return result; + var message = "Source matching failed at document id [" + i + "]. " + result.getMessage(); + return MatchResult.noMatch(message); } } @@ -105,12 +99,20 @@ private MatchResult compareSource(Map> actual, Map matchWithGenericMatcher(actualValues, expectedValues) - ); + // There are cases when field values are stored in ignored source + // so we try to match them as is first and then apply field specific matcher. + // This is temporary, we should be able to tell when source is exact using mappings. + // See #111916. + var genericMatchResult = matchWithGenericMatcher(actualValues, expectedValues); + if (genericMatchResult.isMatch()) { + return genericMatchResult; + } - if (fieldMatch.isMatch() == false) { - var message = "Source documents don't match for field [" + name + "]: " + fieldMatch.getMessage(); + var matchIncludingFieldSpecificMatchers = matchWithFieldSpecificMatcher(name, actualValues, expectedValues).orElse( + genericMatchResult + ); + if (matchIncludingFieldSpecificMatchers.isMatch() == false) { + var message = "Source documents don't match for field [" + name + "]: " + matchIncludingFieldSpecificMatchers.getMessage(); return MatchResult.noMatch(message); } } @@ -130,11 +132,11 @@ private Optional matchWithFieldSpecificMatcher(String fieldName, Li ); } - // Dynamic mapping, nothing to do - return Optional.empty(); + // Field is dynamically mapped + return dynamicFieldMatcher.match(actualValues, expectedValues); } - var actualFieldType = (String) actualFieldMapping.get("type"); + var actualFieldType = (String) actualFieldMapping.mappingParameters().get("type"); if (actualFieldType == null) { throw new IllegalStateException("Field type is missing from leaf field Leaf field [" + fieldName + "] mapping parameters"); } @@ -143,7 +145,7 @@ private Optional matchWithFieldSpecificMatcher(String fieldName, Li if (expectedFieldMapping == null) { throw new IllegalStateException("Leaf field [" + fieldName + "] is present in actual mapping but absent in expected mapping"); } else { - var expectedFieldType = expectedFieldMapping.get("type"); + var expectedFieldType = expectedFieldMapping.mappingParameters().get("type"); if (Objects.equals(actualFieldType, expectedFieldType) == false) { throw new IllegalStateException( "Leaf field [" @@ -157,15 +159,29 @@ private Optional matchWithFieldSpecificMatcher(String fieldName, Li } } + if (sourceMatchesExactly(expectedFieldMapping, expectedValues)) { + return Optional.empty(); + } + var fieldSpecificMatcher = fieldSpecificMatchers.get(actualFieldType); if (fieldSpecificMatcher == null) { return Optional.empty(); } - MatchResult matched = fieldSpecificMatcher.match(actualValues, expectedValues, expectedFieldMapping, actualFieldMapping); + MatchResult matched = fieldSpecificMatcher.match( + actualValues, + expectedValues, + actualFieldMapping.mappingParameters(), + expectedFieldMapping.mappingParameters() + ); return Optional.of(matched); } + // Checks for scenarios when source is stored exactly and therefore can be compared without special logic. + private boolean sourceMatchesExactly(MappingTransforms.FieldMapping mapping, List expectedValues) { + return mapping.parentMappingParameters().stream().anyMatch(m -> m.getOrDefault("enabled", "true").equals("false")); + } + private MatchResult matchWithGenericMatcher(List actualValues, List expectedValues) { var genericListMatcher = new ListEqualMatcher( actualMappings, @@ -179,9 +195,4 @@ private MatchResult matchWithGenericMatcher(List actualValues, List List normalizeValues(List values) { return Collections.emptyList(); } + return normalizeValues(values, Function.identity()); + } + + public static List normalizeValues(List values, Function transform) { + if (values == null) { + return Collections.emptyList(); + } + // Synthetic source modifications: // * null values are not present // * duplicates are removed - return new ArrayList<>(values.stream().filter(v -> v != null && Objects.equals(v, "null") == false).collect(Collectors.toSet())); + return new ArrayList<>( + values.stream().filter(v -> v != null && Objects.equals(v, "null") == false).map(transform).collect(Collectors.toSet()) + ); } private static void descend(String pathFromRoot, Map currentLevel, Map> flattened) { diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java index 615c0006a4ce6..3f3a70a94de2a 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java @@ -35,10 +35,10 @@ import org.elasticsearch.datastreams.action.CreateDataStreamTransportAction; import org.elasticsearch.datastreams.action.DataStreamsStatsTransportAction; import org.elasticsearch.datastreams.action.DeleteDataStreamTransportAction; -import org.elasticsearch.datastreams.action.GetDataStreamsTransportAction; import org.elasticsearch.datastreams.action.MigrateToDataStreamTransportAction; import org.elasticsearch.datastreams.action.ModifyDataStreamsTransportAction; import org.elasticsearch.datastreams.action.PromoteDataStreamTransportAction; +import org.elasticsearch.datastreams.action.TransportGetDataStreamsAction; import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleErrorStore; import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleService; import org.elasticsearch.datastreams.lifecycle.action.DeleteDataStreamLifecycleAction; @@ -218,7 +218,7 @@ public Collection createComponents(PluginServices services) { List> actions = new ArrayList<>(); actions.add(new ActionHandler<>(CreateDataStreamAction.INSTANCE, CreateDataStreamTransportAction.class)); actions.add(new ActionHandler<>(DeleteDataStreamAction.INSTANCE, DeleteDataStreamTransportAction.class)); - actions.add(new ActionHandler<>(GetDataStreamAction.INSTANCE, GetDataStreamsTransportAction.class)); + actions.add(new ActionHandler<>(GetDataStreamAction.INSTANCE, TransportGetDataStreamsAction.class)); actions.add(new ActionHandler<>(DataStreamsStatsAction.INSTANCE, DataStreamsStatsTransportAction.class)); actions.add(new ActionHandler<>(MigrateToDataStreamAction.INSTANCE, MigrateToDataStreamTransportAction.class)); actions.add(new ActionHandler<>(PromoteDataStreamAction.INSTANCE, PromoteDataStreamTransportAction.class)); diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java similarity index 83% rename from modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportAction.java rename to modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java index dcca32355082b..647001d2c9ef5 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java @@ -11,11 +11,13 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.datastreams.DataStreamsActionUtil; +import org.elasticsearch.action.datastreams.DataStreamsStatsAction; import org.elasticsearch.action.datastreams.GetDataStreamAction; import org.elasticsearch.action.datastreams.GetDataStreamAction.Response.IndexProperties; import org.elasticsearch.action.datastreams.GetDataStreamAction.Response.ManagedBy; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; @@ -30,7 +32,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; @@ -43,31 +45,35 @@ import java.time.Instant; import java.util.ArrayList; +import java.util.Arrays; import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import static org.elasticsearch.index.IndexSettings.PREFER_ILM_SETTING; -public class GetDataStreamsTransportAction extends TransportMasterNodeReadAction< +public class TransportGetDataStreamsAction extends TransportMasterNodeReadAction< GetDataStreamAction.Request, GetDataStreamAction.Response> { - private static final Logger LOGGER = LogManager.getLogger(GetDataStreamsTransportAction.class); + private static final Logger LOGGER = LogManager.getLogger(TransportGetDataStreamsAction.class); private final SystemIndices systemIndices; private final ClusterSettings clusterSettings; private final DataStreamGlobalRetentionSettings globalRetentionSettings; + private final Client client; @Inject - public GetDataStreamsTransportAction( + public TransportGetDataStreamsAction( TransportService transportService, ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, SystemIndices systemIndices, - DataStreamGlobalRetentionSettings globalRetentionSettings + DataStreamGlobalRetentionSettings globalRetentionSettings, + Client client ) { super( GetDataStreamAction.NAME, @@ -78,11 +84,12 @@ public GetDataStreamsTransportAction( GetDataStreamAction.Request::new, indexNameExpressionResolver, GetDataStreamAction.Response::new, - EsExecutors.DIRECT_EXECUTOR_SERVICE + transportService.getThreadPool().executor(ThreadPool.Names.MANAGEMENT) ); this.systemIndices = systemIndices; this.globalRetentionSettings = globalRetentionSettings; clusterSettings = clusterService.getClusterSettings(); + this.client = client; } @Override @@ -92,9 +99,42 @@ protected void masterOperation( ClusterState state, ActionListener listener ) throws Exception { - listener.onResponse( - innerOperation(state, request, indexNameExpressionResolver, systemIndices, clusterSettings, globalRetentionSettings) - ); + if (request.verbose()) { + DataStreamsStatsAction.Request req = new DataStreamsStatsAction.Request(); + req.indices(request.indices()); + client.execute(DataStreamsStatsAction.INSTANCE, req, new ActionListener<>() { + @Override + public void onResponse(DataStreamsStatsAction.Response response) { + final Map maxTimestamps = Arrays.stream(response.getDataStreams()) + .collect( + Collectors.toMap( + DataStreamsStatsAction.DataStreamStats::getDataStream, + DataStreamsStatsAction.DataStreamStats::getMaximumTimestamp + ) + ); + listener.onResponse( + innerOperation( + state, + request, + indexNameExpressionResolver, + systemIndices, + clusterSettings, + globalRetentionSettings, + maxTimestamps + ) + ); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } else { + listener.onResponse( + innerOperation(state, request, indexNameExpressionResolver, systemIndices, clusterSettings, globalRetentionSettings, null) + ); + } } static GetDataStreamAction.Response innerOperation( @@ -103,7 +143,8 @@ static GetDataStreamAction.Response innerOperation( IndexNameExpressionResolver indexNameExpressionResolver, SystemIndices systemIndices, ClusterSettings clusterSettings, - DataStreamGlobalRetentionSettings globalRetentionSettings + DataStreamGlobalRetentionSettings globalRetentionSettings, + @Nullable Map maxTimestamps ) { List dataStreams = getDataStreams(state, indexNameExpressionResolver, request); List dataStreamInfos = new ArrayList<>(dataStreams.size()); @@ -216,7 +257,8 @@ public int compareTo(IndexInfo o) { ilmPolicyName, timeSeries, backingIndicesSettingsValues, - indexTemplatePreferIlmValue + indexTemplatePreferIlmValue, + maxTimestamps == null ? null : maxTimestamps.get(dataStream.getName()) ) ); } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java index 0cb29dbcf5b2f..99d4f8bb7cd28 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java @@ -296,13 +296,13 @@ public void close() { @Override public void triggered(SchedulerEngine.Event event) { - if (event.getJobName().equals(LIFECYCLE_JOB_NAME)) { + if (event.jobName().equals(LIFECYCLE_JOB_NAME)) { if (this.isMaster) { logger.trace( "Data stream lifecycle job triggered: {}, {}, {}", - event.getJobName(), - event.getScheduledTime(), - event.getTriggeredTime() + event.jobName(), + event.scheduledTime(), + event.triggeredTime() ); run(clusterService.state()); dslHealthInfoPublisher.publishDslErrorEntries(new ActionListener<>() { @@ -819,7 +819,7 @@ private Index maybeExecuteRollover(ClusterState state, DataStream dataStream, bo RolloverRequest rolloverRequest = getDefaultRolloverRequest( rolloverConfiguration, dataStream.getName(), - dataStream.getLifecycle().getEffectiveDataRetention(dataStream.isSystem() ? null : globalRetentionSettings.get()), + dataStream.getLifecycle().getEffectiveDataRetention(globalRetentionSettings.get(), dataStream.isInternal()), rolloverFailureStore ); transportActionsDeduplicator.executeOnce( @@ -879,7 +879,7 @@ Set maybeExecuteRetention(ClusterState state, DataStream dataStream, Set< Set indicesToBeRemoved = new HashSet<>(); // We know that there is lifecycle and retention because there are indices to be deleted assert dataStream.getLifecycle() != null; - TimeValue effectiveDataRetention = dataStream.getLifecycle().getEffectiveDataRetention(globalRetention); + TimeValue effectiveDataRetention = dataStream.getLifecycle().getEffectiveDataRetention(globalRetention, dataStream.isInternal()); for (Index index : backingIndicesOlderThanRetention) { if (indicesToExcludeForRemainingRun.contains(index) == false) { IndexMetadata backingIndex = metadata.index(index); diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleStatsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleStatsAction.java index 6e930defd4e0b..71f07c8cac668 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleStatsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleStatsAction.java @@ -76,7 +76,7 @@ public Response(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { out.writeOptionalVLong(runDuration); out.writeOptionalVLong(timeBetweenStarts); - out.writeCollection(dataStreamStats, (o, v) -> v.writeTo(o)); + out.writeCollection(dataStreamStats, StreamOutput::writeWriteable); } public Long getRunDuration() { diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java index 855b1713e5ec2..0ffb5809c2f0f 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/TransportExplainDataStreamLifecycleAction.java @@ -103,7 +103,7 @@ protected void masterOperation( ExplainIndexDataStreamLifecycle explainIndexDataStreamLifecycle = new ExplainIndexDataStreamLifecycle( index, true, - parentDataStream.isSystem(), + parentDataStream.isInternal(), idxMetadata.getCreationDate(), rolloverInfo == null ? null : rolloverInfo.getTime(), generationDate, diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestExplainDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestExplainDataStreamLifecycleAction.java index f44e59d0278c3..82350130e57af 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestExplainDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestExplainDataStreamLifecycleAction.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.datastreams.lifecycle.ExplainDataStreamLifecycleAction; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.common.Strings; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -19,6 +20,7 @@ import org.elasticsearch.rest.action.RestRefCountedChunkedToXContentListener; import java.util.List; +import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout; @@ -56,4 +58,9 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient public boolean allowSystemIndexAccessByDefault() { return true; } + + @Override + public Set supportedCapabilities() { + return Set.of(DataStreamLifecycle.EFFECTIVE_RETENTION_REST_API_CAPABILITY); + } } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestGetDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestGetDataStreamLifecycleAction.java index 94724f6778013..ee325ed9655be 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestGetDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestGetDataStreamLifecycleAction.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.datastreams.lifecycle.GetDataStreamLifecycleAction; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.common.Strings; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -19,6 +20,7 @@ import org.elasticsearch.rest.action.RestRefCountedChunkedToXContentListener; import java.util.List; +import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; @@ -54,4 +56,9 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli public boolean allowSystemIndexAccessByDefault() { return true; } + + @Override + public Set supportedCapabilities() { + return Set.of(DataStreamLifecycle.EFFECTIVE_RETENTION_REST_API_CAPABILITY, "data_stream_global_retention"); + } } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java index 5acb59841d6a6..29cda588bc26b 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/rest/RestGetDataStreamsAction.java @@ -10,7 +10,9 @@ import org.elasticsearch.action.datastreams.GetDataStreamAction; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestUtils; @@ -19,6 +21,7 @@ import org.elasticsearch.rest.action.RestToXContentListener; import java.util.List; +import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; @@ -43,6 +46,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli ); getDataStreamsRequest.includeDefaults(request.paramAsBoolean("include_defaults", false)); getDataStreamsRequest.indicesOptions(IndicesOptions.fromRequest(request, getDataStreamsRequest.indicesOptions())); + getDataStreamsRequest.verbose(request.paramAsBoolean("verbose", false)); return channel -> client.execute(GetDataStreamAction.INSTANCE, getDataStreamsRequest, new RestToXContentListener<>(channel)); } @@ -50,4 +54,27 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli public boolean allowSystemIndexAccessByDefault() { return true; } + + @Override + public Set supportedCapabilities() { + return Set.of(DataStreamLifecycle.EFFECTIVE_RETENTION_REST_API_CAPABILITY); + } + + @Override + public Set supportedQueryParameters() { + return Sets.union( + RestRequest.INTERNAL_MARKER_REQUEST_PARAMETERS, + Set.of( + "name", + "include_defaults", + "timeout", + "master_timeout", + IndicesOptions.WildcardOptions.EXPAND_WILDCARDS, + IndicesOptions.ConcreteTargetOptions.IGNORE_UNAVAILABLE, + IndicesOptions.WildcardOptions.ALLOW_NO_INDICES, + IndicesOptions.GatekeeperOptions.IGNORE_THROTTLED, + "verbose" + ) + ); + } } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/TimestampFieldMapperServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/TimestampFieldMapperServiceTests.java index 97959fa385241..eb35c44d30331 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/TimestampFieldMapperServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/TimestampFieldMapperServiceTests.java @@ -61,7 +61,7 @@ public void testGetTimestampFieldTypeForTsdbDataStream() throws IOException { DocWriteResponse indexResponse = indexDoc(); var indicesService = getInstanceFromNode(IndicesService.class); - var result = indicesService.getTimestampFieldType(indexResponse.getShardId().getIndex()); + var result = indicesService.getTimestampFieldTypeInfo(indexResponse.getShardId().getIndex()); assertThat(result, notNullValue()); } @@ -70,7 +70,7 @@ public void testGetTimestampFieldTypeForDataStream() throws IOException { DocWriteResponse indexResponse = indexDoc(); var indicesService = getInstanceFromNode(IndicesService.class); - var result = indicesService.getTimestampFieldType(indexResponse.getShardId().getIndex()); + var result = indicesService.getTimestampFieldTypeInfo(indexResponse.getShardId().getIndex()); assertThat(result, nullValue()); } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsRequestTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsRequestTests.java index 58bb3919d5a7c..824bec21b16e9 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsRequestTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsRequestTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.datastreams.action; import org.elasticsearch.action.datastreams.GetDataStreamAction.Request; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.test.AbstractWireSerializingTestCase; @@ -20,7 +21,7 @@ protected Writeable.Reader instanceReader() { @Override protected Request createTestInstance() { - return new Request(TEST_REQUEST_TIMEOUT, switch (randomIntBetween(1, 4)) { + var req = new Request(TEST_REQUEST_TIMEOUT, switch (randomIntBetween(1, 4)) { case 1 -> generateRandomStringArray(3, 8, false, false); case 2 -> { String[] parameters = generateRandomStringArray(3, 8, false, false); @@ -32,11 +33,40 @@ protected Request createTestInstance() { case 3 -> new String[] { "*" }; default -> null; }); + req.verbose(randomBoolean()); + return req; } @Override protected Request mutateInstance(Request instance) { - return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929 + var indices = instance.indices(); + var indicesOpts = instance.indicesOptions(); + var includeDefaults = instance.includeDefaults(); + var verbose = instance.verbose(); + switch (randomIntBetween(0, 3)) { + case 0 -> indices = randomValueOtherThan(indices, () -> generateRandomStringArray(3, 8, false, false)); + case 1 -> indicesOpts = randomValueOtherThan( + indicesOpts, + () -> IndicesOptions.fromOptions( + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean() + ) + ); + case 2 -> includeDefaults = includeDefaults == false; + case 3 -> verbose = verbose == false; + } + var newReq = new Request(instance.masterNodeTimeout(), indices); + newReq.includeDefaults(includeDefaults); + newReq.indicesOptions(indicesOpts); + newReq.verbose(verbose); + return newReq; } } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java index 4059127b5eb85..d4860de73213b 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; @@ -104,7 +105,8 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti null, null, indexSettingsValues, - false + false, + null ); Response response = new Response(List.of(dataStreamInfo)); XContentBuilder contentBuilder = XContentFactory.jsonBuilder(); @@ -206,7 +208,8 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti null, null, indexSettingsValues, - false + false, + null ); Response response = new Response(List.of(dataStreamInfo)); XContentBuilder contentBuilder = XContentFactory.jsonBuilder(); @@ -283,7 +286,8 @@ private Response.DataStreamInfo mutateInstance(Response.DataStreamInfo instance) var timeSeries = instance.getTimeSeries(); var indexSettings = instance.getIndexSettingsValues(); var templatePreferIlm = instance.templatePreferIlmValue(); - switch (randomIntBetween(0, 6)) { + var maximumTimestamp = instance.getMaximumTimestamp(); + switch (randomIntBetween(0, 7)) { case 0 -> dataStream = randomValueOtherThan(dataStream, DataStreamTestHelper::randomInstance); case 1 -> status = randomValueOtherThan(status, () -> randomFrom(ClusterHealthStatus.values())); case 2 -> indexTemplate = randomBoolean() && indexTemplate != null ? null : randomAlphaOfLengthBetween(2, 10); @@ -305,8 +309,20 @@ private Response.DataStreamInfo mutateInstance(Response.DataStreamInfo instance) ) ); case 6 -> templatePreferIlm = templatePreferIlm ? false : true; + case 7 -> maximumTimestamp = (maximumTimestamp == null) + ? randomNonNegativeLong() + : (usually() ? randomValueOtherThan(maximumTimestamp, ESTestCase::randomNonNegativeLong) : null); } - return new Response.DataStreamInfo(dataStream, status, indexTemplate, ilmPolicyName, timeSeries, indexSettings, templatePreferIlm); + return new Response.DataStreamInfo( + dataStream, + status, + indexTemplate, + ilmPolicyName, + timeSeries, + indexSettings, + templatePreferIlm, + maximumTimestamp + ); } private List> generateRandomTimeSeries() { @@ -342,7 +358,8 @@ private Response.DataStreamInfo generateRandomDataStreamInfo() { randomAlphaOfLengthBetween(2, 10), timeSeries != null ? new Response.TimeSeries(timeSeries) : null, generateRandomIndexSettingsValues(), - randomBoolean() + randomBoolean(), + usually() ? randomNonNegativeLong() : null ); } } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java similarity index 91% rename from modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportActionTests.java rename to modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java index 80d867ec7745e..e167e15576240 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsTransportActionTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java @@ -41,7 +41,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; -public class GetDataStreamsTransportActionTests extends ESTestCase { +public class TransportGetDataStreamsActionTests extends ESTestCase { private final IndexNameExpressionResolver resolver = TestIndexNameExpressionResolver.newInstance(); private final SystemIndices systemIndices = new SystemIndices(List.of()); @@ -54,7 +54,7 @@ public void testGetDataStream() { final String dataStreamName = "my-data-stream"; ClusterState cs = getClusterStateWithDataStreams(List.of(new Tuple<>(dataStreamName, 1)), List.of()); GetDataStreamAction.Request req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStreamName }); - List dataStreams = GetDataStreamsTransportAction.getDataStreams(cs, resolver, req); + List dataStreams = TransportGetDataStreamsAction.getDataStreams(cs, resolver, req); assertThat(dataStreams, transformedItemsMatch(DataStream::getName, contains(dataStreamName))); } @@ -69,19 +69,19 @@ public void testGetDataStreamsWithWildcards() { TEST_REQUEST_TIMEOUT, new String[] { dataStreamNames[1].substring(0, 5) + "*" } ); - List dataStreams = GetDataStreamsTransportAction.getDataStreams(cs, resolver, req); + List dataStreams = TransportGetDataStreamsAction.getDataStreams(cs, resolver, req); assertThat(dataStreams, transformedItemsMatch(DataStream::getName, contains(dataStreamNames[1]))); req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "*" }); - dataStreams = GetDataStreamsTransportAction.getDataStreams(cs, resolver, req); + dataStreams = TransportGetDataStreamsAction.getDataStreams(cs, resolver, req); assertThat(dataStreams, transformedItemsMatch(DataStream::getName, contains(dataStreamNames[1], dataStreamNames[0]))); req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, (String[]) null); - dataStreams = GetDataStreamsTransportAction.getDataStreams(cs, resolver, req); + dataStreams = TransportGetDataStreamsAction.getDataStreams(cs, resolver, req); assertThat(dataStreams, transformedItemsMatch(DataStream::getName, contains(dataStreamNames[1], dataStreamNames[0]))); req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "matches-none*" }); - dataStreams = GetDataStreamsTransportAction.getDataStreams(cs, resolver, req); + dataStreams = TransportGetDataStreamsAction.getDataStreams(cs, resolver, req); assertThat(dataStreams, empty()); } @@ -96,21 +96,21 @@ public void testGetDataStreamsWithoutWildcards() { TEST_REQUEST_TIMEOUT, new String[] { dataStreamNames[0], dataStreamNames[1] } ); - List dataStreams = GetDataStreamsTransportAction.getDataStreams(cs, resolver, req); + List dataStreams = TransportGetDataStreamsAction.getDataStreams(cs, resolver, req); assertThat(dataStreams, transformedItemsMatch(DataStream::getName, contains(dataStreamNames[1], dataStreamNames[0]))); req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStreamNames[1] }); - dataStreams = GetDataStreamsTransportAction.getDataStreams(cs, resolver, req); + dataStreams = TransportGetDataStreamsAction.getDataStreams(cs, resolver, req); assertThat(dataStreams, transformedItemsMatch(DataStream::getName, contains(dataStreamNames[1]))); req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStreamNames[0] }); - dataStreams = GetDataStreamsTransportAction.getDataStreams(cs, resolver, req); + dataStreams = TransportGetDataStreamsAction.getDataStreams(cs, resolver, req); assertThat(dataStreams, transformedItemsMatch(DataStream::getName, contains(dataStreamNames[0]))); GetDataStreamAction.Request req2 = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { "foo" }); IndexNotFoundException e = expectThrows( IndexNotFoundException.class, - () -> GetDataStreamsTransportAction.getDataStreams(cs, resolver, req2) + () -> TransportGetDataStreamsAction.getDataStreams(cs, resolver, req2) ); assertThat(e.getMessage(), containsString("no such index [foo]")); } @@ -121,7 +121,7 @@ public void testGetNonexistentDataStream() { GetDataStreamAction.Request req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] { dataStreamName }); IndexNotFoundException e = expectThrows( IndexNotFoundException.class, - () -> GetDataStreamsTransportAction.getDataStreams(cs, resolver, req) + () -> TransportGetDataStreamsAction.getDataStreams(cs, resolver, req) ); assertThat(e.getMessage(), containsString("no such index [" + dataStreamName + "]")); } @@ -160,13 +160,14 @@ public void testGetTimeSeriesDataStream() { } var req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {}); - var response = GetDataStreamsTransportAction.innerOperation( + var response = TransportGetDataStreamsAction.innerOperation( state, req, resolver, systemIndices, ClusterSettings.createBuiltInClusterSettings(), - dataStreamGlobalRetentionSettings + dataStreamGlobalRetentionSettings, + null ); assertThat( response.getDataStreams(), @@ -190,13 +191,14 @@ public void testGetTimeSeriesDataStream() { mBuilder.remove(dataStream.getIndices().get(1).getName()); state = ClusterState.builder(state).metadata(mBuilder).build(); } - response = GetDataStreamsTransportAction.innerOperation( + response = TransportGetDataStreamsAction.innerOperation( state, req, resolver, systemIndices, ClusterSettings.createBuiltInClusterSettings(), - dataStreamGlobalRetentionSettings + dataStreamGlobalRetentionSettings, + null ); assertThat( response.getDataStreams(), @@ -240,13 +242,14 @@ public void testGetTimeSeriesDataStreamWithOutOfOrderIndices() { } var req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {}); - var response = GetDataStreamsTransportAction.innerOperation( + var response = TransportGetDataStreamsAction.innerOperation( state, req, resolver, systemIndices, ClusterSettings.createBuiltInClusterSettings(), - dataStreamGlobalRetentionSettings + dataStreamGlobalRetentionSettings, + null ); assertThat( response.getDataStreams(), @@ -283,13 +286,14 @@ public void testGetTimeSeriesMixedDataStream() { } var req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {}); - var response = GetDataStreamsTransportAction.innerOperation( + var response = TransportGetDataStreamsAction.innerOperation( state, req, resolver, systemIndices, ClusterSettings.createBuiltInClusterSettings(), - dataStreamGlobalRetentionSettings + dataStreamGlobalRetentionSettings, + null ); var name1 = DataStream.getDefaultBackingIndexName("ds-1", 1, instant.toEpochMilli()); @@ -328,13 +332,14 @@ public void testPassingGlobalRetention() { } var req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {}); - var response = GetDataStreamsTransportAction.innerOperation( + var response = TransportGetDataStreamsAction.innerOperation( state, req, resolver, systemIndices, ClusterSettings.createBuiltInClusterSettings(), - dataStreamGlobalRetentionSettings + dataStreamGlobalRetentionSettings, + null ); assertThat(response.getGlobalRetention(), nullValue()); DataStreamGlobalRetention globalRetention = new DataStreamGlobalRetention( @@ -353,13 +358,14 @@ public void testPassingGlobalRetention() { ), DataStreamFactoryRetention.emptyFactoryRetention() ); - response = GetDataStreamsTransportAction.innerOperation( + response = TransportGetDataStreamsAction.innerOperation( state, req, resolver, systemIndices, ClusterSettings.createBuiltInClusterSettings(), - withGlobalRetentionSettings + withGlobalRetentionSettings, + null ); assertThat(response.getGlobalRetention(), equalTo(globalRetention)); } diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml index ec0c82365a681..7d375d14f8d3a 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml @@ -1243,3 +1243,65 @@ setup: - do: indices.delete_index_template: name: match-all-hidden-template + +--- +"Get maximum_timestamp for the data stream": + - requires: + capabilities: + - method: GET + path: /_data_stream + parameters: [verbose] + test_runner_features: [allowed_warnings, capabilities] + reason: Verbose flag required for the test + + - do: + allowed_warnings: + - "index template [my-template] has index patterns [data-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" + indices.put_index_template: + name: my-template + body: + index_patterns: [data-*] + data_stream: {} + + - do: + indices.create_data_stream: + name: data-stream1 + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: data-stream1 + verbose: false + + - match: { data_streams.0.name: data-stream1 } + - is_false: data_streams.0.maximum_timestamp + + - do: + indices.get_data_stream: + name: data-stream1 + verbose: true + + - match: { data_streams.0.name: data-stream1 } + # 0 because no documents have been indexed yet + - match: { data_streams.0.maximum_timestamp: 0 } + + - do: + index: + index: data-stream1 + refresh: true + body: + '@timestamp': '2020-12-12' + foo: bar + + - do: + indices.get_data_stream: + name: data-stream1 + verbose: true + + - match: { data_streams.0.name: data-stream1 } + - match: { data_streams.0.maximum_timestamp: 1607731200000 } + + - do: + indices.delete_data_stream: + name: data-stream1 + - is_true: acknowledged diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml index 991504b27f65f..af3204ed443ab 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml @@ -748,3 +748,43 @@ teardown: indices.delete: index: .fs-logs-foobar-* - is_true: acknowledged + +--- +"Version conflicts are not redirected to failure store": + - requires: + cluster_features: ["gte_v8.16.0"] + reason: "Redirecting version conflicts to the failure store is considered a bug fixed in 8.16" + test_runner_features: [allowed_warnings, contains] + + - do: + allowed_warnings: + - "index template [generic_logs_template] has index patterns [logs-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [generic_logs_template] will take precedence during new index creation" + indices.put_index_template: + name: generic_logs_template + body: + index_patterns: logs-* + data_stream: + failure_store: true + template: + settings: + number_of_shards: 1 + number_of_replicas: 1 + mappings: + properties: + '@timestamp': + type: date + count: + type: long + + - do: + bulk: + refresh: true + body: + - '{ "create": { "_index": "logs-foobar", "_id": "1" } }' + - '{ "@timestamp": "2022-01-01", "baz": "quick", "a": "brown", "b": "fox" }' + - '{ "create": { "_index": "logs-foobar", "_id": "1" } }' + - '{ "@timestamp": "2022-01-01", "baz": "lazy", "a": "dog" }' + - is_true: errors + - match: { items.1.create._index: '/\.ds-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { items.1.create.status: 409 } + - match: { items.1.create.error.type: version_conflict_engine_exception} diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml index 1cf44312ae7d5..4bf6ccfbfa7ce 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml @@ -1,8 +1,8 @@ setup: - - skip: - features: allowed_warnings - cluster_features: ["gte_v8.11.0"] - reason: "Data stream lifecycles only supported in 8.11+" + - requires: + cluster_features: [ "gte_v8.11.0" ] + reason: "Data stream lifecycle was released as tech preview in 8.11" + test_runner_features: allowed_warnings - do: allowed_warnings: - "index template [my-lifecycle] has index patterns [data-stream-with-lifecycle] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-lifecycle] will take precedence during new index creation" @@ -25,6 +25,7 @@ setup: body: index_patterns: [simple-data-stream1] template: + lifecycle: {} mappings: properties: '@timestamp': @@ -39,27 +40,93 @@ setup: name: simple-data-stream1 --- -"Get data stream lifecycle": +teardown: + - requires: + reason: "Global retention was exposed in 8.16+" + test_runner_features: [ capabilities ] + capabilities: + - method: GET + path: /_data_stream/{index}/_lifecycle + capabilities: [ 'data_stream_global_retention' ] + - do: + cluster.put_settings: + body: + persistent: + data_streams.lifecycle.retention.max: null + data_streams.lifecycle.retention.default: null +--- +"Get data stream lifecycle": + - requires: + reason: "Global retention was exposed in 8.16+" + test_runner_features: [ capabilities ] + capabilities: + - method: GET + path: /_data_stream/{index}/_lifecycle + capabilities: [ 'data_stream_global_retention' ] - do: indices.get_data_lifecycle: name: "data-stream-with-lifecycle" - length: { data_streams: 1} - match: { data_streams.0.name: data-stream-with-lifecycle } - match: { data_streams.0.lifecycle.data_retention: '10d' } + - match: { data_streams.0.lifecycle.effective_retention: '10d' } - match: { data_streams.0.lifecycle.enabled: true} + - match: { global_retention: {} } --- -"Get data stream with default lifecycle": - - skip: - awaits_fix: https://github.com/elastic/elasticsearch/pull/100187 +"Get data stream with default lifecycle configuration": + - requires: + reason: "Global retention was exposed in 8.16+" + test_runner_features: [ capabilities ] + capabilities: + - method: GET + path: /_data_stream/{index}/_lifecycle + capabilities: [ 'data_stream_global_retention' ] + - do: + indices.get_data_lifecycle: + name: "simple-data-stream1" + - length: { data_streams: 1} + - match: { data_streams.0.name: simple-data-stream1 } + - match: { data_streams.0.lifecycle.enabled: true} + - is_false: data_streams.0.lifecycle.effective_retention + - match: { global_retention: {} } +--- +"Get data stream with global retention": + - requires: + reason: "Global retention was exposed in 8.16+" + test_runner_features: [ capabilities ] + capabilities: + - method: GET + path: /_data_stream/{index}/_lifecycle + capabilities: [ 'data_stream_global_retention' ] + - do: + cluster.put_settings: + body: + persistent: + data_streams.lifecycle.retention.default: "7d" + data_streams.lifecycle.retention.max: "9d" - do: indices.get_data_lifecycle: name: "simple-data-stream1" - length: { data_streams: 1} - match: { data_streams.0.name: simple-data-stream1 } - match: { data_streams.0.lifecycle.enabled: true} + - match: { data_streams.0.lifecycle.effective_retention: '7d'} + - match: { global_retention.default_retention: '7d' } + - match: { global_retention.max_retention: '9d' } + + - do: + indices.get_data_lifecycle: + name: "data-stream-with-lifecycle" + - length: { data_streams: 1 } + - match: { data_streams.0.name: data-stream-with-lifecycle } + - match: { data_streams.0.lifecycle.data_retention: '10d' } + - match: { data_streams.0.lifecycle.effective_retention: '9d' } + - match: { data_streams.0.lifecycle.enabled: true } + - match: { global_retention.default_retention: '7d' } + - match: { global_retention.max_retention: '9d' } --- "Put data stream lifecycle": diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/30_not_found.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/30_not_found.yml index 24d0a5649a619..a3252496ef592 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/30_not_found.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/30_not_found.yml @@ -1,8 +1,8 @@ setup: - - skip: - features: allowed_warnings - cluster_features: ["gte_v8.11.0"] - reason: "Data stream lifecycle was GA in 8.11" + - requires: + cluster_features: [ "gte_v8.11.0" ] + reason: "Data stream lifecycle was released as tech preview in 8.11" + test_runner_features: allowed_warnings - do: allowed_warnings: - "index template [my-lifecycle] has index patterns [my-data-stream-1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-lifecycle] will take precedence during new index creation" diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/40_effective_retention.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/40_effective_retention.yml new file mode 100644 index 0000000000000..ef36f283fe237 --- /dev/null +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/40_effective_retention.yml @@ -0,0 +1,104 @@ +setup: + - requires: + cluster_features: [ "gte_v8.11.0" ] + reason: "Data stream lifecycle was released as tech preview in 8.11" + test_runner_features: allowed_warnings + - do: + allowed_warnings: + - "index template [template-with-lifecycle] has index patterns [managed-data-stream] matching patterns from existing older templates [global] with patterns (global => [*]); this template [template-with-lifecycle] will take precedence during new index creation" + indices.put_index_template: + name: template-with-lifecycle + body: + index_patterns: [ managed-data-stream ] + template: + settings: + index.number_of_replicas: 0 + lifecycle: + data_retention: "30d" + data_stream: { } + - do: + indices.create_data_stream: + name: managed-data-stream +--- +teardown: + - do: + cluster.put_settings: + body: + persistent: + data_streams.lifecycle.retention.max: null + data_streams.lifecycle.retention.default: null + +--- +"Retrieve effective retention via the data stream API": + - requires: + reason: "Effective retention was exposed in 8.16+" + test_runner_features: [capabilities] + capabilities: + - method: GET + path: /_data_stream/{index} + capabilities: [ 'data_stream_lifecycle_effective_retention' ] + - do: + indices.get_data_stream: + name: "managed-data-stream" + - match: { data_streams.0.name: managed-data-stream } + - match: { data_streams.0.lifecycle.data_retention: '30d' } + - match: { data_streams.0.lifecycle.effective_retention: '30d'} + - match: { data_streams.0.lifecycle.retention_determined_by: 'data_stream_configuration'} + +--- +"Retrieve effective retention with explain": + - requires: + reason: "Effective retention was exposed in 8.16+" + test_runner_features: [capabilities] + capabilities: + - method: GET + path: /{index}/_lifecycle/explain + capabilities: [ 'data_stream_lifecycle_effective_retention' ] + - do: + cluster.put_settings: + body: + persistent: + data_streams.lifecycle.retention.max: "7d" + - is_true: acknowledged + - do: + indices.get_data_stream: + name: "managed-data-stream" + - match: { data_streams.0.name: managed-data-stream } + - set: + data_streams.0.indices.0.index_name: backing_index + + - do: + indices.explain_data_lifecycle: + index: managed-data-stream + include_defaults: true + - match: { indices.$backing_index.managed_by_lifecycle: true } + - match: { indices.$backing_index.lifecycle.data_retention: '30d' } + - match: { indices.$backing_index.lifecycle.effective_retention: '7d' } + - match: { indices.$backing_index.lifecycle.retention_determined_by: 'max_global_retention' } + +--- +"Retrieve effective retention with data stream lifecycle": + - requires: + reason: "Effective retention was exposed in 8.16+" + test_runner_features: [capabilities] + capabilities: + - method: GET + path: /_data_stream/{index}/_lifecycle + capabilities: [ 'data_stream_lifecycle_effective_retention' ] + - do: + indices.put_data_lifecycle: + name: "managed-data-stream" + body: {} + - is_true: acknowledged + - do: + cluster.put_settings: + body: + persistent: + data_streams.lifecycle.retention.default: "7d" + - do: + indices.get_data_lifecycle: + name: "managed-data-stream" + - length: { data_streams: 1} + - match: { data_streams.0.name: managed-data-stream } + - match: { data_streams.0.lifecycle.effective_retention: '7d' } + - match: { data_streams.0.lifecycle.retention_determined_by: 'default_global_retention' } diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle index 90d52de6f0fff..d7100745680ba 100644 --- a/modules/ingest-common/build.gradle +++ b/modules/ingest-common/build.gradle @@ -5,6 +5,8 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ +import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask + apply plugin: 'elasticsearch.internal-yaml-rest-test' apply plugin: 'elasticsearch.yaml-rest-compat-test' apply plugin: 'elasticsearch.internal-cluster-test' @@ -29,7 +31,7 @@ restResources { } } -tasks.named('yamlRestTest') { +tasks.withType(StandaloneRestIntegTestTask).configureEach { usesDefaultDistribution() } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CommunityIdProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CommunityIdProcessor.java index 27ef5a10dd5c2..0377da53846d5 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CommunityIdProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CommunityIdProcessor.java @@ -225,7 +225,7 @@ private static Flow buildFlow( } flow.protocol = Transport.fromObject(protocol); - switch (flow.protocol) { + switch (flow.protocol.getType()) { case Tcp, Udp, Sctp -> { flow.sourcePort = parseIntFromObjectOrString(sourcePort.get(), "source port"); if (flow.sourcePort < 1 || flow.sourcePort > 65535) { @@ -336,12 +336,12 @@ public CommunityIdProcessor create( */ public static final class Flow { - private static final List TRANSPORTS_WITH_PORTS = List.of( - Transport.Tcp, - Transport.Udp, - Transport.Sctp, - Transport.Icmp, - Transport.IcmpIpV6 + private static final List TRANSPORTS_WITH_PORTS = List.of( + Transport.Type.Tcp, + Transport.Type.Udp, + Transport.Type.Sctp, + Transport.Type.Icmp, + Transport.Type.IcmpIpV6 ); InetAddress source; @@ -362,20 +362,21 @@ boolean isOrdered() { } byte[] toBytes() { - boolean hasPort = TRANSPORTS_WITH_PORTS.contains(protocol); + Transport.Type protoType = protocol.getType(); + boolean hasPort = TRANSPORTS_WITH_PORTS.contains(protoType); int len = source.getAddress().length + destination.getAddress().length + 2 + (hasPort ? 4 : 0); ByteBuffer bb = ByteBuffer.allocate(len); boolean isOneWay = false; - if (protocol == Transport.Icmp || protocol == Transport.IcmpIpV6) { + if (protoType == Transport.Type.Icmp || protoType == Transport.Type.IcmpIpV6) { // ICMP protocols populate port fields with ICMP data - Integer equivalent = IcmpType.codeEquivalent(icmpType, protocol == Transport.IcmpIpV6); + Integer equivalent = IcmpType.codeEquivalent(icmpType, protoType == Transport.Type.IcmpIpV6); isOneWay = equivalent == null; sourcePort = icmpType; destinationPort = equivalent == null ? icmpCode : equivalent; } - boolean keepOrder = isOrdered() || ((protocol == Transport.Icmp || protocol == Transport.IcmpIpV6) && isOneWay); + boolean keepOrder = isOrdered() || ((protoType == Transport.Type.Icmp || protoType == Transport.Type.IcmpIpV6) && isOneWay); bb.put(keepOrder ? source.getAddress() : destination.getAddress()); bb.put(keepOrder ? destination.getAddress() : source.getAddress()); bb.put(toUint16(protocol.getTransportNumber() << 8)); @@ -397,39 +398,63 @@ String toCommunityId(byte[] seed) { } } - public enum Transport { - Icmp(1), - Igmp(2), - Tcp(6), - Udp(17), - Gre(47), - IcmpIpV6(58), - Eigrp(88), - Ospf(89), - Pim(103), - Sctp(132); - - private final int transportNumber; + static class Transport { + public enum Type { + Unknown(-1), + Icmp(1), + Igmp(2), + Tcp(6), + Udp(17), + Gre(47), + IcmpIpV6(58), + Eigrp(88), + Ospf(89), + Pim(103), + Sctp(132); + + private final int transportNumber; + + private static final Map TRANSPORT_NAMES; + + static { + TRANSPORT_NAMES = new HashMap<>(); + TRANSPORT_NAMES.put("icmp", Icmp); + TRANSPORT_NAMES.put("igmp", Igmp); + TRANSPORT_NAMES.put("tcp", Tcp); + TRANSPORT_NAMES.put("udp", Udp); + TRANSPORT_NAMES.put("gre", Gre); + TRANSPORT_NAMES.put("ipv6-icmp", IcmpIpV6); + TRANSPORT_NAMES.put("icmpv6", IcmpIpV6); + TRANSPORT_NAMES.put("eigrp", Eigrp); + TRANSPORT_NAMES.put("ospf", Ospf); + TRANSPORT_NAMES.put("pim", Pim); + TRANSPORT_NAMES.put("sctp", Sctp); + } - private static final Map TRANSPORT_NAMES; + Type(int transportNumber) { + this.transportNumber = transportNumber; + } - static { - TRANSPORT_NAMES = new HashMap<>(); - TRANSPORT_NAMES.put("icmp", Icmp); - TRANSPORT_NAMES.put("igmp", Igmp); - TRANSPORT_NAMES.put("tcp", Tcp); - TRANSPORT_NAMES.put("udp", Udp); - TRANSPORT_NAMES.put("gre", Gre); - TRANSPORT_NAMES.put("ipv6-icmp", IcmpIpV6); - TRANSPORT_NAMES.put("icmpv6", IcmpIpV6); - TRANSPORT_NAMES.put("eigrp", Eigrp); - TRANSPORT_NAMES.put("ospf", Ospf); - TRANSPORT_NAMES.put("pim", Pim); - TRANSPORT_NAMES.put("sctp", Sctp); + public int getTransportNumber() { + return transportNumber; + } } - Transport(int transportNumber) { + private Type type; + private int transportNumber; + + Transport(int transportNumber, Type type) { // Change constructor to public this.transportNumber = transportNumber; + this.type = type; + } + + Transport(Type type) { // Change constructor to public + this.transportNumber = type.getTransportNumber(); + this.type = type; + } + + public Type getType() { + return this.type; } public int getTransportNumber() { @@ -437,19 +462,26 @@ public int getTransportNumber() { } public static Transport fromNumber(int transportNumber) { - return switch (transportNumber) { - case 1 -> Icmp; - case 2 -> Igmp; - case 6 -> Tcp; - case 17 -> Udp; - case 47 -> Gre; - case 58 -> IcmpIpV6; - case 88 -> Eigrp; - case 89 -> Ospf; - case 103 -> Pim; - case 132 -> Sctp; - default -> throw new IllegalArgumentException("unknown transport protocol number [" + transportNumber + "]"); + if (transportNumber < 0 || transportNumber >= 255) { + // transport numbers range https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml + throw new IllegalArgumentException("invalid transport protocol number [" + transportNumber + "]"); + } + + Type type = switch (transportNumber) { + case 1 -> Type.Icmp; + case 2 -> Type.Igmp; + case 6 -> Type.Tcp; + case 17 -> Type.Udp; + case 47 -> Type.Gre; + case 58 -> Type.IcmpIpV6; + case 88 -> Type.Eigrp; + case 89 -> Type.Ospf; + case 103 -> Type.Pim; + case 132 -> Type.Sctp; + default -> Type.Unknown; }; + + return new Transport(transportNumber, type); } public static Transport fromObject(Object o) { @@ -457,8 +489,8 @@ public static Transport fromObject(Object o) { return fromNumber(number.intValue()); } else if (o instanceof String protocolStr) { // check if matches protocol name - if (TRANSPORT_NAMES.containsKey(protocolStr.toLowerCase(Locale.ROOT))) { - return TRANSPORT_NAMES.get(protocolStr.toLowerCase(Locale.ROOT)); + if (Type.TRANSPORT_NAMES.containsKey(protocolStr.toLowerCase(Locale.ROOT))) { + return new Transport(Type.TRANSPORT_NAMES.get(protocolStr.toLowerCase(Locale.ROOT))); } // check if convertible to protocol number diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorTests.java index ca9b3f3d81bd9..3848f4531adcb 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorTests.java @@ -166,14 +166,30 @@ public void testBeatsProtocolNumber() throws Exception { testCommunityIdProcessor(event, "1:D3t8Q1aFA6Ev0A/AO4i9PnU3AeI="); } - public void testBeatsIanaNumber() throws Exception { + public void testBeatsIanaNumberProtocolTCP() throws Exception { @SuppressWarnings("unchecked") var network = (Map) event.get("network"); network.remove("transport"); - network.put("iana_number", CommunityIdProcessor.Transport.Tcp.getTransportNumber()); + network.put("iana_number", CommunityIdProcessor.Transport.Type.Tcp.getTransportNumber()); testCommunityIdProcessor(event, "1:LQU9qZlK+B5F3KDmev6m5PMibrg="); } + public void testBeatsIanaNumberProtocolIPv4() throws Exception { + @SuppressWarnings("unchecked") + var network = (Map) event.get("network"); + network.put("iana_number", "4"); + network.remove("transport"); + @SuppressWarnings("unchecked") + var source = (Map) event.get("source"); + source.put("ip", "192.168.1.2"); + source.remove("port"); + @SuppressWarnings("unchecked") + var destination = (Map) event.get("destination"); + destination.put("ip", "10.1.2.3"); + destination.remove("port"); + testCommunityIdProcessor(event, "1:KXQzmk3bdsvD6UXj7dvQ4bM6Zvw="); + } + public void testIpv6() throws Exception { @SuppressWarnings("unchecked") var source = (Map) event.get("source"); @@ -201,10 +217,10 @@ public void testStringAndNumber() throws Exception { @SuppressWarnings("unchecked") var network = (Map) event.get("network"); network.remove("transport"); - network.put("iana_number", CommunityIdProcessor.Transport.Tcp.getTransportNumber()); + network.put("iana_number", CommunityIdProcessor.Transport.Type.Tcp.getTransportNumber()); testCommunityIdProcessor(event, "1:LQU9qZlK+B5F3KDmev6m5PMibrg="); - network.put("iana_number", Integer.toString(CommunityIdProcessor.Transport.Tcp.getTransportNumber())); + network.put("iana_number", Integer.toString(CommunityIdProcessor.Transport.Type.Tcp.getTransportNumber())); testCommunityIdProcessor(event, "1:LQU9qZlK+B5F3KDmev6m5PMibrg="); // protocol number @@ -359,8 +375,13 @@ private void testCommunityIdProcessor(Map source, int seed, Stri } public void testTransportEnum() { - for (CommunityIdProcessor.Transport t : CommunityIdProcessor.Transport.values()) { - assertThat(CommunityIdProcessor.Transport.fromNumber(t.getTransportNumber()), equalTo(t)); + for (CommunityIdProcessor.Transport.Type t : CommunityIdProcessor.Transport.Type.values()) { + if (t == CommunityIdProcessor.Transport.Type.Unknown) { + expectThrows(IllegalArgumentException.class, () -> CommunityIdProcessor.Transport.fromNumber(t.getTransportNumber())); + continue; + } + + assertThat(CommunityIdProcessor.Transport.fromNumber(t.getTransportNumber()).getType(), equalTo(t)); } } diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle index 5bdb6da5c7b29..bc5bb165cd0d2 100644 --- a/modules/ingest-geoip/build.gradle +++ b/modules/ingest-geoip/build.gradle @@ -88,3 +88,7 @@ tasks.named("yamlRestTestV7CompatTransform").configure { task -> task.skipTestsByFilePattern("**/ingest_geoip/20_geoip_processor.yml", "from 8.0 yaml rest tests use geoip test fixture and default geoip are no longer packaged. In 7.x yaml tests used default databases which makes tests results very different, so skipping these tests") // task.skipTest("lang_mustache/50_multi_search_template/Multi-search template with errors", "xxx") } + +artifacts { + restTests(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test")) +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTests.java index 203ecaea72c0e..1676ce14698a9 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderTests.java @@ -40,6 +40,7 @@ import org.elasticsearch.telemetry.metric.MeterRegistry; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.client.NoOpClient; +import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentType; import org.hamcrest.Matchers; @@ -86,7 +87,11 @@ public void setup() throws IOException { "e4a3411cdd7b21eaf18675da5a7f9f360d33c6882363b2c19c38715834c9e836 GeoIP2-City_20240709.tar.gz".getBytes(StandardCharsets.UTF_8) ); clusterService = mock(ClusterService.class); - threadPool = new ThreadPool(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "test").build(), MeterRegistry.NOOP); + threadPool = new ThreadPool( + Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "test").build(), + MeterRegistry.NOOP, + new DefaultBuiltInExecutorBuilders() + ); when(clusterService.getClusterSettings()).thenReturn( new ClusterSettings(Settings.EMPTY, Set.of(GeoIpDownloaderTaskExecutor.POLL_INTERVAL_SETTING)) ); diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java index 984bd37181fe7..f213868fb65a1 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTests.java @@ -45,6 +45,7 @@ import org.elasticsearch.telemetry.metric.MeterRegistry; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.client.NoOpClient; +import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; @@ -92,7 +93,11 @@ public void setup() throws IOException { httpClient = mock(HttpClient.class); when(httpClient.getBytes(anyString())).thenReturn("[]".getBytes(StandardCharsets.UTF_8)); clusterService = mock(ClusterService.class); - threadPool = new ThreadPool(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "test").build(), MeterRegistry.NOOP); + threadPool = new ThreadPool( + Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "test").build(), + MeterRegistry.NOOP, + new DefaultBuiltInExecutorBuilders() + ); when(clusterService.getClusterSettings()).thenReturn( new ClusterSettings( Settings.EMPTY, diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScoreScript.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScoreScript.java index 159851affd004..622a1bd4afd25 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScoreScript.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScoreScript.java @@ -42,6 +42,12 @@ public boolean needs_score() { return needsScores; } + @Override + public boolean needs_termStats() { + // _termStats is not available for expressions + return false; + } + @Override public ScoreScript newInstance(final DocReader reader) throws IOException { // Use DocReader to get the leaf context while transitioning to DocReader for Painless. DocReader for expressions should follow. diff --git a/modules/lang-mustache/src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml b/modules/lang-mustache/src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml index 109bc8888889f..de9b3a0ec9bc2 100644 --- a/modules/lang-mustache/src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml +++ b/modules/lang-mustache/src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml @@ -114,14 +114,14 @@ setup: - match: { responses.0.hits.total: 2 } - match: { responses.1.error.root_cause.0.type: x_content_e_o_f_exception } - - match: { responses.1.error.root_cause.0.reason: "/Unexpected.end.of.input/" } + - match: { responses.1.error.root_cause.0.reason: "/\\[1:22\\].Unexpected.end.of.file/" } - match: { responses.2.hits.total: 1 } - match: { responses.3.error.root_cause.0.type: parsing_exception } - match: { responses.3.error.root_cause.0.reason: "/unknown.query.\\[unknown\\]/" } - match: { responses.4.error.root_cause.0.type: illegal_argument_exception } - match: { responses.4.error.root_cause.0.reason: "[rest_total_hits_as_int] cannot be used if the tracking of total hits is not accurate, got 1" } - match: { responses.0.status: 200 } - - match: { responses.1.status: 500 } + - match: { responses.1.status: 400 } - match: { responses.2.status: 200 } - match: { responses.3.status: 400 } diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt index 1e9e9f40985cb..045905c358cd2 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/java.util.txt @@ -684,6 +684,24 @@ class java.util.Hashtable { def clone() } +class java.util.HexFormat { + HexFormat of() + HexFormat ofDelimiter(String) + HexFormat withDelimiter(String) + HexFormat withPrefix(String) + HexFormat withSuffix(String) + HexFormat withUpperCase() + HexFormat withLowerCase() + String delimiter() + String prefix() + String suffix() + boolean isUpperCase() + String formatHex(byte[]) + String formatHex(byte[],int,int) + byte[] parseHex(CharSequence) + byte[] parseHex(CharSequence,int,int) +} + class java.util.IdentityHashMap { () (Map) diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt index 5082d5f1c7bdb..0dab7dcbadfb5 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt @@ -13,6 +13,23 @@ class org.elasticsearch.script.ScoreScript @no_import { class org.elasticsearch.script.ScoreScript$Factory @no_import { } +class org.elasticsearch.script.StatsSummary { + double getMin() + double getMax() + double getAverage() + double getSum() + long getCount() +} + +class org.elasticsearch.script.ScriptTermStats { + int uniqueTermsCount() + int matchedTermsCount() + StatsSummary docFreq() + StatsSummary totalTermFreq() + StatsSummary termFreq() + StatsSummary termPositions() +} + static_import { double saturation(double, double) from_class org.elasticsearch.script.ScoreScriptUtils double sigmoid(double, double, double) from_class org.elasticsearch.script.ScoreScriptUtils diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScoreScriptTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScoreScriptTests.java new file mode 100644 index 0000000000000..08b55fdf3bcc3 --- /dev/null +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScoreScriptTests.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.painless; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.ScoreScript; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.test.ESSingleNodeTestCase; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static java.util.Collections.emptyMap; +import static org.elasticsearch.painless.ScriptTestCase.PAINLESS_BASE_WHITELIST; + +public class ScoreScriptTests extends ESSingleNodeTestCase { + /** + * Test that needTermStats() is reported correctly depending on whether _termStats is used + */ + public void testNeedsTermStats() { + IndexService index = createIndex("test", Settings.EMPTY, "type", "d", "type=double"); + + Map, List> contexts = new HashMap<>(); + List whitelists = new ArrayList<>(PAINLESS_BASE_WHITELIST); + whitelists.add(WhitelistLoader.loadFromResourceFiles(PainlessPlugin.class, "org.elasticsearch.script.score.txt")); + contexts.put(ScoreScript.CONTEXT, whitelists); + PainlessScriptEngine service = new PainlessScriptEngine(Settings.EMPTY, contexts); + + SearchExecutionContext searchExecutionContext = index.newSearchExecutionContext(0, 0, null, () -> 0, null, emptyMap()); + + ScoreScript.Factory factory = service.compile(null, "1.2", ScoreScript.CONTEXT, Collections.emptyMap()); + ScoreScript.LeafFactory ss = factory.newFactory(Collections.emptyMap(), searchExecutionContext.lookup()); + assertFalse(ss.needs_termStats()); + + factory = service.compile(null, "doc['d'].value", ScoreScript.CONTEXT, Collections.emptyMap()); + ss = factory.newFactory(Collections.emptyMap(), searchExecutionContext.lookup()); + assertFalse(ss.needs_termStats()); + + factory = service.compile(null, "1/_termStats.totalTermFreq().getAverage()", ScoreScript.CONTEXT, Collections.emptyMap()); + ss = factory.newFactory(Collections.emptyMap(), searchExecutionContext.lookup()); + assertTrue(ss.needs_termStats()); + + factory = service.compile(null, "doc['d'].value * _termStats.docFreq().getSum()", ScoreScript.CONTEXT, Collections.emptyMap()); + ss = factory.newFactory(Collections.emptyMap(), searchExecutionContext.lookup()); + assertTrue(ss.needs_termStats()); + } +} diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/190_term_statistics_script_score.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/190_term_statistics_script_score.yml new file mode 100644 index 0000000000000..f82b844f01588 --- /dev/null +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/190_term_statistics_script_score.yml @@ -0,0 +1,612 @@ +setup: + - requires: + cluster_features: ["script.term_stats"] + reason: "support for term stats has been added in 8.16" + + - do: + indices.create: + index: test-index + body: + settings: + number_of_shards: "2" + mappings: + properties: + title: + type: text + genre: + type: text + fields: + keyword: + type: keyword + + - do: + index: { refresh: true, index: test-index, id: "1", routing: 0, body: {"title": "Star wars", "genre": "Sci-fi"} } + - do: + index: { refresh: true, index: test-index, id: "2", routing: 1, body: {"title": "Star trek", "genre": "Sci-fi"} } + - do: + index: { refresh: true, index: test-index, id: "3", routing: 1, body: {"title": "Rambo", "genre": "War movie"} } + - do: + index: { refresh: true, index: test-index, id: "4", routing: 1, body: {"title": "Rambo II", "genre": "War movie"} } + +--- +"match query: uniqueTermsCount without DFS": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + script_score: + query: { match: { "title": "Star wars" } } + script: + source: "return _termStats.uniqueTermsCount()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 2 } + - match: { hits.hits.1._score: 2 } + +--- +"match query: uniqueTermsCount with DFS": + - do: + search: + search_type: dfs_query_then_fetch + rest_total_hits_as_int: true + index: test-index + body: + query: + script_score: + query: { match: { "title": "Star wars" } } + script: + source: "return _termStats.uniqueTermsCount()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 2 } + - match: { hits.hits.1._score: 2 } + +--- +"match query: matchedTermsCount without DFS": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + script_score: + query: { match: { "title": "Star wars" } } + script: + source: "return _termStats.matchedTermsCount()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 2 } + - match: { hits.hits.1._score: 1 } + +--- +"match query: matchedTermsCount with DFS": + - do: + search: + rest_total_hits_as_int: true + search_type: dfs_query_then_fetch + index: test-index + body: + query: + script_score: + query: { match: { "title": "Star wars" } } + script: + source: "return _termStats.matchedTermsCount()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 2 } + - match: { hits.hits.1._score: 1 } + +--- +"match query: docFreq min without DFS": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + script_score: + query: { match: { "title": "Star wars" } } + script: + source: "return _termStats.docFreq().getMin()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 1 } + - match: { hits.hits.1._score: 0 } + +--- +"match query: docFreq min with DFS": + - do: + search: + rest_total_hits_as_int: true + search_type: dfs_query_then_fetch + index: test-index + body: + query: + script_score: + query: { match: { "title": "Star wars" } } + script: + source: "return _termStats.docFreq().getMin()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 1 } + - match: { hits.hits.1._score: 1 } + +--- +"match query: docFreq max without DFS": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + script_score: + query: { match: { "title": "Star wars" } } + script: + source: "return _termStats.docFreq().getMax()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 1 } + - match: { hits.hits.1._score: 1 } + +--- +"match query: docFreq max with DFS": + - do: + search: + rest_total_hits_as_int: true + search_type: dfs_query_then_fetch + index: test-index + body: + query: + script_score: + query: { match: { "title": "Star wars" } } + script: + source: "return _termStats.docFreq().getMax()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 2 } + - match: { hits.hits.1._score: 2 } + +--- +"match query: totalTermFreq sum without DFS": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + script_score: + query: { match: { "title": "Star wars" } } + script: + source: "return _termStats.totalTermFreq().getSum()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 2 } + - match: { hits.hits.1._score: 1 } + +--- +"match query: totalTermFreq sum with DFS": + - do: + search: + rest_total_hits_as_int: true + search_type: dfs_query_then_fetch + index: test-index + body: + query: + script_score: + query: { match: { "title": "Star wars" } } + script: + source: "return _termStats.totalTermFreq().getSum()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 3 } + - match: { hits.hits.1._score: 3 } + +--- +"match query: termFreq sum without DFS": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + script_score: + query: { match: { "title": "Star wars" } } + script: + source: "return _termStats.termFreq().getSum()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 2 } + - match: { hits.hits.1._score: 1 } + +--- +"match query: termFreq sum with DFS": + - do: + search: + rest_total_hits_as_int: true + search_type: dfs_query_then_fetch + index: test-index + body: + query: + script_score: + query: { match: { "title": "Star wars" } } + script: + source: "return _termStats.termFreq().getSum()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 2 } + - match: { hits.hits.1._score: 1 } + +--- +"match query: termPositions avg without DFS": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + script_score: + query: { match: { "title": "Star wars" } } + script: + source: "return _termStats.termPositions().getAverage()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 1.5 } + - match: { hits.hits.1._score: 1 } + +--- +"match query: termPositions avg with DFS": + - do: + search: + rest_total_hits_as_int: true + search_type: dfs_query_then_fetch + index: test-index + body: + query: + script_score: + query: { match: { "title": "Star wars" } } + script: + source: "return _termStats.termPositions().getAverage()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 1.5 } + - match: { hits.hits.1._score: 1 } + +--- +"term query: uniqueTermsCount without DFS": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + script_score: + query: { term: { "genre.keyword": "Sci-fi" } } + script: + source: "return _termStats.uniqueTermsCount()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 1 } + - match: { hits.hits.1._score: 1 } + +--- +"term query: uniqueTermsCount with DFS": + - do: + search: + search_type: dfs_query_then_fetch + rest_total_hits_as_int: true + index: test-index + body: + query: + script_score: + query: { term: { "genre.keyword": "Sci-fi" } } + script: + source: "return _termStats.uniqueTermsCount()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 1 } + - match: { hits.hits.1._score: 1 } + +--- +"term query: matchedTermsCount without DFS": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + script_score: + query: { term: { "genre.keyword": "Sci-fi" } } + script: + source: "return _termStats.matchedTermsCount()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 1 } + - match: { hits.hits.1._score: 1 } + +--- +"term query: matchedTermsCount with DFS": + - do: + search: + rest_total_hits_as_int: true + search_type: dfs_query_then_fetch + index: test-index + body: + query: + script_score: + query: { term: { "genre.keyword": "Sci-fi" } } + script: + source: "return _termStats.matchedTermsCount()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 1 } + - match: { hits.hits.1._score: 1 } + +--- +"term query: docFreq min without DFS": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + script_score: + query: { term: { "genre.keyword": "Sci-fi" } } + script: + source: "return _termStats.docFreq().getMin()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 1 } + - match: { hits.hits.1._score: 1 } + +--- +"term query: docFreq min with DFS": + - do: + search: + rest_total_hits_as_int: true + search_type: dfs_query_then_fetch + index: test-index + body: + query: + script_score: + query: { term: { "genre.keyword": "Sci-fi" } } + script: + source: "return _termStats.docFreq().getMin()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 2 } + - match: { hits.hits.1._score: 2 } + +--- +"term query: docFreq max without DFS": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + script_score: + query: { term: { "genre.keyword": "Sci-fi" } } + script: + source: "return _termStats.docFreq().getMax()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 1 } + - match: { hits.hits.1._score: 1 } + +--- +"term query: docFreq max with DFS": + - do: + search: + rest_total_hits_as_int: true + search_type: dfs_query_then_fetch + index: test-index + body: + query: + script_score: + query: { term: { "genre.keyword": "Sci-fi" } } + script: + source: "return _termStats.docFreq().getMax()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 2 } + - match: { hits.hits.1._score: 2 } + +--- +"term query: totalTermFreq sum without DFS": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + script_score: + query: { term: { "genre.keyword": "Sci-fi" } } + script: + source: "return _termStats.totalTermFreq().getSum()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 1 } + - match: { hits.hits.1._score: 1 } + +--- +"term query: totalTermFreq sum with DFS": + - do: + search: + rest_total_hits_as_int: true + search_type: dfs_query_then_fetch + index: test-index + body: + query: + script_score: + query: { term: { "genre.keyword": "Sci-fi" } } + script: + source: "return _termStats.totalTermFreq().getSum()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 2 } + - match: { hits.hits.1._score: 2 } + +--- +"term query: termFreq sum without DFS": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + script_score: + query: { term: { "genre.keyword": "Sci-fi" } } + script: + source: "return _termStats.termFreq().getSum()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 1 } + - match: { hits.hits.1._score: 1 } + +--- +"term query: termFreq sum with DFS": + - do: + search: + rest_total_hits_as_int: true + search_type: dfs_query_then_fetch + index: test-index + body: + query: + script_score: + query: { term: { "genre.keyword": "Sci-fi" } } + script: + source: "return _termStats.termFreq().getSum()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 1 } + - match: { hits.hits.1._score: 1 } + +--- +"term query: termPositions avg without DFS": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + script_score: + query: { term: { "genre.keyword": "Sci-fi" } } + script: + source: "return _termStats.termPositions().getAverage()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 0 } + - match: { hits.hits.1._score: 0 } + +--- +"term query: termPositions avg with DFS": + - do: + search: + rest_total_hits_as_int: true + search_type: dfs_query_then_fetch + index: test-index + body: + query: + script_score: + query: { term: { "genre.keyword": "Sci-fi" } } + script: + source: "return _termStats.termPositions().getAverage()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 0 } + - match: { hits.hits.1._score: 0 } + +--- +"Complex bool query: uniqueTermsCount": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + script_score: + query: + bool: + must: + match: { "title": "star wars" } + should: + term: { "genre.keyword": "Sci-fi" } + filter: + match: { "genre" : "sci"} + must_not: + term: { "genre.keyword": "War" } + script: + source: "return _termStats.uniqueTermsCount()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 4 } + - match: { hits.hits.1._score: 4 } + + +--- +"match_all query: uniqueTermsCount": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + script_score: + query: + match_all: {} + script: + source: "return _termStats.uniqueTermsCount()" + - match: { hits.total: 4 } + - match: { hits.hits.0._score: 0 } + - match: { hits.hits.1._score: 0 } + - match: { hits.hits.2._score: 0 } + - match: { hits.hits.3._score: 0 } + +--- +"match_all query: docFreq": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + script_score: + query: + match_all: {} + script: + source: "return _termStats.docFreq().getMax()" + - match: { hits.total: 4 } + - match: { hits.hits.0._score: 0 } + - match: { hits.hits.1._score: 0 } + - match: { hits.hits.2._score: 0 } + - match: { hits.hits.3._score: 0 } + +--- +"match_all query: totalTermFreq": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + script_score: + query: + match_all: {} + script: + source: "return _termStats.totalTermFreq().getSum()" + - match: { hits.total: 4 } + - match: { hits.hits.0._score: 0 } + - match: { hits.hits.1._score: 0 } + - match: { hits.hits.2._score: 0 } + - match: { hits.hits.3._score: 0 } + +--- +"match_all query: termFreq": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + script_score: + query: + match_all: {} + script: + source: "return _termStats.termFreq().getMax()" + - match: { hits.total: 4 } + - match: { hits.hits.0._score: 0 } + - match: { hits.hits.1._score: 0 } + - match: { hits.hits.2._score: 0 } + - match: { hits.hits.3._score: 0 } + +--- +"match_all query: termPositions": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + script_score: + query: + match_all: {} + script: + source: "return _termStats.termPositions().getSum()" + - match: { hits.total: 4 } + - match: { hits.hits.0._score: 0 } + - match: { hits.hits.1._score: 0 } + - match: { hits.hits.2._score: 0 } + - match: { hits.hits.3._score: 0 } diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/191_term_statistics_function_score.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/191_term_statistics_function_score.yml new file mode 100644 index 0000000000000..de4d6530f4a92 --- /dev/null +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/191_term_statistics_function_score.yml @@ -0,0 +1,680 @@ +setup: + - requires: + cluster_features: ["script.term_stats"] + reason: "support for term stats has been added in 8.16" + + - do: + indices.create: + index: test-index + body: + settings: + number_of_shards: "2" + mappings: + properties: + title: + type: text + genre: + type: text + fields: + keyword: + type: keyword + + - do: + index: { refresh: true, index: test-index, id: "1", routing: 0, body: {"title": "Star wars", "genre": "Sci-fi"} } + - do: + index: { refresh: true, index: test-index, id: "2", routing: 1, body: {"title": "Star trek", "genre": "Sci-fi"} } + - do: + index: { refresh: true, index: test-index, id: "3", routing: 1, body: {"title": "Rambo", "genre": "War movie"} } + - do: + index: { refresh: true, index: test-index, id: "4", routing: 1, body: {"title": "Rambo II", "genre": "War movie"} } + +--- +"match query: uniqueTermsCount without DFS": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + function_score: + boost_mode: replace + query: { match: { "title": "Star wars" } } + script_score: + script: + source: "return _termStats.uniqueTermsCount()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 2 } + - match: { hits.hits.1._score: 2 } + +--- +"match query: uniqueTermsCount with DFS": + - do: + search: + search_type: dfs_query_then_fetch + rest_total_hits_as_int: true + index: test-index + body: + query: + function_score: + boost_mode: replace + query: { match: { "title": "Star wars" } } + script_score: + script: + source: "return _termStats.uniqueTermsCount()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 2 } + - match: { hits.hits.1._score: 2 } + +--- +"match query: matchedTermsCount without DFS": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + function_score: + boost_mode: replace + query: { match: { "title": "Star wars" } } + script_score: + script: + source: "return _termStats.matchedTermsCount()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 2 } + - match: { hits.hits.1._score: 1 } + +--- +"match query: matchedTermsCount with DFS": + - do: + search: + rest_total_hits_as_int: true + search_type: dfs_query_then_fetch + index: test-index + body: + query: + function_score: + boost_mode: replace + query: { match: { "title": "Star wars" } } + script_score: + script: + source: "return _termStats.matchedTermsCount()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 2 } + - match: { hits.hits.1._score: 1 } + +--- +"match query: docFreq min without DFS": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + function_score: + boost_mode: replace + query: { match: { "title": "Star wars" } } + script_score: + script: + source: "return _termStats.docFreq().getMin()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 1 } + - match: { hits.hits.1._score: 0 } + +--- +"match query: docFreq min with DFS": + - do: + search: + rest_total_hits_as_int: true + search_type: dfs_query_then_fetch + index: test-index + body: + query: + function_score: + boost_mode: replace + query: { match: { "title": "Star wars" } } + script_score: + script: + source: "return _termStats.docFreq().getMin()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 1 } + - match: { hits.hits.1._score: 1 } + +--- +"match query: docFreq max without DFS": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + function_score: + boost_mode: replace + query: { match: { "title": "Star wars" } } + script_score: + script: + source: "return _termStats.docFreq().getMax()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 1 } + - match: { hits.hits.1._score: 1 } + +--- +"match query: docFreq max with DFS": + - do: + search: + rest_total_hits_as_int: true + search_type: dfs_query_then_fetch + index: test-index + body: + query: + function_score: + boost_mode: replace + query: { match: { "title": "Star wars" } } + script_score: + script: + source: "return _termStats.docFreq().getMax()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 2 } + - match: { hits.hits.1._score: 2 } + +--- +"match query: totalTermFreq sum without DFS": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + function_score: + boost_mode: replace + query: { match: { "title": "Star wars" } } + script_score: + script: + source: "return _termStats.totalTermFreq().getSum()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 2 } + - match: { hits.hits.1._score: 1 } + +--- +"match query: totalTermFreq sum with DFS": + - do: + search: + rest_total_hits_as_int: true + search_type: dfs_query_then_fetch + index: test-index + body: + query: + function_score: + boost_mode: replace + query: { match: { "title": "Star wars" } } + script_score: + script: + source: "return _termStats.totalTermFreq().getSum()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 3 } + - match: { hits.hits.1._score: 3 } + +--- +"match query: termFreq sum without DFS": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + function_score: + boost_mode: replace + query: { match: { "title": "Star wars" } } + script_score: + script: + source: "return _termStats.termFreq().getSum()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 2 } + - match: { hits.hits.1._score: 1 } + +--- +"match query: termFreq sum with DFS": + - do: + search: + rest_total_hits_as_int: true + search_type: dfs_query_then_fetch + index: test-index + body: + query: + function_score: + boost_mode: replace + query: { match: { "title": "Star wars" } } + script_score: + script: + source: "return _termStats.termFreq().getSum()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 2 } + - match: { hits.hits.1._score: 1 } + +--- +"match query: termPositions avg without DFS": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + function_score: + boost_mode: replace + query: { match: { "title": "Star wars" } } + script_score: + script: + source: "return _termStats.termPositions().getAverage()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 1.5 } + - match: { hits.hits.1._score: 1 } + +--- +"match query: termPositions avg with DFS": + - do: + search: + rest_total_hits_as_int: true + search_type: dfs_query_then_fetch + index: test-index + body: + query: + function_score: + boost_mode: replace + query: { match: { "title": "Star wars" } } + script_score: + script: + source: "return _termStats.termPositions().getAverage()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 1.5 } + - match: { hits.hits.1._score: 1 } + +--- +"term query: uniqueTermsCount without DFS": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + function_score: + boost_mode: replace + query: { term: { "genre.keyword": "Sci-fi" } } + script_score: + script: + source: "return _termStats.uniqueTermsCount()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 1 } + - match: { hits.hits.1._score: 1 } + +--- +"term query: uniqueTermsCount with DFS": + - do: + search: + search_type: dfs_query_then_fetch + rest_total_hits_as_int: true + index: test-index + body: + query: + function_score: + boost_mode: replace + query: { term: { "genre.keyword": "Sci-fi" } } + script_score: + script: + source: "return _termStats.uniqueTermsCount()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 1 } + - match: { hits.hits.1._score: 1 } + +--- +"term query: matchedTermsCount without DFS": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + function_score: + boost_mode: replace + query: { term: { "genre.keyword": "Sci-fi" } } + script_score: + script: + source: "return _termStats.matchedTermsCount()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 1 } + - match: { hits.hits.1._score: 1 } + +--- +"term query: matchedTermsCount with DFS": + - do: + search: + rest_total_hits_as_int: true + search_type: dfs_query_then_fetch + index: test-index + body: + query: + function_score: + boost_mode: replace + query: { term: { "genre.keyword": "Sci-fi" } } + script_score: + script: + source: "return _termStats.matchedTermsCount()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 1 } + - match: { hits.hits.1._score: 1 } + +--- +"term query: docFreq min without DFS": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + function_score: + boost_mode: replace + query: { term: { "genre.keyword": "Sci-fi" } } + script_score: + script: + source: "return _termStats.docFreq().getMin()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 1 } + - match: { hits.hits.1._score: 1 } + +--- +"term query: docFreq min with DFS": + - do: + search: + rest_total_hits_as_int: true + search_type: dfs_query_then_fetch + index: test-index + body: + query: + function_score: + boost_mode: replace + query: { term: { "genre.keyword": "Sci-fi" } } + script_score: + script: + source: "return _termStats.docFreq().getMin()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 2 } + - match: { hits.hits.1._score: 2 } + +--- +"term query: docFreq max without DFS": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + function_score: + boost_mode: replace + query: { term: { "genre.keyword": "Sci-fi" } } + script_score: + script: + source: "return _termStats.docFreq().getMax()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 1 } + - match: { hits.hits.1._score: 1 } + +--- +"term query: docFreq max with DFS": + - do: + search: + rest_total_hits_as_int: true + search_type: dfs_query_then_fetch + index: test-index + body: + query: + function_score: + boost_mode: replace + query: { term: { "genre.keyword": "Sci-fi" } } + script_score: + script: + source: "return _termStats.docFreq().getMax()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 2 } + - match: { hits.hits.1._score: 2 } + +--- +"term query: totalTermFreq sum without DFS": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + function_score: + boost_mode: replace + query: { term: { "genre.keyword": "Sci-fi" } } + script_score: + script: + source: "return _termStats.totalTermFreq().getSum()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 1 } + - match: { hits.hits.1._score: 1 } + +--- +"term query: totalTermFreq sum with DFS": + - do: + search: + rest_total_hits_as_int: true + search_type: dfs_query_then_fetch + index: test-index + body: + query: + function_score: + boost_mode: replace + query: { term: { "genre.keyword": "Sci-fi" } } + script_score: + script: + source: "return _termStats.totalTermFreq().getSum()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 2 } + - match: { hits.hits.1._score: 2 } + +--- +"term query: termFreq sum without DFS": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + function_score: + boost_mode: replace + query: { term: { "genre.keyword": "Sci-fi" } } + script_score: + script: + source: "return _termStats.termFreq().getSum()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 1 } + - match: { hits.hits.1._score: 1 } + +--- +"term query: termFreq sum with DFS": + - do: + search: + rest_total_hits_as_int: true + search_type: dfs_query_then_fetch + index: test-index + body: + query: + function_score: + boost_mode: replace + query: { term: { "genre.keyword": "Sci-fi" } } + script_score: + script: + source: "return _termStats.termFreq().getSum()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 1 } + - match: { hits.hits.1._score: 1 } + +--- +"term query: termPositions avg without DFS": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + function_score: + boost_mode: replace + query: { term: { "genre.keyword": "Sci-fi" } } + script_score: + script: + source: "return _termStats.termPositions().getAverage()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 0 } + - match: { hits.hits.1._score: 0 } + +--- +"term query: termPositions avg with DFS": + - do: + search: + rest_total_hits_as_int: true + search_type: dfs_query_then_fetch + index: test-index + body: + query: + function_score: + boost_mode: replace + query: { term: { "genre.keyword": "Sci-fi" } } + script_score: + script: + source: "return _termStats.termPositions().getAverage()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 0 } + - match: { hits.hits.1._score: 0 } + +--- +"Complex bool query: uniqueTermsCount": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + function_score: + boost_mode: replace + query: + bool: + must: + match: { "title": "star wars" } + should: + term: { "genre.keyword": "Sci-fi" } + filter: + match: { "genre" : "sci"} + must_not: + term: { "genre.keyword": "War" } + script_score: + script: + source: "return _termStats.uniqueTermsCount()" + - match: { hits.total: 2 } + - match: { hits.hits.0._score: 4 } + - match: { hits.hits.1._score: 4 } + + +--- +"match_all query: uniqueTermsCount": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + function_score: + boost_mode: replace + query: + match_all: {} + script_score: + script: + source: "return _termStats.uniqueTermsCount()" + - match: { hits.total: 4 } + - match: { hits.hits.0._score: 0 } + - match: { hits.hits.1._score: 0 } + - match: { hits.hits.2._score: 0 } + - match: { hits.hits.3._score: 0 } + +--- +"match_all query: docFreq": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + function_score: + boost_mode: replace + query: + match_all: {} + script_score: + script: + source: "return _termStats.docFreq().getMax()" + - match: { hits.total: 4 } + - match: { hits.hits.0._score: 0 } + - match: { hits.hits.1._score: 0 } + - match: { hits.hits.2._score: 0 } + - match: { hits.hits.3._score: 0 } + +--- +"match_all query: totalTermFreq": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + function_score: + boost_mode: replace + query: + match_all: {} + script_score: + script: + source: "return _termStats.totalTermFreq().getSum()" + - match: { hits.total: 4 } + - match: { hits.hits.0._score: 0 } + - match: { hits.hits.1._score: 0 } + - match: { hits.hits.2._score: 0 } + - match: { hits.hits.3._score: 0 } + +--- +"match_all query: termFreq": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + function_score: + boost_mode: replace + query: + match_all: {} + script_score: + script: + source: "return _termStats.termFreq().getMax()" + - match: { hits.total: 4 } + - match: { hits.hits.0._score: 0 } + - match: { hits.hits.1._score: 0 } + - match: { hits.hits.2._score: 0 } + - match: { hits.hits.3._score: 0 } + +--- +"match_all query: termPositions": + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + function_score: + boost_mode: replace + query: + match_all: {} + script_score: + script: + source: "return _termStats.termPositions().getSum()" + - match: { hits.total: 4 } + - match: { hits.hits.0._score: 0 } + - match: { hits.hits.1._score: 0 } + - match: { hits.hits.2._score: 0 } + - match: { hits.hits.3._score: 0 } diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/65_runtime_doc_values.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/65_runtime_doc_values.yml index 148b8e55e1a4a..b5190a579f62d 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/65_runtime_doc_values.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/65_runtime_doc_values.yml @@ -12,7 +12,7 @@ setup: script: source: | for (date in field('date')) { - emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT)); + emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH)); } total_value_double: type: double @@ -55,7 +55,7 @@ setup: source: | if (doc.containsKey('date')) { for (date in doc['date']) { - emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT)); + emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH)); } } doc_total_value_double: @@ -737,7 +737,7 @@ setup: script: source: | for (date in field('date')) { - emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT)); + emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH)); } sort: [ { rank: asc } ] script_fields: @@ -758,7 +758,7 @@ setup: script: source: | for (date in field('date')) { - emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT)); + emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH)); } sort: [ { rank: asc } ] script_fields: @@ -924,7 +924,7 @@ setup: source: | if (doc.containsKey('date')) { for (date in doc['date']) { - emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT)); + emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH)); } } sort: [ { rank: asc } ] @@ -947,7 +947,7 @@ setup: source: | if (doc.containsKey('date')) { for (date in doc['date']) { - emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT)); + emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH)); } } sort: [ { rank: asc } ] @@ -1133,7 +1133,7 @@ setup: script: source: | for (date in field('date')) { - emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT)); + emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH)); } sort: [ { rank: asc } ] script_fields: @@ -1156,7 +1156,7 @@ setup: script: source: | for (date in field('date')) { - emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT)); + emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH)); } sort: [ { rank: asc } ] script_fields: @@ -1337,7 +1337,7 @@ setup: source: | if (doc.containsKey('date')) { for (date in doc['date']) { - emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT)); + emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH)); } } sort: [ { rank: asc } ] @@ -1362,7 +1362,7 @@ setup: source: | if (doc.containsKey('date')) { for (date in doc['date']) { - emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT)); + emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH)); } } sort: [ { rank: asc } ] diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java index 2808dae31239c..f41d365f305bd 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java @@ -352,7 +352,7 @@ private static int getLevels(int treeLevels, double precisionInMeters, int defau public LegacyGeoShapeFieldMapper build(MapperBuilderContext context) { LegacyGeoShapeParser parser = new LegacyGeoShapeParser(); GeoShapeFieldType ft = buildFieldType(parser, context); - return new LegacyGeoShapeFieldMapper(leafName(), ft, multiFieldsBuilder.build(this, context), copyTo, parser, this); + return new LegacyGeoShapeFieldMapper(leafName(), ft, builderParams(this, context), parser, this); } } @@ -537,20 +537,18 @@ public PrefixTreeStrategy resolvePrefixTreeStrategy(String strategyName) { public LegacyGeoShapeFieldMapper( String simpleName, MappedFieldType mappedFieldType, - MultiFields multiFields, - CopyTo copyTo, + BuilderParams builderParams, LegacyGeoShapeParser parser, Builder builder ) { super( simpleName, mappedFieldType, + builderParams, builder.ignoreMalformed.get(), builder.coerce.get(), builder.ignoreZValue.get(), builder.orientation.get(), - multiFields, - copyTo, parser ); this.indexCreatedVersion = builder.indexCreatedVersion; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index 899cc42fea1e0..d6225674c7626 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -139,13 +139,11 @@ private MatchOnlyTextFieldType buildFieldType(MapperBuilderContext context) { @Override public MatchOnlyTextFieldMapper build(MapperBuilderContext context) { MatchOnlyTextFieldType tft = buildFieldType(context); - MultiFields multiFields = multiFieldsBuilder.build(this, context); return new MatchOnlyTextFieldMapper( leafName(), Defaults.FIELD_TYPE, tft, - multiFields, - copyTo, + builderParams(this, context), context.isSourceSynthetic(), this ); @@ -382,12 +380,11 @@ private MatchOnlyTextFieldMapper( String simpleName, FieldType fieldType, MatchOnlyTextFieldType mappedFieldType, - MultiFields multiFields, - CopyTo copyTo, + BuilderParams builderParams, boolean storeSource, Builder builder ) { - super(simpleName, mappedFieldType, multiFields, copyTo, false, null); + super(simpleName, mappedFieldType, builderParams); assert mappedFieldType.getTextSearchInfo().isTokenized(); assert mappedFieldType.hasDocValues() == false; this.fieldType = freezeAndDeduplicateFieldType(fieldType); @@ -442,12 +439,12 @@ protected SyntheticSourceMode syntheticSourceMode() { @Override public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - if (copyTo.copyToFields().isEmpty() != true) { + if (copyTo().copyToFields().isEmpty() != true) { throw new IllegalArgumentException( "field [" + fullPath() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" ); } - return new StringStoredFieldFieldLoader(fieldType().storedFieldNameForSyntheticSource(), leafName(), null) { + return new StringStoredFieldFieldLoader(fieldType().storedFieldNameForSyntheticSource(), leafName()) { @Override protected void write(XContentBuilder b, Object value) throws IOException { b.value((String) value); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java index bd3845e1ee18a..0b475641e4290 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java @@ -98,8 +98,7 @@ public RankFeatureFieldMapper build(MapperBuilderContext context) { positiveScoreImpact.getValue(), nullValue.getValue() ), - multiFieldsBuilder.build(this, context), - copyTo, + builderParams(this, context), positiveScoreImpact.getValue(), nullValue.getValue() ); @@ -172,12 +171,11 @@ public Query termQuery(Object value, SearchExecutionContext context) { private RankFeatureFieldMapper( String simpleName, MappedFieldType mappedFieldType, - MultiFields multiFields, - CopyTo copyTo, + BuilderParams builderParams, boolean positiveScoreImpact, Float nullValue ) { - super(simpleName, mappedFieldType, multiFields, copyTo, false, null); + super(simpleName, mappedFieldType, builderParams); this.positiveScoreImpact = positiveScoreImpact; this.nullValue = nullValue; } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java index e6cb3010f9960..5b1d35ec03c0e 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java @@ -66,8 +66,7 @@ public RankFeaturesFieldMapper build(MapperBuilderContext context) { return new RankFeaturesFieldMapper( leafName(), new RankFeaturesFieldType(context.buildFullName(leafName()), meta.getValue(), positiveScoreImpact.getValue()), - multiFieldsBuilder.build(this, context), - copyTo, + builderParams(this, context), positiveScoreImpact.getValue() ); } @@ -122,11 +121,10 @@ private static String indexedValueForSearch(Object value) { private RankFeaturesFieldMapper( String simpleName, MappedFieldType mappedFieldType, - MultiFields multiFields, - CopyTo copyTo, + BuilderParams builderParams, boolean positiveScoreImpact ) { - super(simpleName, mappedFieldType, multiFields, copyTo, false, null); + super(simpleName, mappedFieldType, builderParams); this.positiveScoreImpact = positiveScoreImpact; } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index c346a7d669149..4e46105bd0534 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -197,14 +197,7 @@ public ScaledFloatFieldMapper build(MapperBuilderContext context) { metric.getValue(), indexMode ); - return new ScaledFloatFieldMapper( - leafName(), - type, - multiFieldsBuilder.build(this, context), - copyTo, - context.isSourceSynthetic(), - this - ); + return new ScaledFloatFieldMapper(leafName(), type, builderParams(this, context), context.isSourceSynthetic(), this); } } @@ -470,12 +463,11 @@ public String toString() { private ScaledFloatFieldMapper( String simpleName, ScaledFloatFieldType mappedFieldType, - MultiFields multiFields, - CopyTo copyTo, + BuilderParams builderParams, boolean isSourceSynthetic, Builder builder ) { - super(simpleName, mappedFieldType, multiFields, copyTo); + super(simpleName, mappedFieldType, builderParams); this.isSourceSynthetic = isSourceSynthetic; this.indexed = builder.indexed.getValue(); this.hasDocValues = builder.hasDocValues.getValue(); @@ -728,7 +720,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + "] doesn't support synthetic source because it doesn't have doc values" ); } - if (copyTo.copyToFields().isEmpty() != true) { + if (copyTo().copyToFields().isEmpty() != true) { throw new IllegalArgumentException( "field [" + fullPath() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" ); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java index d521f9b2d2a31..57ac8fdfbb023 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapper.java @@ -262,11 +262,10 @@ public SearchAsYouTypeFieldMapper build(MapperBuilderContext context) { return new SearchAsYouTypeFieldMapper( leafName(), ft, - copyTo, + builderParams(this, context), indexAnalyzers, prefixFieldMapper, shingleFieldMappers, - multiFieldsBuilder.build(this, context), this ); } @@ -498,7 +497,7 @@ static final class PrefixFieldMapper extends FieldMapper { final FieldType fieldType; PrefixFieldMapper(FieldType fieldType, PrefixFieldType mappedFieldType) { - super(mappedFieldType.name(), mappedFieldType, MultiFields.empty(), CopyTo.empty()); + super(mappedFieldType.name(), mappedFieldType, BuilderParams.empty()); this.fieldType = Mapper.freezeAndDeduplicateFieldType(fieldType); } @@ -537,7 +536,7 @@ static final class ShingleFieldMapper extends FieldMapper { private final FieldType fieldType; ShingleFieldMapper(FieldType fieldType, ShingleFieldType mappedFieldtype) { - super(mappedFieldtype.name(), mappedFieldtype, MultiFields.empty(), CopyTo.empty()); + super(mappedFieldtype.name(), mappedFieldtype, BuilderParams.empty()); this.fieldType = freezeAndDeduplicateFieldType(fieldType); } @@ -672,14 +671,13 @@ public SpanQuery spanPrefixQuery(String value, SpanMultiTermQueryWrapper.SpanRew public SearchAsYouTypeFieldMapper( String simpleName, SearchAsYouTypeFieldType mappedFieldType, - CopyTo copyTo, + BuilderParams builderParams, Map indexAnalyzers, PrefixFieldMapper prefixField, ShingleFieldMapper[] shingleFields, - MultiFields multiFields, Builder builder ) { - super(simpleName, mappedFieldType, multiFields, copyTo, false, null); + super(simpleName, mappedFieldType, builderParams); this.prefixField = prefixField; this.shingleFields = shingleFields; this.maxShingleSize = builder.maxShingleSize.getValue(); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java index 9db677ddddffa..fa0a96a548a97 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java @@ -87,7 +87,7 @@ public TokenCountFieldMapper build(MapperBuilderContext context) { nullValue.getValue(), meta.getValue() ); - return new TokenCountFieldMapper(leafName(), ft, multiFieldsBuilder.build(this, context), copyTo, this); + return new TokenCountFieldMapper(leafName(), ft, builderParams(this, context), this); } } @@ -135,14 +135,8 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) private final boolean enablePositionIncrements; private final Integer nullValue; - protected TokenCountFieldMapper( - String simpleName, - MappedFieldType defaultFieldType, - MultiFields multiFields, - CopyTo copyTo, - Builder builder - ) { - super(simpleName, defaultFieldType, multiFields, copyTo); + protected TokenCountFieldMapper(String simpleName, MappedFieldType defaultFieldType, BuilderParams builderParams, Builder builder) { + super(simpleName, defaultFieldType, builderParams); this.analyzer = builder.analyzer.getValue(); this.enablePositionIncrements = builder.enablePositionIncrements.getValue(); this.nullValue = builder.nullValue.getValue(); diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java index 7e9b6916e99d4..f6392f32a88d6 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java @@ -88,7 +88,7 @@ public Object valueForDisplay(Object value) { } protected ParentIdFieldMapper(String name, boolean eagerGlobalOrdinals) { - super(name, new ParentIdFieldType(name, eagerGlobalOrdinals), MultiFields.empty(), CopyTo.empty(), false, null); + super(name, new ParentIdFieldType(name, eagerGlobalOrdinals), BuilderParams.empty()); } @Override diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java index dc760c0b07b71..ccb67f5c51acf 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -210,7 +210,7 @@ protected ParentJoinFieldMapper( boolean eagerGlobalOrdinals, List relations ) { - super(simpleName, mappedFieldType, MultiFields.empty(), CopyTo.empty(), false, null); + super(simpleName, mappedFieldType, BuilderParams.empty()); this.parentIdFields = parentIdFields; this.eagerGlobalOrdinals = eagerGlobalOrdinals; this.relations = relations; diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java index ad936a5491b69..576ea4dbd5d23 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java @@ -137,8 +137,6 @@ protected Parameter[] getParameters() { @Override public PercolatorFieldMapper build(MapperBuilderContext context) { PercolatorFieldType fieldType = new PercolatorFieldType(context.buildFullName(leafName()), meta.getValue()); - // TODO should percolator even allow multifields? - MultiFields multiFields = multiFieldsBuilder.build(this, context); context = context.createChildContext(leafName(), null); KeywordFieldMapper extractedTermsField = createExtractQueryFieldBuilder( EXTRACTED_TERMS_FIELD_NAME, @@ -165,8 +163,7 @@ public PercolatorFieldMapper build(MapperBuilderContext context) { return new PercolatorFieldMapper( leafName(), fieldType, - multiFields, - copyTo, + builderParams(this, context), searchExecutionContext, extractedTermsField, extractionResultField, @@ -375,8 +372,7 @@ static Tuple, Map>> extractTermsAndRanges(In PercolatorFieldMapper( String simpleName, MappedFieldType mappedFieldType, - MultiFields multiFields, - CopyTo copyTo, + BuilderParams builderParams, Supplier searchExecutionContext, KeywordFieldMapper queryTermsField, KeywordFieldMapper extractionResultField, @@ -387,7 +383,7 @@ static Tuple, Map>> extractTermsAndRanges(In IndexVersion indexCreatedVersion, Supplier clusterTransportVersion ) { - super(simpleName, mappedFieldType, multiFields, copyTo); + super(simpleName, mappedFieldType, builderParams); this.searchExecutionContext = searchExecutionContext; this.queryTermsField = queryTermsField; this.extractionResultField = extractionResultField; diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/ReindexPluginMetricsIT.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/ReindexPluginMetricsIT.java new file mode 100644 index 0000000000000..e7d26b0808a48 --- /dev/null +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/index/reindex/ReindexPluginMetricsIT.java @@ -0,0 +1,216 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.reindex; + +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.PluginsService; +import org.elasticsearch.reindex.BulkIndexByScrollResponseMatcher; +import org.elasticsearch.reindex.ReindexPlugin; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.telemetry.Measurement; +import org.elasticsearch.telemetry.TestTelemetryPlugin; +import org.elasticsearch.test.ESIntegTestCase; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.reindex.DeleteByQueryMetrics.DELETE_BY_QUERY_TIME_HISTOGRAM; +import static org.elasticsearch.reindex.ReindexMetrics.REINDEX_TIME_HISTOGRAM; +import static org.elasticsearch.reindex.UpdateByQueryMetrics.UPDATE_BY_QUERY_TIME_HISTOGRAM; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.hamcrest.Matchers.equalTo; + +@ESIntegTestCase.ClusterScope(numDataNodes = 0, numClientNodes = 0, scope = ESIntegTestCase.Scope.TEST) +public class ReindexPluginMetricsIT extends ESIntegTestCase { + @Override + protected Collection> nodePlugins() { + return Arrays.asList(ReindexPlugin.class, TestTelemetryPlugin.class); + } + + protected ReindexRequestBuilder reindex() { + return new ReindexRequestBuilder(client()); + } + + protected UpdateByQueryRequestBuilder updateByQuery() { + return new UpdateByQueryRequestBuilder(client()); + } + + protected DeleteByQueryRequestBuilder deleteByQuery() { + return new DeleteByQueryRequestBuilder(client()); + } + + public static BulkIndexByScrollResponseMatcher matcher() { + return new BulkIndexByScrollResponseMatcher(); + } + + public void testReindexMetrics() throws Exception { + final String dataNodeName = internalCluster().startNode(); + + indexRandom( + true, + prepareIndex("source").setId("1").setSource("foo", "a"), + prepareIndex("source").setId("2").setSource("foo", "a"), + prepareIndex("source").setId("3").setSource("foo", "b"), + prepareIndex("source").setId("4").setSource("foo", "c") + ); + assertHitCount(prepareSearch("source").setSize(0), 4); + + final TestTelemetryPlugin testTelemetryPlugin = internalCluster().getInstance(PluginsService.class, dataNodeName) + .filterPlugins(TestTelemetryPlugin.class) + .findFirst() + .orElseThrow(); + + // Copy all the docs + reindex().source("source").destination("dest").get(); + // Use assertBusy to wait for all threads to complete so we get deterministic results + assertBusy(() -> { + testTelemetryPlugin.collect(); + List measurements = testTelemetryPlugin.getLongHistogramMeasurement(REINDEX_TIME_HISTOGRAM); + assertThat(measurements.size(), equalTo(1)); + }); + + // Now none of them + createIndex("none"); + reindex().source("source").destination("none").filter(termQuery("foo", "no_match")).get(); + assertBusy(() -> { + testTelemetryPlugin.collect(); + List measurements = testTelemetryPlugin.getLongHistogramMeasurement(REINDEX_TIME_HISTOGRAM); + assertThat(measurements.size(), equalTo(2)); + }); + + // Now half of them + reindex().source("source").destination("dest_half").filter(termQuery("foo", "a")).get(); + assertBusy(() -> { + testTelemetryPlugin.collect(); + List measurements = testTelemetryPlugin.getLongHistogramMeasurement(REINDEX_TIME_HISTOGRAM); + assertThat(measurements.size(), equalTo(3)); + }); + + // Limit with maxDocs + reindex().source("source").destination("dest_size_one").maxDocs(1).get(); + assertBusy(() -> { + testTelemetryPlugin.collect(); + List measurements = testTelemetryPlugin.getLongHistogramMeasurement(REINDEX_TIME_HISTOGRAM); + assertThat(measurements.size(), equalTo(4)); + }); + } + + public void testDeleteByQueryMetrics() throws Exception { + final String dataNodeName = internalCluster().startNode(); + + indexRandom( + true, + prepareIndex("test").setId("1").setSource("foo", "a"), + prepareIndex("test").setId("2").setSource("foo", "a"), + prepareIndex("test").setId("3").setSource("foo", "b"), + prepareIndex("test").setId("4").setSource("foo", "c"), + prepareIndex("test").setId("5").setSource("foo", "d"), + prepareIndex("test").setId("6").setSource("foo", "e"), + prepareIndex("test").setId("7").setSource("foo", "f") + ); + + assertHitCount(prepareSearch("test").setSize(0), 7); + + final TestTelemetryPlugin testTelemetryPlugin = internalCluster().getInstance(PluginsService.class, dataNodeName) + .filterPlugins(TestTelemetryPlugin.class) + .findFirst() + .orElseThrow(); + + // Deletes two docs that matches "foo:a" + deleteByQuery().source("test").filter(termQuery("foo", "a")).refresh(true).get(); + assertBusy(() -> { + testTelemetryPlugin.collect(); + List measurements = testTelemetryPlugin.getLongHistogramMeasurement(DELETE_BY_QUERY_TIME_HISTOGRAM); + assertThat(measurements.size(), equalTo(1)); + }); + + // Deletes the two first docs with limit by size + DeleteByQueryRequestBuilder request = deleteByQuery().source("test").filter(QueryBuilders.matchAllQuery()).size(2).refresh(true); + request.source().addSort("foo.keyword", SortOrder.ASC); + request.get(); + assertBusy(() -> { + testTelemetryPlugin.collect(); + List measurements = testTelemetryPlugin.getLongHistogramMeasurement(DELETE_BY_QUERY_TIME_HISTOGRAM); + assertThat(measurements.size(), equalTo(2)); + }); + + // Deletes but match no docs + deleteByQuery().source("test").filter(termQuery("foo", "no_match")).refresh(true).get(); + assertBusy(() -> { + testTelemetryPlugin.collect(); + List measurements = testTelemetryPlugin.getLongHistogramMeasurement(DELETE_BY_QUERY_TIME_HISTOGRAM); + assertThat(measurements.size(), equalTo(3)); + }); + + // Deletes all remaining docs + deleteByQuery().source("test").filter(QueryBuilders.matchAllQuery()).refresh(true).get(); + assertBusy(() -> { + testTelemetryPlugin.collect(); + List measurements = testTelemetryPlugin.getLongHistogramMeasurement(DELETE_BY_QUERY_TIME_HISTOGRAM); + assertThat(measurements.size(), equalTo(4)); + }); + } + + public void testUpdateByQueryMetrics() throws Exception { + final String dataNodeName = internalCluster().startNode(); + + indexRandom( + true, + prepareIndex("test").setId("1").setSource("foo", "a"), + prepareIndex("test").setId("2").setSource("foo", "a"), + prepareIndex("test").setId("3").setSource("foo", "b"), + prepareIndex("test").setId("4").setSource("foo", "c") + ); + assertHitCount(prepareSearch("test").setSize(0), 4); + assertEquals(1, client().prepareGet("test", "1").get().getVersion()); + assertEquals(1, client().prepareGet("test", "4").get().getVersion()); + + final TestTelemetryPlugin testTelemetryPlugin = internalCluster().getInstance(PluginsService.class, dataNodeName) + .filterPlugins(TestTelemetryPlugin.class) + .findFirst() + .orElseThrow(); + + // Reindex all the docs + updateByQuery().source("test").refresh(true).get(); + assertBusy(() -> { + testTelemetryPlugin.collect(); + List measurements = testTelemetryPlugin.getLongHistogramMeasurement(UPDATE_BY_QUERY_TIME_HISTOGRAM); + assertThat(measurements.size(), equalTo(1)); + }); + + // Now none of them + updateByQuery().source("test").filter(termQuery("foo", "no_match")).refresh(true).get(); + assertBusy(() -> { + testTelemetryPlugin.collect(); + List measurements = testTelemetryPlugin.getLongHistogramMeasurement(UPDATE_BY_QUERY_TIME_HISTOGRAM); + assertThat(measurements.size(), equalTo(2)); + }); + + // Now half of them + updateByQuery().source("test").filter(termQuery("foo", "a")).refresh(true).get(); + assertBusy(() -> { + testTelemetryPlugin.collect(); + List measurements = testTelemetryPlugin.getLongHistogramMeasurement(UPDATE_BY_QUERY_TIME_HISTOGRAM); + assertThat(measurements.size(), equalTo(3)); + }); + + // Limit with size + UpdateByQueryRequestBuilder request = updateByQuery().source("test").size(3).refresh(true); + request.source().addSort("foo.keyword", SortOrder.ASC); + request.get(); + assertBusy(() -> { + testTelemetryPlugin.collect(); + List measurements = testTelemetryPlugin.getLongHistogramMeasurement(UPDATE_BY_QUERY_TIME_HISTOGRAM); + assertThat(measurements.size(), equalTo(4)); + }); + } +} diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/DeleteByQueryMetrics.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/DeleteByQueryMetrics.java new file mode 100644 index 0000000000000..2cedf0d5f5823 --- /dev/null +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/DeleteByQueryMetrics.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.reindex; + +import org.elasticsearch.telemetry.metric.LongHistogram; +import org.elasticsearch.telemetry.metric.MeterRegistry; + +public class DeleteByQueryMetrics { + public static final String DELETE_BY_QUERY_TIME_HISTOGRAM = "es.delete_by_query.duration.histogram"; + + private final LongHistogram deleteByQueryTimeSecsHistogram; + + public DeleteByQueryMetrics(MeterRegistry meterRegistry) { + this( + meterRegistry.registerLongHistogram(DELETE_BY_QUERY_TIME_HISTOGRAM, "Time taken to execute Delete by Query request", "seconds") + ); + } + + private DeleteByQueryMetrics(LongHistogram deleteByQueryTimeSecsHistogram) { + this.deleteByQueryTimeSecsHistogram = deleteByQueryTimeSecsHistogram; + } + + public long recordTookTime(long tookTime) { + deleteByQueryTimeSecsHistogram.record(tookTime); + return tookTime; + } +} diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexMetrics.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexMetrics.java new file mode 100644 index 0000000000000..3025357aa6538 --- /dev/null +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexMetrics.java @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.reindex; + +import org.elasticsearch.telemetry.metric.LongHistogram; +import org.elasticsearch.telemetry.metric.MeterRegistry; + +public class ReindexMetrics { + + public static final String REINDEX_TIME_HISTOGRAM = "es.reindex.duration.histogram"; + + private final LongHistogram reindexTimeSecsHistogram; + + public ReindexMetrics(MeterRegistry meterRegistry) { + this(meterRegistry.registerLongHistogram(REINDEX_TIME_HISTOGRAM, "Time to reindex by search", "millis")); + } + + private ReindexMetrics(LongHistogram reindexTimeSecsHistogram) { + this.reindexTimeSecsHistogram = reindexTimeSecsHistogram; + } + + public long recordTookTime(long tookTime) { + reindexTimeSecsHistogram.record(tookTime); + return tookTime; + } +} diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java index 1a40f77250e5f..3169d4c4ee1fb 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexPlugin.java @@ -34,7 +34,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.function.Predicate; import java.util.function.Supplier; @@ -85,8 +84,11 @@ public List getRestHandlers( @Override public Collection createComponents(PluginServices services) { - return Collections.singletonList( - new ReindexSslConfig(services.environment().settings(), services.environment(), services.resourceWatcherService()) + return List.of( + new ReindexSslConfig(services.environment().settings(), services.environment(), services.resourceWatcherService()), + new ReindexMetrics(services.telemetryProvider().getMeterRegistry()), + new UpdateByQueryMetrics(services.telemetryProvider().getMeterRegistry()), + new DeleteByQueryMetrics(services.telemetryProvider().getMeterRegistry()) ); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java index dbe1968bb076a..cb393a42f52a1 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/Reindexer.java @@ -37,6 +37,7 @@ import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.VersionType; @@ -65,6 +66,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiFunction; import java.util.function.LongSupplier; @@ -82,19 +84,22 @@ public class Reindexer { private final ThreadPool threadPool; private final ScriptService scriptService; private final ReindexSslConfig reindexSslConfig; + private final ReindexMetrics reindexMetrics; Reindexer( ClusterService clusterService, Client client, ThreadPool threadPool, ScriptService scriptService, - ReindexSslConfig reindexSslConfig + ReindexSslConfig reindexSslConfig, + @Nullable ReindexMetrics reindexMetrics ) { this.clusterService = clusterService; this.client = client; this.threadPool = threadPool; this.scriptService = scriptService; this.reindexSslConfig = reindexSslConfig; + this.reindexMetrics = reindexMetrics; } public void initTask(BulkByScrollTask task, ReindexRequest request, ActionListener listener) { @@ -102,6 +107,8 @@ public void initTask(BulkByScrollTask task, ReindexRequest request, ActionListen } public void execute(BulkByScrollTask task, ReindexRequest request, Client bulkClient, ActionListener listener) { + long startTime = System.nanoTime(); + BulkByScrollParallelizationHelper.executeSlicedAction( task, request, @@ -122,7 +129,12 @@ public void execute(BulkByScrollTask task, ReindexRequest request, Client bulkCl clusterService.state(), reindexSslConfig, request, - listener + ActionListener.runAfter(listener, () -> { + long elapsedTime = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - startTime); + if (reindexMetrics != null) { + reindexMetrics.recordTookTime(elapsedTime); + } + }) ); searchAction.start(); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportDeleteByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportDeleteByQueryAction.java index 755587feb47d3..53381c33d7f78 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportDeleteByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportDeleteByQueryAction.java @@ -15,6 +15,7 @@ import org.elasticsearch.client.internal.ParentTaskAssigningClient; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.reindex.BulkByScrollResponse; import org.elasticsearch.index.reindex.BulkByScrollTask; import org.elasticsearch.index.reindex.DeleteByQueryAction; @@ -25,12 +26,15 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import java.util.concurrent.TimeUnit; + public class TransportDeleteByQueryAction extends HandledTransportAction { private final ThreadPool threadPool; private final Client client; private final ScriptService scriptService; private final ClusterService clusterService; + private final DeleteByQueryMetrics deleteByQueryMetrics; @Inject public TransportDeleteByQueryAction( @@ -39,18 +43,21 @@ public TransportDeleteByQueryAction( Client client, TransportService transportService, ScriptService scriptService, - ClusterService clusterService + ClusterService clusterService, + @Nullable DeleteByQueryMetrics deleteByQueryMetrics ) { super(DeleteByQueryAction.NAME, transportService, actionFilters, DeleteByQueryRequest::new, EsExecutors.DIRECT_EXECUTOR_SERVICE); this.threadPool = threadPool; this.client = client; this.scriptService = scriptService; this.clusterService = clusterService; + this.deleteByQueryMetrics = deleteByQueryMetrics; } @Override public void doExecute(Task task, DeleteByQueryRequest request, ActionListener listener) { BulkByScrollTask bulkByScrollTask = (BulkByScrollTask) task; + long startTime = System.nanoTime(); BulkByScrollParallelizationHelper.startSlicedAction( request, bulkByScrollTask, @@ -64,8 +71,20 @@ public void doExecute(Task task, DeleteByQueryRequest request, ActionListener { + long elapsedTime = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - startTime); + if (deleteByQueryMetrics != null) { + deleteByQueryMetrics.recordTookTime(elapsedTime); + } + }) + ).start(); } ); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java index a86af2ca2b83e..821a137ac7566 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportReindexAction.java @@ -19,6 +19,7 @@ import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.reindex.BulkByScrollResponse; import org.elasticsearch.index.reindex.BulkByScrollTask; import org.elasticsearch.index.reindex.ReindexAction; @@ -53,7 +54,8 @@ public TransportReindexAction( AutoCreateIndex autoCreateIndex, Client client, TransportService transportService, - ReindexSslConfig sslConfig + ReindexSslConfig sslConfig, + @Nullable ReindexMetrics reindexMetrics ) { this( ReindexAction.NAME, @@ -66,7 +68,8 @@ public TransportReindexAction( autoCreateIndex, client, transportService, - sslConfig + sslConfig, + reindexMetrics ); } @@ -81,12 +84,13 @@ protected TransportReindexAction( AutoCreateIndex autoCreateIndex, Client client, TransportService transportService, - ReindexSslConfig sslConfig + ReindexSslConfig sslConfig, + @Nullable ReindexMetrics reindexMetrics ) { super(name, transportService, actionFilters, ReindexRequest::new, EsExecutors.DIRECT_EXECUTOR_SERVICE); this.client = client; this.reindexValidator = new ReindexValidator(settings, clusterService, indexNameExpressionResolver, autoCreateIndex); - this.reindexer = new Reindexer(clusterService, client, threadPool, scriptService, sslConfig); + this.reindexer = new Reindexer(clusterService, client, threadPool, scriptService, sslConfig, reindexMetrics); } @Override diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportUpdateByQueryAction.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportUpdateByQueryAction.java index fc0bfa3c8a214..997d4d32fe042 100644 --- a/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportUpdateByQueryAction.java +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/TransportUpdateByQueryAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.reindex.BulkByScrollResponse; import org.elasticsearch.index.reindex.BulkByScrollTask; import org.elasticsearch.index.reindex.ScrollableHitSource; @@ -35,6 +36,7 @@ import org.elasticsearch.transport.TransportService; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.function.BiFunction; import java.util.function.LongSupplier; @@ -44,6 +46,7 @@ public class TransportUpdateByQueryAction extends HandledTransportAction listener) { BulkByScrollTask bulkByScrollTask = (BulkByScrollTask) task; + long startTime = System.nanoTime(); BulkByScrollParallelizationHelper.startSlicedAction( request, bulkByScrollTask, @@ -78,8 +84,21 @@ protected void doExecute(Task task, UpdateByQueryRequest request, ActionListener clusterService.localNode(), bulkByScrollTask ); - new AsyncIndexBySearchAction(bulkByScrollTask, logger, assigningClient, threadPool, scriptService, request, state, listener) - .start(); + new AsyncIndexBySearchAction( + bulkByScrollTask, + logger, + assigningClient, + threadPool, + scriptService, + request, + state, + ActionListener.runAfter(listener, () -> { + long elapsedTime = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - startTime); + if (updateByQueryMetrics != null) { + updateByQueryMetrics.recordTookTime(elapsedTime); + } + }) + ).start(); } ); } diff --git a/modules/reindex/src/main/java/org/elasticsearch/reindex/UpdateByQueryMetrics.java b/modules/reindex/src/main/java/org/elasticsearch/reindex/UpdateByQueryMetrics.java new file mode 100644 index 0000000000000..6ca52769a1ba9 --- /dev/null +++ b/modules/reindex/src/main/java/org/elasticsearch/reindex/UpdateByQueryMetrics.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.reindex; + +import org.elasticsearch.telemetry.metric.LongHistogram; +import org.elasticsearch.telemetry.metric.MeterRegistry; + +public class UpdateByQueryMetrics { + public static final String UPDATE_BY_QUERY_TIME_HISTOGRAM = "es.update_by_query.duration.histogram"; + + private final LongHistogram updateByQueryTimeSecsHistogram; + + public UpdateByQueryMetrics(MeterRegistry meterRegistry) { + this( + meterRegistry.registerLongHistogram(UPDATE_BY_QUERY_TIME_HISTOGRAM, "Time taken to execute Update by Query request", "seconds") + ); + } + + private UpdateByQueryMetrics(LongHistogram updateByQueryTimeSecsHistogram) { + this.updateByQueryTimeSecsHistogram = updateByQueryTimeSecsHistogram; + } + + public long recordTookTime(long tookTime) { + updateByQueryTimeSecsHistogram.record(tookTime); + return tookTime; + } +} diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/DeleteByQueryMetricsTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/DeleteByQueryMetricsTests.java new file mode 100644 index 0000000000000..58adc6aebaa9b --- /dev/null +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/DeleteByQueryMetricsTests.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.reindex; + +import org.elasticsearch.telemetry.InstrumentType; +import org.elasticsearch.telemetry.Measurement; +import org.elasticsearch.telemetry.RecordingMeterRegistry; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.util.List; + +import static org.elasticsearch.reindex.DeleteByQueryMetrics.DELETE_BY_QUERY_TIME_HISTOGRAM; + +public class DeleteByQueryMetricsTests extends ESTestCase { + private RecordingMeterRegistry recordingMeterRegistry; + private DeleteByQueryMetrics metrics; + + @Before + public void createMetrics() { + recordingMeterRegistry = new RecordingMeterRegistry(); + metrics = new DeleteByQueryMetrics(recordingMeterRegistry); + } + + public void testRecordTookTime() { + int secondsTaken = randomIntBetween(1, 50); + metrics.recordTookTime(secondsTaken); + List measurements = recordingMeterRegistry.getRecorder() + .getMeasurements(InstrumentType.LONG_HISTOGRAM, DELETE_BY_QUERY_TIME_HISTOGRAM); + assertEquals(measurements.size(), 1); + assertEquals(measurements.get(0).getLong(), secondsTaken); + } +} diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexMetricsTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexMetricsTests.java new file mode 100644 index 0000000000000..4711530585817 --- /dev/null +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexMetricsTests.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.reindex; + +import org.elasticsearch.telemetry.InstrumentType; +import org.elasticsearch.telemetry.Measurement; +import org.elasticsearch.telemetry.RecordingMeterRegistry; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.util.List; + +import static org.elasticsearch.reindex.ReindexMetrics.REINDEX_TIME_HISTOGRAM; + +public class ReindexMetricsTests extends ESTestCase { + + private RecordingMeterRegistry recordingMeterRegistry; + private ReindexMetrics metrics; + + @Before + public void createMetrics() { + recordingMeterRegistry = new RecordingMeterRegistry(); + metrics = new ReindexMetrics(recordingMeterRegistry); + } + + public void testRecordTookTime() { + int secondsTaken = randomIntBetween(1, 50); + metrics.recordTookTime(secondsTaken); + List measurements = recordingMeterRegistry.getRecorder() + .getMeasurements(InstrumentType.LONG_HISTOGRAM, REINDEX_TIME_HISTOGRAM); + assertEquals(measurements.size(), 1); + assertEquals(measurements.get(0).getLong(), secondsTaken); + } +} diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryMetricsTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryMetricsTests.java new file mode 100644 index 0000000000000..548d18d202984 --- /dev/null +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryMetricsTests.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.reindex; + +import org.elasticsearch.telemetry.InstrumentType; +import org.elasticsearch.telemetry.Measurement; +import org.elasticsearch.telemetry.RecordingMeterRegistry; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.util.List; + +import static org.elasticsearch.reindex.UpdateByQueryMetrics.UPDATE_BY_QUERY_TIME_HISTOGRAM; + +public class UpdateByQueryMetricsTests extends ESTestCase { + + private RecordingMeterRegistry recordingMeterRegistry; + private UpdateByQueryMetrics metrics; + + @Before + public void createMetrics() { + recordingMeterRegistry = new RecordingMeterRegistry(); + metrics = new UpdateByQueryMetrics(recordingMeterRegistry); + } + + public void testRecordTookTime() { + int secondsTaken = randomIntBetween(1, 50); + metrics.recordTookTime(secondsTaken); + List measurements = recordingMeterRegistry.getRecorder() + .getMeasurements(InstrumentType.LONG_HISTOGRAM, UPDATE_BY_QUERY_TIME_HISTOGRAM); + assertEquals(measurements.size(), 1); + assertEquals(measurements.get(0).getLong(), secondsTaken); + } +} diff --git a/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWithScriptTests.java b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWithScriptTests.java index 876ddefda161b..c4d591f804750 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWithScriptTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryWithScriptTests.java @@ -60,6 +60,7 @@ protected TransportUpdateByQueryAction.AsyncIndexBySearchAction action(ScriptSer null, transportService, scriptService, + null, null ); return new TransportUpdateByQueryAction.AsyncIndexBySearchAction( diff --git a/modules/repository-azure/build.gradle b/modules/repository-azure/build.gradle index 9c63304e8267b..6334e5ae6a195 100644 --- a/modules/repository-azure/build.gradle +++ b/modules/repository-azure/build.gradle @@ -24,16 +24,16 @@ versions << [ dependencies { // Microsoft - api "com.azure:azure-core-http-netty:1.15.1" - api "com.azure:azure-core:1.50.0" - api "com.azure:azure-identity:1.13.1" - api "com.azure:azure-json:1.1.0" - api "com.azure:azure-storage-blob:12.26.1" - api "com.azure:azure-storage-common:12.26.0" - api "com.azure:azure-storage-internal-avro:12.11.1" - api "com.azure:azure-xml:1.0.0" + api "com.azure:azure-core-http-netty:1.15.3" + api "com.azure:azure-core:1.51.0" + api "com.azure:azure-identity:1.13.2" + api "com.azure:azure-json:1.2.0" + api "com.azure:azure-storage-blob:12.27.1" + api "com.azure:azure-storage-common:12.26.1" + api "com.azure:azure-storage-internal-avro:12.12.1" + api "com.azure:azure-xml:1.1.0" api "com.microsoft.azure:msal4j-persistence-extension:1.3.0" - api "com.microsoft.azure:msal4j:1.16.1" + api "com.microsoft.azure:msal4j:1.16.2" // Jackson api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" @@ -57,7 +57,7 @@ dependencies { api "org.reactivestreams:reactive-streams:1.0.4" // Others - api "com.fasterxml.woodstox:woodstox-core:6.4.0" + api "com.fasterxml.woodstox:woodstox-core:6.7.0" api "com.github.stephenc.jcip:jcip-annotations:1.0-1" api "com.nimbusds:content-type:2.3" api "com.nimbusds:lang-tag:1.7" @@ -69,7 +69,7 @@ dependencies { api "net.java.dev.jna:jna:${versions.jna}" // Maven says 5.14.0 but this aligns with the Elasticsearch-wide version api "net.minidev:accessors-smart:2.5.0" api "net.minidev:json-smart:2.5.0" - api "org.codehaus.woodstox:stax2-api:4.2.1" + api "org.codehaus.woodstox:stax2-api:4.2.2" api "org.ow2.asm:asm:9.3" runtimeOnly "com.google.crypto.tink:tink:1.14.0" diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java index 388474acc75ea..c8c0b15db5ebe 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java @@ -26,6 +26,7 @@ import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.function.Function; import static org.elasticsearch.core.Strings.format; @@ -175,4 +176,9 @@ protected ByteSizeValue chunkSize() { public boolean isReadOnly() { return readonly; } + + @Override + protected Set getExtraUsageFeatures() { + return storageService.getExtraUsageFeatures(Repository.CLIENT_NAME.get(getMetadata().settings())); + } } diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java index 0d6cd7bf3d246..09088004759a8 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageService.java @@ -24,6 +24,7 @@ import java.net.Proxy; import java.net.URL; import java.util.Map; +import java.util.Set; import java.util.function.BiConsumer; import static java.util.Collections.emptyMap; @@ -165,4 +166,15 @@ public void refreshSettings(Map clientsSettings) { this.storageSettings = Map.copyOf(clientsSettings); // clients are built lazily by {@link client(String, LocationMode)} } + + /** + * For Azure repositories, we report the different kinds of credentials in use in the telemetry. + */ + public Set getExtraUsageFeatures(String clientName) { + try { + return getClientSettings(clientName).credentialsUsageFeatures(); + } catch (Exception e) { + return Set.of(); + } + } } diff --git a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java index b3e8dd8898bea..2333a1fdb9e93 100644 --- a/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java +++ b/modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureStorageSettings.java @@ -29,6 +29,7 @@ import java.util.HashMap; import java.util.Locale; import java.util.Map; +import java.util.Set; final class AzureStorageSettings { @@ -130,6 +131,7 @@ final class AzureStorageSettings { private final int maxRetries; private final Proxy proxy; private final boolean hasCredentials; + private final Set credentialsUsageFeatures; private AzureStorageSettings( String account, @@ -150,6 +152,12 @@ private AzureStorageSettings( this.endpointSuffix = endpointSuffix; this.timeout = timeout; this.maxRetries = maxRetries; + this.credentialsUsageFeatures = Strings.hasText(key) ? Set.of("uses_key_credentials") + : Strings.hasText(sasToken) ? Set.of("uses_sas_token") + : SocketAccess.doPrivilegedException(() -> System.getenv("AZURE_FEDERATED_TOKEN_FILE")) == null + ? Set.of("uses_default_credentials", "uses_managed_identity") + : Set.of("uses_default_credentials", "uses_workload_identity"); + // Register the proxy if we have any // Validate proxy settings if (proxyType.equals(Proxy.Type.DIRECT) && ((proxyPort != 0) || Strings.hasText(proxyHost))) { @@ -366,4 +374,8 @@ private String deriveURIFromSettings(boolean isPrimary) { throw new IllegalArgumentException(e); } } + + public Set credentialsUsageFeatures() { + return credentialsUsageFeatures; + } } diff --git a/modules/repository-azure/src/yamlRestTest/resources/rest-api-spec/test/repository_azure/20_repository.yml b/modules/repository-azure/src/yamlRestTest/resources/rest-api-spec/test/repository_azure/20_repository.yml index 299183f26d9dc..a4a7d0b22a0ed 100644 --- a/modules/repository-azure/src/yamlRestTest/resources/rest-api-spec/test/repository_azure/20_repository.yml +++ b/modules/repository-azure/src/yamlRestTest/resources/rest-api-spec/test/repository_azure/20_repository.yml @@ -235,6 +235,19 @@ setup: snapshot: missing wait_for_completion: true +--- +"Usage stats": + - requires: + cluster_features: + - repositories.supports_usage_stats + reason: requires this feature + + - do: + cluster.stats: {} + + - gte: { repositories.azure.count: 1 } + - gte: { repositories.azure.read_write: 1 } + --- teardown: diff --git a/modules/repository-gcs/src/yamlRestTest/resources/rest-api-spec/test/repository_gcs/20_repository.yml b/modules/repository-gcs/src/yamlRestTest/resources/rest-api-spec/test/repository_gcs/20_repository.yml index 68d61be4983c5..e8c34a4b6a20b 100644 --- a/modules/repository-gcs/src/yamlRestTest/resources/rest-api-spec/test/repository_gcs/20_repository.yml +++ b/modules/repository-gcs/src/yamlRestTest/resources/rest-api-spec/test/repository_gcs/20_repository.yml @@ -232,6 +232,19 @@ setup: snapshot: missing wait_for_completion: true +--- +"Usage stats": + - requires: + cluster_features: + - repositories.supports_usage_stats + reason: requires this feature + + - do: + cluster.stats: {} + + - gte: { repositories.gcs.count: 1 } + - gte: { repositories.gcs.read_write: 1 } + --- teardown: diff --git a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java index 1ab370ad203fc..7916bb5942fc7 100644 --- a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java +++ b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java @@ -85,6 +85,7 @@ import java.util.stream.StreamSupport; import static org.elasticsearch.repositories.RepositoriesMetrics.METRIC_REQUESTS_TOTAL; +import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.getRepositoryDataBlobName; import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomNonDataPurpose; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; @@ -428,12 +429,7 @@ public void testEnforcedCooldownPeriod() throws IOException { ); repository.blobStore() .blobContainer(repository.basePath()) - .writeBlobAtomic( - randomNonDataPurpose(), - BlobStoreRepository.INDEX_FILE_PREFIX + modifiedRepositoryData.getGenId(), - serialized, - true - ); + .writeBlobAtomic(randomNonDataPurpose(), getRepositoryDataBlobName(modifiedRepositoryData.getGenId()), serialized, true); final String newSnapshotName = "snapshot-new"; final long beforeThrottledSnapshot = repository.threadPool().relativeTimeInNanos(); diff --git a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml b/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml index 77870697f93ae..e88a0861ec01c 100644 --- a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml +++ b/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml @@ -345,6 +345,19 @@ setup: snapshot: missing wait_for_completion: true +--- +"Usage stats": + - requires: + cluster_features: + - repositories.supports_usage_stats + reason: requires this feature + + - do: + cluster.stats: {} + + - gte: { repositories.s3.count: 1 } + - gte: { repositories.s3.read_write: 1 } + --- teardown: diff --git a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml b/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml index 4a62d6183470d..501af980e17e3 100644 --- a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml +++ b/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml @@ -256,6 +256,19 @@ setup: snapshot: missing wait_for_completion: true +--- +"Usage stats": + - requires: + cluster_features: + - repositories.supports_usage_stats + reason: requires this feature + + - do: + cluster.stats: {} + + - gte: { repositories.s3.count: 1 } + - gte: { repositories.s3.read_write: 1 } + --- teardown: diff --git a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml b/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml index e24ff1ad0e559..129f0ba5d7588 100644 --- a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml +++ b/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml @@ -256,6 +256,19 @@ setup: snapshot: missing wait_for_completion: true +--- +"Usage stats": + - requires: + cluster_features: + - repositories.supports_usage_stats + reason: requires this feature + + - do: + cluster.stats: {} + + - gte: { repositories.s3.count: 1 } + - gte: { repositories.s3.read_write: 1 } + --- teardown: diff --git a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml b/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml index 9c332cc7d9301..de334b4b3df96 100644 --- a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml +++ b/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml @@ -256,6 +256,19 @@ setup: snapshot: missing wait_for_completion: true +--- +"Usage stats": + - requires: + cluster_features: + - repositories.supports_usage_stats + reason: requires this feature + + - do: + cluster.stats: {} + + - gte: { repositories.s3.count: 1 } + - gte: { repositories.s3.read_write: 1 } + --- teardown: diff --git a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/60_repository_sts_credentials.yml b/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/60_repository_sts_credentials.yml index 24c2b2b1741d6..09a8526017960 100644 --- a/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/60_repository_sts_credentials.yml +++ b/modules/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/60_repository_sts_credentials.yml @@ -257,6 +257,19 @@ setup: snapshot: missing wait_for_completion: true +--- +"Usage stats": + - requires: + cluster_features: + - repositories.supports_usage_stats + reason: requires this feature + + - do: + cluster.stats: {} + + - gte: { repositories.s3.count: 1 } + - gte: { repositories.s3.read_write: 1 } + --- teardown: diff --git a/modules/repository-url/build.gradle b/modules/repository-url/build.gradle index 3537d430e212b..3fe2f9d9bae42 100644 --- a/modules/repository-url/build.gradle +++ b/modules/repository-url/build.gradle @@ -33,6 +33,11 @@ dependencies { internalClusterTestImplementation project(':test:fixtures:url-fixture') } +tasks.named("yamlRestTestV7CompatTransform").configure { task -> + task.skipTest("repository_url/10_basic/Restore with repository-url using file://", "Error message has changed") + task.skipTest("repository_url/10_basic/Restore with repository-url using http://", "Error message has changed") +} + tasks.named("thirdPartyAudit").configure { ignoreMissingClasses( 'javax.servlet.ServletContextEvent', diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/10_keyword.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/10_keyword.yml index 7bd7b6c7779e2..11214907eb17e 100644 --- a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/10_keyword.yml +++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/10_keyword.yml @@ -12,7 +12,7 @@ setup: day_of_week: type: keyword script: | - emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT)); + emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH)); # Test fetching from _source day_of_week_from_source: type: keyword @@ -75,7 +75,7 @@ setup: - match: {sensor.mappings.runtime.day_of_week.type: keyword } - match: sensor.mappings.runtime.day_of_week.script.source: | - emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT)); + emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH)); - match: {sensor.mappings.runtime.day_of_week.script.lang: painless } # --- TODO get field mappings needs to be adapted @@ -90,7 +90,7 @@ setup: # type: keyword # script: # source: | -# emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT)); +# emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH)); # lang: painless # meta: {} # diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/13_keyword_calculated_at_index.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/13_keyword_calculated_at_index.yml index 1c10a017a5c33..4bedfa3e923a8 100644 --- a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/13_keyword_calculated_at_index.yml +++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/13_keyword_calculated_at_index.yml @@ -21,7 +21,7 @@ setup: day_of_week: type: keyword script: | - emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT)); + emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH)); # Test fetching from _source day_of_week_from_source: type: keyword @@ -74,7 +74,7 @@ setup: - match: {sensor.mappings.properties.day_of_week.type: keyword } - match: sensor.mappings.properties.day_of_week.script.source: | - emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT)); + emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH)); - match: {sensor.mappings.properties.day_of_week.script.lang: painless } --- diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/40_runtime_mappings.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/40_runtime_mappings.yml index 0e7d0b78bba47..b6acc7a18345a 100644 --- a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/40_runtime_mappings.yml +++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/40_runtime_mappings.yml @@ -34,7 +34,7 @@ setup: day_of_week: type: keyword script: - source: "emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))" + source: "emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))" - match: {indices: ["test-1"]} - length: {fields.timestamp: 1} @@ -78,7 +78,7 @@ setup: day_of_week: type: keyword script: - source: "emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))" + source: "emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))" - match: {indices: ["test-1", "test-2"]} - length: {fields.day_of_week: 1} diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/80_multiple_indices.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/80_multiple_indices.yml index 0c571975098b2..dc52350a25a75 100644 --- a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/80_multiple_indices.yml +++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/80_multiple_indices.yml @@ -12,7 +12,7 @@ setup: day_of_week: type: keyword script: | - emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT)); + emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH)); tomorrow: type: date script: diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java index 3e74a74dbd49c..ce7704e6e040c 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.mocksocket.MockSocket; import org.elasticsearch.telemetry.metric.MeterRegistry; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.DefaultBuiltInExecutorBuilders; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportSettings; import org.junit.After; @@ -52,7 +53,7 @@ public class Netty4SizeHeaderFrameDecoderTests extends ESTestCase { @Before public void startThreadPool() { - threadPool = new ThreadPool(settings, MeterRegistry.NOOP); + threadPool = new ThreadPool(settings, MeterRegistry.NOOP, new DefaultBuiltInExecutorBuilders()); NetworkService networkService = new NetworkService(Collections.emptyList()); PageCacheRecycler recycler = new MockPageCacheRecycler(Settings.EMPTY); nettyTransport = new Netty4Transport( diff --git a/muted-tests.yml b/muted-tests.yml index f480938c24a13..6600ae65d5809 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -1,25 +1,7 @@ tests: -- class: "org.elasticsearch.xpack.textstructure.structurefinder.TimestampFormatFinderTests" - issue: "https://github.com/elastic/elasticsearch/issues/108855" - method: "testGuessIsDayFirstFromLocale" -- class: "org.elasticsearch.test.rest.ClientYamlTestSuiteIT" - issue: "https://github.com/elastic/elasticsearch/issues/108857" - method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}" - class: "org.elasticsearch.upgrades.SearchStatesIT" issue: "https://github.com/elastic/elasticsearch/issues/108991" method: "testCanMatch" -- class: "org.elasticsearch.upgrades.MlTrainedModelsUpgradeIT" - issue: "https://github.com/elastic/elasticsearch/issues/108993" - method: "testTrainedModelInference" -- class: "org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT" - issue: "https://github.com/elastic/elasticsearch/issues/109188" - method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}" -- class: "org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT" - issue: "https://github.com/elastic/elasticsearch/issues/109189" - method: "test {p0=esql/70_locale/Date format with Italian locale}" -- class: "org.elasticsearch.xpack.test.rest.XPackRestIT" - issue: "https://github.com/elastic/elasticsearch/issues/109200" - method: "test {p0=esql/70_locale/Date format with Italian locale}" - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT method: test {yaml=reference/esql/esql-async-query-api/line_17} issue: https://github.com/elastic/elasticsearch/issues/109260 @@ -71,9 +53,6 @@ tests: - class: org.elasticsearch.nativeaccess.VectorSystemPropertyTests method: testSystemPropertyDisabled issue: https://github.com/elastic/elasticsearch/issues/110949 -- class: org.elasticsearch.xpack.esql.spatial.SpatialPushDownGeoPointIT - method: testPushedDownQueriesSingleValue - issue: https://github.com/elastic/elasticsearch/issues/111084 - class: org.elasticsearch.multi_node.GlobalCheckpointSyncActionIT issue: https://github.com/elastic/elasticsearch/issues/111124 - class: org.elasticsearch.cluster.PrevalidateShardPathIT @@ -85,9 +64,6 @@ tests: - class: org.elasticsearch.xpack.security.authc.oidc.OpenIdConnectAuthIT method: testAuthenticateWithImplicitFlow issue: https://github.com/elastic/elasticsearch/issues/111191 -- class: org.elasticsearch.action.admin.indices.create.SplitIndexIT - method: testSplitIndexPrimaryTerm - issue: https://github.com/elastic/elasticsearch/issues/111282 - class: org.elasticsearch.xpack.ml.integration.DatafeedJobsRestIT issue: https://github.com/elastic/elasticsearch/issues/111319 - class: org.elasticsearch.xpack.ml.integration.InferenceIngestInputConfigIT @@ -125,57 +101,109 @@ tests: - class: org.elasticsearch.xpack.restart.CoreFullClusterRestartIT method: testSnapshotRestore {cluster=UPGRADED} issue: https://github.com/elastic/elasticsearch/issues/111799 -- class: org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT - method: "test {p0=esql/26_aggs_bucket/friendlier BUCKET interval hourly: #110916}" - issue: https://github.com/elastic/elasticsearch/issues/111901 -- class: org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT - method: "test {p0=esql/26_aggs_bucket/friendlier BUCKET interval: monthly #110916}" - issue: https://github.com/elastic/elasticsearch/issues/111902 -- class: org.elasticsearch.xpack.sql.qa.security.JdbcCsvSpecIT - method: test {agg-ordering.testHistogramDateTimeWithCountAndOrder_1} - issue: https://github.com/elastic/elasticsearch/issues/111918 -- class: org.elasticsearch.xpack.sql.qa.security.JdbcCsvSpecIT - method: test {agg-ordering.testHistogramDateTimeWithCountAndOrder_2} - issue: https://github.com/elastic/elasticsearch/issues/111919 -- class: org.elasticsearch.xpack.sql.qa.single_node.JdbcCsvSpecIT - method: test {agg-ordering.testHistogramDateTimeWithCountAndOrder_2} - issue: https://github.com/elastic/elasticsearch/issues/111919 -- class: org.elasticsearch.xpack.sql.qa.single_node.JdbcCsvSpecIT - method: test {date.testDateParseHaving} - issue: https://github.com/elastic/elasticsearch/issues/111921 -- class: org.elasticsearch.xpack.sql.qa.single_node.JdbcCsvSpecIT - method: test {datetime.testDateTimeParseHaving} - issue: https://github.com/elastic/elasticsearch/issues/111922 -- class: org.elasticsearch.xpack.sql.qa.single_node.JdbcCsvSpecIT - method: test {agg-ordering.testHistogramDateTimeWithCountAndOrder_1} - issue: https://github.com/elastic/elasticsearch/issues/111918 -- class: org.elasticsearch.xpack.sql.qa.single_node.JdbcCsvSpecIT - issue: https://github.com/elastic/elasticsearch/issues/111923 -- class: org.elasticsearch.xpack.sql.qa.multi_cluster_with_security.JdbcCsvSpecIT - method: test {datetime.testDateTimeParseHaving} - issue: https://github.com/elastic/elasticsearch/issues/111922 -- class: org.elasticsearch.xpack.sql.qa.multi_cluster_with_security.JdbcCsvSpecIT - method: test {agg-ordering.testHistogramDateTimeWithCountAndOrder_1} - issue: https://github.com/elastic/elasticsearch/issues/111918 -- class: org.elasticsearch.xpack.sql.qa.multi_cluster_with_security.JdbcCsvSpecIT - method: test {date.testDateParseHaving} - issue: https://github.com/elastic/elasticsearch/issues/111921 -- class: org.elasticsearch.xpack.sql.qa.multi_cluster_with_security.JdbcCsvSpecIT - method: test {agg-ordering.testHistogramDateTimeWithCountAndOrder_2} - issue: https://github.com/elastic/elasticsearch/issues/111919 -- class: org.elasticsearch.xpack.sql.qa.multi_cluster_with_security.JdbcCsvSpecIT - issue: https://github.com/elastic/elasticsearch/issues/111923 -- class: org.elasticsearch.xpack.sql.qa.security.JdbcCsvSpecIT - issue: https://github.com/elastic/elasticsearch/issues/111923 -- class: org.elasticsearch.xpack.esql.qa.mixed.FieldExtractorIT - method: testScaledFloat - issue: https://github.com/elastic/elasticsearch/issues/112003 -- class: org.elasticsearch.xpack.esql.qa.single_node.RestEsqlIT - method: testForceSleepsProfile {SYNC} - issue: https://github.com/elastic/elasticsearch/issues/112039 -- class: org.elasticsearch.xpack.esql.qa.single_node.RestEsqlIT - method: testForceSleepsProfile {ASYNC} - issue: https://github.com/elastic/elasticsearch/issues/112049 +- class: org.elasticsearch.xpack.inference.InferenceRestIT + method: test {p0=inference/80_random_rerank_retriever/Random rerank retriever predictably shuffles results} + issue: https://github.com/elastic/elasticsearch/issues/111999 +- class: org.elasticsearch.xpack.ml.integration.MlJobIT + method: testDeleteJobAfterMissingIndex + issue: https://github.com/elastic/elasticsearch/issues/112088 +- class: org.elasticsearch.xpack.test.rest.XPackRestIT + method: test {p0=transform/preview_transforms/Test preview transform latest} + issue: https://github.com/elastic/elasticsearch/issues/112144 +- class: org.elasticsearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT + issue: https://github.com/elastic/elasticsearch/issues/112147 +- class: org.elasticsearch.smoketest.WatcherYamlRestIT + method: test {p0=watcher/usage/10_basic/Test watcher usage stats output} + issue: https://github.com/elastic/elasticsearch/issues/112189 +- class: org.elasticsearch.xpack.test.rest.XPackRestIT + method: test {p0=ml/inference_processor/Test create processor with missing mandatory fields} + issue: https://github.com/elastic/elasticsearch/issues/112191 +- class: org.elasticsearch.xpack.ml.integration.MlJobIT + method: testDeleteJobAsync + issue: https://github.com/elastic/elasticsearch/issues/112212 +- class: org.elasticsearch.search.retriever.RankDocRetrieverBuilderIT + method: testRankDocsRetrieverWithCollapse + issue: https://github.com/elastic/elasticsearch/issues/112254 +- class: org.elasticsearch.search.ccs.CCSUsageTelemetryIT + issue: https://github.com/elastic/elasticsearch/issues/112324 +- class: org.elasticsearch.datastreams.logsdb.qa.StandardVersusLogsIndexModeRandomDataChallengeRestIT + method: testMatchAllQuery + issue: https://github.com/elastic/elasticsearch/issues/112374 +- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT + method: test {yaml=reference/rest-api/watcher/put-watch/line_120} + issue: https://github.com/elastic/elasticsearch/issues/99517 +- class: org.elasticsearch.xpack.ml.integration.MlJobIT + method: testMultiIndexDelete + issue: https://github.com/elastic/elasticsearch/issues/112381 +- class: org.elasticsearch.xpack.searchablesnapshots.cache.shared.NodesCachesStatsIntegTests + method: testNodesCachesStats + issue: https://github.com/elastic/elasticsearch/issues/112384 +- class: org.elasticsearch.action.admin.cluster.stats.CCSTelemetrySnapshotTests + method: testToXContent + issue: https://github.com/elastic/elasticsearch/issues/112325 +- class: org.elasticsearch.search.retriever.RankDocRetrieverBuilderIT + method: testRankDocsRetrieverWithNestedQuery + issue: https://github.com/elastic/elasticsearch/issues/112421 +- class: org.elasticsearch.indices.mapping.UpdateMappingIntegrationIT + issue: https://github.com/elastic/elasticsearch/issues/112423 +- class: org.elasticsearch.xpack.esql.expression.function.aggregate.SpatialCentroidTests + method: "testAggregateIntermediate {TestCase= #2}" + issue: https://github.com/elastic/elasticsearch/issues/112461 +- class: org.elasticsearch.xpack.esql.expression.function.aggregate.SpatialCentroidTests + method: testAggregateIntermediate {TestCase=} + issue: https://github.com/elastic/elasticsearch/issues/112463 +- class: org.elasticsearch.xpack.esql.action.ManyShardsIT + method: testRejection + issue: https://github.com/elastic/elasticsearch/issues/112406 +- class: org.elasticsearch.xpack.esql.action.ManyShardsIT + method: testConcurrentQueries + issue: https://github.com/elastic/elasticsearch/issues/112424 +- class: org.elasticsearch.xpack.inference.external.http.RequestBasedTaskRunnerTests + method: testLoopOneAtATime + issue: https://github.com/elastic/elasticsearch/issues/112471 +- class: org.elasticsearch.ingest.geoip.IngestGeoIpClientYamlTestSuiteIT + issue: https://github.com/elastic/elasticsearch/issues/111497 +- class: org.elasticsearch.smoketest.SmokeTestIngestWithAllDepsClientYamlTestSuiteIT + method: test {yaml=ingest/80_ingest_simulate/Test ingest simulate with reroute and mapping validation from templates} + issue: https://github.com/elastic/elasticsearch/issues/112575 +- class: org.elasticsearch.script.mustache.LangMustacheClientYamlTestSuiteIT + method: test {yaml=lang_mustache/50_multi_search_template/Multi-search template with errors} + issue: https://github.com/elastic/elasticsearch/issues/112580 +- class: org.elasticsearch.xpack.security.authc.kerberos.SimpleKdcLdapServerTests + method: testClientServiceMutualAuthentication + issue: https://github.com/elastic/elasticsearch/issues/112529 +- class: org.elasticsearch.search.basic.SearchWhileRelocatingIT + method: testSearchAndRelocateConcurrentlyRandomReplicas + issue: https://github.com/elastic/elasticsearch/issues/112515 +- class: org.elasticsearch.xpack.test.rest.XPackRestIT + method: test {p0=terms_enum/10_basic/Test search after on unconfigured constant keyword field} + issue: https://github.com/elastic/elasticsearch/issues/112624 +- class: org.elasticsearch.xpack.esql.EsqlAsyncSecurityIT + method: testIndexPatternErrorMessageComparison_ESQL_SearchDSL + issue: https://github.com/elastic/elasticsearch/issues/112630 +- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT + method: test {yaml=simulate.ingest/10_basic/Test mapping validation from templates} + issue: https://github.com/elastic/elasticsearch/issues/112633 +- class: org.elasticsearch.compute.aggregation.blockhash.BlockHashTests + method: testBytesRefLongHashHugeCombinatorialExplosion {forcePackedHash=false} + issue: https://github.com/elastic/elasticsearch/issues/112442 +- class: org.elasticsearch.compute.aggregation.blockhash.BlockHashTests + method: testBytesRefLongHashHugeCombinatorialExplosion {forcePackedHash=true} + issue: https://github.com/elastic/elasticsearch/issues/112443 +- class: org.elasticsearch.xpack.ml.integration.MlJobIT + method: testPutJob_GivenFarequoteConfig + issue: https://github.com/elastic/elasticsearch/issues/112382 +- class: org.elasticsearch.xpack.eql.EqlClientYamlIT + issue: https://github.com/elastic/elasticsearch/issues/112617 +- class: org.elasticsearch.xpack.security.authc.kerberos.KerberosTicketValidatorTests + method: testWhenKeyTabWithInvalidContentFailsValidation + issue: https://github.com/elastic/elasticsearch/issues/112631 +- class: org.elasticsearch.xpack.security.authc.kerberos.KerberosTicketValidatorTests + method: testValidKebrerosTicket + issue: https://github.com/elastic/elasticsearch/issues/112632 +- class: org.elasticsearch.xpack.security.authc.kerberos.KerberosTicketValidatorTests + method: testKerbTicketGeneratedForDifferentServerFailsValidation + issue: https://github.com/elastic/elasticsearch/issues/112639 # Examples: # diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java index 2d27447b618e9..7a0caf56d6066 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java @@ -336,15 +336,7 @@ public ICUCollationKeywordFieldMapper build(MapperBuilderContext context) { ignoreAbove.getValue(), meta.getValue() ); - return new ICUCollationKeywordFieldMapper( - leafName(), - buildFieldType(), - ft, - multiFieldsBuilder.build(this, context), - copyTo, - collator, - this - ); + return new ICUCollationKeywordFieldMapper(leafName(), buildFieldType(), ft, builderParams(this, context), collator, this); } } @@ -474,12 +466,11 @@ protected ICUCollationKeywordFieldMapper( String simpleName, FieldType fieldType, MappedFieldType mappedFieldType, - MultiFields multiFields, - CopyTo copyTo, + BuilderParams builderParams, Collator collator, Builder builder ) { - super(simpleName, mappedFieldType, multiFields, copyTo, false, null); + super(simpleName, mappedFieldType, builderParams); assert collator.isFrozen(); this.fieldType = freezeAndDeduplicateFieldType(fieldType); this.params = builder.collatorParams(); diff --git a/plugins/examples/custom-processor/build.gradle b/plugins/examples/custom-processor/build.gradle new file mode 100644 index 0000000000000..69da64d8ebe86 --- /dev/null +++ b/plugins/examples/custom-processor/build.gradle @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +apply plugin: 'elasticsearch.esplugin' +apply plugin: 'elasticsearch.yaml-rest-test' + +esplugin { + name 'custom-processor' + description 'An example plugin showing how to register a custom ingest processor' + classname 'org.elasticsearch.example.customprocessor.ExampleProcessorPlugin' + licenseFile rootProject.file('SSPL-1.0+ELASTIC-LICENSE-2.0.txt') + noticeFile rootProject.file('NOTICE.txt') +} + +dependencies { + yamlRestTestRuntimeOnly "org.apache.logging.log4j:log4j-core:${log4jVersion}" +} diff --git a/plugins/examples/custom-processor/src/main/java/org/elasticsearch/example/customprocessor/ExampleProcessorPlugin.java b/plugins/examples/custom-processor/src/main/java/org/elasticsearch/example/customprocessor/ExampleProcessorPlugin.java new file mode 100644 index 0000000000000..1ba145a92ca7d --- /dev/null +++ b/plugins/examples/custom-processor/src/main/java/org/elasticsearch/example/customprocessor/ExampleProcessorPlugin.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.example.customprocessor; + +import org.elasticsearch.ingest.Processor; +import org.elasticsearch.plugins.IngestPlugin; +import org.elasticsearch.plugins.Plugin; + +import java.util.Map; + +public class ExampleProcessorPlugin extends Plugin implements IngestPlugin { + + @Override + public Map getProcessors(Processor.Parameters parameters) { + return Map.of(ExampleRepeatProcessor.TYPE, new ExampleRepeatProcessor.Factory()); + } +} diff --git a/plugins/examples/custom-processor/src/main/java/org/elasticsearch/example/customprocessor/ExampleRepeatProcessor.java b/plugins/examples/custom-processor/src/main/java/org/elasticsearch/example/customprocessor/ExampleRepeatProcessor.java new file mode 100644 index 0000000000000..f0f942459281a --- /dev/null +++ b/plugins/examples/custom-processor/src/main/java/org/elasticsearch/example/customprocessor/ExampleRepeatProcessor.java @@ -0,0 +1,53 @@ +package org.elasticsearch.example.customprocessor; + +import org.elasticsearch.ingest.AbstractProcessor; +import org.elasticsearch.ingest.ConfigurationUtils; +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.Processor; + +import java.util.Map; + +/** + * Example of adding an ingest processor with a plugin. + */ +public class ExampleRepeatProcessor extends AbstractProcessor { + public static final String TYPE = "repeat"; + public static final String FIELD_KEY_NAME = "field"; + + private final String field; + + ExampleRepeatProcessor(String tag, String description, String field) { + super(tag, description); + this.field = field; + } + + @Override + public IngestDocument execute(IngestDocument document) { + Object val = document.getFieldValue(field, Object.class, true); + + if (val instanceof String string) { + String repeated = string.concat(string); + document.setFieldValue(field, repeated); + } + return document; + } + + @Override + public String getType() { + return TYPE; + } + + public static class Factory implements Processor.Factory { + + @Override + public ExampleRepeatProcessor create( + Map registry, + String tag, + String description, + Map config + ) { + String field = ConfigurationUtils.readStringProperty(TYPE, tag, config, FIELD_KEY_NAME); + return new ExampleRepeatProcessor(tag, description, field); + } + } +} diff --git a/plugins/examples/custom-processor/src/yamlRestTest/java/org/elasticsearch/example/customprocessor/ExampleProcessorClientYamlTestSuiteIT.java b/plugins/examples/custom-processor/src/yamlRestTest/java/org/elasticsearch/example/customprocessor/ExampleProcessorClientYamlTestSuiteIT.java new file mode 100644 index 0000000000000..ac08df358fe5e --- /dev/null +++ b/plugins/examples/custom-processor/src/yamlRestTest/java/org/elasticsearch/example/customprocessor/ExampleProcessorClientYamlTestSuiteIT.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.example.customprocessor; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; + +/** + * {@link ExampleProcessorClientYamlTestSuiteIT} executes the plugin's REST API integration tests. + *

+ * The tests can be executed using the command: ./gradlew :custom-processor:yamlRestTest + *

+ * This class extends {@link ESClientYamlSuiteTestCase}, which takes care of parsing the YAML files + * located in the src/yamlRestTest/resources/rest-api-spec/test/ directory and validates them against the + * custom REST API definition files located in src/yamlRestTest/resources/rest-api-spec/api/. + *

+ * Once validated, {@link ESClientYamlSuiteTestCase} executes the REST tests against a single node + * integration cluster which has the plugin already installed by the Gradle build script. + *

+ */ +public class ExampleProcessorClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + + public ExampleProcessorClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + // The test executes all the test candidates by default + // see ESClientYamlSuiteTestCase.REST_TESTS_SUITE + return ESClientYamlSuiteTestCase.createParameters(); + } +} diff --git a/plugins/examples/custom-processor/src/yamlRestTest/resources/rest-api-spec/test/customprocessor/10_basic.yml b/plugins/examples/custom-processor/src/yamlRestTest/resources/rest-api-spec/test/customprocessor/10_basic.yml new file mode 100644 index 0000000000000..40f5835fe9760 --- /dev/null +++ b/plugins/examples/custom-processor/src/yamlRestTest/resources/rest-api-spec/test/customprocessor/10_basic.yml @@ -0,0 +1,15 @@ +"Custom processor is present": + - do: + ingest.put_pipeline: + id: pipeline1 + body: > + { + "processors": [ + { + "repeat" : { + "field": "test" + } + } + ] + } + - match: { acknowledged: true } diff --git a/plugins/examples/custom-processor/src/yamlRestTest/resources/rest-api-spec/test/customprocessor/20_process_document.yml b/plugins/examples/custom-processor/src/yamlRestTest/resources/rest-api-spec/test/customprocessor/20_process_document.yml new file mode 100644 index 0000000000000..7e8bc2e0a2d78 --- /dev/null +++ b/plugins/examples/custom-processor/src/yamlRestTest/resources/rest-api-spec/test/customprocessor/20_process_document.yml @@ -0,0 +1,59 @@ +setup: + - do: + ingest.put_pipeline: + id: pipeline1 + body: > + { + "processors": [ + { + "repeat" : { + "field": "to_repeat" + } + } + ] + } +--- +teardown: + - do: + ingest.delete_pipeline: + id: pipeline1 + ignore: 404 + + - do: + indices.delete: + index: index1 + ignore: 404 +--- +"Process document": + # index a document with field to be processed + - do: + index: + id: doc1 + index: index1 + pipeline: pipeline1 + body: { to_repeat: "foo" } + - match: { result: "created" } + + # validate document is processed + - do: + get: + index: index1 + id: doc1 + - match: { _source: { to_repeat: "foofoo" } } +--- +"Does not process document without field": + # index a document without field to be processed + - do: + index: + id: doc1 + index: index1 + pipeline: pipeline1 + body: { field1: "foo" } + - match: { result: "created" } + + # validate document is not processed + - do: + get: + index: index1 + id: doc1 + - match: { _source: { field1: "foo" } } diff --git a/plugins/examples/custom-settings/src/test/java/org/elasticsearch/example/customsettings/ExampleCustomSettingsConfigTests.java b/plugins/examples/custom-settings/src/test/java/org/elasticsearch/example/customsettings/ExampleCustomSettingsConfigTests.java index f5e205500d23d..23a134d58b8ad 100644 --- a/plugins/examples/custom-settings/src/test/java/org/elasticsearch/example/customsettings/ExampleCustomSettingsConfigTests.java +++ b/plugins/examples/custom-settings/src/test/java/org/elasticsearch/example/customsettings/ExampleCustomSettingsConfigTests.java @@ -17,7 +17,7 @@ *

* It's a JUnit test class that extends {@link ESTestCase} which provides useful methods for testing. *

- * The tests can be executed in the IDE or using the command: ./gradlew :example-plugins:custom-settings:test + * The tests can be executed in the IDE or using the command: ./gradlew :custom-settings:test */ public class ExampleCustomSettingsConfigTests extends ESTestCase { diff --git a/plugins/examples/custom-settings/src/yamlRestTest/java/org/elasticsearch/example/customsettings/ExampleCustomSettingsClientYamlTestSuiteIT.java b/plugins/examples/custom-settings/src/yamlRestTest/java/org/elasticsearch/example/customsettings/ExampleCustomSettingsClientYamlTestSuiteIT.java index 40a8af569b33e..9377cc7afd47a 100644 --- a/plugins/examples/custom-settings/src/yamlRestTest/java/org/elasticsearch/example/customsettings/ExampleCustomSettingsClientYamlTestSuiteIT.java +++ b/plugins/examples/custom-settings/src/yamlRestTest/java/org/elasticsearch/example/customsettings/ExampleCustomSettingsClientYamlTestSuiteIT.java @@ -15,7 +15,7 @@ /** * {@link ExampleCustomSettingsClientYamlTestSuiteIT} executes the plugin's REST API integration tests. *

- * The tests can be executed using the command: ./gradlew :example-plugins:custom-settings:yamlRestTest + * The tests can be executed using the command: ./gradlew :custom-settings:yamlRestTest *

* This class extends {@link ESClientYamlSuiteTestCase}, which takes care of parsing the YAML files * located in the src/yamlRestTest/resources/rest-api-spec/test/ directory and validates them against the diff --git a/plugins/examples/gradle/wrapper/gradle-wrapper.jar b/plugins/examples/gradle/wrapper/gradle-wrapper.jar index 7454180f2ae88..a4b76b9530d66 100644 Binary files a/plugins/examples/gradle/wrapper/gradle-wrapper.jar and b/plugins/examples/gradle/wrapper/gradle-wrapper.jar differ diff --git a/plugins/examples/gradle/wrapper/gradle-wrapper.properties b/plugins/examples/gradle/wrapper/gradle-wrapper.properties index efe2ff3449216..9036682bf0f0c 100644 --- a/plugins/examples/gradle/wrapper/gradle-wrapper.properties +++ b/plugins/examples/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionSha256Sum=258e722ec21e955201e31447b0aed14201765a3bfbae296a46cf60b70e66db70 -distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-all.zip +distributionSha256Sum=682b4df7fe5accdca84a4d1ef6a3a6ab096b3efd5edf7de2bd8c758d95a93703 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-all.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/plugins/examples/gradlew b/plugins/examples/gradlew index 1b6c787337ffb..f5feea6d6b116 100755 --- a/plugins/examples/gradlew +++ b/plugins/examples/gradlew @@ -15,6 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# SPDX-License-Identifier: Apache-2.0 +# ############################################################################## # @@ -55,7 +57,7 @@ # Darwin, MinGW, and NonStop. # # (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt # within the Gradle project. # # You can find Gradle at https://github.com/gradle/gradle/. @@ -80,13 +82,12 @@ do esac done -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit - -APP_NAME="Gradle" +# This is normally unused +# shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s +' "$PWD" ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -133,22 +134,29 @@ location of your Java installation." fi else JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac case $MAX_FD in #( '' | soft) :;; #( *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -193,11 +201,15 @@ if "$cygwin" || "$msys" ; then done fi -# Collect all arguments for the java command; -# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of -# shell script including quotes and variable substitutions, so put them in -# double quotes to make sure that they get re-expanded; and -# * put everything else in single quotes, so that it's not re-expanded. + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ @@ -205,6 +217,12 @@ set -- \ org.gradle.wrapper.GradleWrapperMain \ "$@" +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + # Use "xargs" to parse quoted args. # # With -n1 it outputs one arg per line, with the quotes and backslashes removed. diff --git a/plugins/examples/gradlew.bat b/plugins/examples/gradlew.bat index ac1b06f93825d..9b42019c7915b 100644 --- a/plugins/examples/gradlew.bat +++ b/plugins/examples/gradlew.bat @@ -13,8 +13,10 @@ @rem See the License for the specific language governing permissions and @rem limitations under the License. @rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem -@if "%DEBUG%" == "" @echo off +@if "%DEBUG%"=="" @echo off @rem ########################################################################## @rem @rem Gradle startup script for Windows @@ -25,7 +27,8 @@ if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @@ -40,13 +43,13 @@ if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto execute +if %ERRORLEVEL% equ 0 goto execute -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail @@ -56,11 +59,11 @@ set JAVA_EXE=%JAVA_HOME%/bin/java.exe if exist "%JAVA_EXE%" goto execute -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail @@ -75,13 +78,15 @@ set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar :end @rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd +if %ERRORLEVEL% equ 0 goto mainEnd :fail rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% :mainEnd if "%OS%"=="Windows_NT" endlocal diff --git a/plugins/examples/rest-handler/src/yamlRestTest/java/org/elasticsearch/example/resthandler/ExampleRestHandlerClientYamlTestSuiteIT.java b/plugins/examples/rest-handler/src/yamlRestTest/java/org/elasticsearch/example/resthandler/ExampleRestHandlerClientYamlTestSuiteIT.java index e6d5ac688cce2..9ebfc5ebfe9de 100644 --- a/plugins/examples/rest-handler/src/yamlRestTest/java/org/elasticsearch/example/resthandler/ExampleRestHandlerClientYamlTestSuiteIT.java +++ b/plugins/examples/rest-handler/src/yamlRestTest/java/org/elasticsearch/example/resthandler/ExampleRestHandlerClientYamlTestSuiteIT.java @@ -15,7 +15,7 @@ /** * {@link ExampleRestHandlerClientYamlTestSuiteIT} executes the plugin's REST API integration tests. *

- * The tests can be executed using the command: ./gradlew :example-plugins:rest-handler:yamlRestTest + * The tests can be executed using the command: ./gradlew :rest-handler:yamlRestTest *

* This class extends {@link ESClientYamlSuiteTestCase}, which takes care of parsing the YAML files * located in the src/yamlRestTest/resources/rest-api-spec/test/ directory and validates them against the diff --git a/plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java b/plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java index dc429538fec3b..7e177c4b7b6a0 100644 --- a/plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java +++ b/plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java @@ -124,6 +124,11 @@ public boolean needs_score() { return false; // Return true if the script needs the score } + @Override + public boolean needs_termStats() { + return false; // Return true if the script needs term statistics via get_termStats() + } + @Override public ScoreScript newInstance(DocReader docReader) throws IOException { diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index dac8e051f25f8..4f077fdcde069 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -150,7 +150,6 @@ private AnnotatedTextFieldType buildFieldType(FieldType fieldType, MapperBuilder @Override public AnnotatedTextFieldMapper build(MapperBuilderContext context) { - MultiFields multiFields = multiFieldsBuilder.build(this, context); FieldType fieldType = TextParams.buildFieldType(() -> true, store, indexOptions, norms, termVectors); if (fieldType.indexOptions() == IndexOptions.NONE) { throw new IllegalArgumentException("[" + CONTENT_TYPE + "] fields must be indexed"); @@ -162,12 +161,12 @@ public AnnotatedTextFieldMapper build(MapperBuilderContext context) { ); } } + BuilderParams builderParams = builderParams(this, context); return new AnnotatedTextFieldMapper( leafName(), fieldType, - buildFieldType(fieldType, context, multiFields), - multiFields, - copyTo, + buildFieldType(fieldType, context, builderParams.multiFields()), + builderParams, this ); } @@ -523,11 +522,10 @@ protected AnnotatedTextFieldMapper( String simpleName, FieldType fieldType, AnnotatedTextFieldType mappedFieldType, - MultiFields multiFields, - CopyTo copyTo, + BuilderParams builderParams, Builder builder ) { - super(simpleName, mappedFieldType, multiFields, copyTo); + super(simpleName, mappedFieldType, builderParams); assert fieldType.tokenized(); this.fieldType = freezeAndDeduplicateFieldType(fieldType); this.builder = builder; @@ -578,13 +576,13 @@ protected SyntheticSourceMode syntheticSourceMode() { @Override public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - if (copyTo.copyToFields().isEmpty() != true) { + if (copyTo().copyToFields().isEmpty() != true) { throw new IllegalArgumentException( "field [" + fullPath() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" ); } if (fieldType.stored()) { - return new StringStoredFieldFieldLoader(fullPath(), leafName(), null) { + return new StringStoredFieldFieldLoader(fullPath(), leafName()) { @Override protected void write(XContentBuilder b, Object value) throws IOException { b.value((String) value); diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java index 0b29bc9062917..979ca842ef346 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java @@ -57,8 +57,7 @@ public Murmur3FieldMapper build(MapperBuilderContext context) { return new Murmur3FieldMapper( leafName(), new Murmur3FieldType(context.buildFullName(leafName()), stored.getValue(), meta.getValue()), - multiFieldsBuilder.build(this, context), - copyTo + builderParams(this, context) ); } } @@ -94,8 +93,8 @@ public Query termQuery(Object value, SearchExecutionContext context) { } } - protected Murmur3FieldMapper(String simpleName, MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo) { - super(simpleName, mappedFieldType, multiFields, copyTo); + protected Murmur3FieldMapper(String simpleName, MappedFieldType mappedFieldType, BuilderParams builderParams) { + super(simpleName, mappedFieldType, builderParams); } @Override diff --git a/qa/ccs-common-rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml b/qa/ccs-common-rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml index e35282bb6bfde..05d90d582e7f2 100644 --- a/qa/ccs-common-rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml +++ b/qa/ccs-common-rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml @@ -13,7 +13,7 @@ setup: day_of_week: type: keyword script: - source: "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))" + source: "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))" - do: bulk: refresh: true diff --git a/qa/ccs-common-rest/src/yamlRestTest/resources/rest-api-spec/test/eql/20_runtime_mappings.yml b/qa/ccs-common-rest/src/yamlRestTest/resources/rest-api-spec/test/eql/20_runtime_mappings.yml index 58462786f9a2f..1c1a39a7bc1ac 100644 --- a/qa/ccs-common-rest/src/yamlRestTest/resources/rest-api-spec/test/eql/20_runtime_mappings.yml +++ b/qa/ccs-common-rest/src/yamlRestTest/resources/rest-api-spec/test/eql/20_runtime_mappings.yml @@ -9,7 +9,7 @@ setup: day_of_week: type: keyword script: - source: "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))" + source: "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))" - do: bulk: refresh: true diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml index a42b987a9bddd..1a77019914283 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml @@ -212,3 +212,88 @@ setup: - match: { docs.1.doc._index: "index" } - match: { docs.1.doc._source.field1: "BAR" } - match: { docs.1.doc.executed_pipelines: ["my-pipeline"] } + +--- +"Test ingest simulate with reroute and mapping validation from templates": + + - skip: + features: headers + + - requires: + cluster_features: ["simulate.mapping.validation.templates"] + reason: "ingest simulate index mapping validation added in 8.16" + + - do: + headers: + Content-Type: application/json + ingest.put_pipeline: + id: "reroute-pipeline" + body: > + { + "processors": [ + { + "reroute": { + "destination": "second-index" + } + } + ] + } + - match: { acknowledged: true } + + - do: + indices.put_index_template: + name: first-index-template + body: + index_patterns: first-index* + template: + settings: + default_pipeline: "reroute-pipeline" + mappings: + dynamic: strict + properties: + foo: + type: text + + - do: + indices.put_index_template: + name: second-index-template + body: + index_patterns: second-index* + template: + mappings: + dynamic: strict + properties: + bar: + type: text + + - do: + headers: + Content-Type: application/json + simulate.ingest: + body: > + { + "docs": [ + { + "_index": "first-index", + "_id": "id", + "_source": { + "foo": "bar" + } + }, + { + "_index": "first-index", + "_id": "id", + "_source": { + "bar": "foo" + } + } + ] + } + - length: { docs: 2 } + - match: { docs.0.doc._index: "second-index" } + - match: { docs.0.doc._source.foo: "bar" } + - match: { docs.0.doc.error.type: "strict_dynamic_mapping_exception" } + - match: { docs.0.doc.error.reason: "[1:8] mapping set to strict, dynamic introduction of [foo] within [_doc] is not allowed" } + - match: { docs.1.doc._index: "second-index" } + - match: { docs.1.doc._source.bar: "foo" } + - not_exists: docs.1.doc.error diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream.json index 59cd8521f275e..2a95e2552bb33 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream.json @@ -51,6 +51,10 @@ "master_timeout":{ "type":"time", "description":"Specify timeout for connection to master" + }, + "verbose":{ + "type":"boolean", + "description":"Whether the maximum timestamp for each data stream should be calculated and returned (default: false)" } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.resolve_index.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.resolve_index.json index 4ea78bfd45460..e27e3a0450bff 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.resolve_index.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.resolve_index.json @@ -37,6 +37,16 @@ ], "default":"open", "description":"Whether wildcard expressions should get expanded to open or closed indices (default: open)" + }, + "ignore_unavailable":{ + "type":"boolean", + "description":"Whether specified concrete indices should be ignored when unavailable (missing or closed)", + "default":false + }, + "allow_no_indices":{ + "type":"boolean", + "description":"Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)", + "default":true } } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_geoip_database.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_geoip_database.json index 07f9e37740279..5447ea1e5a4e3 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_geoip_database.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_geoip_database.json @@ -7,7 +7,8 @@ "stability":"stable", "visibility":"public", "headers":{ - "accept": [ "application/json"] + "accept": [ "application/json"], + "content_type": ["application/json"] }, "url":{ "paths":[ diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/README.asciidoc b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/README.asciidoc index 5716afdd205c0..0ddac662e73ef 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/README.asciidoc +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/README.asciidoc @@ -138,7 +138,7 @@ other test runners to skip tests if they do not support the capabilities API yet path: /_api parameters: [param1, param2] capabilities: [cap1, cap2] - test_runner_feature: [capabilities] + test_runner_features: [capabilities] reason: Capability required to run test - do: ... test definitions ... diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.health/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.health/10_basic.yml index 504b7c8f9b1b6..063c614859a3f 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.health/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.health/10_basic.yml @@ -1,32 +1,45 @@ --- "Help": + - requires: + capabilities: + - method: GET + path: /_cluster/health + capabilities: [ unassigned_pri_shard_count ] + test_runner_features: capabilities + reason: Capability required to run test - do: cat.health: help: true - match: $body: | - /^ epoch .+ \n - timestamp .+ \n - cluster .+ \n - status .+ \n - node.total .+ \n - node.data .+ \n - shards .+ \n - pri .+ \n - relo .+ \n - init .+ \n - unassign .+ \n - pending_tasks .+ \n - max_task_wait_time .+ \n - active_shards_percent .+ \n - + /^ epoch .+\n + timestamp .+\n + cluster .+\n + status .+\n + node.total .+\n + node.data .+\n + shards .+\n + pri .+\n + relo .+\n + init .+\n + unassign .+\n + unassign.pri .+\n + pending_tasks .+\n + max_task_wait_time .+\n + active_shards_percent .+\n $/ --- "Empty cluster": - + - requires: + capabilities: + - method: GET + path: /_cluster/health + capabilities: [ unassigned_pri_shard_count ] + test_runner_features: capabilities + reason: Capability required to run test - do: cat.health: {} @@ -44,6 +57,7 @@ \d+ \s+ # relo \d+ \s+ # init \d+ \s+ # unassign + \d+ \s+ # unassign.pri \d+ \s+ # pending_tasks (-|\d+(?:[.]\d+)?m?s) \s+ # max task waiting time \d+\.\d+% # active shards percent @@ -54,7 +68,13 @@ --- "With ts parameter": - + - requires: + capabilities: + - method: GET + path: /_cluster/health + capabilities: [ unassigned_pri_shard_count ] + test_runner_features: capabilities + reason: Capability required to run test - do: cat.health: ts: false @@ -71,6 +91,7 @@ \d+ \s+ # relo \d+ \s+ # init \d+ \s+ # unassign + \d+ \s+ # unassign.pri \d+ \s+ # pending_tasks (-|\d+(?:[.]\d+)?m?s) \s+ # max task waiting time \d+\.\d+% # active shards percent diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/10_basic.yml index a01b68e96bbd2..c1c9741eb5fa3 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/10_basic.yml @@ -1,21 +1,38 @@ --- "cluster health basic test": + - requires: + capabilities: + - method: GET + path: /_cluster/health + capabilities: [ unassigned_pri_shard_count ] + test_runner_features: capabilities + reason: Capability required to run test + - do: cluster.health: {} - is_true: cluster_name - is_false: timed_out - - gte: { number_of_nodes: 1 } - - gte: { number_of_data_nodes: 1 } - - match: { active_primary_shards: 0 } - - match: { active_shards: 0 } - - match: { relocating_shards: 0 } - - match: { initializing_shards: 0 } - - match: { unassigned_shards: 0 } - - gte: { number_of_pending_tasks: 0 } + - gte: { number_of_nodes: 1 } + - gte: { number_of_data_nodes: 1 } + - match: { active_primary_shards: 0 } + - match: { active_shards: 0 } + - match: { relocating_shards: 0 } + - match: { initializing_shards: 0 } + - match: { unassigned_shards: 0 } + - match: { unassigned_primary_shards: 0 } + - gte: { number_of_pending_tasks: 0 } --- "cluster health basic test, one index": + - requires: + capabilities: + - method: GET + path: /_cluster/health + capabilities: [ unassigned_pri_shard_count ] + test_runner_features: capabilities + reason: Capability required to run test + - do: indices.create: index: test_index @@ -31,17 +48,26 @@ - is_true: cluster_name - is_false: timed_out - - gte: { number_of_nodes: 1 } - - gte: { number_of_data_nodes: 1 } - - gt: { active_primary_shards: 0 } - - gt: { active_shards: 0 } - - gte: { relocating_shards: 0 } - - match: { initializing_shards: 0 } - - match: { unassigned_shards: 0 } - - gte: { number_of_pending_tasks: 0 } + - gte: { number_of_nodes: 1 } + - gte: { number_of_data_nodes: 1 } + - gt: { active_primary_shards: 0 } + - gt: { active_shards: 0 } + - gte: { relocating_shards: 0 } + - match: { initializing_shards: 0 } + - match: { unassigned_shards: 0 } + - match: { unassigned_primary_shards: 0 } + - gte: { number_of_pending_tasks: 0 } --- "cluster health basic test, one index with wait for active shards": + - requires: + capabilities: + - method: GET + path: /_cluster/health + capabilities: [ unassigned_pri_shard_count ] + test_runner_features: capabilities + reason: Capability required to run test + - do: indices.create: index: test_index @@ -57,17 +83,26 @@ - is_true: cluster_name - is_false: timed_out - - gte: { number_of_nodes: 1 } - - gte: { number_of_data_nodes: 1 } - - gt: { active_primary_shards: 0 } - - gt: { active_shards: 0 } - - gte: { relocating_shards: 0 } - - match: { initializing_shards: 0 } - - match: { unassigned_shards: 0 } - - gte: { number_of_pending_tasks: 0 } + - gte: { number_of_nodes: 1 } + - gte: { number_of_data_nodes: 1 } + - gt: { active_primary_shards: 0 } + - gt: { active_shards: 0 } + - gte: { relocating_shards: 0 } + - match: { initializing_shards: 0 } + - match: { unassigned_shards: 0 } + - match: { unassigned_primary_shards: 0 } + - gte: { number_of_pending_tasks: 0 } --- "cluster health basic test, one index with wait for all active shards": + - requires: + capabilities: + - method: GET + path: /_cluster/health + capabilities: [ unassigned_pri_shard_count ] + test_runner_features: capabilities + reason: Capability required to run test + - do: indices.create: index: test_index @@ -83,16 +118,18 @@ - is_true: cluster_name - is_false: timed_out - - gte: { number_of_nodes: 1 } - - gte: { number_of_data_nodes: 1 } - - gt: { active_primary_shards: 0 } - - gt: { active_shards: 0 } - - gte: { relocating_shards: 0 } - - match: { initializing_shards: 0 } - - match: { unassigned_shards: 0 } - - gte: { number_of_pending_tasks: 0 } + - gte: { number_of_nodes: 1 } + - gte: { number_of_data_nodes: 1 } + - gt: { active_primary_shards: 0 } + - gt: { active_shards: 0 } + - gte: { relocating_shards: 0 } + - match: { initializing_shards: 0 } + - match: { unassigned_shards: 0 } + - match: { unassigned_primary_shards: 0 } + - gte: { number_of_pending_tasks: 0 } --- + "cluster health basic test, one index with wait for no initializing shards": - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/20_request_timeout.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/20_request_timeout.yml index 66a7cb2b48dbd..5b3103f416a72 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/20_request_timeout.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.health/20_request_timeout.yml @@ -1,5 +1,12 @@ --- "cluster health request timeout on waiting for nodes": + - requires: + capabilities: + - method: GET + path: /_cluster/health + capabilities: [ unassigned_pri_shard_count ] + test_runner_features: capabilities + reason: Capability required to run test - do: catch: request_timeout cluster.health: @@ -8,17 +15,25 @@ - is_true: cluster_name - is_true: timed_out - - gte: { number_of_nodes: 1 } - - gte: { number_of_data_nodes: 1 } - - match: { active_primary_shards: 0 } - - match: { active_shards: 0 } - - match: { relocating_shards: 0 } - - match: { initializing_shards: 0 } - - match: { unassigned_shards: 0 } - - gte: { number_of_pending_tasks: 0 } + - gte: { number_of_nodes: 1 } + - gte: { number_of_data_nodes: 1 } + - match: { active_primary_shards: 0 } + - match: { active_shards: 0 } + - match: { relocating_shards: 0 } + - match: { initializing_shards: 0 } + - match: { unassigned_shards: 0 } + - match: { unassigned_primary_shards: 0 } + - gte: { number_of_pending_tasks: 0 } --- "cluster health request timeout waiting for active shards": + - requires: + capabilities: + - method: GET + path: /_cluster/health + capabilities: [ unassigned_pri_shard_count ] + test_runner_features: capabilities + reason: Capability required to run test - do: catch: request_timeout cluster.health: @@ -27,11 +42,12 @@ - is_true: cluster_name - is_true: timed_out - - gte: { number_of_nodes: 1 } - - gte: { number_of_data_nodes: 1 } - - match: { active_primary_shards: 0 } - - match: { active_shards: 0 } - - match: { relocating_shards: 0 } - - match: { initializing_shards: 0 } - - match: { unassigned_shards: 0 } - - gte: { number_of_pending_tasks: 0 } + - gte: { number_of_nodes: 1 } + - gte: { number_of_data_nodes: 1 } + - match: { active_primary_shards: 0 } + - match: { active_shards: 0 } + - match: { relocating_shards: 0 } + - match: { initializing_shards: 0 } + - match: { unassigned_shards: 0 } + - match: { unassigned_primary_shards: 0 } + - gte: { number_of_pending_tasks: 0 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/91_metrics_no_subobjects.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/91_metrics_no_subobjects.yml index 94c19a4d69e17..5881ec83ebe85 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/91_metrics_no_subobjects.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/91_metrics_no_subobjects.yml @@ -1,25 +1,28 @@ --- "Metrics object indexing": - requires: - test_runner_features: allowed_warnings_regex + test_runner_features: [ "allowed_warnings", "allowed_warnings_regex" ] cluster_features: ["gte_v8.3.0"] reason: added in 8.3.0 - do: - indices.put_template: + allowed_warnings: + - "index template [test] has index patterns [test-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test] will take precedence during new index creation" + indices.put_index_template: name: test body: index_patterns: test-* - mappings: - dynamic_templates: - - no_subobjects: - match: metrics - mapping: - type: object - subobjects: false - properties: - host.name: - type: keyword + template: + mappings: + dynamic_templates: + - no_subobjects: + match: metrics + mapping: + type: object + subobjects: false + properties: + host.name: + type: keyword - do: allowed_warnings_regex: @@ -65,20 +68,23 @@ --- "Root without subobjects": - requires: - test_runner_features: allowed_warnings_regex + test_runner_features: [ "allowed_warnings", "allowed_warnings_regex" ] cluster_features: ["gte_v8.3.0"] reason: added in 8.3.0 - do: - indices.put_template: + allowed_warnings: + - "index template [test] has index patterns [test-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test] will take precedence during new index creation" + indices.put_index_template: name: test body: index_patterns: test-* - mappings: - subobjects: false - properties: - host.name: - type: keyword + template: + mappings: + subobjects: false + properties: + host.name: + type: keyword - do: allowed_warnings_regex: @@ -124,27 +130,30 @@ --- "Metrics object indexing with synthetic source": - requires: - test_runner_features: allowed_warnings_regex + test_runner_features: [ "allowed_warnings", "allowed_warnings_regex" ] cluster_features: ["gte_v8.4.0"] reason: added in 8.4.0 - do: - indices.put_template: + allowed_warnings: + - "index template [test] has index patterns [test-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test] will take precedence during new index creation" + indices.put_index_template: name: test body: index_patterns: test-* - mappings: - _source: - mode: synthetic - dynamic_templates: - - no_subobjects: - match: metrics - mapping: - type: object - subobjects: false - properties: - host.name: - type: keyword + template: + mappings: + _source: + mode: synthetic + dynamic_templates: + - no_subobjects: + match: metrics + mapping: + type: object + subobjects: false + properties: + host.name: + type: keyword - do: allowed_warnings_regex: @@ -191,22 +200,25 @@ --- "Root without subobjects with synthetic source": - requires: - test_runner_features: allowed_warnings_regex + test_runner_features: [ "allowed_warnings", "allowed_warnings_regex" ] cluster_features: ["gte_v8.4.0"] reason: added in 8.4.0 - do: - indices.put_template: + allowed_warnings: + - "index template [test] has index patterns [test-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test] will take precedence during new index creation" + indices.put_index_template: name: test body: index_patterns: test-* - mappings: - _source: - mode: synthetic - subobjects: false - properties: - host.name: - type: keyword + template: + mappings: + _source: + mode: synthetic + subobjects: false + properties: + host.name: + type: keyword - do: allowed_warnings_regex: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/92_metrics_auto_subobjects.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/92_metrics_auto_subobjects.yml new file mode 100644 index 0000000000000..414c24cfffd7d --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/92_metrics_auto_subobjects.yml @@ -0,0 +1,262 @@ +--- +"Metrics object indexing": + - requires: + test_runner_features: [ "allowed_warnings", "allowed_warnings_regex" ] + cluster_features: ["mapper.subobjects_auto"] + reason: requires supporting subobjects auto setting + + - do: + allowed_warnings: + - "index template [test] has index patterns [test-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test] will take precedence during new index creation" + indices.put_index_template: + name: test + body: + index_patterns: test-* + template: + mappings: + dynamic_templates: + - no_subobjects: + match: metrics + mapping: + type: object + subobjects: auto + properties: + host.name: + type: keyword + + - do: + allowed_warnings_regex: + - "index \\[test-1\\] matches multiple legacy templates \\[global, test\\], composable templates will only match a single template" + index: + index: test-1 + id: 1 + refresh: true + body: + { metrics.host.name: localhost, metrics.host.id: 1, metrics.time: 10, metrics.time.max: 100, metrics.time.min: 1 } + + - do: + field_caps: + index: test-1 + fields: metrics* + - match: {fields.metrics\.host\.id.long.searchable: true} + - match: {fields.metrics\.host\.id.long.aggregatable: true} + - match: {fields.metrics\.host\.name.keyword.searchable: true} + - match: {fields.metrics\.host\.name.keyword.aggregatable: true} + - match: {fields.metrics\.time.long.searchable: true} + - match: {fields.metrics\.time.long.aggregatable: true} + - match: {fields.metrics\.time\.max.long.searchable: true} + - match: {fields.metrics\.time\.max.long.aggregatable: true} + - match: {fields.metrics\.time\.min.long.searchable: true} + - match: {fields.metrics\.time\.min.long.aggregatable: true} + + - do: + get: + index: test-1 + id: 1 + - match: {_index: "test-1"} + - match: {_id: "1"} + - match: {_version: 1} + - match: {found: true} + - match: + _source: + metrics.host.name: localhost + metrics.host.id: 1 + metrics.time: 10 + metrics.time.max: 100 + metrics.time.min: 1 + +--- +"Root with metrics": + - requires: + test_runner_features: [ "allowed_warnings", "allowed_warnings_regex" ] + cluster_features: ["mapper.subobjects_auto"] + reason: requires supporting subobjects auto setting + + - do: + allowed_warnings: + - "index template [test] has index patterns [test-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test] will take precedence during new index creation" + indices.put_index_template: + name: test + body: + index_patterns: test-* + template: + mappings: + subobjects: auto + properties: + host.name: + type: keyword + + - do: + allowed_warnings_regex: + - "index \\[test-1\\] matches multiple legacy templates \\[global, test\\], composable templates will only match a single template" + index: + index: test-1 + id: 1 + refresh: true + body: + { host.name: localhost, host.id: 1, time: 10, time.max: 100, time.min: 1 } + + - do: + field_caps: + index: test-1 + fields: [host*, time*] + - match: {fields.host\.name.keyword.searchable: true} + - match: {fields.host\.name.keyword.aggregatable: true} + - match: {fields.host\.id.long.searchable: true} + - match: {fields.host\.id.long.aggregatable: true} + - match: {fields.time.long.searchable: true} + - match: {fields.time.long.aggregatable: true} + - match: {fields.time\.max.long.searchable: true} + - match: {fields.time\.max.long.aggregatable: true} + - match: {fields.time\.min.long.searchable: true} + - match: {fields.time\.min.long.aggregatable: true} + + - do: + get: + index: test-1 + id: 1 + - match: {_index: "test-1"} + - match: {_id: "1"} + - match: {_version: 1} + - match: {found: true} + - match: + _source: + host.name: localhost + host.id: 1 + time: 10 + time.max: 100 + time.min: 1 + +--- +"Metrics object indexing with synthetic source": + - requires: + test_runner_features: [ "allowed_warnings", "allowed_warnings_regex" ] + cluster_features: ["mapper.subobjects_auto"] + reason: added in 8.4.0 + + - do: + allowed_warnings: + - "index template [test] has index patterns [test-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test] will take precedence during new index creation" + indices.put_index_template: + name: test + body: + index_patterns: test-* + template: + mappings: + _source: + mode: synthetic + dynamic_templates: + - no_subobjects: + match: metrics + mapping: + type: object + subobjects: auto + properties: + host.name: + type: keyword + + - do: + allowed_warnings_regex: + - "index \\[test-1\\] matches multiple legacy templates \\[global, test\\], composable templates will only match a single template" + index: + index: test-1 + id: 1 + refresh: true + body: + { metrics.host.name: localhost, metrics.host.id: 1, metrics.time: 10, metrics.time.max: 100, metrics.time.min: 1 } + + - do: + field_caps: + index: test-1 + fields: metrics* + - match: {fields.metrics\.host\.id.long.searchable: true} + - match: {fields.metrics\.host\.id.long.aggregatable: true} + - match: {fields.metrics\.host\.name.keyword.searchable: true} + - match: {fields.metrics\.host\.name.keyword.aggregatable: true} + - match: {fields.metrics\.time.long.searchable: true} + - match: {fields.metrics\.time.long.aggregatable: true} + - match: {fields.metrics\.time\.max.long.searchable: true} + - match: {fields.metrics\.time\.max.long.aggregatable: true} + - match: {fields.metrics\.time\.min.long.searchable: true} + - match: {fields.metrics\.time\.min.long.aggregatable: true} + + - do: + get: + index: test-1 + id: 1 + - match: {_index: "test-1"} + - match: {_id: "1"} + - match: {_version: 1} + - match: {found: true} + - match: + _source: + metrics: + host.name: localhost + host.id: 1 + time: 10 + time.max: 100 + time.min: 1 + +--- +"Root without subobjects with synthetic source": + - requires: + test_runner_features: [ "allowed_warnings", "allowed_warnings_regex" ] + cluster_features: ["mapper.subobjects_auto"] + reason: added in 8.4.0 + + - do: + allowed_warnings: + - "index template [test] has index patterns [test-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test] will take precedence during new index creation" + indices.put_index_template: + name: test + body: + index_patterns: test-* + template: + mappings: + _source: + mode: synthetic + subobjects: auto + properties: + host.name: + type: keyword + + - do: + allowed_warnings_regex: + - "index \\[test-1\\] matches multiple legacy templates \\[global, test\\], composable templates will only match a single template" + index: + index: test-1 + id: 1 + refresh: true + body: + { host.name: localhost, host.id: 1, time: 10, time.max: 100, time.min: 1 } + + - do: + field_caps: + index: test-1 + fields: [host*, time*] + - match: {fields.host\.name.keyword.searchable: true} + - match: {fields.host\.name.keyword.aggregatable: true} + - match: {fields.host\.id.long.searchable: true} + - match: {fields.host\.id.long.aggregatable: true} + - match: {fields.time.long.searchable: true} + - match: {fields.time.long.aggregatable: true} + - match: {fields.time\.max.long.searchable: true} + - match: {fields.time\.max.long.aggregatable: true} + - match: {fields.time\.min.long.searchable: true} + - match: {fields.time\.min.long.aggregatable: true} + + - do: + get: + index: test-1 + id: 1 + - match: {_index: "test-1"} + - match: {_id: "1"} + - match: {_version: 1} + - match: {found: true} + - match: + _source: + host.name: localhost + host.id: 1 + time: 10 + time.max: 100 + time.min: 1 diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml index e51074ee55270..265aec75dc9c2 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml @@ -446,260 +446,6 @@ mixed disabled and enabled objects: - match: { hits.hits.0._source.path.to.bad.value: false } ---- -object array: - - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: requires tracking ignored source - - - do: - indices.create: - index: test - body: - mappings: - _source: - mode: synthetic - properties: - id: - type: integer - regular: - properties: - span: - properties: - id: - type: keyword - trace: - properties: - id: - type: keyword - stored: - store_array_source: true - properties: - span: - properties: - id: - type: keyword - trace: - properties: - id: - type: keyword - - - do: - bulk: - index: test - refresh: true - body: - - '{ "create": { } }' - - '{ "id": 1, "regular": [ { "trace": { "id": "a" }, "span": { "id": "1" } }, { "trace": { "id": "b" }, "span": { "id": "1" } } ] }' - - '{ "create": { } }' - - '{ "id": 2, "stored": [ { "trace": { "id": "a" }, "span": { "id": "1" } }, { "trace": { "id": "b" }, "span": { "id": "1" } } ] }' - - - do: - search: - index: test - sort: id - - - length: { hits.hits.0._source.regular: 2 } - - match: { hits.hits.0._source.regular.span.id: "1" } - - match: { hits.hits.0._source.regular.trace.id: [ "a", "b" ] } - - - length: { hits.hits.1._source.stored: 2 } - - match: { hits.hits.1._source.stored.0.trace.id: a } - - match: { hits.hits.1._source.stored.0.span.id: "1" } - - match: { hits.hits.1._source.stored.1.trace.id: b } - - match: { hits.hits.1._source.stored.1.span.id: "1" } - - ---- -object array within array: - - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: requires tracking ignored source - - - do: - indices.create: - index: test - body: - mappings: - _source: - mode: synthetic - properties: - stored: - store_array_source: true - properties: - path: - store_array_source: true - properties: - to: - properties: - trace: - type: keyword - - - do: - bulk: - index: test - refresh: true - body: - - '{ "create": { } }' - - '{ "stored": [ { "path": [{ "to": { "trace": "A" } }, { "to": { "trace": "B" } } ] }, { "path": { "to": { "trace": "C" } } } ] }' - - - do: - search: - index: test - - - length: { hits.hits.0._source.stored: 2 } - - match: { hits.hits.0._source.stored.0.path.0.to.trace: A } - - match: { hits.hits.0._source.stored.0.path.1.to.trace: B } - - match: { hits.hits.0._source.stored.1.path.to.trace: C } - - ---- -no object array: - - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: requires tracking ignored source - - - do: - indices.create: - index: test - body: - mappings: - _source: - mode: synthetic - properties: - stored: - store_array_source: true - properties: - span: - properties: - id: - type: keyword - trace: - properties: - id: - type: keyword - - - do: - bulk: - index: test - refresh: true - body: - - '{ "create": { } }' - - '{ "stored": { "trace": { "id": "a" }, "span": { "id": "b" } } }' - - - do: - search: - index: test - - - match: { hits.hits.0._source.stored.trace.id: a } - - match: { hits.hits.0._source.stored.span.id: b } - - ---- -field ordering in object array: - - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: requires tracking ignored source - - - do: - indices.create: - index: test - body: - mappings: - _source: - mode: synthetic - properties: - a: - type: keyword - b: - store_array_source: true - properties: - aa: - type: keyword - bb: - type: keyword - c: - type: keyword - d: - store_array_source: true - properties: - aa: - type: keyword - bb: - type: keyword - - - do: - bulk: - index: test - refresh: true - body: - - '{ "create": { } }' - - '{ "c": 1, "d": [ { "bb": 10, "aa": 20 }, { "aa": 30, "bb": 40 } ], "a": 2, "b": [ { "bb": 100, "aa": 200 }, { "aa": 300, "bb": 400 } ] }' - - - do: - search: - index: test - - - length: { hits.hits.0._source: 4 } - - match: { hits.hits.0._source: { "a": "2", "b": [ { "bb": 100, "aa": 200 }, { "aa": 300, "bb": 400 } ], "c": "1", "d": [ { "bb": 10, "aa": 20 }, { "aa": 30, "bb": 40 } ] } } - - ---- -nested object array next to other fields: - - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: requires tracking ignored source - - - do: - indices.create: - index: test - body: - mappings: - _source: - mode: synthetic - properties: - a: - type: keyword - b: - properties: - c: - store_array_source: true - properties: - aa: - type: keyword - bb: - type: keyword - d: - properties: - aa: - type: keyword - bb: - type: keyword - e: - type: keyword - f: - type: keyword - - - do: - bulk: - index: test - refresh: true - body: - - '{ "create": { } }' - - '{ "a": 1, "b": { "c": [ { "bb": 10, "aa": 20 }, { "aa": 30, "bb": 40 } ], "d": [ { "bb": 100, "aa": 200 }, { "aa": 300, "bb": 400 } ], "e": 1000 }, "f": 2000 }' - - - do: - search: - index: test - - - match: { hits.hits.0._source.a: "1" } - - match: { hits.hits.0._source.b.c: [{ "bb": 10, "aa": 20 }, { "aa": 30, "bb": 40 }] } - - match: { hits.hits.0._source.b.d.aa: [ "200", "300" ] } - - match: { hits.hits.0._source.b.d.bb: [ "100", "400" ] } - - match: { hits.hits.0._source.b.e: "1000" } - - match: { hits.hits.0._source.f: "2000" } - - --- object with dynamic override: - requires: @@ -1158,10 +904,10 @@ doubly nested object: --- -nested object with stored array: +subobjects auto: - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: requires tracking ignored source + cluster_features: ["mapper.subobjects_auto"] + reason: requires tracking ignored source and supporting subobjects auto setting - do: indices.create: @@ -1170,63 +916,36 @@ nested object with stored array: mappings: _source: mode: synthetic + subobjects: auto properties: - name: - type: keyword - nested_array_regular: - type: nested - nested_array_stored: - type: nested + id: + type: integer + regular: + properties: + span: + properties: + id: + type: keyword + trace: + properties: + id: + type: keyword + stored: store_array_source: true - - - do: - bulk: - index: test - refresh: true - body: - - '{ "create": { } }' - - '{ "name": "A", "nested_array_regular": [ { "b": [ { "c": 10 }, { "c": 100 } ] }, { "b": [ { "c": 20 }, { "c": 200 } ] } ] }' - - '{ "create": { } }' - - '{ "name": "B", "nested_array_stored": [ { "b": [ { "c": 10 }, { "c": 100 } ] }, { "b": [ { "c": 20 }, { "c": 200 } ] } ] }' - - - match: { errors: false } - - - do: - search: - index: test - sort: name - - match: { hits.total.value: 2 } - - match: { hits.hits.0._source.name: A } - - match: { hits.hits.0._source.nested_array_regular.0.b.c: [ 10, 100] } - - match: { hits.hits.0._source.nested_array_regular.1.b.c: [ 20, 200] } - - match: { hits.hits.1._source.name: B } - - match: { hits.hits.1._source.nested_array_stored.0.b.0.c: 10 } - - match: { hits.hits.1._source.nested_array_stored.0.b.1.c: 100 } - - match: { hits.hits.1._source.nested_array_stored.1.b.0.c: 20 } - - match: { hits.hits.1._source.nested_array_stored.1.b.1.c: 200 } - ---- -empty nested object sorted as a first document: - - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: requires tracking ignored source - - - do: - indices.create: - index: test - body: - settings: - index: - sort.field: "name" - sort.order: "asc" - mappings: - _source: - mode: synthetic - properties: - name: - type: keyword + properties: + span: + properties: + id: + type: keyword + trace: + properties: + id: + type: keyword nested: type: nested + auto_obj: + type: object + subobjects: auto - do: bulk: @@ -1234,19 +953,40 @@ empty nested object sorted as a first document: refresh: true body: - '{ "create": { } }' - - '{ "name": "B", "nested": { "a": "b" } }' + - '{ "id": 1, "foo": 10, "foo.bar": 100, "regular": [ { "trace": { "id": "a" }, "span": { "id": "1" } }, { "trace": { "id": "b" }, "span": { "id": "1" } } ] }' - '{ "create": { } }' - - '{ "name": "A" }' + - '{ "id": 2, "foo": 20, "foo.bar": 200, "stored": [ { "trace": { "id": "a" }, "span": { "id": "1" } }, { "trace": { "id": "b" }, "span": { "id": "1" } } ] }' + - '{ "create": { } }' + - '{ "id": 3, "foo": 30, "foo.bar": 300, "nested": [ { "a": 10, "b": 20 }, { "a": 100, "b": 200 } ] }' + - '{ "create": { } }' + - '{ "id": 4, "auto_obj": { "foo": 40, "foo.bar": 400 } }' - match: { errors: false } - do: search: index: test - sort: name - - - match: { hits.total.value: 2 } - - match: { hits.hits.0._source.name: A } - - match: { hits.hits.1._source.name: B } - - match: { hits.hits.1._source.nested.a: "b" } + sort: id + - match: { hits.hits.0._source.id: 1 } + - match: { hits.hits.0._source.foo: 10 } + - match: { hits.hits.0._source.foo\.bar: 100 } + - match: { hits.hits.0._source.regular.span.id: "1" } + - match: { hits.hits.0._source.regular.trace.id: [ "a", "b" ] } + - match: { hits.hits.1._source.id: 2 } + - match: { hits.hits.1._source.foo: 20 } + - match: { hits.hits.1._source.foo\.bar: 200 } + - match: { hits.hits.1._source.stored.0.trace.id: a } + - match: { hits.hits.1._source.stored.0.span.id: "1" } + - match: { hits.hits.1._source.stored.1.trace.id: b } + - match: { hits.hits.1._source.stored.1.span.id: "1" } + - match: { hits.hits.2._source.id: 3 } + - match: { hits.hits.2._source.foo: 30 } + - match: { hits.hits.2._source.foo\.bar: 300 } + - match: { hits.hits.2._source.nested.0.a: 10 } + - match: { hits.hits.2._source.nested.0.b: 20 } + - match: { hits.hits.2._source.nested.1.a: 100 } + - match: { hits.hits.2._source.nested.1.b: 200 } + - match: { hits.hits.3._source.id: 4 } + - match: { hits.hits.3._source.auto_obj.foo: 40 } + - match: { hits.hits.3._source.auto_obj.foo\.bar: 400 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml new file mode 100644 index 0000000000000..917f0540c4dd4 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml @@ -0,0 +1,732 @@ +--- +object param - object array: + - requires: + cluster_features: ["mapper.track_ignored_source"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + id: + type: integer + regular: + properties: + span: + properties: + id: + type: keyword + trace: + properties: + id: + type: keyword + stored: + store_array_source: true + properties: + span: + properties: + id: + type: keyword + trace: + properties: + id: + type: keyword + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "id": 1, "regular": [ { "trace": { "id": "a" }, "span": { "id": "1" } }, { "trace": { "id": "b" }, "span": { "id": "1" } } ] }' + - '{ "create": { } }' + - '{ "id": 2, "stored": [ { "trace": { "id": "a" }, "span": { "id": "1" } }, { "trace": { "id": "b" }, "span": { "id": "1" } } ] }' + + - do: + search: + index: test + sort: id + + - length: { hits.hits.0._source.regular: 2 } + - match: { hits.hits.0._source.regular.span.id: "1" } + - match: { hits.hits.0._source.regular.trace.id: [ "a", "b" ] } + + - length: { hits.hits.1._source.stored: 2 } + - match: { hits.hits.1._source.stored.0.trace.id: a } + - match: { hits.hits.1._source.stored.0.span.id: "1" } + - match: { hits.hits.1._source.stored.1.trace.id: b } + - match: { hits.hits.1._source.stored.1.span.id: "1" } + + +--- +object param - object array within array: + - requires: + cluster_features: ["mapper.track_ignored_source"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + stored: + store_array_source: true + properties: + path: + store_array_source: true + properties: + to: + properties: + trace: + type: keyword + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "stored": [ { "path": [{ "to": { "trace": "A" } }, { "to": { "trace": "B" } } ] }, { "path": { "to": { "trace": "C" } } } ] }' + + - do: + search: + index: test + + - length: { hits.hits.0._source.stored: 2 } + - match: { hits.hits.0._source.stored.0.path.0.to.trace: A } + - match: { hits.hits.0._source.stored.0.path.1.to.trace: B } + - match: { hits.hits.0._source.stored.1.path.to.trace: C } + + +--- +object param - no object array: + - requires: + cluster_features: ["mapper.track_ignored_source"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + stored: + store_array_source: true + properties: + span: + properties: + id: + type: keyword + trace: + properties: + id: + type: keyword + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "stored": { "trace": { "id": "a" }, "span": { "id": "b" } } }' + + - do: + search: + index: test + + - match: { hits.hits.0._source.stored.trace.id: a } + - match: { hits.hits.0._source.stored.span.id: b } + + +--- +object param - field ordering in object array: + - requires: + cluster_features: ["mapper.track_ignored_source"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + a: + type: keyword + b: + store_array_source: true + properties: + aa: + type: keyword + bb: + type: keyword + c: + type: keyword + d: + store_array_source: true + properties: + aa: + type: keyword + bb: + type: keyword + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "c": 1, "d": [ { "bb": 10, "aa": 20 }, { "aa": 30, "bb": 40 } ], "a": 2, "b": [ { "bb": 100, "aa": 200 }, { "aa": 300, "bb": 400 } ] }' + + - do: + search: + index: test + + - length: { hits.hits.0._source: 4 } + - match: { hits.hits.0._source: { "a": "2", "b": [ { "bb": 100, "aa": 200 }, { "aa": 300, "bb": 400 } ], "c": "1", "d": [ { "bb": 10, "aa": 20 }, { "aa": 30, "bb": 40 } ] } } + + +--- +object param - nested object array next to other fields: + - requires: + cluster_features: ["mapper.track_ignored_source"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + a: + type: keyword + b: + properties: + c: + store_array_source: true + properties: + aa: + type: keyword + bb: + type: keyword + d: + properties: + aa: + type: keyword + bb: + type: keyword + e: + type: keyword + f: + type: keyword + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "a": 1, "b": { "c": [ { "bb": 10, "aa": 20 }, { "aa": 30, "bb": 40 } ], "d": [ { "bb": 100, "aa": 200 }, { "aa": 300, "bb": 400 } ], "e": 1000 }, "f": 2000 }' + + - do: + search: + index: test + + - match: { hits.hits.0._source.a: "1" } + - match: { hits.hits.0._source.b.c: [{ "bb": 10, "aa": 20 }, { "aa": 30, "bb": 40 }] } + - match: { hits.hits.0._source.b.d.aa: [ "200", "300" ] } + - match: { hits.hits.0._source.b.d.bb: [ "100", "400" ] } + - match: { hits.hits.0._source.b.e: "1000" } + - match: { hits.hits.0._source.f: "2000" } + + +--- +object param - nested object with stored array: + - requires: + cluster_features: ["mapper.track_ignored_source"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + name: + type: keyword + nested_array_regular: + type: nested + nested_array_stored: + type: nested + store_array_source: true + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "name": "A", "nested_array_regular": [ { "b": [ { "c": 10 }, { "c": 100 } ] }, { "b": [ { "c": 20 }, { "c": 200 } ] } ] }' + - '{ "create": { } }' + - '{ "name": "B", "nested_array_stored": [ { "b": [ { "c": 10 }, { "c": 100 } ] }, { "b": [ { "c": 20 }, { "c": 200 } ] } ] }' + + - match: { errors: false } + + - do: + search: + index: test + sort: name + - match: { hits.total.value: 2 } + - match: { hits.hits.0._source.name: A } + - match: { hits.hits.0._source.nested_array_regular.0.b.c: [ 10, 100] } + - match: { hits.hits.0._source.nested_array_regular.1.b.c: [ 20, 200] } + - match: { hits.hits.1._source.name: B } + - match: { hits.hits.1._source.nested_array_stored.0.b.0.c: 10 } + - match: { hits.hits.1._source.nested_array_stored.0.b.1.c: 100 } + - match: { hits.hits.1._source.nested_array_stored.1.b.0.c: 20 } + - match: { hits.hits.1._source.nested_array_stored.1.b.1.c: 200 } + + +--- +# 112156 +stored field under object with store_array_source: + - requires: + cluster_features: ["mapper.source.synthetic_source_stored_fields_advance_fix"] + reason: requires bug fix to be implemented + + - do: + indices.create: + index: test + body: + settings: + index: + sort.field: "name" + sort.order: "asc" + mappings: + _source: + mode: synthetic + properties: + name: + type: keyword + obj: + store_array_source: true + properties: + foo: + type: keyword + store: true + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "name": "B", "obj": null }' + - '{ "create": { } }' + - '{ "name": "A", "obj": [ { "foo": "hello_from_the_other_side" } ] }' + + - match: { errors: false } + + - do: + search: + index: test + sort: name + + - match: { hits.total.value: 2 } + - match: { hits.hits.0._source.name: A } + - match: { hits.hits.0._source.obj: [ { "foo": "hello_from_the_other_side" } ] } + - match: { hits.hits.1._source.name: B } + - match: { hits.hits.1._source.obj: null } + + +--- +index param - root arrays: + - requires: + cluster_features: ["mapper.synthetic_source_keep"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping: + synthetic_source_keep: arrays + mappings: + _source: + mode: synthetic + properties: + id: + type: integer + leaf: + type: integer + obj: + properties: + span: + properties: + id: + type: keyword + trace: + properties: + id: + type: keyword + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "id": 1, "leaf": [30, 20, 10], "obj": [ { "trace": { "id": "a" }, "span": { "id": "1" } }, { "trace": { "id": "b" }, "span": { "id": "1" } } ] }' + - '{ "create": { } }' + - '{ "id": 2, "leaf": [130, 120, 110], "obj": [ { "trace": { "id": "aa" }, "span": { "id": "2" } }, { "trace": { "id": "bb" }, "span": { "id": "2" } } ] }' + + - do: + search: + index: test + sort: id + + - match: { hits.hits.0._source.id: 1 } + - match: { hits.hits.0._source.leaf: [30, 20, 10] } + - length: { hits.hits.0._source.obj: 2 } + - match: { hits.hits.0._source.obj.0.trace.id: a } + - match: { hits.hits.0._source.obj.0.span.id: "1" } + - match: { hits.hits.0._source.obj.1.trace.id: b } + - match: { hits.hits.0._source.obj.1.span.id: "1" } + + - match: { hits.hits.1._source.id: 2 } + - match: { hits.hits.1._source.leaf: [ 130, 120, 110 ] } + - length: { hits.hits.1._source.obj: 2 } + - match: { hits.hits.1._source.obj.0.trace.id: aa } + - match: { hits.hits.1._source.obj.0.span.id: "2" } + - match: { hits.hits.1._source.obj.1.trace.id: bb } + - match: { hits.hits.1._source.obj.1.span.id: "2" } + + +--- +index param - dynamic root arrays: + - requires: + cluster_features: ["mapper.synthetic_source_keep"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping: + synthetic_source_keep: arrays + mappings: + _source: + mode: synthetic + properties: + id: + type: integer + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "id": 1, "leaf": [30, 20, 10], "obj": [ { "trace": { "id": "a" }, "span": { "id": "1" } }, { "trace": { "id": "b" }, "span": { "id": "1" } } ] }' + - '{ "create": { } }' + - '{ "id": 2, "leaf": [130, 120, 110], "obj": [ { "trace": { "id": "aa" }, "span": { "id": "2" } }, { "trace": { "id": "bb" }, "span": { "id": "2" } } ] }' + + - do: + search: + index: test + sort: id + + - match: { hits.hits.0._source.id: 1 } + - match: { hits.hits.0._source.leaf: [30, 20, 10] } + - length: { hits.hits.0._source.obj: 2 } + - match: { hits.hits.0._source.obj.0.trace.id: a } + - match: { hits.hits.0._source.obj.0.span.id: "1" } + - match: { hits.hits.0._source.obj.1.trace.id: b } + - match: { hits.hits.0._source.obj.1.span.id: "1" } + + - match: { hits.hits.1._source.id: 2 } + - match: { hits.hits.1._source.leaf: [ 130, 120, 110 ] } + - length: { hits.hits.1._source.obj: 2 } + - match: { hits.hits.1._source.obj.0.trace.id: aa } + - match: { hits.hits.1._source.obj.0.span.id: "2" } + - match: { hits.hits.1._source.obj.1.trace.id: bb } + - match: { hits.hits.1._source.obj.1.span.id: "2" } + + +--- +index param - object array within array: + - requires: + cluster_features: ["mapper.synthetic_source_keep"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping: + synthetic_source_keep: arrays + mappings: + _source: + mode: synthetic + properties: + stored: + properties: + path: + properties: + to: + properties: + trace: + type: keyword + values: + type: integer + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "stored": [ { "path": [{ "to": { "trace": "A", "values": [2, 1] } }, { "to": { "trace": "B", "values": [2, 1] } } ] }, { "path": { "to": { "trace": "C", "values": 3 } } } ] }' + + - do: + search: + index: test + + - length: { hits.hits.0._source.stored: 2 } + - match: { hits.hits.0._source.stored.0.path.0.to.trace: A } + - match: { hits.hits.0._source.stored.0.path.0.to.values: [2, 1] } + - match: { hits.hits.0._source.stored.0.path.1.to.trace: B } + - match: { hits.hits.0._source.stored.0.path.1.to.values: [2, 1] } + - match: { hits.hits.0._source.stored.1.path.to.trace: C } + - match: { hits.hits.0._source.stored.1.path.to.values: 3 } + + +--- +index param - no object array: + - requires: + cluster_features: ["mapper.synthetic_source_keep"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping: + synthetic_source_keep: arrays + mappings: + _source: + mode: synthetic + properties: + stored: + properties: + span: + properties: + id: + type: keyword + trace: + properties: + id: + type: keyword + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "stored": { "trace": { "id": "a" }, "span": { "id": "b" } } }' + + - do: + search: + index: test + + - match: { hits.hits.0._source.stored.trace.id: a } + - match: { hits.hits.0._source.stored.span.id: b } + + +--- +index param - field ordering: + - requires: + cluster_features: ["mapper.synthetic_source_keep"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping: + synthetic_source_keep: arrays + mappings: + _source: + mode: synthetic + properties: + a: + type: keyword + b: + properties: + aa: + type: keyword + bb: + type: keyword + c: + type: keyword + d: + properties: + aa: + type: keyword + bb: + type: keyword + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "c": [30, 20, 10], "d": [ { "bb": 10, "aa": 20 }, { "aa": 30, "bb": 40 } ], "a": 2, "b": [ { "bb": 100, "aa": 200 }, { "aa": 300, "bb": 400 } ] }' + + - do: + search: + index: test + + - length: { hits.hits.0._source: 4 } + - match: { hits.hits.0._source: { "a": "2", "b": [ { "bb": 100, "aa": 200 }, { "aa": 300, "bb": 400 } ], "c": [30, 20, 10], "d": [ { "bb": 10, "aa": 20 }, { "aa": 30, "bb": 40 } ] } } + + +--- +index param - nested arrays: + - requires: + cluster_features: ["mapper.synthetic_source_keep"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping: + synthetic_source_keep: arrays + mappings: + _source: + mode: synthetic + properties: + a: + type: keyword + b: + properties: + c: + properties: + aa: + type: keyword + bb: + type: keyword + d: + type: integer + e: + type: keyword + f: + type: keyword + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "a": 1, "b": { "c": [ { "bb": 10, "aa": 20 }, { "aa": 30, "bb": 40 } ], "d": [ 300, 200, 100 ], "e": 1000 }, "f": 2000 }' + - '{ "create": { } }' + - '{ "a": 11, "b": { "c": [ { "bb": 110, "aa": 120 }, { "aa": 130, "bb": 140 } ], "d": [ 1300, 1200, 1100 ], "e": 11000 }, "f": 12000 }' + + + - do: + search: + index: test + sort: a + + - match: { hits.hits.0._source.a: "1" } + - match: { hits.hits.0._source.b.c: [{ "bb": 10, "aa": 20 }, { "aa": 30, "bb": 40 }] } + - match: { hits.hits.0._source.b.d: [ 300, 200, 100 ] } + - match: { hits.hits.0._source.b.e: "1000" } + - match: { hits.hits.0._source.f: "2000" } + + - match: { hits.hits.1._source.a: "11" } + - match: { hits.hits.1._source.b.c: [ { "bb": 110, "aa": 120 }, { "aa": 130, "bb": 140 } ] } + - match: { hits.hits.1._source.b.d: [ 1300, 1200, 1100 ] } + - match: { hits.hits.1._source.b.e: "11000" } + - match: { hits.hits.1._source.f: "12000" } + +--- +index param - nested object with stored array: + - requires: + cluster_features: ["mapper.synthetic_source_keep"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + settings: + index: + mapping: + synthetic_source_keep: arrays + mappings: + _source: + mode: synthetic + properties: + name: + type: keyword + nested: + type: nested + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "name": "A", "nested": [ { "b": [ { "c": 10 }, { "c": 100 } ] }, { "b": [ { "c": 20 }, { "c": 200 } ] } ] }' + - '{ "create": { } }' + - '{ "name": "B", "nested": [ { "b": [ { "c": 30 }, { "c": 300 } ] }, { "b": [ { "c": 40 }, { "c": 400 } ] } ] }' + + - match: { errors: false } + + - do: + search: + index: test + sort: name + - match: { hits.total.value: 2 } + - match: { hits.hits.0._source.name: A } + - match: { hits.hits.0._source.nested.0.b.0.c: 10 } + - match: { hits.hits.0._source.nested.0.b.1.c: 100 } + - match: { hits.hits.0._source.nested.1.b.0.c: 20 } + - match: { hits.hits.0._source.nested.1.b.1.c: 200 } + - match: { hits.hits.1._source.name: B } + - match: { hits.hits.1._source.nested.0.b.0.c: 30 } + - match: { hits.hits.1._source.nested.0.b.1.c: 300 } + - match: { hits.hits.1._source.nested.1.b.0.c: 40 } + - match: { hits.hits.1._source.nested.1.b.1.c: 400 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml index 45bcf64f98945..3d82539944a97 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml @@ -449,6 +449,115 @@ index: test-generic - match: { test-generic.mappings.properties.parent.properties.child\.grandchild.type: "keyword" } + +--- +"Composable index templates that include subobjects: auto at root": + - requires: + cluster_features: ["mapper.subobjects_auto"] + reason: "https://github.com/elastic/elasticsearch/issues/96768 fixed at 8.11.0" + test_runner_features: "allowed_warnings" + + - do: + cluster.put_component_template: + name: test-subobjects + body: + template: + mappings: + subobjects: auto + properties: + message: + enabled: false + + - do: + cluster.put_component_template: + name: test-field + body: + template: + mappings: + properties: + parent.subfield: + type: keyword + + - do: + allowed_warnings: + - "index template [test-composable-template] has index patterns [test-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-template] will take precedence during new index creation" + indices.put_index_template: + name: test-composable-template + body: + index_patterns: + - test-* + composed_of: + - test-subobjects + - test-field + - is_true: acknowledged + + - do: + indices.create: + index: test-generic + + - do: + indices.get_mapping: + index: test-generic + - match: { test-generic.mappings.properties.parent\.subfield.type: "keyword" } + - match: { test-generic.mappings.properties.message.type: "object" } + +--- +"Composable index templates that include subobjects: auto on arbitrary field": + - requires: + cluster_features: ["mapper.subobjects_auto"] + reason: "https://github.com/elastic/elasticsearch/issues/96768 fixed at 8.11.0" + test_runner_features: "allowed_warnings" + + - do: + cluster.put_component_template: + name: test-subobjects + body: + template: + mappings: + properties: + parent: + type: object + subobjects: auto + properties: + message: + enabled: false + + - do: + cluster.put_component_template: + name: test-subfield + body: + template: + mappings: + properties: + parent: + properties: + child.grandchild: + type: keyword + + - do: + allowed_warnings: + - "index template [test-composable-template] has index patterns [test-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-template] will take precedence during new index creation" + indices.put_index_template: + name: test-composable-template + body: + index_patterns: + - test-* + composed_of: + - test-subobjects + - test-subfield + - is_true: acknowledged + + - do: + indices.create: + index: test-generic + + - do: + indices.get_mapping: + index: test-generic + - match: { test-generic.mappings.properties.parent.properties.child\.grandchild.type: "keyword" } + - match: { test-generic.mappings.properties.parent.properties.message.type: "object" } + + --- "Composition of component templates with different legal field mappings": - skip: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.split/40_routing_partition_size.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.split/40_routing_partition_size.yml index 80a8ccf0d1063..11ffbe1d8464d 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.split/40_routing_partition_size.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.split/40_routing_partition_size.yml @@ -16,22 +16,22 @@ more than 1: - do: index: index: source - id: 1 - routing: 1 + id: "1" + routing: "1" body: { "foo": "hello world" } - do: index: index: source - id: 2 - routing: 2 + id: "2" + routing: "2" body: { "foo": "hello world 2" } - do: index: index: source - id: 3 - routing: 3 + id: "3" + routing: "3" body: { "foo": "hello world 3" } # make it read-only @@ -66,8 +66,8 @@ more than 1: - do: get: index: target - routing: 1 - id: 1 + routing: "1" + id: "1" - match: { _index: target } - match: { _id: "1" } @@ -76,8 +76,8 @@ more than 1: - do: get: index: target - routing: 2 - id: 2 + routing: "2" + id: "2" - match: { _index: target } - match: { _id: "2" } @@ -86,8 +86,8 @@ more than 1: - do: get: index: target - routing: 3 - id: 3 + routing: "3" + id: "3" - match: { _index: target } - match: { _id: "3" } @@ -117,22 +117,22 @@ exactly 1: - do: index: index: source - id: 1 - routing: 1 + id: "1" + routing: "1" body: { "foo": "hello world" } - do: index: index: source - id: 2 - routing: 2 + id: "2" + routing: "2" body: { "foo": "hello world 2" } - do: index: index: source - id: 3 - routing: 3 + id: "3" + routing: "3" body: { "foo": "hello world 3" } # make it read-only @@ -167,8 +167,8 @@ exactly 1: - do: get: index: target - routing: 1 - id: 1 + routing: "1" + id: "1" - match: { _index: target } - match: { _id: "1" } @@ -177,8 +177,8 @@ exactly 1: - do: get: index: target - routing: 2 - id: 2 + routing: "2" + id: "2" - match: { _index: target } - match: { _id: "2" } @@ -187,8 +187,8 @@ exactly 1: - do: get: index: target - routing: 3 - id: 3 + routing: "3" + id: "3" - match: { _index: target } - match: { _id: "3" } @@ -221,22 +221,22 @@ nested: - do: index: index: source - id: 1 - routing: 1 + id: "1" + routing: "1" body: { "foo": "hello world", "n": [{"foo": "goodbye world"}, {"foo": "more words"}] } - do: index: index: source - id: 2 - routing: 2 + id: "2" + routing: "2" body: { "foo": "hello world 2" } - do: index: index: source - id: 3 - routing: 3 + id: "3" + routing: "3" body: { "foo": "hello world 3" } # make it read-only @@ -271,8 +271,8 @@ nested: - do: get: index: target - routing: 1 - id: 1 + routing: "1" + id: "1" - match: { _index: target } - match: { _id: "1" } @@ -281,8 +281,8 @@ nested: - do: get: index: target - routing: 2 - id: 2 + routing: "2" + id: "2" - match: { _index: target } - match: { _id: "2" } @@ -291,8 +291,8 @@ nested: - do: get: index: target - routing: 3 - id: 3 + routing: "3" + id: "3" - match: { _index: target } - match: { _id: "3" } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.split/50_routing_required.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.split/50_routing_required.yml index 38bf9d72ef8ff..4c8d7736631c9 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.split/50_routing_required.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.split/50_routing_required.yml @@ -15,22 +15,22 @@ routing required: - do: index: index: source - id: 1 - routing: 1 + id: "1" + routing: "1" body: { "foo": "hello world" } - do: index: index: source - id: 2 - routing: 2 + id: "2" + routing: "2" body: { "foo": "hello world 2" } - do: index: index: source - id: 3 - routing: 3 + id: "3" + routing: "3" body: { "foo": "hello world 3" } # make it read-only @@ -65,8 +65,8 @@ routing required: - do: get: index: target - routing: 1 - id: 1 + routing: "1" + id: "1" - match: { _index: target } - match: { _id: "1" } @@ -75,8 +75,8 @@ routing required: - do: get: index: target - routing: 2 - id: 2 + routing: "2" + id: "2" - match: { _index: target } - match: { _id: "2" } @@ -85,8 +85,8 @@ routing required: - do: get: index: target - routing: 3 - id: 3 + routing: "3" + id: "3" - match: { _index: target } - match: { _id: "3" } @@ -122,22 +122,22 @@ nested: - do: index: index: source - id: 1 - routing: 1 + id: "1" + routing: "1" body: { "foo": "hello world", "n": [{"foo": "goodbye world"}, {"foo": "more words"}] } - do: index: index: source - id: 2 - routing: 2 + id: "2" + routing: "2" body: { "foo": "hello world 2" } - do: index: index: source - id: 3 - routing: 3 + id: "3" + routing: "3" body: { "foo": "hello world 3" } # make it read-only @@ -172,8 +172,8 @@ nested: - do: get: index: target - routing: 1 - id: 1 + routing: "1" + id: "1" - match: { _index: target } - match: { _id: "1" } @@ -182,8 +182,8 @@ nested: - do: get: index: target - routing: 2 - id: 2 + routing: "2" + id: "2" - match: { _index: target } - match: { _id: "2" } @@ -192,8 +192,8 @@ nested: - do: get: index: target - routing: 3 - id: 3 + routing: "3" + id: "3" - match: { _index: target } - match: { _id: "3" } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml index 2935c0c1c41b5..ff17a92ed0fcc 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml @@ -46,6 +46,94 @@ keyword: docs.1._source: kwd: bar +--- +keyword with normalizer: + - requires: + cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] + reason: support for normalizer on keyword fields + - do: + indices.create: + index: test-keyword-with-normalizer + body: + settings: + analysis: + normalizer: + lowercase: + type: custom + filter: + - lowercase + mappings: + _source: + mode: synthetic + properties: + keyword: + type: keyword + normalizer: lowercase + keyword_with_ignore_above: + type: keyword + normalizer: lowercase + ignore_above: 10 + keyword_without_doc_values: + type: keyword + normalizer: lowercase + doc_values: false + + - do: + index: + index: test-keyword-with-normalizer + id: 1 + body: + keyword: "the Quick Brown Fox jumps over the lazy Dog" + keyword_with_ignore_above: "the Quick Brown Fox jumps over the lazy Dog" + keyword_without_doc_values: "the Quick Brown Fox jumps over the lazy Dog" + + - do: + index: + index: test-keyword-with-normalizer + id: 2 + body: + keyword: "The five BOXING wizards jump Quickly" + keyword_with_ignore_above: "The five BOXING wizards jump Quickly" + keyword_without_doc_values: "The five BOXING wizards jump Quickly" + + - do: + index: + index: test-keyword-with-normalizer + id: 3 + body: + keyword: [ "May the FORCE be with You!", "Do or Do Not, There is no Try" ] + keyword_with_ignore_above: [ "May the FORCE be with You!", "Do or Do Not, There is no Try" ] + keyword_without_doc_values: [ "May the FORCE be with You!", "Do or Do Not, There is no Try" ] + + - do: + mget: + index: test-keyword-with-normalizer + body: + ids: [ 1, 2, 3 ] + - match: { docs.0._index: "test-keyword-with-normalizer" } + - match: { docs.0._id: "1" } + - match: + docs.0._source: + keyword: "the Quick Brown Fox jumps over the lazy Dog" + keyword_with_ignore_above: "the Quick Brown Fox jumps over the lazy Dog" + keyword_without_doc_values: "the Quick Brown Fox jumps over the lazy Dog" + + - match: { docs.1._index: "test-keyword-with-normalizer" } + - match: { docs.1._id: "2" } + - match: + docs.1._source: + keyword: "The five BOXING wizards jump Quickly" + keyword_with_ignore_above: "The five BOXING wizards jump Quickly" + keyword_without_doc_values: "The five BOXING wizards jump Quickly" + + - match: { docs.2._index: "test-keyword-with-normalizer" } + - match: { docs.2._id: "3" } + - match: + docs.2._source: + keyword: [ "May the FORCE be with You!", "Do or Do Not, There is no Try" ] + keyword_with_ignore_above: [ "May the FORCE be with You!", "Do or Do Not, There is no Try" ] + keyword_without_doc_values: [ "May the FORCE be with You!", "Do or Do Not, There is no Try" ] + --- stored text: - requires: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml index c4815304e0799..7c345b7d4d3ac 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/180_locale_dependent_mapping.yml @@ -11,26 +11,26 @@ date_field: type: date format: "E, d MMM yyyy HH:mm:ss Z" - locale: "de" + locale: "fr" - do: bulk: refresh: true body: - '{"index": {"_index": "test_index", "_id": "1"}}' - - '{"date_field": "Mi, 06 Dez 2000 02:55:00 -0800"}' + - '{"date_field": "mer., 6 déc. 2000 02:55:00 -0800"}' - '{"index": {"_index": "test_index", "_id": "2"}}' - - '{"date_field": "Do, 07 Dez 2000 02:55:00 -0800"}' + - '{"date_field": "jeu., 7 déc. 2000 02:55:00 -0800"}' - do: search: rest_total_hits_as_int: true index: test_index - body: {"query" : {"range" : {"date_field" : {"gte": "Di, 05 Dez 2000 02:55:00 -0800", "lte": "Do, 07 Dez 2000 00:00:00 -0800"}}}} + body: {"query" : {"range" : {"date_field" : {"gte": "mar., 5 déc. 2000 02:55:00 -0800", "lte": "jeu., 7 déc. 2000 00:00:00 -0800"}}}} - match: { hits.total: 1 } - do: search: rest_total_hits_as_int: true index: test_index - body: {"query" : {"range" : {"date_field" : {"gte": "Di, 05 Dez 2000 02:55:00 -0800", "lte": "Fr, 08 Dez 2000 00:00:00 -0800"}}}} + body: {"query" : {"range" : {"date_field" : {"gte": "mar., 5 déc. 2000 02:55:00 -0800", "lte": "ven., 8 déc. 2000 00:00:00 -0800"}}}} - match: { hits.total: 2 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml index 703f2a0352fbd..c120bed2d369d 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml @@ -1125,3 +1125,55 @@ fetch geo_point: - match: { hits.hits.0.fields.root\.keyword.0: 'parent' } - match: { hits.hits.0.fields.root\.subfield.0: 'child' } - match: { hits.hits.0.fields.root\.subfield\.keyword.0: 'child' } + +--- +"Test with subobjects: auto": + - requires: + cluster_features: "mapper.subobjects_auto" + reason: requires support for subobjects auto setting + + - do: + indices.create: + index: test + body: + mappings: + subobjects: auto + properties: + message: + type: object + subobjects: auto + enabled: false + + - do: + index: + index: test + refresh: true + body: > + { + "root": "parent", + "root.subfield": "child", + "message": { + "foo": 10, + "foo.bar": 20 + } + } + - match: {result: "created"} + + - do: + search: + index: test + body: + query: + term: + root.subfield: + value: 'child' + fields: + - field: 'root*' + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields.root.0: 'parent' } + - match: { hits.hits.0.fields.root\.keyword.0: 'parent' } + - match: { hits.hits.0.fields.root\.subfield.0: 'child' } + - match: { hits.hits.0.fields.root\.subfield\.keyword.0: 'child' } + - is_false: hits.hits.0.fields.message + - match: { hits.hits.0._source.message.foo: 10 } + - match: { hits.hits.0._source.message.foo\.bar: 20 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/simulate.ingest/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/simulate.ingest/10_basic.yml index 5928dce2c104e..da0f00d960534 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/simulate.ingest/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/simulate.ingest/10_basic.yml @@ -258,6 +258,280 @@ setup: - not_exists: docs.1.doc.error --- +"Test mapping validation from templates": + + - skip: + features: + - headers + - allowed_warnings + + - requires: + cluster_features: ["simulate.mapping.validation.templates"] + reason: "ingest simulate index mapping validation added in 8.16" + + - do: + indices.put_template: + name: v1_template + body: + index_patterns: v1_strict_nonexistent* + mappings: + dynamic: strict + properties: + foo: + type: text + + - do: + allowed_warnings: + - "index template [v2_template] has index patterns [v2_strict_nonexistent*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [v2_template] will take precedence during new index creation" + indices.put_index_template: + name: v2_template + body: + index_patterns: v2_strict_nonexistent* + template: + mappings: + dynamic: strict + properties: + foo: + type: text + + - do: + allowed_warnings: + - "index template [v2_hidden_template] has index patterns [v2_strict_hidden_nonexistent*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [v2_hidden_template] will take precedence during new index creation" + indices.put_index_template: + name: v2_hidden_template + body: + index_patterns: v2_strict_hidden_nonexistent* + template: + settings: + index: + hidden: true + mappings: + dynamic: strict + properties: + foo: + type: text + + - do: + headers: + Content-Type: application/json + simulate.ingest: + body: > + { + "docs": [ + { + "_index": "v1_strict_nonexistent_index", + "_id": "id", + "_source": { + "foob": "bar" + } + }, + { + "_index": "v1_strict_nonexistent_index", + "_id": "id", + "_source": { + "foo": "rab" + } + } + ], + "pipeline_substitutions": { + "my-pipeline": { + "processors": [ + ] + } + } + } + - length: { docs: 2 } + - match: { docs.0.doc._source.foob: "bar" } + - match: { docs.0.doc.error.type: "strict_dynamic_mapping_exception" } + - match: { docs.0.doc.error.reason: "[1:9] mapping set to strict, dynamic introduction of [foob] within [_doc] is not allowed" } + - match: { docs.1.doc._source.foo: "rab" } + - not_exists: docs.1.doc.error + + - do: + headers: + Content-Type: application/json + simulate.ingest: + body: > + { + "docs": [ + { + "_index": "v2_strict_nonexistent_index", + "_id": "id", + "_source": { + "foob": "bar" + } + }, + { + "_index": "v2_strict_nonexistent_index", + "_id": "id", + "_source": { + "foo": "rab" + } + }, + { + "_index": "v2_strict_hidden_nonexistent_index", + "_id": "id", + "_source": { + "foob": "bar" + } + } + ], + "pipeline_substitutions": { + "my-pipeline": { + "processors": [ + ] + } + } + } + - length: { docs: 3 } + - match: { docs.0.doc._source.foob: "bar" } + - match: { docs.0.doc.error.type: "strict_dynamic_mapping_exception" } + - match: { docs.0.doc.error.reason: "[1:9] mapping set to strict, dynamic introduction of [foob] within [_doc] is not allowed" } + - match: { docs.1.doc._source.foo: "rab" } + - not_exists: docs.1.doc.error + - match: { docs.2.doc._source.foob: "bar" } + - match: { docs.2.doc.error.type: "strict_dynamic_mapping_exception" } + - match: { docs.2.doc.error.reason: "[1:9] mapping set to strict, dynamic introduction of [foob] within [_doc] is not allowed" } + +--- +"Test mapping validation for data streams from templates": + + - skip: + features: + - headers + - allowed_warnings + + - requires: + cluster_features: ["simulate.mapping.validation.templates"] + reason: "ingest simulate index mapping validation added in 8.16" + + - do: + allowed_warnings: + - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" + indices.put_index_template: + name: my-template1 + body: + index_patterns: [simple-data-stream1] + template: + settings: + index.number_of_replicas: 1 + mappings: + dynamic: strict + properties: + foo: + type: text + data_stream: {} + + - do: + allowed_warnings: + - "index template [my-hidden-template1] has index patterns [simple-hidden-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-hidden-template1] will take precedence during new index creation" + indices.put_index_template: + name: my-hidden-template1 + body: + index_patterns: [simple-hidden-data-stream1] + template: + settings: + index.number_of_replicas: 1 + mappings: + dynamic: strict + properties: + foo: + type: text + data_stream: + hidden: true + + - do: + headers: + Content-Type: application/json + simulate.ingest: + body: > + { + "docs": [ + { + "_index": "simple-data-stream1", + "_id": "id", + "_source": { + "@timestamp": "2020-12-12", + "foob": "bar" + } + }, + { + "_index": "simple-data-stream1", + "_id": "id", + "_source": { + "@timestamp": "2020-12-12", + "foo": "rab" + } + }, + { + "_index": "simple-hidden-data-stream1", + "_id": "id", + "_source": { + "@timestamp": "2020-12-12", + "foob": "bar" + } + } + ], + "pipeline_substitutions": { + "my-pipeline": { + "processors": [ + ] + } + } + } + - length: { docs: 3 } + - match: { docs.0.doc._source.foob: "bar" } + - match: { docs.0.doc.error.type: "strict_dynamic_mapping_exception" } + - match: { docs.0.doc.error.reason: "[1:35] mapping set to strict, dynamic introduction of [foob] within [_doc] is not allowed" } + - match: { docs.1.doc._source.foo: "rab" } + - not_exists: docs.1.doc.error + - match: { docs.2.doc._source.foob: "bar" } + - match: { docs.2.doc.error.type: "strict_dynamic_mapping_exception" } + - match: { docs.2.doc.error.reason: "[1:35] mapping set to strict, dynamic introduction of [foob] within [_doc] is not allowed" } + + - do: + indices.create_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + + - do: + headers: + Content-Type: application/json + simulate.ingest: + body: > + { + "docs": [ + { + "_index": "simple-data-stream1", + "_id": "id", + "_source": { + "@timestamp": "2020-12-12", + "foob": "bar" + } + }, + { + "_index": "simple-data-stream1", + "_id": "id", + "_source": { + "@timestamp": "2020-12-12", + "foo": "rab" + } + } + ], + "pipeline_substitutions": { + "my-pipeline": { + "processors": [ + ] + } + } + } + - length: { docs: 2 } + - match: { docs.0.doc._source.foob: "bar" } + - match: { docs.0.doc.error.type: "strict_dynamic_mapping_exception" } + - match: { docs.0.doc.error.reason: "[1:35] mapping set to strict, dynamic introduction of [foob] within [_doc] is not allowed" } + - match: { docs.1.doc._source.foo: "rab" } + - not_exists: docs.1.doc.error +--- "Test index templates with pipelines": - skip: diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java index 27fd54c39cc95..22549a1562dcd 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java @@ -276,6 +276,7 @@ public void testSplitIndexPrimaryTerm() throws Exception { .put(indexSettings()) .put("number_of_shards", numberOfShards) .put("index.number_of_routing_shards", numberOfTargetShards) + .put("index.routing.rebalance.enable", EnableAllocationDecider.Rebalance.NONE) ).get(); ensureGreen(TimeValue.timeValueSeconds(120)); // needs more than the default to allocate many shards diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java new file mode 100644 index 0000000000000..4a56a6ce8ddb6 --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java @@ -0,0 +1,278 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.bulk; + +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; +import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.ingest.SimulateIndexResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class TransportSimulateBulkActionIT extends ESIntegTestCase { + @SuppressWarnings("unchecked") + public void testMappingValidationIndexExists() { + /* + * This test simulates a BulkRequest of two documents into an existing index. Then we make sure the index contains no documents, and + * that the index's mapping in the cluster state has not been updated with the two new field. + */ + String indexName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + String mapping = """ + { + "_doc":{ + "dynamic":"strict", + "properties":{ + "foo1":{ + "type":"text" + } + } + } + } + """; + indicesAdmin().create(new CreateIndexRequest(indexName).mapping(mapping)).actionGet(); + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.add(new IndexRequest(indexName).source(""" + { + "foo1": "baz" + } + """, XContentType.JSON).id(randomUUID())); + bulkRequest.add(new IndexRequest(indexName).source(""" + { + "foo3": "baz" + } + """, XContentType.JSON).id(randomUUID())); + BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); + assertThat(response.getItems().length, equalTo(2)); + assertThat(response.getItems()[0].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[0].getResponse()).getException()); + assertThat(response.getItems()[1].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertThat( + ((SimulateIndexResponse) response.getItems()[1].getResponse()).getException().getMessage(), + containsString("mapping set to strict, dynamic introduction of") + ); + indicesAdmin().refresh(new RefreshRequest(indexName)).actionGet(); + SearchResponse searchResponse = client().search(new SearchRequest(indexName)).actionGet(); + assertThat(searchResponse.getHits().getTotalHits().value, equalTo(0L)); + searchResponse.decRef(); + ClusterStateResponse clusterStateResponse = admin().cluster().state(new ClusterStateRequest()).actionGet(); + Map indexMapping = clusterStateResponse.getState().metadata().index(indexName).mapping().sourceAsMap(); + Map fields = (Map) indexMapping.get("properties"); + assertThat(fields.size(), equalTo(1)); + } + + public void testMappingValidationIndexDoesNotExistsNoTemplate() { + /* + * This test simulates a BulkRequest of two documents into an index that does not exist. There is no template (other than the + * mapping-less "random-index-template" created by the parent class), so we expect no mapping validation failure. + */ + String indexName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.add(new IndexRequest(indexName).source(""" + { + "foo1": "baz" + } + """, XContentType.JSON).id(randomUUID())); + bulkRequest.add(new IndexRequest(indexName).source(""" + { + "foo3": "baz" + } + """, XContentType.JSON).id(randomUUID())); + BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); + assertThat(response.getItems().length, equalTo(2)); + assertThat(response.getItems()[0].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[0].getResponse()).getException()); + assertThat(response.getItems()[1].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[1].getResponse()).getException()); + } + + public void testMappingValidationIndexDoesNotExistsV2Template() throws IOException { + /* + * This test simulates a BulkRequest of two documents into an index that does not exist. The index matches a v2 index template. It + * has strict mappings and one of our documents has it as a field not in the mapping, so we expect a mapping validation error. + */ + String indexName = "my-index-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + String mappingString = """ + { + "_doc":{ + "dynamic":"strict", + "properties":{ + "foo1":{ + "type":"text" + } + } + } + } + """; + CompressedXContent mapping = CompressedXContent.fromJSON(mappingString); + Template template = new Template(Settings.EMPTY, mapping, null); + ComposableIndexTemplate composableIndexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of("my-index-*")) + .template(template) + .build(); + TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request("test"); + request.indexTemplate(composableIndexTemplate); + + client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.add(new IndexRequest(indexName).source(""" + { + "foo1": "baz" + } + """, XContentType.JSON).id(randomUUID())); + bulkRequest.add(new IndexRequest(indexName).source(""" + { + "foo3": "baz" + } + """, XContentType.JSON).id(randomUUID())); + BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); + assertThat(response.getItems().length, equalTo(2)); + assertThat(response.getItems()[0].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[0].getResponse()).getException()); + assertThat(response.getItems()[1].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertThat( + ((SimulateIndexResponse) response.getItems()[1].getResponse()).getException().getMessage(), + containsString("mapping set to strict, dynamic introduction of") + ); + } + + public void testMappingValidationIndexDoesNotExistsV1Template() { + /* + * This test simulates a BulkRequest of two documents into an index that does not exist. The index matches a v1 index template. It + * has a mapping that defines "foo1" as an integer field and one of our documents has it as a string, so we expect a mapping + * validation exception. + */ + String indexName = "my-index-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + indicesAdmin().putTemplate( + new PutIndexTemplateRequest("test-template").patterns(List.of("my-index-*")).mapping("foo1", "type=integer") + ).actionGet(); + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.add(new IndexRequest(indexName).source(""" + { + "foo1": "baz" + } + """, XContentType.JSON).id(randomUUID())); + bulkRequest.add(new IndexRequest(indexName).source(""" + { + "foo3": "baz" + } + """, XContentType.JSON).id(randomUUID())); + BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); + assertThat(response.getItems().length, equalTo(2)); + assertThat(response.getItems()[0].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertThat( + ((SimulateIndexResponse) response.getItems()[0].getResponse()).getException().getMessage(), + containsString("failed to parse field [foo1] of type [integer] ") + ); + assertNull(((SimulateIndexResponse) response.getItems()[1].getResponse()).getException()); + assertThat(response.getItems()[1].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + } + + public void testMappingValidationIndexDoesNotExistsDataStream() throws IOException { + /* + * This test simulates a BulkRequest of two documents into an index that does not exist. The index matches a v2 index template. It + * has strict mappings and one of our documents has it as a field not in the mapping, so we expect a mapping validation error. + */ + String indexName = "my-data-stream-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + String mappingString = """ + { + "_doc":{ + "dynamic":"strict", + "properties":{ + "foo1":{ + "type":"text" + } + } + } + } + """; + CompressedXContent mapping = CompressedXContent.fromJSON(mappingString); + Template template = new Template(Settings.EMPTY, mapping, null); + ComposableIndexTemplate.DataStreamTemplate dataStreamTemplate = new ComposableIndexTemplate.DataStreamTemplate(); + ComposableIndexTemplate composableIndexTemplate = ComposableIndexTemplate.builder() + .indexPatterns(List.of("my-data-stream-*")) + .dataStreamTemplate(dataStreamTemplate) + .template(template) + .build(); + TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request("test"); + request.indexTemplate(composableIndexTemplate); + + client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); + { + // First, try with no @timestamp to make sure we're picking up data-stream-specific templates + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.add(new IndexRequest(indexName).source(""" + { + "foo1": "baz" + } + """, XContentType.JSON).id(randomUUID())); + bulkRequest.add(new IndexRequest(indexName).source(""" + { + "foo3": "baz" + } + """, XContentType.JSON).id(randomUUID())); + BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); + assertThat(response.getItems().length, equalTo(2)); + assertThat(response.getItems()[0].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertThat( + ((SimulateIndexResponse) response.getItems()[0].getResponse()).getException().getMessage(), + containsString("data stream timestamp field [@timestamp] is missing") + ); + assertThat(response.getItems()[1].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertThat( + ((SimulateIndexResponse) response.getItems()[1].getResponse()).getException().getMessage(), + containsString("mapping set to strict, dynamic introduction of") + ); + } + { + // Now with @timestamp + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.add(new IndexRequest(indexName).source(""" + { + "@timestamp": "2024-08-27", + "foo1": "baz" + } + """, XContentType.JSON).id(randomUUID())); + bulkRequest.add(new IndexRequest(indexName).source(""" + { + "@timestamp": "2024-08-27", + "foo3": "baz" + } + """, XContentType.JSON).id(randomUUID())); + BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); + assertThat(response.getItems().length, equalTo(2)); + assertThat(response.getItems()[0].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[0].getResponse()).getException()); + assertThat(response.getItems()[1].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertThat( + ((SimulateIndexResponse) response.getItems()[1].getResponse()).getException().getMessage(), + containsString("mapping set to strict, dynamic introduction of") + ); + } + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/search/PointInTimeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/search/PointInTimeIT.java index a9a5bb074c9ac..da2dfc50d7fe9 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/search/PointInTimeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/search/PointInTimeIT.java @@ -10,12 +10,16 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.action.NoShardAvailableActionException; +import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteUtils; import org.elasticsearch.action.admin.indices.stats.CommonStats; +import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.allocation.command.AllocateEmptyPrimaryAllocationCommand; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; @@ -54,11 +58,14 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailuresAndResponse; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; import static org.hamcrest.Matchers.arrayWithSize; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.everyItem; +import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.in; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.not; public class PointInTimeIT extends ESIntegTestCase { @@ -84,7 +91,7 @@ public void testBasic() { prepareIndex("test").setId(id).setSource("value", i).get(); } refresh("test"); - BytesReference pitId = openPointInTime(new String[] { "test" }, TimeValue.timeValueMinutes(2)); + BytesReference pitId = openPointInTime(new String[] { "test" }, TimeValue.timeValueMinutes(2)).getPointInTimeId(); assertResponse(prepareSearch().setPointInTime(new PointInTimeBuilder(pitId)), resp1 -> { assertThat(resp1.pointInTimeId(), equalTo(pitId)); assertHitCount(resp1, numDocs); @@ -130,7 +137,7 @@ public void testMultipleIndices() { prepareIndex(index).setId(id).setSource("value", i).get(); } refresh(); - BytesReference pitId = openPointInTime(new String[] { "*" }, TimeValue.timeValueMinutes(2)); + BytesReference pitId = openPointInTime(new String[] { "*" }, TimeValue.timeValueMinutes(2)).getPointInTimeId(); try { int moreDocs = randomIntBetween(10, 50); assertNoFailuresAndResponse(prepareSearch().setPointInTime(new PointInTimeBuilder(pitId)), resp -> { @@ -212,7 +219,7 @@ public void testRelocation() throws Exception { prepareIndex("test").setId(Integer.toString(i)).setSource("value", i).get(); } refresh(); - BytesReference pitId = openPointInTime(new String[] { "test" }, TimeValue.timeValueMinutes(2)); + BytesReference pitId = openPointInTime(new String[] { "test" }, TimeValue.timeValueMinutes(2)).getPointInTimeId(); try { assertNoFailuresAndResponse(prepareSearch().setPointInTime(new PointInTimeBuilder(pitId)), resp -> { assertHitCount(resp, numDocs); @@ -264,7 +271,7 @@ public void testPointInTimeNotFound() throws Exception { prepareIndex("index").setId(id).setSource("value", i).get(); } refresh(); - BytesReference pit = openPointInTime(new String[] { "index" }, TimeValue.timeValueSeconds(5)); + BytesReference pit = openPointInTime(new String[] { "index" }, TimeValue.timeValueSeconds(5)).getPointInTimeId(); assertNoFailuresAndResponse(prepareSearch().setPointInTime(new PointInTimeBuilder(pit)), resp1 -> { assertHitCount(resp1, index1); if (rarely()) { @@ -305,7 +312,7 @@ public void testIndexNotFound() { prepareIndex("index-2").setId(id).setSource("value", i).get(); } refresh(); - BytesReference pit = openPointInTime(new String[] { "index-*" }, TimeValue.timeValueMinutes(2)); + BytesReference pit = openPointInTime(new String[] { "index-*" }, TimeValue.timeValueMinutes(2)).getPointInTimeId(); try { assertNoFailuresAndResponse( prepareSearch().setPointInTime(new PointInTimeBuilder(pit)), @@ -348,7 +355,7 @@ public void testCanMatch() throws Exception { assertAcked(prepareCreate("test").setSettings(settings).setMapping(""" {"properties":{"created_date":{"type": "date", "format": "yyyy-MM-dd"}}}""")); ensureGreen("test"); - BytesReference pitId = openPointInTime(new String[] { "test*" }, TimeValue.timeValueMinutes(2)); + BytesReference pitId = openPointInTime(new String[] { "test*" }, TimeValue.timeValueMinutes(2)).getPointInTimeId(); try { for (String node : internalCluster().nodesInclude("test")) { for (IndexService indexService : internalCluster().getInstance(IndicesService.class, node)) { @@ -415,7 +422,7 @@ public void testPartialResults() throws Exception { prepareIndex(randomFrom("test-2")).setId(Integer.toString(i)).setSource("value", i).get(); } refresh(); - BytesReference pitId = openPointInTime(new String[] { "test-*" }, TimeValue.timeValueMinutes(2)); + BytesReference pitId = openPointInTime(new String[] { "test-*" }, TimeValue.timeValueMinutes(2)).getPointInTimeId(); try { assertNoFailuresAndResponse(prepareSearch().setPointInTime(new PointInTimeBuilder(pitId)), resp -> { assertHitCount(resp, numDocs1 + numDocs2); @@ -447,7 +454,7 @@ public void testPITTiebreak() throws Exception { } } refresh("index-*"); - BytesReference pit = openPointInTime(new String[] { "index-*" }, TimeValue.timeValueHours(1)); + BytesReference pit = openPointInTime(new String[] { "index-*" }, TimeValue.timeValueHours(1)).getPointInTimeId(); try { for (int size = 1; size <= numIndex; size++) { SortOrder order = randomBoolean() ? SortOrder.ASC : SortOrder.DESC; @@ -532,6 +539,176 @@ public void testOpenPITConcurrentShardRequests() throws Exception { } } + public void testMissingShardsWithPointInTime() throws Exception { + final Settings nodeAttributes = Settings.builder().put("node.attr.foo", "bar").build(); + final String masterNode = internalCluster().startMasterOnlyNode(nodeAttributes); + List dataNodes = internalCluster().startDataOnlyNodes(2, nodeAttributes); + + final String index = "my_test_index"; + // tried to have randomIntBetween(3, 10) but having more shards than 3 was taking forever and throwing timeouts + final int numShards = 3; + final int numReplicas = 0; + // create an index with numShards shards and 0 replicas + createIndex( + index, + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicas) + .put("index.routing.allocation.require.foo", "bar") + .build() + ); + + // index some documents + int numDocs = randomIntBetween(10, 50); + for (int i = 0; i < numDocs; i++) { + String id = Integer.toString(i); + prepareIndex(index).setId(id).setSource("value", i).get(); + } + refresh(index); + + // create a PIT when all shards are present + OpenPointInTimeResponse pointInTimeResponse = openPointInTime(new String[] { index }, TimeValue.timeValueMinutes(1)); + try { + // ensure that the PIT created has all the shards there + assertThat(numShards, equalTo(pointInTimeResponse.getTotalShards())); + assertThat(numShards, equalTo(pointInTimeResponse.getSuccessfulShards())); + assertThat(0, equalTo(pointInTimeResponse.getFailedShards())); + assertThat(0, equalTo(pointInTimeResponse.getSkippedShards())); + + // make a request using the above PIT + assertResponse( + prepareSearch().setQuery(new MatchAllQueryBuilder()) + .setPointInTime(new PointInTimeBuilder(pointInTimeResponse.getPointInTimeId())), + resp -> { + // ensure that al docs are returned + assertThat(resp.pointInTimeId(), equalTo(pointInTimeResponse.getPointInTimeId())); + assertHitCount(resp, numDocs); + } + ); + + // pick up a random data node to shut down + final String randomDataNode = randomFrom(dataNodes); + + // find which shards to relocate + final String nodeId = admin().cluster().prepareNodesInfo(randomDataNode).get().getNodes().get(0).getNode().getId(); + Set shardsToRelocate = new HashSet<>(); + for (ShardStats stats : admin().indices().prepareStats(index).get().getShards()) { + if (nodeId.equals(stats.getShardRouting().currentNodeId())) { + shardsToRelocate.add(stats.getShardRouting().shardId().id()); + } + } + + final int shardsRemoved = shardsToRelocate.size(); + + // shut down the random data node + internalCluster().stopNode(randomDataNode); + + // ensure that the index is Red + ensureRed(index); + + // verify that not all documents can now be retrieved + assertResponse(prepareSearch().setQuery(new MatchAllQueryBuilder()), resp -> { + assertThat(resp.getSuccessfulShards(), equalTo(numShards - shardsRemoved)); + assertThat(resp.getFailedShards(), equalTo(shardsRemoved)); + assertNotNull(resp.getHits().getTotalHits()); + assertThat(resp.getHits().getTotalHits().value, lessThan((long) numDocs)); + }); + + // create a PIT when some shards are missing + OpenPointInTimeResponse pointInTimeResponseOneNodeDown = openPointInTime( + new String[] { index }, + TimeValue.timeValueMinutes(10), + true + ); + try { + // assert that some shards are indeed missing from PIT + assertThat(pointInTimeResponseOneNodeDown.getTotalShards(), equalTo(numShards)); + assertThat(pointInTimeResponseOneNodeDown.getSuccessfulShards(), equalTo(numShards - shardsRemoved)); + assertThat(pointInTimeResponseOneNodeDown.getFailedShards(), equalTo(shardsRemoved)); + assertThat(pointInTimeResponseOneNodeDown.getSkippedShards(), equalTo(0)); + + // ensure that the response now contains fewer documents than the total number of indexed documents + assertResponse( + prepareSearch().setQuery(new MatchAllQueryBuilder()) + .setPointInTime(new PointInTimeBuilder(pointInTimeResponseOneNodeDown.getPointInTimeId())), + resp -> { + assertThat(resp.getSuccessfulShards(), equalTo(numShards - shardsRemoved)); + assertThat(resp.getFailedShards(), equalTo(shardsRemoved)); + assertThat(resp.pointInTimeId(), equalTo(pointInTimeResponseOneNodeDown.getPointInTimeId())); + assertNotNull(resp.getHits().getTotalHits()); + assertThat(resp.getHits().getTotalHits().value, lessThan((long) numDocs)); + } + ); + + // add another node to the cluster and re-allocate the shards + final String newNodeName = internalCluster().startDataOnlyNode(nodeAttributes); + try { + for (int shardId : shardsToRelocate) { + ClusterRerouteUtils.reroute(client(), new AllocateEmptyPrimaryAllocationCommand(index, shardId, newNodeName, true)); + } + ensureGreen(TimeValue.timeValueMinutes(2), index); + + // index some more documents + for (int i = numDocs; i < numDocs * 2; i++) { + String id = Integer.toString(i); + prepareIndex(index).setId(id).setSource("value", i).get(); + } + refresh(index); + + // ensure that we now see at least numDocs results from the updated index + assertResponse(prepareSearch().setQuery(new MatchAllQueryBuilder()), resp -> { + assertThat(resp.getSuccessfulShards(), equalTo(numShards)); + assertThat(resp.getFailedShards(), equalTo(0)); + assertNotNull(resp.getHits().getTotalHits()); + assertThat(resp.getHits().getTotalHits().value, greaterThan((long) numDocs)); + }); + + // ensure that when using the previously created PIT, we'd see the same number of documents as before regardless of the + // newly indexed documents + assertResponse( + prepareSearch().setQuery(new MatchAllQueryBuilder()) + .setPointInTime(new PointInTimeBuilder(pointInTimeResponseOneNodeDown.getPointInTimeId())), + resp -> { + assertThat(resp.pointInTimeId(), equalTo(pointInTimeResponseOneNodeDown.getPointInTimeId())); + assertThat(resp.getTotalShards(), equalTo(numShards)); + assertThat(resp.getSuccessfulShards(), equalTo(numShards - shardsRemoved)); + assertThat(resp.getFailedShards(), equalTo(shardsRemoved)); + assertThat(resp.getShardFailures().length, equalTo(shardsRemoved)); + for (var failure : resp.getShardFailures()) { + assertTrue(shardsToRelocate.contains(failure.shardId())); + assertThat(failure.getCause(), instanceOf(NoShardAvailableActionException.class)); + } + assertNotNull(resp.getHits().getTotalHits()); + // we expect less documents as the newly indexed ones should not be part of the PIT + assertThat(resp.getHits().getTotalHits().value, lessThan((long) numDocs)); + } + ); + + Exception exc = expectThrows( + Exception.class, + () -> prepareSearch().setQuery(new MatchAllQueryBuilder()) + .setPointInTime(new PointInTimeBuilder(pointInTimeResponseOneNodeDown.getPointInTimeId())) + .setAllowPartialSearchResults(false) + .get() + ); + assertThat(exc.getCause().getMessage(), containsString("missing shards")); + + } finally { + internalCluster().stopNode(newNodeName); + } + } finally { + closePointInTime(pointInTimeResponseOneNodeDown.getPointInTimeId()); + } + + } finally { + closePointInTime(pointInTimeResponse.getPointInTimeId()); + internalCluster().stopNode(masterNode); + for (String dataNode : dataNodes) { + internalCluster().stopNode(dataNode); + } + } + } + @SuppressWarnings({ "rawtypes", "unchecked" }) private void assertPagination(PointInTimeBuilder pit, int expectedNumDocs, int size, SortBuilder... sorts) throws Exception { Set seen = new HashSet<>(); @@ -590,10 +767,14 @@ private void assertPagination(PointInTimeBuilder pit, int expectedNumDocs, int s assertThat(seen.size(), equalTo(expectedNumDocs)); } - private BytesReference openPointInTime(String[] indices, TimeValue keepAlive) { - OpenPointInTimeRequest request = new OpenPointInTimeRequest(indices).keepAlive(keepAlive); - final OpenPointInTimeResponse response = client().execute(TransportOpenPointInTimeAction.TYPE, request).actionGet(); - return response.getPointInTimeId(); + private OpenPointInTimeResponse openPointInTime(String[] indices, TimeValue keepAlive) { + return openPointInTime(indices, keepAlive, false); + } + + private OpenPointInTimeResponse openPointInTime(String[] indices, TimeValue keepAlive, boolean allowPartialSearchResults) { + OpenPointInTimeRequest request = new OpenPointInTimeRequest(indices).keepAlive(keepAlive) + .allowPartialSearchResults(allowPartialSearchResults); + return client().execute(TransportOpenPointInTimeAction.TYPE, request).actionGet(); } private void closePointInTime(BytesReference readerId) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java index 428e116ecd1ca..88d934973fc49 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/search/SearchProgressActionListenerIT.java @@ -25,7 +25,6 @@ import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.elasticsearch.test.junit.annotations.TestIssueLogging; import java.util.ArrayList; import java.util.Arrays; @@ -41,10 +40,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.lessThan; -@TestIssueLogging( - issueUrl = "https://github.com/elastic/elasticsearch/issues/109830", - value = "org.elasticsearch.action.search:TRACE," + "org.elasticsearch.search.SearchService:TRACE" -) public class SearchProgressActionListenerIT extends ESSingleNodeTestCase { private List shards; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/BackgroundRetentionLeaseSyncActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/BackgroundRetentionLeaseSyncActionIT.java new file mode 100644 index 0000000000000..0bab5be245ecf --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/BackgroundRetentionLeaseSyncActionIT.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.seqno; + +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.test.ESIntegTestCase; + +import java.util.stream.Stream; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) +public class BackgroundRetentionLeaseSyncActionIT extends ESIntegTestCase { + + public void testActionCompletesWhenReplicaCircuitBreakersAreAtCapacity() throws Exception { + internalCluster().startMasterOnlyNodes(1); + String primary = internalCluster().startDataOnlyNode(); + assertAcked( + prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + ) + ); + + String replica = internalCluster().startDataOnlyNode(); + ensureGreen("test"); + + try (var ignored = fullyAllocateCircuitBreakerOnNode(replica, CircuitBreaker.IN_FLIGHT_REQUESTS)) { + final ClusterState state = internalCluster().clusterService().state(); + final Index testIndex = resolveIndex("test"); + final ShardId testIndexShardZero = new ShardId(testIndex, 0); + final String testLeaseId = "test-lease/123"; + RetentionLeases newLeases = addTestLeaseToRetentionLeases(primary, testIndex, testLeaseId); + internalCluster().getInstance(RetentionLeaseSyncer.class, primary) + .backgroundSync( + testIndexShardZero, + state.routingTable().shardRoutingTable(testIndexShardZero).primaryShard().allocationId().getId(), + state.term(), + newLeases + ); + + // Wait for test lease to appear on replica + IndicesService replicaIndicesService = internalCluster().getInstance(IndicesService.class, replica); + assertBusy(() -> { + RetentionLeases retentionLeases = replicaIndicesService.indexService(testIndex).getShard(0).getRetentionLeases(); + assertTrue(retentionLeases.contains(testLeaseId)); + }); + } + } + + private static RetentionLeases addTestLeaseToRetentionLeases(String primaryNodeName, Index index, String leaseId) { + IndicesService primaryIndicesService = internalCluster().getInstance(IndicesService.class, primaryNodeName); + RetentionLeases currentLeases = primaryIndicesService.indexService(index).getShard(0).getRetentionLeases(); + RetentionLease newLease = new RetentionLease(leaseId, 0, System.currentTimeMillis(), "test source"); + return new RetentionLeases( + currentLeases.primaryTerm(), + currentLeases.version() + 1, + Stream.concat(currentLeases.leases().stream(), Stream.of(newLease)).toList() + ); + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/RetentionLeaseSyncActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/RetentionLeaseSyncActionIT.java new file mode 100644 index 0000000000000..2d8f455792172 --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/seqno/RetentionLeaseSyncActionIT.java @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.seqno; + +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.replication.ReplicationResponse; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.index.IndexingPressure; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.test.ESIntegTestCase; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) +public class RetentionLeaseSyncActionIT extends ESIntegTestCase { + + public void testActionCompletesWhenReplicaCircuitBreakersAreAtCapacity() { + internalCluster().startMasterOnlyNodes(1); + String primary = internalCluster().startDataOnlyNode(); + assertAcked( + prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + ) + ); + + String replica = internalCluster().startDataOnlyNode(); + ensureGreen("test"); + + try (var ignored = fullyAllocateCircuitBreakerOnNode(replica, CircuitBreaker.IN_FLIGHT_REQUESTS)) { + assertThatRetentionLeaseSyncCompletesSuccessfully(primary); + } + } + + public void testActionCompletesWhenPrimaryIndexingPressureIsAtCapacity() { + internalCluster().startMasterOnlyNodes(1); + String primary = internalCluster().startDataOnlyNode(); + assertAcked( + prepareCreate("test").setSettings( + Settings.builder() + .put(indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + ) + ); + + String replica = internalCluster().startDataOnlyNode(); + ensureGreen("test"); + + try (Releasable ignored = fullyAllocatePrimaryIndexingCapacityOnNode(primary)) { + assertThatRetentionLeaseSyncCompletesSuccessfully(primary); + } + } + + private static void assertThatRetentionLeaseSyncCompletesSuccessfully(String primaryNodeName) { + RetentionLeaseSyncer instance = internalCluster().getInstance(RetentionLeaseSyncer.class, primaryNodeName); + PlainActionFuture retentionLeaseSyncResult = new PlainActionFuture<>(); + ClusterState state = internalCluster().clusterService().state(); + ShardId testIndexShardZero = new ShardId(resolveIndex("test"), 0); + ShardRouting primaryShard = state.routingTable().shardRoutingTable(testIndexShardZero).primaryShard(); + instance.sync( + testIndexShardZero, + primaryShard.allocationId().getId(), + state.term(), + RetentionLeases.EMPTY, + retentionLeaseSyncResult + ); + safeGet(retentionLeaseSyncResult); + } + + /** + * Fully allocate primary indexing capacity on a node + * + * @param targetNode The name of the node on which to allocate + * @return A {@link Releasable} which will release the capacity when closed + */ + private static Releasable fullyAllocatePrimaryIndexingCapacityOnNode(String targetNode) { + return internalCluster().getInstance(IndexingPressure.class, targetNode) + .markPrimaryOperationStarted( + 1, + IndexingPressure.MAX_INDEXING_BYTES.get(internalCluster().getInstance(Settings.class, targetNode)).getBytes() + 1, + true + ); + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreCorruptionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreCorruptionIT.java index 422696d6b61c6..702eba9b2bfb8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreCorruptionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreCorruptionIT.java @@ -8,7 +8,7 @@ package org.elasticsearch.repositories.blobstore; -import org.apache.lucene.tests.mockfile.ExtrasFS; +import org.apache.logging.log4j.Level; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; @@ -23,18 +23,12 @@ import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase; import org.elasticsearch.snapshots.SnapshotState; -import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockLog; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; import org.junit.Before; -import java.io.IOException; -import java.nio.file.FileVisitResult; import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; -import java.util.Base64; import java.util.List; public class BlobStoreCorruptionIT extends AbstractSnapshotIntegTestCase { @@ -57,7 +51,7 @@ public void testCorruptionDetection() throws Exception { flushAndRefresh(indexName); createSnapshot(repositoryName, snapshotName, List.of(indexName)); - final var corruptedFile = corruptRandomFile(repositoryRootPath); + final var corruptedFile = BlobStoreCorruptionUtils.corruptRandomFile(repositoryRootPath); final var corruptedFileType = RepositoryFileType.getRepositoryFileType(repositoryRootPath, corruptedFile); final var corruptionDetectors = new ArrayList, ?>>(); @@ -74,17 +68,51 @@ public void testCorruptionDetection() throws Exception { // detect corruption by taking another snapshot if (corruptedFileType == RepositoryFileType.SHARD_GENERATION) { - corruptionDetectors.add(exceptionListener -> { - logger.info("--> taking another snapshot"); - client().admin() - .cluster() - .prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, repositoryName, randomIdentifier()) - .setWaitForCompletion(true) - .execute(exceptionListener.map(createSnapshotResponse -> { - assertNotEquals(SnapshotState.SUCCESS, createSnapshotResponse.getSnapshotInfo().state()); - return new ElasticsearchException("create-snapshot failed as expected"); - })); - }); + if (Files.exists(corruptedFile)) { + corruptionDetectors.add(exceptionListener -> { + logger.info("--> taking another snapshot"); + client().admin() + .cluster() + .prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, repositoryName, randomIdentifier()) + .setWaitForCompletion(true) + .execute(exceptionListener.map(createSnapshotResponse -> { + assertNotEquals(SnapshotState.SUCCESS, createSnapshotResponse.getSnapshotInfo().state()); + return new ElasticsearchException("create-snapshot failed as expected"); + })); + }); + } else { + corruptionDetectors.add(exceptionListener -> { + logger.info("--> taking another snapshot"); + final var mockLog = MockLog.capture(BlobStoreRepository.class); + mockLog.addExpectation( + new MockLog.SeenEventExpectation( + "fallback message", + "org.elasticsearch.repositories.blobstore.BlobStoreRepository", + Level.ERROR, + "index [*] shard generation [*] in [" + + repositoryName + + "][*] not found - falling back to reading all shard snapshots" + ) + ); + mockLog.addExpectation( + new MockLog.SeenEventExpectation( + "shard blobs list", + "org.elasticsearch.repositories.blobstore.BlobStoreRepository", + Level.ERROR, + "read shard snapshots [*] due to missing shard generation [*] for index [*] in [" + repositoryName + "][*]" + ) + ); + client().admin() + .cluster() + .prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, repositoryName, randomIdentifier()) + .setWaitForCompletion(true) + .execute(ActionListener.releaseAfter(exceptionListener.map(createSnapshotResponse -> { + assertEquals(SnapshotState.SUCCESS, createSnapshotResponse.getSnapshotInfo().state()); + mockLog.assertAllExpectationsMatched(); + return new ElasticsearchException("create-snapshot logged errors as expected"); + }), mockLog)); + }); + } } // detect corruption by restoring the snapshot @@ -126,61 +154,4 @@ public void testCorruptionDetection() throws Exception { logger.info(Strings.format("--> corrupted [%s] and caught exception", corruptedFile), exception); } } - - private static Path corruptRandomFile(Path repositoryRootPath) throws IOException { - final var corruptedFileType = getRandomCorruptibleFileType(); - final var corruptedFile = getRandomFileToCorrupt(repositoryRootPath, corruptedFileType); - if (randomBoolean()) { - logger.info("--> deleting [{}]", corruptedFile); - Files.delete(corruptedFile); - } else { - corruptFileContents(corruptedFile); - } - return corruptedFile; - } - - private static void corruptFileContents(Path fileToCorrupt) throws IOException { - final var oldFileContents = Files.readAllBytes(fileToCorrupt); - logger.info("--> contents of [{}] before corruption: [{}]", fileToCorrupt, Base64.getEncoder().encodeToString(oldFileContents)); - final byte[] newFileContents = new byte[randomBoolean() ? oldFileContents.length : between(0, oldFileContents.length)]; - System.arraycopy(oldFileContents, 0, newFileContents, 0, newFileContents.length); - if (newFileContents.length == oldFileContents.length) { - final var corruptionPosition = between(0, newFileContents.length - 1); - newFileContents[corruptionPosition] = randomValueOtherThan(oldFileContents[corruptionPosition], ESTestCase::randomByte); - logger.info( - "--> updating byte at position [{}] from [{}] to [{}]", - corruptionPosition, - oldFileContents[corruptionPosition], - newFileContents[corruptionPosition] - ); - } else { - logger.info("--> truncating file from length [{}] to length [{}]", oldFileContents.length, newFileContents.length); - } - Files.write(fileToCorrupt, newFileContents); - logger.info("--> contents of [{}] after corruption: [{}]", fileToCorrupt, Base64.getEncoder().encodeToString(newFileContents)); - } - - private static RepositoryFileType getRandomCorruptibleFileType() { - return randomValueOtherThanMany( - // these blob types do not have reliable corruption detection, so we must skip them - t -> t == RepositoryFileType.ROOT_INDEX_N || t == RepositoryFileType.ROOT_INDEX_LATEST, - () -> randomFrom(RepositoryFileType.values()) - ); - } - - private static Path getRandomFileToCorrupt(Path repositoryRootPath, RepositoryFileType corruptedFileType) throws IOException { - final var corruptibleFiles = new ArrayList(); - Files.walkFileTree(repositoryRootPath, new SimpleFileVisitor<>() { - @Override - public FileVisitResult visitFile(Path filePath, BasicFileAttributes attrs) throws IOException { - if (ExtrasFS.isExtra(filePath.getFileName().toString()) == false - && RepositoryFileType.getRepositoryFileType(repositoryRootPath, filePath) == corruptedFileType) { - corruptibleFiles.add(filePath); - } - return super.visitFile(filePath, attrs); - } - }); - return randomFrom(corruptibleFiles); - } - } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryCleanupIT.java b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryCleanupIT.java index 01b01fdf5fcde..3eb05aa36b1b5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryCleanupIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryCleanupIT.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.util.concurrent.ExecutionException; +import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.getRepositoryDataBlobName; import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomNonDataPurpose; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFutureThrows; import static org.hamcrest.Matchers.containsString; @@ -151,12 +152,7 @@ public void testCleanupOldIndexN() throws ExecutionException, InterruptedExcepti createOldIndexNFuture, () -> repository.blobStore() .blobContainer(repository.basePath()) - .writeBlob( - randomNonDataPurpose(), - BlobStoreRepository.INDEX_FILE_PREFIX + generation, - new BytesArray(new byte[1]), - true - ) + .writeBlob(randomNonDataPurpose(), getRepositoryDataBlobName(generation), new BytesArray(new byte[1]), true) ) ); createOldIndexNFuture.get(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSUsageTelemetryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSUsageTelemetryIT.java new file mode 100644 index 0000000000000..8b7f69df9fcc3 --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CCSUsageTelemetryIT.java @@ -0,0 +1,733 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.ccs; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.admin.cluster.stats.CCSTelemetrySnapshot; +import org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.Result; +import org.elasticsearch.action.search.ClosePointInTimeRequest; +import org.elasticsearch.action.search.OpenPointInTimeRequest; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.TransportClosePointInTimeAction; +import org.elasticsearch.action.search.TransportOpenPointInTimeAction; +import org.elasticsearch.action.search.TransportSearchAction; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.common.util.FeatureFlag; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.builder.PointInTimeBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.query.SlowRunningQueryBuilder; +import org.elasticsearch.search.query.ThrowingQueryBuilder; +import org.elasticsearch.search.retriever.MinimalCompoundRetrieverIT; +import org.elasticsearch.search.retriever.RetrieverBuilder; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.test.AbstractMultiClustersTestCase; +import org.elasticsearch.test.InternalTestCluster; +import org.elasticsearch.usage.UsageService; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.rules.TestRule; +import org.junit.runner.Description; +import org.junit.runners.model.Statement; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.ASYNC_FEATURE; +import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.MRT_FEATURE; +import static org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.WILDCARD_FEATURE; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; +import static org.hamcrest.Matchers.equalTo; + +public class CCSUsageTelemetryIT extends AbstractMultiClustersTestCase { + private static final Logger LOGGER = LogManager.getLogger(CCSUsageTelemetryIT.class); + private static final String REMOTE1 = "cluster-a"; + private static final String REMOTE2 = "cluster-b"; + private static final FeatureFlag CCS_TELEMETRY_FEATURE_FLAG = new FeatureFlag("ccs_telemetry"); + + @Override + protected boolean reuseClusters() { + return false; + } + + @Override + protected Collection remoteClusterAlias() { + return List.of(REMOTE1, REMOTE2); + } + + @Rule + public SkipUnavailableRule skipOverride = new SkipUnavailableRule(REMOTE1, REMOTE2); + + @BeforeClass + protected static void skipIfTelemetryDisabled() { + assumeTrue("Skipping test as CCS_TELEMETRY_FEATURE_FLAG is disabled", CCS_TELEMETRY_FEATURE_FLAG.isEnabled()); + } + + @Override + protected Map skipUnavailableForRemoteClusters() { + var map = skipOverride.getMap(); + LOGGER.info("Using skip_unavailable map: [{}]", map); + return map; + } + + @Override + protected Collection> nodePlugins(String clusterAlias) { + return CollectionUtils.appendToCopy(super.nodePlugins(clusterAlias), CrossClusterSearchIT.TestQueryBuilderPlugin.class); + } + + private SearchRequest makeSearchRequest(String... indices) { + SearchRequest searchRequest = new SearchRequest(indices); + searchRequest.allowPartialSearchResults(false); + searchRequest.setBatchedReduceSize(randomIntBetween(3, 20)); + searchRequest.setCcsMinimizeRoundtrips(randomBoolean()); + if (randomBoolean()) { + searchRequest.setPreFilterShardSize(1); + } + searchRequest.source(new SearchSourceBuilder().query(new MatchAllQueryBuilder()).size(10)); + return searchRequest; + } + + /** + * Run search request and get telemetry from it + */ + private CCSTelemetrySnapshot getTelemetryFromSearch(SearchRequest searchRequest) throws ExecutionException, InterruptedException { + // We want to send search to a specific node (we don't care which one) so that we could + // collect the CCS telemetry from it later + String nodeName = cluster(LOCAL_CLUSTER).getRandomNodeName(); + // We don't care here too much about the response, we just want to trigger the telemetry collection. + // So we check it's not null and leave the rest to other tests. + assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull); + return getTelemetrySnapshot(nodeName); + } + + private CCSTelemetrySnapshot getTelemetryFromFailedSearch(SearchRequest searchRequest) throws Exception { + // We want to send search to a specific node (we don't care which one) so that we could + // collect the CCS telemetry from it later + String nodeName = cluster(LOCAL_CLUSTER).getRandomNodeName(); + PlainActionFuture queryFuture = new PlainActionFuture<>(); + cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest, queryFuture); + assertBusy(() -> assertTrue(queryFuture.isDone())); + + // We expect failure, but we don't care too much which failure it is in this test + ExecutionException ee = expectThrows(ExecutionException.class, queryFuture::get); + assertNotNull(ee.getCause()); + + return getTelemetrySnapshot(nodeName); + } + + /** + * Create search request for indices and get telemetry from it + */ + private CCSTelemetrySnapshot getTelemetryFromSearch(String... indices) throws ExecutionException, InterruptedException { + return getTelemetryFromSearch(makeSearchRequest(indices)); + } + + /** + * Search on all remotes + */ + public void testAllRemotesSearch() throws ExecutionException, InterruptedException { + Map testClusterInfo = setupClusters(); + String localIndex = (String) testClusterInfo.get("local.index"); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + + SearchRequest searchRequest = makeSearchRequest(localIndex, "*:" + remoteIndex); + boolean minimizeRoundtrips = TransportSearchAction.shouldMinimizeRoundtrips(searchRequest); + + String nodeName = cluster(LOCAL_CLUSTER).getRandomNodeName(); + assertResponse( + cluster(LOCAL_CLUSTER).client(nodeName) + .filterWithHeader(Map.of(Task.X_ELASTIC_PRODUCT_ORIGIN_HTTP_HEADER, "kibana")) + .search(searchRequest), + Assert::assertNotNull + ); + CCSTelemetrySnapshot telemetry = getTelemetrySnapshot(nodeName); + + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(1L)); + assertThat(telemetry.getFailureReasons().size(), equalTo(0)); + assertThat(telemetry.getTook().count(), equalTo(1L)); + assertThat(telemetry.getTookMrtTrue().count(), equalTo(minimizeRoundtrips ? 1L : 0L)); + assertThat(telemetry.getTookMrtFalse().count(), equalTo(minimizeRoundtrips ? 0L : 1L)); + assertThat(telemetry.getRemotesPerSearchAvg(), equalTo(2.0)); + assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L)); + assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(0L)); + assertThat(telemetry.getClientCounts().size(), equalTo(1)); + assertThat(telemetry.getClientCounts().get("kibana"), equalTo(1L)); + if (minimizeRoundtrips) { + assertThat(telemetry.getFeatureCounts().get(MRT_FEATURE), equalTo(1L)); + } else { + assertThat(telemetry.getFeatureCounts().get(MRT_FEATURE), equalTo(null)); + } + assertThat(telemetry.getFeatureCounts().get(ASYNC_FEATURE), equalTo(null)); + + var perCluster = telemetry.getByRemoteCluster(); + assertThat(perCluster.size(), equalTo(3)); + for (String clusterAlias : remoteClusterAlias()) { + var clusterTelemetry = perCluster.get(clusterAlias); + assertThat(clusterTelemetry.getCount(), equalTo(1L)); + assertThat(clusterTelemetry.getSkippedCount(), equalTo(0L)); + assertThat(clusterTelemetry.getTook().count(), equalTo(1L)); + } + + // another search + assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull); + telemetry = getTelemetrySnapshot(nodeName); + assertThat(telemetry.getTotalCount(), equalTo(2L)); + assertThat(telemetry.getSuccessCount(), equalTo(2L)); + assertThat(telemetry.getFailureReasons().size(), equalTo(0)); + assertThat(telemetry.getTook().count(), equalTo(2L)); + assertThat(telemetry.getTookMrtTrue().count(), equalTo(minimizeRoundtrips ? 2L : 0L)); + assertThat(telemetry.getTookMrtFalse().count(), equalTo(minimizeRoundtrips ? 0L : 2L)); + assertThat(telemetry.getRemotesPerSearchAvg(), equalTo(2.0)); + assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L)); + assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(0L)); + assertThat(telemetry.getClientCounts().size(), equalTo(1)); + assertThat(telemetry.getClientCounts().get("kibana"), equalTo(1L)); + perCluster = telemetry.getByRemoteCluster(); + assertThat(perCluster.size(), equalTo(3)); + for (String clusterAlias : remoteClusterAlias()) { + var clusterTelemetry = perCluster.get(clusterAlias); + assertThat(clusterTelemetry.getCount(), equalTo(2L)); + assertThat(clusterTelemetry.getSkippedCount(), equalTo(0L)); + assertThat(clusterTelemetry.getTook().count(), equalTo(2L)); + } + } + + /** + * Search on a specific remote + */ + public void testOneRemoteSearch() throws ExecutionException, InterruptedException { + Map testClusterInfo = setupClusters(); + String localIndex = (String) testClusterInfo.get("local.index"); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + + // Make request to cluster a + SearchRequest searchRequest = makeSearchRequest(localIndex, REMOTE1 + ":" + remoteIndex); + String nodeName = cluster(LOCAL_CLUSTER).getRandomNodeName(); + assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull); + CCSTelemetrySnapshot telemetry = getTelemetrySnapshot(nodeName); + var perCluster = telemetry.getByRemoteCluster(); + assertThat(perCluster.size(), equalTo(2)); + assertThat(perCluster.get(REMOTE1).getCount(), equalTo(1L)); + assertThat(perCluster.get(REMOTE1).getTook().count(), equalTo(1L)); + assertThat(perCluster.get(REMOTE2), equalTo(null)); + assertThat(telemetry.getClientCounts().size(), equalTo(0)); + + // Make request to cluster b + searchRequest = makeSearchRequest(localIndex, REMOTE2 + ":" + remoteIndex); + assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull); + telemetry = getTelemetrySnapshot(nodeName); + assertThat(telemetry.getTotalCount(), equalTo(2L)); + assertThat(telemetry.getSuccessCount(), equalTo(2L)); + perCluster = telemetry.getByRemoteCluster(); + assertThat(perCluster.size(), equalTo(3)); + assertThat(perCluster.get(REMOTE1).getCount(), equalTo(1L)); + assertThat(perCluster.get(REMOTE1).getTook().count(), equalTo(1L)); + assertThat(perCluster.get(REMOTE2).getCount(), equalTo(1L)); + assertThat(perCluster.get(REMOTE2).getTook().count(), equalTo(1L)); + } + + /** + * Local search should not produce any telemetry at all + */ + public void testLocalOnlySearch() throws ExecutionException, InterruptedException { + Map testClusterInfo = setupClusters(); + String localIndex = (String) testClusterInfo.get("local.index"); + + CCSTelemetrySnapshot telemetry = getTelemetryFromSearch(localIndex); + assertThat(telemetry.getTotalCount(), equalTo(0L)); + } + + /** + * Search on remotes only, without local index + */ + public void testRemoteOnlySearch() throws ExecutionException, InterruptedException { + Map testClusterInfo = setupClusters(); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + + CCSTelemetrySnapshot telemetry = getTelemetryFromSearch("*:" + remoteIndex); + var perCluster = telemetry.getByRemoteCluster(); + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(1L)); + assertThat(telemetry.getFailureReasons().size(), equalTo(0)); + assertThat(telemetry.getTook().count(), equalTo(1L)); + assertThat(perCluster.size(), equalTo(2)); + assertThat(telemetry.getClientCounts().size(), equalTo(0)); + assertThat(perCluster.get(REMOTE1).getCount(), equalTo(1L)); + assertThat(perCluster.get(REMOTE1).getSkippedCount(), equalTo(0L)); + assertThat(perCluster.get(REMOTE1).getTook().count(), equalTo(1L)); + assertThat(perCluster.get(REMOTE2).getCount(), equalTo(1L)); + assertThat(perCluster.get(REMOTE2).getSkippedCount(), equalTo(0L)); + assertThat(perCluster.get(REMOTE2).getTook().count(), equalTo(1L)); + } + + /** + * Count wildcard searches. Only wildcards in index names (not in cluster names) are counted. + */ + public void testWildcardSearch() throws ExecutionException, InterruptedException { + Map testClusterInfo = setupClusters(); + String localIndex = (String) testClusterInfo.get("local.index"); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + + SearchRequest searchRequest = makeSearchRequest(localIndex, "*:" + remoteIndex); + String nodeName = cluster(LOCAL_CLUSTER).getRandomNodeName(); + assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull); + CCSTelemetrySnapshot telemetry = getTelemetrySnapshot(nodeName); + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getFeatureCounts().get(WILDCARD_FEATURE), equalTo(null)); + + searchRequest = makeSearchRequest("*", REMOTE1 + ":" + remoteIndex); + assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull); + telemetry = getTelemetrySnapshot(nodeName); + assertThat(telemetry.getTotalCount(), equalTo(2L)); + assertThat(telemetry.getFeatureCounts().get(WILDCARD_FEATURE), equalTo(1L)); + + searchRequest = makeSearchRequest(localIndex, REMOTE2 + ":*"); + assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull); + telemetry = getTelemetrySnapshot(nodeName); + assertThat(telemetry.getTotalCount(), equalTo(3L)); + assertThat(telemetry.getFeatureCounts().get(WILDCARD_FEATURE), equalTo(2L)); + + // Wildcards in cluster name do not count + searchRequest = makeSearchRequest(localIndex, "*:" + remoteIndex); + assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull); + telemetry = getTelemetrySnapshot(nodeName); + assertThat(telemetry.getTotalCount(), equalTo(4L)); + assertThat(telemetry.getFeatureCounts().get(WILDCARD_FEATURE), equalTo(2L)); + + // Wildcard in the middle of the index name counts + searchRequest = makeSearchRequest(localIndex, REMOTE2 + ":rem*"); + assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull); + telemetry = getTelemetrySnapshot(nodeName); + assertThat(telemetry.getTotalCount(), equalTo(5L)); + assertThat(telemetry.getFeatureCounts().get(WILDCARD_FEATURE), equalTo(3L)); + + // Wildcard only counted once per search + searchRequest = makeSearchRequest("*", REMOTE1 + ":rem*", REMOTE2 + ":remote*"); + assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull); + telemetry = getTelemetrySnapshot(nodeName); + assertThat(telemetry.getTotalCount(), equalTo(6L)); + assertThat(telemetry.getFeatureCounts().get(WILDCARD_FEATURE), equalTo(4L)); + } + + /** + * Test complete search failure + */ + public void testFailedSearch() throws Exception { + Map testClusterInfo = setupClusters(); + String localIndex = (String) testClusterInfo.get("local.index"); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + + SearchRequest searchRequest = makeSearchRequest(localIndex, "*:" + remoteIndex); + // shardId -1 means to throw the Exception on all shards, so should result in complete search failure + ThrowingQueryBuilder queryBuilder = new ThrowingQueryBuilder(randomLong(), new IllegalStateException("index corrupted"), -1); + searchRequest.source(new SearchSourceBuilder().query(queryBuilder).size(10)); + searchRequest.allowPartialSearchResults(true); + + CCSTelemetrySnapshot telemetry = getTelemetryFromFailedSearch(searchRequest); + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(0L)); + assertThat(telemetry.getTook().count(), equalTo(0L)); + assertThat(telemetry.getTookMrtTrue().count(), equalTo(0L)); + assertThat(telemetry.getTookMrtFalse().count(), equalTo(0L)); + Map expectedFailures = Map.of(Result.UNKNOWN.getName(), 1L); + assertThat(telemetry.getFailureReasons(), equalTo(expectedFailures)); + } + + /** + * Search when all the remotes failed and skipped + */ + public void testSkippedAllRemotesSearch() throws Exception { + Map testClusterInfo = setupClusters(); + String localIndex = (String) testClusterInfo.get("local.index"); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + + SearchRequest searchRequest = makeSearchRequest(localIndex, "*:" + remoteIndex); + // throw Exception on all shards of remoteIndex, but not against localIndex + ThrowingQueryBuilder queryBuilder = new ThrowingQueryBuilder( + randomLong(), + new IllegalStateException("index corrupted"), + remoteIndex + ); + searchRequest.source(new SearchSourceBuilder().query(queryBuilder).size(10)); + searchRequest.allowPartialSearchResults(true); + + String nodeName = cluster(LOCAL_CLUSTER).getRandomNodeName(); + assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull); + + CCSTelemetrySnapshot telemetry = getTelemetrySnapshot(nodeName); + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(1L)); + // Note that this counts how many searches had skipped remotes, not how many remotes are skipped + assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(1L)); + // Still count the remote that failed + assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L)); + assertThat(telemetry.getTook().count(), equalTo(1L)); + // Each remote will have its skipped count bumped + var perCluster = telemetry.getByRemoteCluster(); + assertThat(perCluster.size(), equalTo(3)); + for (String remote : remoteClusterAlias()) { + assertThat(perCluster.get(remote).getCount(), equalTo(0L)); + assertThat(perCluster.get(remote).getSkippedCount(), equalTo(1L)); + assertThat(perCluster.get(remote).getTook().count(), equalTo(0L)); + } + } + + public void testSkippedOneRemoteSearch() throws Exception { + Map testClusterInfo = setupClusters(); + String localIndex = (String) testClusterInfo.get("local.index"); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + + // Remote1 will fail, Remote2 will just do nothing but it counts as success + SearchRequest searchRequest = makeSearchRequest(localIndex, REMOTE1 + ":" + remoteIndex, REMOTE2 + ":" + "nosuchindex*"); + // throw Exception on all shards of remoteIndex, but not against localIndex + ThrowingQueryBuilder queryBuilder = new ThrowingQueryBuilder( + randomLong(), + new IllegalStateException("index corrupted"), + remoteIndex + ); + searchRequest.source(new SearchSourceBuilder().query(queryBuilder).size(10)); + searchRequest.allowPartialSearchResults(true); + + String nodeName = cluster(LOCAL_CLUSTER).getRandomNodeName(); + assertResponse(cluster(LOCAL_CLUSTER).client(nodeName).search(searchRequest), Assert::assertNotNull); + + CCSTelemetrySnapshot telemetry = getTelemetrySnapshot(nodeName); + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(1L)); + // Note that this counts how many searches had skipped remotes, not how many remotes are skipped + assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(1L)); + // Still count the remote that failed + assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L)); + assertThat(telemetry.getTook().count(), equalTo(1L)); + // Each remote will have its skipped count bumped + var perCluster = telemetry.getByRemoteCluster(); + assertThat(perCluster.size(), equalTo(3)); + // This one is skipped + assertThat(perCluster.get(REMOTE1).getCount(), equalTo(0L)); + assertThat(perCluster.get(REMOTE1).getSkippedCount(), equalTo(1L)); + assertThat(perCluster.get(REMOTE1).getTook().count(), equalTo(0L)); + // This one is OK + assertThat(perCluster.get(REMOTE2).getCount(), equalTo(1L)); + assertThat(perCluster.get(REMOTE2).getSkippedCount(), equalTo(0L)); + assertThat(perCluster.get(REMOTE2).getTook().count(), equalTo(1L)); + } + + /** + * Test what happens if remote times out - it should be skipped + */ + public void testRemoteTimesOut() throws Exception { + Map testClusterInfo = setupClusters(); + String localIndex = (String) testClusterInfo.get("local.index"); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + + SearchRequest searchRequest = makeSearchRequest(localIndex, REMOTE1 + ":" + remoteIndex); + // This works only with minimize_roundtrips enabled, since otherwise timed out shards will be counted as + // partial failure, and we disable partial results.. + searchRequest.setCcsMinimizeRoundtrips(true); + + TimeValue searchTimeout = new TimeValue(500, TimeUnit.MILLISECONDS); + // query builder that will sleep for the specified amount of time in the query phase + SlowRunningQueryBuilder slowRunningQueryBuilder = new SlowRunningQueryBuilder(searchTimeout.millis() * 5, remoteIndex); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(slowRunningQueryBuilder).timeout(searchTimeout); + searchRequest.source(sourceBuilder); + + CCSTelemetrySnapshot telemetry = getTelemetryFromSearch(searchRequest); + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(1L)); + assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(1L)); + assertThat(telemetry.getRemotesPerSearchMax(), equalTo(1L)); + var perCluster = telemetry.getByRemoteCluster(); + assertThat(perCluster.size(), equalTo(2)); + assertThat(perCluster.get(REMOTE1).getCount(), equalTo(0L)); + assertThat(perCluster.get(REMOTE1).getSkippedCount(), equalTo(1L)); + assertThat(perCluster.get(REMOTE1).getTook().count(), equalTo(0L)); + assertThat(perCluster.get(REMOTE2), equalTo(null)); + } + + /** + * Test what happens if remote times out and there's no local - it should be skipped + */ + public void testRemoteOnlyTimesOut() throws Exception { + Map testClusterInfo = setupClusters(); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + + SearchRequest searchRequest = makeSearchRequest(REMOTE1 + ":" + remoteIndex); + // This works only with minimize_roundtrips enabled, since otherwise timed out shards will be counted as + // partial failure, and we disable partial results... + searchRequest.setCcsMinimizeRoundtrips(true); + + TimeValue searchTimeout = new TimeValue(100, TimeUnit.MILLISECONDS); + // query builder that will sleep for the specified amount of time in the query phase + SlowRunningQueryBuilder slowRunningQueryBuilder = new SlowRunningQueryBuilder(searchTimeout.millis() * 5, remoteIndex); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(slowRunningQueryBuilder).timeout(searchTimeout); + searchRequest.source(sourceBuilder); + + CCSTelemetrySnapshot telemetry = getTelemetryFromSearch(searchRequest); + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(1L)); + assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(1L)); + assertThat(telemetry.getRemotesPerSearchMax(), equalTo(1L)); + var perCluster = telemetry.getByRemoteCluster(); + assertThat(perCluster.size(), equalTo(1)); + assertThat(perCluster.get(REMOTE1).getCount(), equalTo(0L)); + assertThat(perCluster.get(REMOTE1).getSkippedCount(), equalTo(1L)); + assertThat(perCluster.get(REMOTE1).getTook().count(), equalTo(0L)); + assertThat(perCluster.get(REMOTE2), equalTo(null)); + } + + @SkipOverride(aliases = { REMOTE1 }) + public void testRemoteTimesOutFailure() throws Exception { + Map testClusterInfo = setupClusters(); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + + SearchRequest searchRequest = makeSearchRequest(REMOTE1 + ":" + remoteIndex); + + TimeValue searchTimeout = new TimeValue(100, TimeUnit.MILLISECONDS); + // query builder that will sleep for the specified amount of time in the query phase + SlowRunningQueryBuilder slowRunningQueryBuilder = new SlowRunningQueryBuilder(searchTimeout.millis() * 5, remoteIndex); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(slowRunningQueryBuilder).timeout(searchTimeout); + searchRequest.source(sourceBuilder); + + CCSTelemetrySnapshot telemetry = getTelemetryFromFailedSearch(searchRequest); + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(0L)); + // Failure is not skipping + assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(0L)); + // Still count the remote that failed + assertThat(telemetry.getRemotesPerSearchMax(), equalTo(1L)); + assertThat(telemetry.getTook().count(), equalTo(0L)); + Map expectedFailure = Map.of(Result.TIMEOUT.getName(), 1L); + assertThat(telemetry.getFailureReasons(), equalTo(expectedFailure)); + // No per-cluster data on total failure + assertThat(telemetry.getByRemoteCluster().size(), equalTo(0)); + } + + /** + * Search when all the remotes failed and not skipped + */ + @SkipOverride(aliases = { REMOTE1, REMOTE2 }) + public void testFailedAllRemotesSearch() throws Exception { + Map testClusterInfo = setupClusters(); + String localIndex = (String) testClusterInfo.get("local.index"); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + + SearchRequest searchRequest = makeSearchRequest(localIndex, "*:" + remoteIndex); + // throw Exception on all shards of remoteIndex, but not against localIndex + ThrowingQueryBuilder queryBuilder = new ThrowingQueryBuilder( + randomLong(), + new IllegalStateException("index corrupted"), + remoteIndex + ); + searchRequest.source(new SearchSourceBuilder().query(queryBuilder).size(10)); + + CCSTelemetrySnapshot telemetry = getTelemetryFromFailedSearch(searchRequest); + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(0L)); + // Failure is not skipping + assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(0L)); + // Still count the remote that failed + assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L)); + assertThat(telemetry.getTook().count(), equalTo(0L)); + Map expectedFailure = Map.of(Result.REMOTES_UNAVAILABLE.getName(), 1L); + assertThat(telemetry.getFailureReasons(), equalTo(expectedFailure)); + // No per-cluster data on total failure + assertThat(telemetry.getByRemoteCluster().size(), equalTo(0)); + } + + /** + * Test that we're still counting remote search even if remote cluster has no such index + */ + public void testRemoteHasNoIndex() throws Exception { + Map testClusterInfo = setupClusters(); + String localIndex = (String) testClusterInfo.get("local.index"); + + CCSTelemetrySnapshot telemetry = getTelemetryFromSearch(localIndex, REMOTE1 + ":" + "no_such_index*"); + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(1L)); + var perCluster = telemetry.getByRemoteCluster(); + assertThat(perCluster.size(), equalTo(2)); + assertThat(perCluster.get(REMOTE1).getCount(), equalTo(1L)); + assertThat(perCluster.get(REMOTE1).getTook().count(), equalTo(1L)); + assertThat(perCluster.get(REMOTE2), equalTo(null)); + } + + /** + * Test that we're still counting remote search even if remote cluster has no such index + */ + @SkipOverride(aliases = { REMOTE1 }) + public void testRemoteHasNoIndexFailure() throws Exception { + SearchRequest searchRequest = makeSearchRequest(REMOTE1 + ":no_such_index"); + CCSTelemetrySnapshot telemetry = getTelemetryFromFailedSearch(searchRequest); + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(0L)); + var perCluster = telemetry.getByRemoteCluster(); + assertThat(perCluster.size(), equalTo(0)); + Map expectedFailure = Map.of(Result.NOT_FOUND.getName(), 1L); + assertThat(telemetry.getFailureReasons(), equalTo(expectedFailure)); + } + + public void testPITSearch() throws ExecutionException, InterruptedException { + Map testClusterInfo = setupClusters(); + String localIndex = (String) testClusterInfo.get("local.index"); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + + OpenPointInTimeRequest openPITRequest = new OpenPointInTimeRequest(localIndex, "*:" + remoteIndex).keepAlive( + TimeValue.timeValueMinutes(5) + ); + String nodeName = cluster(LOCAL_CLUSTER).getRandomNodeName(); + var client = cluster(LOCAL_CLUSTER).client(nodeName); + BytesReference pitID = client.execute(TransportOpenPointInTimeAction.TYPE, openPITRequest).actionGet().getPointInTimeId(); + SearchRequest searchRequest = new SearchRequest().source( + new SearchSourceBuilder().pointInTimeBuilder(new PointInTimeBuilder(pitID).setKeepAlive(TimeValue.timeValueMinutes(5))) + .sort("@timestamp") + .size(10) + ); + searchRequest.setCcsMinimizeRoundtrips(randomBoolean()); + + assertResponse(client.search(searchRequest), Assert::assertNotNull); + // do it again + assertResponse(client.search(searchRequest), Assert::assertNotNull); + client.execute(TransportClosePointInTimeAction.TYPE, new ClosePointInTimeRequest(pitID)).actionGet(); + CCSTelemetrySnapshot telemetry = getTelemetrySnapshot(nodeName); + + assertThat(telemetry.getTotalCount(), equalTo(2L)); + assertThat(telemetry.getSuccessCount(), equalTo(2L)); + } + + public void testCompoundRetrieverSearch() throws ExecutionException, InterruptedException { + RetrieverBuilder compoundRetriever = new MinimalCompoundRetrieverIT.CompoundRetriever(Collections.emptyList()); + Map testClusterInfo = setupClusters(); + String localIndex = (String) testClusterInfo.get("local.index"); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + + SearchRequest searchRequest = makeSearchRequest(localIndex, "*:" + remoteIndex); + searchRequest.source(new SearchSourceBuilder().retriever(compoundRetriever)); + + CCSTelemetrySnapshot telemetry = getTelemetryFromSearch(searchRequest); + assertThat(telemetry.getTotalCount(), equalTo(1L)); + assertThat(telemetry.getSuccessCount(), equalTo(1L)); + } + + private CCSTelemetrySnapshot getTelemetrySnapshot(String nodeName) { + var usage = cluster(LOCAL_CLUSTER).getInstance(UsageService.class, nodeName); + return usage.getCcsUsageHolder().getCCSTelemetrySnapshot(); + } + + private Map setupClusters() { + String localIndex = "demo"; + int numShardsLocal = randomIntBetween(2, 10); + Settings localSettings = indexSettings(numShardsLocal, randomIntBetween(0, 1)).build(); + assertAcked( + client(LOCAL_CLUSTER).admin() + .indices() + .prepareCreate(localIndex) + .setSettings(localSettings) + .setMapping("@timestamp", "type=date", "f", "type=text") + ); + indexDocs(client(LOCAL_CLUSTER), localIndex); + + String remoteIndex = "prod"; + int numShardsRemote = randomIntBetween(2, 10); + for (String clusterAlias : remoteClusterAlias()) { + final InternalTestCluster remoteCluster = cluster(clusterAlias); + remoteCluster.ensureAtLeastNumDataNodes(randomIntBetween(1, 3)); + assertAcked( + client(clusterAlias).admin() + .indices() + .prepareCreate(remoteIndex) + .setSettings(indexSettings(numShardsRemote, randomIntBetween(0, 1))) + .setMapping("@timestamp", "type=date", "f", "type=text") + ); + assertFalse( + client(clusterAlias).admin() + .cluster() + .prepareHealth(remoteIndex) + .setWaitForYellowStatus() + .setTimeout(TimeValue.timeValueSeconds(10)) + .get() + .isTimedOut() + ); + indexDocs(client(clusterAlias), remoteIndex); + } + + Map clusterInfo = new HashMap<>(); + clusterInfo.put("local.index", localIndex); + clusterInfo.put("remote.index", remoteIndex); + return clusterInfo; + } + + private int indexDocs(Client client, String index) { + int numDocs = between(5, 20); + for (int i = 0; i < numDocs; i++) { + client.prepareIndex(index).setSource("f", "v", "@timestamp", randomNonNegativeLong()).get(); + } + client.admin().indices().prepareRefresh(index).get(); + return numDocs; + } + + /** + * Annotation to mark specific cluster in a test as not to be skipped when unavailable + */ + @Retention(RetentionPolicy.RUNTIME) + @Target(ElementType.METHOD) + @interface SkipOverride { + String[] aliases(); + } + + /** + * Test rule to process skip annotations + */ + static class SkipUnavailableRule implements TestRule { + private final Map skipMap; + + SkipUnavailableRule(String... clusterAliases) { + this.skipMap = Arrays.stream(clusterAliases).collect(Collectors.toMap(Function.identity(), alias -> true)); + } + + public Map getMap() { + return skipMap; + } + + @Override + public Statement apply(Statement base, Description description) { + // Check for annotation named "SkipOverride" and set the overrides accordingly + var aliases = description.getAnnotation(SkipOverride.class); + if (aliases != null) { + for (String alias : aliases.aliases()) { + skipMap.put(alias, false); + } + } + return base; + } + + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java index 076158ee22037..cc272042d5384 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java @@ -89,6 +89,7 @@ import static org.hamcrest.Matchers.array; import static org.hamcrest.Matchers.arrayContainingInAnyOrder; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasKey; @@ -711,6 +712,63 @@ public void testCancel() throws Exception { } } + public void testIndexMode() throws Exception { + Map indexModes = new HashMap<>(); + // metrics + { + final String metricsMapping = """ + { + "properties": { + "@timestamp": { "type": "date" }, + "hostname": { "type": "keyword", "time_series_dimension": true }, + "request_count" : { "type" : "long", "time_series_metric" : "counter" }, + "cluster": {"type": "keyword"} + } + } + """; + Settings settings = Settings.builder().put("mode", "time_series").putList("routing_path", List.of("hostname")).build(); + int numIndices = between(1, 5); + for (int i = 0; i < numIndices; i++) { + assertAcked(indicesAdmin().prepareCreate("test_metrics_" + i).setSettings(settings).setMapping(metricsMapping).get()); + indexModes.put("test_metrics_" + i, IndexMode.TIME_SERIES); + assertAcked(indicesAdmin().prepareCreate("test_old_metrics_" + i).setMapping(metricsMapping).get()); + indexModes.put("test_old_metrics_" + i, IndexMode.STANDARD); + } + } + // logsdb + { + final String logsMapping = """ + { + "properties": { + "@timestamp": { "type": "date" }, + "hostname": { "type": "keyword"}, + "request_count" : { "type" : "long"}, + "cluster": {"type": "keyword"} + } + } + """; + Settings settings = Settings.builder().put("mode", "logsdb").build(); + int numIndices = between(1, 5); + for (int i = 0; i < numIndices; i++) { + assertAcked(indicesAdmin().prepareCreate("test_logs_" + i).setSettings(settings).setMapping(logsMapping).get()); + indexModes.put("test_logs_" + i, IndexMode.LOGSDB); + assertAcked(indicesAdmin().prepareCreate("test_old_logs_" + i).setMapping(logsMapping).get()); + indexModes.put("test_old_logs_" + i, IndexMode.STANDARD); + } + } + FieldCapabilitiesRequest request = new FieldCapabilitiesRequest(); + request.setMergeResults(false); + request.indices("test_*"); + request.fields(randomFrom("*", "@timestamp", "host*")); + var resp = client().fieldCaps(request).get(); + assertThat(resp.getFailures(), empty()); + Map actualIndexModes = new HashMap<>(); + for (var indexResp : resp.getIndexResponses()) { + actualIndexModes.put(indexResp.getIndexName(), indexResp.getIndexMode()); + } + assertThat(actualIndexModes, equalTo(indexModes)); + } + private void assertIndices(FieldCapabilitiesResponse response, String... indices) { assertNotNull(response.getIndices()); Arrays.sort(indices); @@ -859,7 +917,7 @@ protected String contentType() { @Override public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - return new StringStoredFieldFieldLoader(fullPath(), leafName(), null) { + return new StringStoredFieldFieldLoader(fullPath(), leafName()) { @Override protected void write(XContentBuilder b, Object value) throws IOException { BytesRef ref = (BytesRef) value; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java index ee60888d7a0a8..c59fc0f68c4d4 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/functionscore/ExplainableScriptIT.java @@ -73,6 +73,11 @@ public boolean needs_score() { return false; } + @Override + public boolean needs_termStats() { + return false; + } + @Override public ScoreScript newInstance(DocReader docReader) { return new MyScript(params1, lookup, ((DocValuesDocReader) docReader).getLeafReaderContext()); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/query/SearchQueryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/query/SearchQueryIT.java index 384395bcb78e7..0a30de1bb3741 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/query/SearchQueryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/query/SearchQueryIT.java @@ -1629,14 +1629,8 @@ public void testRangeQueryWithTimeZone() throws Exception { * Test range with a custom locale, e.g. "de" in this case. Documents here mention the day of week * as "Mi" for "Mittwoch (Wednesday" and "Do" for "Donnerstag (Thursday)" and the month in the query * as "Dez" for "Dezember (December)". - * Note: this test currently needs the JVM arg `-Djava.locale.providers=SPI,COMPAT` to be set. - * When running with gradle this is done implicitly through the BuildPlugin, but when running from - * an IDE this might need to be set manually in the run configuration. See also CONTRIBUTING.md section - * on "Configuring IDEs And Running Tests". */ public void testRangeQueryWithLocaleMapping() throws Exception { - assert ("SPI,COMPAT".equals(System.getProperty("java.locale.providers"))) : "`-Djava.locale.providers=SPI,COMPAT` needs to be set"; - assertAcked( prepareCreate("test").setMapping( jsonBuilder().startObject() @@ -1644,7 +1638,7 @@ public void testRangeQueryWithLocaleMapping() throws Exception { .startObject("date_field") .field("type", "date") .field("format", "E, d MMM yyyy HH:mm:ss Z") - .field("locale", "de") + .field("locale", "fr") .endObject() .endObject() .endObject() @@ -1653,19 +1647,19 @@ public void testRangeQueryWithLocaleMapping() throws Exception { indexRandom( true, - prepareIndex("test").setId("1").setSource("date_field", "Mi, 06 Dez 2000 02:55:00 -0800"), - prepareIndex("test").setId("2").setSource("date_field", "Do, 07 Dez 2000 02:55:00 -0800") + prepareIndex("test").setId("1").setSource("date_field", "mer., 6 déc. 2000 02:55:00 -0800"), + prepareIndex("test").setId("2").setSource("date_field", "jeu., 7 déc. 2000 02:55:00 -0800") ); assertHitCount( prepareSearch("test").setQuery( - QueryBuilders.rangeQuery("date_field").gte("Di, 05 Dez 2000 02:55:00 -0800").lte("Do, 07 Dez 2000 00:00:00 -0800") + QueryBuilders.rangeQuery("date_field").gte("mar., 5 déc. 2000 02:55:00 -0800").lte("jeu., 7 déc. 2000 00:00:00 -0800") ), 1L ); assertHitCount( prepareSearch("test").setQuery( - QueryBuilders.rangeQuery("date_field").gte("Di, 05 Dez 2000 02:55:00 -0800").lte("Fr, 08 Dez 2000 00:00:00 -0800") + QueryBuilders.rangeQuery("date_field").gte("mar., 5 déc. 2000 02:55:00 -0800").lte("ven., 8 déc. 2000 00:00:00 -0800") ), 2L ); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/MinimalCompoundRetrieverIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/MinimalCompoundRetrieverIT.java new file mode 100644 index 0000000000000..8c65d28711c1b --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/MinimalCompoundRetrieverIT.java @@ -0,0 +1,198 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.retriever; + +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryRewriteContext; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.test.AbstractMultiClustersTestCase; +import org.elasticsearch.test.InternalTestCluster; +import org.elasticsearch.transport.RemoteClusterAware; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; +import static org.hamcrest.Matchers.equalTo; + +public class MinimalCompoundRetrieverIT extends AbstractMultiClustersTestCase { + + // CrossClusterSearchIT + private static final String REMOTE_CLUSTER = "cluster_a"; + + @Override + protected Collection remoteClusterAlias() { + return List.of(REMOTE_CLUSTER); + } + + @Override + protected Map skipUnavailableForRemoteClusters() { + return Map.of(REMOTE_CLUSTER, randomBoolean()); + } + + @Override + protected boolean reuseClusters() { + return false; + } + + public void testSimpleSearch() throws ExecutionException, InterruptedException { + RetrieverBuilder compoundRetriever = new CompoundRetriever(Collections.emptyList()); + Map testClusterInfo = setupTwoClusters(); + String localIndex = (String) testClusterInfo.get("local.index"); + String remoteIndex = (String) testClusterInfo.get("remote.index"); + SearchRequest searchRequest = new SearchRequest(localIndex, REMOTE_CLUSTER + ":" + remoteIndex); + searchRequest.source(new SearchSourceBuilder().retriever(compoundRetriever)); + assertResponse(client(LOCAL_CLUSTER).search(searchRequest), response -> { + assertNotNull(response); + + SearchResponse.Clusters clusters = response.getClusters(); + assertFalse("search cluster results should NOT be marked as partial", clusters.hasPartialResults()); + assertThat(clusters.getTotal(), equalTo(2)); + assertThat(clusters.getClusterStateCount(SearchResponse.Cluster.Status.SUCCESSFUL), equalTo(2)); + assertThat(clusters.getClusterStateCount(SearchResponse.Cluster.Status.SKIPPED), equalTo(0)); + assertThat(clusters.getClusterStateCount(SearchResponse.Cluster.Status.RUNNING), equalTo(0)); + assertThat(clusters.getClusterStateCount(SearchResponse.Cluster.Status.PARTIAL), equalTo(0)); + assertThat(clusters.getClusterStateCount(SearchResponse.Cluster.Status.FAILED), equalTo(0)); + assertThat(response.getHits().getTotalHits().value, equalTo(testClusterInfo.get("total_docs"))); + }); + } + + private Map setupTwoClusters() { + int totalDocs = 0; + String localIndex = "demo"; + int numShardsLocal = randomIntBetween(2, 10); + Settings localSettings = indexSettings(numShardsLocal, randomIntBetween(0, 1)).build(); + assertAcked( + client(LOCAL_CLUSTER).admin() + .indices() + .prepareCreate(localIndex) + .setSettings(localSettings) + .setMapping("some_field", "type=keyword") + ); + totalDocs += indexDocs(client(LOCAL_CLUSTER), localIndex); + + String remoteIndex = "prod"; + int numShardsRemote = randomIntBetween(2, 10); + final InternalTestCluster remoteCluster = cluster(REMOTE_CLUSTER); + remoteCluster.ensureAtLeastNumDataNodes(randomIntBetween(1, 3)); + assertAcked( + client(REMOTE_CLUSTER).admin() + .indices() + .prepareCreate(remoteIndex) + .setSettings(indexSettings(numShardsRemote, randomIntBetween(0, 1))) + .setMapping("some_field", "type=keyword") + ); + assertFalse( + client(REMOTE_CLUSTER).admin() + .cluster() + .prepareHealth(remoteIndex) + .setWaitForYellowStatus() + .setTimeout(TimeValue.timeValueSeconds(10)) + .get() + .isTimedOut() + ); + totalDocs += indexDocs(client(REMOTE_CLUSTER), remoteIndex); + + String skipUnavailableKey = Strings.format("cluster.remote.%s.skip_unavailable", REMOTE_CLUSTER); + Setting skipUnavailableSetting = cluster(REMOTE_CLUSTER).clusterService().getClusterSettings().get(skipUnavailableKey); + boolean skipUnavailable = (boolean) cluster(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY).clusterService() + .getClusterSettings() + .get(skipUnavailableSetting); + + Map clusterInfo = new HashMap<>(); + clusterInfo.put("local.num_shards", numShardsLocal); + clusterInfo.put("local.index", localIndex); + clusterInfo.put("remote.num_shards", numShardsRemote); + clusterInfo.put("remote.index", remoteIndex); + clusterInfo.put("remote.skip_unavailable", skipUnavailable); + clusterInfo.put("total_docs", (long) totalDocs); + return clusterInfo; + } + + private int indexDocs(Client client, String index) { + int numDocs = between(500, 1200); + for (int i = 0; i < numDocs; i++) { + client.prepareIndex(index).setSource("some_field", i).get(); + } + client.admin().indices().prepareRefresh(index).get(); + return numDocs; + } + + public static class CompoundRetriever extends RetrieverBuilder { + + private final List sources; + + public CompoundRetriever(List sources) { + this.sources = sources; + } + + @Override + public boolean isCompound() { + return true; + } + + @Override + public QueryBuilder topDocsQuery() { + throw new UnsupportedOperationException("should not be called"); + } + + @Override + public RetrieverBuilder rewrite(QueryRewriteContext ctx) throws IOException { + if (ctx.getPointInTimeBuilder() == null) { + throw new IllegalStateException("PIT is required"); + } + if (sources.isEmpty()) { + StandardRetrieverBuilder standardRetrieverBuilder = new StandardRetrieverBuilder(); + standardRetrieverBuilder.queryBuilder = new MatchAllQueryBuilder(); + return standardRetrieverBuilder; + } + return sources.get(0); + } + + @Override + public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) { + throw new UnsupportedOperationException("should not be called"); + } + + @Override + public String getName() { + return "compound_retriever"; + } + + @Override + protected void doToXContent(XContentBuilder builder, Params params) throws IOException { + // no-op + } + + @Override + protected boolean doEquals(Object o) { + return false; + } + + @Override + protected int doHashCode() { + return 0; + } + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RankDocRetrieverBuilderIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RankDocRetrieverBuilderIT.java new file mode 100644 index 0000000000000..fa4cafc66c822 --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RankDocRetrieverBuilderIT.java @@ -0,0 +1,755 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.retriever; + +import org.apache.lucene.search.TotalHits; +import org.apache.lucene.search.join.ScoreMode; +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.search.MultiSearchRequest; +import org.elasticsearch.action.search.MultiSearchResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.TransportMultiSearchAction; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.Maps; +import org.elasticsearch.index.query.InnerHitBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.QueryRewriteContext; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.MockSearchService; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.builder.PointInTimeBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.collapse.CollapseBuilder; +import org.elasticsearch.search.rank.RankDoc; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.NestedSortBuilder; +import org.elasticsearch.search.sort.ScoreSortBuilder; +import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; +import org.junit.Before; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; +import static org.hamcrest.Matchers.equalTo; + +public class RankDocRetrieverBuilderIT extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return List.of(MockSearchService.TestPlugin.class); + } + + public record RetrieverSource(RetrieverBuilder retriever, SearchSourceBuilder source) {} + + private static String INDEX = "test_index"; + private static final String ID_FIELD = "_id"; + private static final String DOC_FIELD = "doc"; + private static final String TEXT_FIELD = "text"; + private static final String VECTOR_FIELD = "vector"; + private static final String TOPIC_FIELD = "topic"; + private static final String LAST_30D_FIELD = "views.last30d"; + private static final String ALL_TIME_FIELD = "views.all"; + + @Before + public void setup() throws Exception { + String mapping = """ + { + "properties": { + "vector": { + "type": "dense_vector", + "dims": 3, + "element_type": "float", + "index": true, + "similarity": "l2_norm", + "index_options": { + "type": "hnsw" + } + }, + "text": { + "type": "text" + }, + "doc": { + "type": "keyword" + }, + "topic": { + "type": "keyword" + }, + "views": { + "type": "nested", + "properties": { + "last30d": { + "type": "integer" + }, + "all": { + "type": "integer" + } + } + } + } + } + """; + createIndex(INDEX, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).build()); + admin().indices().preparePutMapping(INDEX).setSource(mapping, XContentType.JSON).get(); + indexDoc( + INDEX, + "doc_1", + DOC_FIELD, + "doc_1", + TOPIC_FIELD, + "technology", + TEXT_FIELD, + "the quick brown fox jumps over the lazy dog", + LAST_30D_FIELD, + 100 + ); + indexDoc( + INDEX, + "doc_2", + DOC_FIELD, + "doc_2", + TOPIC_FIELD, + "astronomy", + TEXT_FIELD, + "you know, for Search!", + VECTOR_FIELD, + new float[] { 1.0f, 2.0f, 3.0f }, + LAST_30D_FIELD, + 3 + ); + indexDoc(INDEX, "doc_3", DOC_FIELD, "doc_3", TOPIC_FIELD, "technology", VECTOR_FIELD, new float[] { 6.0f, 6.0f, 6.0f }); + indexDoc( + INDEX, + "doc_4", + DOC_FIELD, + "doc_4", + TOPIC_FIELD, + "technology", + TEXT_FIELD, + "aardvark is a really awesome animal, but not very quick", + ALL_TIME_FIELD, + 100, + LAST_30D_FIELD, + 40 + ); + indexDoc(INDEX, "doc_5", DOC_FIELD, "doc_5", TOPIC_FIELD, "science", TEXT_FIELD, "irrelevant stuff"); + indexDoc( + INDEX, + "doc_6", + DOC_FIELD, + "doc_6", + TEXT_FIELD, + "quick quick quick quick search", + VECTOR_FIELD, + new float[] { 10.0f, 30.0f, 100.0f }, + LAST_30D_FIELD, + 15 + ); + indexDoc( + INDEX, + "doc_7", + DOC_FIELD, + "doc_7", + TOPIC_FIELD, + "biology", + TEXT_FIELD, + "dog", + VECTOR_FIELD, + new float[] { 3.0f, 3.0f, 3.0f }, + ALL_TIME_FIELD, + 1000 + ); + refresh(INDEX); + } + + public void testRankDocsRetrieverBasicWithPagination() { + final int rankWindowSize = 100; + SearchSourceBuilder source = new SearchSourceBuilder(); + StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); + // this one retrieves docs 1, 4, and 6 + standard0.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.queryStringQuery("quick").defaultField(TEXT_FIELD)) + .boost(10L); + StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); + // this one retrieves docs 2 and 6 due to prefilter + standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L); + standard1.preFilterQueryBuilders.add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); + // this one retrieves docs 7, 2, 3, and 6 + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( + VECTOR_FIELD, + new float[] { 3.0f, 3.0f, 3.0f }, + null, + 10, + 100, + null + ); + // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and + // resolves ties based on actual score, rank, and then the doc (we're forcing 1 shard for consistent results) + // so ideal rank would be: 6, 2, 1, 4, 7, 3 and with pagination, we'd just omit the first result + source.retriever( + new CompoundRetrieverWithRankDocs( + rankWindowSize, + Arrays.asList( + new RetrieverSource(standard0, null), + new RetrieverSource(standard1, null), + new RetrieverSource(knnRetrieverBuilder, null) + ) + ) + ); + // include some pagination as well + source.from(1); + SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); + ElasticsearchAssertions.assertResponse(req, resp -> { + assertNull(resp.pointInTimeId()); + assertNotNull(resp.getHits().getTotalHits()); + assertThat(resp.getHits().getTotalHits().value, equalTo(6L)); + assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); + assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_2")); + assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_1")); + assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_4")); + assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_7")); + assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_3")); + }); + } + + public void testRankDocsRetrieverWithAggs() { + // same as above, but we only want to bring back the top result from each subsearch + // so that would be 1, 2, and 7 + // and final rank would be (based on score): 2, 1, 7 + // aggs should still account for the same docs as the testRankDocsRetriever test, i.e. all but doc_5 + final int rankWindowSize = 1; + SearchSourceBuilder source = new SearchSourceBuilder(); + StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); + // this one retrieves docs 1, 4, and 6 + standard0.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.queryStringQuery("quick").defaultField(TEXT_FIELD)) + .boost(10L); + StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); + // this one retrieves docs 2 and 6 due to prefilter + standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L); + standard1.preFilterQueryBuilders.add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); + // this one retrieves docs 7, 2, 3, and 6 + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( + VECTOR_FIELD, + new float[] { 3.0f, 3.0f, 3.0f }, + null, + 10, + 100, + null + ); + source.retriever( + new CompoundRetrieverWithRankDocs( + rankWindowSize, + Arrays.asList( + new RetrieverSource(standard0, null), + new RetrieverSource(standard1, null), + new RetrieverSource(knnRetrieverBuilder, null) + ) + ) + ); + source.aggregation(new TermsAggregationBuilder("topic").field(TOPIC_FIELD)); + SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); + ElasticsearchAssertions.assertResponse(req, resp -> { + assertNull(resp.pointInTimeId()); + assertNotNull(resp.getHits().getTotalHits()); + assertThat(resp.getHits().getTotalHits().value, equalTo(1L)); + assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); + assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_2")); + assertNotNull(resp.getAggregations()); + assertNotNull(resp.getAggregations().get("topic")); + Terms terms = resp.getAggregations().get("topic"); + // doc_3 is not part of the final aggs computation as it is only retrieved through the knn retriever + // and is outside of the rank window + assertThat(terms.getBucketByKey("technology").getDocCount(), equalTo(2L)); + assertThat(terms.getBucketByKey("astronomy").getDocCount(), equalTo(1L)); + assertThat(terms.getBucketByKey("biology").getDocCount(), equalTo(1L)); + }); + } + + public void testRankDocsRetrieverWithCollapse() { + final int rankWindowSize = 100; + SearchSourceBuilder source = new SearchSourceBuilder(); + StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); + // this one retrieves docs 1, 4, and 6 + standard0.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.queryStringQuery("quick").defaultField(TEXT_FIELD)) + .boost(10L); + StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); + // this one retrieves docs 2 and 6 due to prefilter + standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L); + standard1.preFilterQueryBuilders.add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); + // this one retrieves docs 7, 2, 3, and 6 + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( + VECTOR_FIELD, + new float[] { 3.0f, 3.0f, 3.0f }, + null, + 10, + 100, + null + ); + // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and + // resolves ties based on actual score, rank, and then the doc (we're forcing 1 shard for consistent results) + // so ideal rank would be: 6, 2, 1, 4, 7, 3 + // with collapsing on topic field we would have 6, 2, 1, 7 + source.retriever( + new CompoundRetrieverWithRankDocs( + rankWindowSize, + Arrays.asList( + new RetrieverSource(standard0, null), + new RetrieverSource(standard1, null), + new RetrieverSource(knnRetrieverBuilder, null) + ) + ) + ); + source.collapse( + new CollapseBuilder(TOPIC_FIELD).setInnerHits( + new InnerHitBuilder("a").addSort(new FieldSortBuilder(DOC_FIELD).order(SortOrder.DESC)).setSize(10) + ) + ); + source.fetchField(TOPIC_FIELD); + SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); + ElasticsearchAssertions.assertResponse(req, resp -> { + assertNull(resp.pointInTimeId()); + assertNotNull(resp.getHits().getTotalHits()); + assertThat(resp.getHits().getTotalHits().value, equalTo(6L)); + assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); + assertThat(resp.getHits().getHits().length, equalTo(4)); + assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_6")); + assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_2")); + assertThat(resp.getHits().getAt(1).field(TOPIC_FIELD).getValue().toString(), equalTo("astronomy")); + assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_1")); + assertThat(resp.getHits().getAt(2).field(TOPIC_FIELD).getValue().toString(), equalTo("technology")); + assertThat(resp.getHits().getAt(2).getInnerHits().get("a").getHits().length, equalTo(3)); + assertThat(resp.getHits().getAt(2).getInnerHits().get("a").getAt(0).getId(), equalTo("doc_4")); + assertThat(resp.getHits().getAt(2).getInnerHits().get("a").getAt(1).getId(), equalTo("doc_3")); + assertThat(resp.getHits().getAt(2).getInnerHits().get("a").getAt(2).getId(), equalTo("doc_1")); + assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_7")); + assertThat(resp.getHits().getAt(3).field(TOPIC_FIELD).getValue().toString(), equalTo("biology")); + }); + } + + public void testRankDocsRetrieverWithCollapseAndAggs() { + // same as above, but we only want to bring back the top result from each subsearch + // so that would be 1, 2, and 7 + // and final rank would be (based on score): 2, 1, 7 + // aggs should still account for the same docs as the testRankDocsRetriever test, i.e. all but doc_5 + final int rankWindowSize = 10; + SearchSourceBuilder source = new SearchSourceBuilder(); + StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); + // this one retrieves docs 1 and 6 as doc_4 is collapsed to doc_1 + standard0.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.queryStringQuery("quick").defaultField(TEXT_FIELD)) + .boost(10L); + standard0.collapseBuilder = new CollapseBuilder(TOPIC_FIELD).setInnerHits( + new InnerHitBuilder("a").addSort(new FieldSortBuilder(DOC_FIELD).order(SortOrder.DESC)).setSize(10) + ); + StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); + // this one retrieves docs 2 and 6 due to prefilter + standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L); + standard1.preFilterQueryBuilders.add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); + // this one retrieves docs 7, 2, 3, and 6 + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( + VECTOR_FIELD, + new float[] { 3.0f, 3.0f, 3.0f }, + null, + 10, + 100, + null + ); + // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and + // resolves ties based on actual score, rank, and then the doc (we're forcing 1 shard for consistent results) + // so ideal rank would be: 6, 2, 1, 4, 7, 3 + source.retriever( + new CompoundRetrieverWithRankDocs( + rankWindowSize, + Arrays.asList( + new RetrieverSource(standard0, null), + new RetrieverSource(standard1, null), + new RetrieverSource(knnRetrieverBuilder, null) + ) + ) + ); + source.aggregation(new TermsAggregationBuilder("topic").field(TOPIC_FIELD)); + SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); + ElasticsearchAssertions.assertResponse(req, resp -> { + assertNull(resp.pointInTimeId()); + assertNotNull(resp.getHits().getTotalHits()); + assertThat(resp.getHits().getTotalHits().value, equalTo(5L)); + assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); + assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_6")); + assertNotNull(resp.getAggregations()); + assertNotNull(resp.getAggregations().get("topic")); + Terms terms = resp.getAggregations().get("topic"); + // doc_3 is not part of the final aggs computation as it is only retrieved through the knn retriever + // and is outside of the rank window + assertThat(terms.getBucketByKey("technology").getDocCount(), equalTo(3L)); + assertThat(terms.getBucketByKey("astronomy").getDocCount(), equalTo(1L)); + assertThat(terms.getBucketByKey("biology").getDocCount(), equalTo(1L)); + }); + } + + public void testRankDocsRetrieverWithNestedQuery() { + final int rankWindowSize = 100; + SearchSourceBuilder source = new SearchSourceBuilder(); + StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); + // this one retrieves docs 1, 4, and 6 + standard0.queryBuilder = QueryBuilders.nestedQuery("views", QueryBuilders.rangeQuery(LAST_30D_FIELD).gt(10L), ScoreMode.Avg) + .innerHit(new InnerHitBuilder("a").addSort(new FieldSortBuilder(DOC_FIELD).order(SortOrder.DESC)).setSize(10)); + StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); + // this one retrieves docs 2 and 6 due to prefilter + standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L); + standard1.preFilterQueryBuilders.add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); + // this one retrieves docs 7, 2, 3, and 6 + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( + VECTOR_FIELD, + new float[] { 3.0f, 3.0f, 3.0f }, + null, + 10, + 100, + null + ); + // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and + // resolves ties based on actual score, rank, and then the doc (we're forcing 1 shard for consistent results) + // so ideal rank would be: 6, 2, 1, 4, 3, 7 + source.retriever( + new CompoundRetrieverWithRankDocs( + rankWindowSize, + Arrays.asList( + new RetrieverSource(standard0, null), + new RetrieverSource(standard1, null), + new RetrieverSource(knnRetrieverBuilder, null) + ) + ) + ); + source.fetchField(TOPIC_FIELD); + SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); + ElasticsearchAssertions.assertResponse(req, resp -> { + assertNull(resp.pointInTimeId()); + assertNotNull(resp.getHits().getTotalHits()); + assertThat(resp.getHits().getTotalHits().value, equalTo(6L)); + assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); + assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_6")); + assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_2")); + assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_1")); + assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_7")); + assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_4")); + assertThat(resp.getHits().getAt(5).getId(), equalTo("doc_3")); + }); + } + + public void testRankDocsRetrieverMultipleCompoundRetrievers() { + final int rankWindowSize = 100; + SearchSourceBuilder source = new SearchSourceBuilder(); + StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); + // this one retrieves docs 1, 4, and 6 + standard0.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.queryStringQuery("quick").defaultField(TEXT_FIELD)) + .boost(10L); + StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); + // this one retrieves docs 2 and 6 due to prefilter + standard1.queryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.termsQuery(ID_FIELD, "doc_2", "doc_3", "doc_6")).boost(20L); + standard1.preFilterQueryBuilders.add(QueryBuilders.queryStringQuery("search").defaultField(TEXT_FIELD)); + // this one retrieves docs 7, 2, 3, and 6 + KnnRetrieverBuilder knnRetrieverBuilder = new KnnRetrieverBuilder( + VECTOR_FIELD, + new float[] { 3.0f, 3.0f, 3.0f }, + null, + 10, + 100, + null + ); + // the compound retriever here produces a score for a doc based on the percentage of the queries that it was matched on and + // resolves ties based on actual score, rank, and then the doc (we're forcing 1 shard for consistent results) + // so ideal rank would be: 6, 2, 1, 4, 7, 3 + CompoundRetrieverWithRankDocs compoundRetriever1 = new CompoundRetrieverWithRankDocs( + rankWindowSize, + Arrays.asList( + new RetrieverSource(standard0, null), + new RetrieverSource(standard1, null), + new RetrieverSource(knnRetrieverBuilder, null) + ) + ); + // simple standard retriever that would have the doc_4 as its first (and only) result + StandardRetrieverBuilder standard2 = new StandardRetrieverBuilder(); + standard2.queryBuilder = QueryBuilders.queryStringQuery("aardvark").defaultField(TEXT_FIELD); + + // combining the two retrievers would bring doc_4 at the top as it would be the only one present in both doc sets + // the rest of the docs would be sorted based on their ranks as they have the same score (1/2) + source.retriever( + new CompoundRetrieverWithRankDocs( + rankWindowSize, + Arrays.asList(new RetrieverSource(compoundRetriever1, null), new RetrieverSource(standard2, null)) + ) + ); + + SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); + ElasticsearchAssertions.assertResponse(req, resp -> { + assertNull(resp.pointInTimeId()); + assertNotNull(resp.getHits().getTotalHits()); + assertThat(resp.getHits().getTotalHits().value, equalTo(6L)); + assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); + assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_4")); + assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_6")); + assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_2")); + assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_1")); + assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_7")); + assertThat(resp.getHits().getAt(5).getId(), equalTo("doc_3")); + }); + } + + public void testRankDocsRetrieverDifferentNestedSorting() { + final int rankWindowSize = 100; + SearchSourceBuilder source = new SearchSourceBuilder(); + StandardRetrieverBuilder standard0 = new StandardRetrieverBuilder(); + // this one retrieves docs 1, 4, 6, 2 + standard0.queryBuilder = QueryBuilders.nestedQuery("views", QueryBuilders.rangeQuery(LAST_30D_FIELD).gt(0), ScoreMode.Avg); + standard0.sortBuilders = List.of( + new FieldSortBuilder(LAST_30D_FIELD).setNestedSort(new NestedSortBuilder("views")).order(SortOrder.DESC) + ); + StandardRetrieverBuilder standard1 = new StandardRetrieverBuilder(); + // this one retrieves docs 4, 7 + standard1.queryBuilder = QueryBuilders.nestedQuery("views", QueryBuilders.rangeQuery(ALL_TIME_FIELD).gt(0), ScoreMode.Avg); + standard1.sortBuilders = List.of( + new FieldSortBuilder(ALL_TIME_FIELD).setNestedSort(new NestedSortBuilder("views")).order(SortOrder.ASC) + ); + + source.retriever( + new CompoundRetrieverWithRankDocs( + rankWindowSize, + Arrays.asList(new RetrieverSource(standard0, null), new RetrieverSource(standard1, null)) + ) + ); + + SearchRequestBuilder req = client().prepareSearch(INDEX).setSource(source); + ElasticsearchAssertions.assertResponse(req, resp -> { + assertNull(resp.pointInTimeId()); + assertNotNull(resp.getHits().getTotalHits()); + assertThat(resp.getHits().getTotalHits().value, equalTo(5L)); + assertThat(resp.getHits().getTotalHits().relation, equalTo(TotalHits.Relation.EQUAL_TO)); + assertThat(resp.getHits().getAt(0).getId(), equalTo("doc_4")); + assertThat(resp.getHits().getAt(1).getId(), equalTo("doc_1")); + assertThat(resp.getHits().getAt(2).getId(), equalTo("doc_7")); + assertThat(resp.getHits().getAt(3).getId(), equalTo("doc_6")); + assertThat(resp.getHits().getAt(4).getId(), equalTo("doc_2")); + }); + } + + class CompoundRetrieverWithRankDocs extends RetrieverBuilder { + + private final List sources; + private final int rankWindowSize; + + private CompoundRetrieverWithRankDocs(int rankWindowSize, List sources) { + this.rankWindowSize = rankWindowSize; + this.sources = Collections.unmodifiableList(sources); + } + + @Override + public boolean isCompound() { + return true; + } + + @Override + public QueryBuilder topDocsQuery() { + throw new UnsupportedOperationException("should not be called"); + } + + @Override + public RetrieverBuilder rewrite(QueryRewriteContext ctx) throws IOException { + if (ctx.getPointInTimeBuilder() == null) { + throw new IllegalStateException("PIT is required"); + } + + // Rewrite prefilters + boolean hasChanged = false; + var newPreFilters = rewritePreFilters(ctx); + hasChanged |= newPreFilters != preFilterQueryBuilders; + + // Rewrite retriever sources + List newRetrievers = new ArrayList<>(); + for (var entry : sources) { + RetrieverBuilder newRetriever = entry.retriever.rewrite(ctx); + if (newRetriever != entry.retriever) { + newRetrievers.add(new RetrieverSource(newRetriever, null)); + hasChanged |= newRetriever != entry.retriever; + } else if (newRetriever == entry.retriever) { + var sourceBuilder = entry.source != null + ? entry.source + : createSearchSourceBuilder(ctx.getPointInTimeBuilder(), newRetriever); + var rewrittenSource = sourceBuilder.rewrite(ctx); + newRetrievers.add(new RetrieverSource(newRetriever, rewrittenSource)); + hasChanged |= rewrittenSource != entry.source; + } + } + if (hasChanged) { + return new CompoundRetrieverWithRankDocs(rankWindowSize, newRetrievers); + } + + // execute searches + final SetOnce results = new SetOnce<>(); + final MultiSearchRequest multiSearchRequest = new MultiSearchRequest(); + for (var entry : sources) { + SearchRequest searchRequest = new SearchRequest().source(entry.source); + // The can match phase can reorder shards, so we disable it to ensure the stable ordering + searchRequest.setPreFilterShardSize(Integer.MAX_VALUE); + multiSearchRequest.add(searchRequest); + } + ctx.registerAsyncAction((client, listener) -> { + client.execute(TransportMultiSearchAction.TYPE, multiSearchRequest, new ActionListener<>() { + @Override + public void onResponse(MultiSearchResponse items) { + List topDocs = new ArrayList<>(); + for (int i = 0; i < items.getResponses().length; i++) { + var item = items.getResponses()[i]; + var rankDocs = getRankDocs(item.getResponse()); + sources.get(i).retriever().setRankDocs(rankDocs); + topDocs.add(rankDocs); + } + results.set(combineResults(topDocs)); + listener.onResponse(null); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + }); + + return new RankDocsRetrieverBuilder( + rankWindowSize, + newRetrievers.stream().map(s -> s.retriever).toList(), + results::get, + newPreFilters + ); + } + + @Override + public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder, boolean compoundUsed) { + throw new UnsupportedOperationException("should not be called"); + } + + @Override + public String getName() { + return "compound_retriever"; + } + + @Override + protected void doToXContent(XContentBuilder builder, Params params) throws IOException { + + } + + @Override + protected boolean doEquals(Object o) { + return false; + } + + @Override + protected int doHashCode() { + return 0; + } + + private RankDoc[] getRankDocs(SearchResponse searchResponse) { + assert searchResponse != null; + int size = Math.min(rankWindowSize, searchResponse.getHits().getHits().length); + RankDoc[] docs = new RankDoc[size]; + for (int i = 0; i < size; i++) { + var hit = searchResponse.getHits().getAt(i); + long sortValue = (long) hit.getRawSortValues()[hit.getRawSortValues().length - 1]; + int doc = decodeDoc(sortValue); + int shardRequestIndex = decodeShardRequestIndex(sortValue); + docs[i] = new RankDoc(doc, hit.getScore(), shardRequestIndex); + docs[i].rank = i + 1; + } + return docs; + } + + public static int decodeDoc(long value) { + return (int) value; + } + + public static int decodeShardRequestIndex(long value) { + return (int) (value >> 32); + } + + record RankDocAndHitRatio(RankDoc rankDoc, float hitRatio) {} + + /** + * Combines the provided {@code rankResults} to return the final top documents. + */ + public RankDoc[] combineResults(List rankResults) { + int totalQueries = rankResults.size(); + final float step = 1.0f / totalQueries; + Map docsToRankResults = Maps.newMapWithExpectedSize(rankWindowSize); + for (var rankResult : rankResults) { + for (RankDoc scoreDoc : rankResult) { + docsToRankResults.compute(new RankDoc.RankKey(scoreDoc.doc, scoreDoc.shardIndex), (key, value) -> { + if (value == null) { + RankDoc res = new RankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex); + res.rank = scoreDoc.rank; + return new RankDocAndHitRatio(res, step); + } else { + RankDoc res = new RankDoc(scoreDoc.doc, Math.max(scoreDoc.score, value.rankDoc.score), scoreDoc.shardIndex); + res.rank = Math.min(scoreDoc.rank, value.rankDoc.rank); + return new RankDocAndHitRatio(res, value.hitRatio + step); + } + }); + } + } + // sort the results based on hit ratio, then doc, then rank, and final tiebreaker is based on smaller doc id + RankDocAndHitRatio[] sortedResults = docsToRankResults.values().toArray(RankDocAndHitRatio[]::new); + Arrays.sort(sortedResults, (RankDocAndHitRatio doc1, RankDocAndHitRatio doc2) -> { + if (doc1.hitRatio != doc2.hitRatio) { + return doc1.hitRatio < doc2.hitRatio ? 1 : -1; + } + if (false == (Float.isNaN(doc1.rankDoc.score) || Float.isNaN(doc2.rankDoc.score)) + && (doc1.rankDoc.score != doc2.rankDoc.score)) { + return doc1.rankDoc.score < doc2.rankDoc.score ? 1 : -1; + } + if (doc1.rankDoc.rank != doc2.rankDoc.rank) { + return doc1.rankDoc.rank < doc2.rankDoc.rank ? -1 : 1; + } + return doc1.rankDoc.doc < doc2.rankDoc.doc ? -1 : 1; + }); + // trim the results if needed, otherwise each shard will always return `rank_window_size` results. + // pagination and all else will happen on the coordinator when combining the shard responses + RankDoc[] topResults = new RankDoc[Math.min(rankWindowSize, sortedResults.length)]; + for (int rank = 0; rank < topResults.length; ++rank) { + topResults[rank] = sortedResults[rank].rankDoc; + topResults[rank].rank = rank + 1; + topResults[rank].score = sortedResults[rank].hitRatio; + } + return topResults; + } + } + + private SearchSourceBuilder createSearchSourceBuilder(PointInTimeBuilder pit, RetrieverBuilder retrieverBuilder) { + var sourceBuilder = new SearchSourceBuilder().pointInTimeBuilder(pit).trackTotalHits(false).size(100); + retrieverBuilder.extractToSearchSourceBuilder(sourceBuilder, false); + + // Record the shard id in the sort result + List> sortBuilders = sourceBuilder.sorts() != null ? new ArrayList<>(sourceBuilder.sorts()) : new ArrayList<>(); + if (sortBuilders.isEmpty()) { + sortBuilders.add(new ScoreSortBuilder()); + } + sortBuilders.add(new FieldSortBuilder(FieldSortBuilder.SHARD_DOC_FIELD_NAME)); + sourceBuilder.sort(sortBuilders); + return sourceBuilder; + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverRewriteIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverRewriteIT.java index 00013a8d396ba..e618a1b75cc4d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverRewriteIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/retriever/RetrieverRewriteIT.java @@ -21,6 +21,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.plugins.Plugin; @@ -141,6 +142,11 @@ private AssertingRetrieverBuilder(RetrieverBuilder innerRetriever) { this.innerRetriever = innerRetriever; } + @Override + public QueryBuilder topDocsQuery() { + return null; + } + @Override public RetrieverBuilder rewrite(QueryRewriteContext ctx) throws IOException { assertNull(ctx.getPointInTimeBuilder()); @@ -200,6 +206,11 @@ public boolean isCompound() { return true; } + @Override + public QueryBuilder topDocsQuery() { + return null; + } + @Override public RetrieverBuilder rewrite(QueryRewriteContext ctx) throws IOException { assertNotNull(ctx.getPointInTimeBuilder()); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java index 71616abf0dcfa..f61cce863ce59 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java @@ -62,6 +62,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.getRepositoryDataBlobName; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileExists; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; @@ -153,7 +154,10 @@ public void testRecreateCorruptedRepositoryDuringSnapshotsFails() throws Excepti Settings repoSettings = getRepositoryMetadata(repoName).settings(); Path repo = PathUtils.get(repoSettings.get("location")); - Files.move(repo.resolve("index-" + repositoryData.getGenId()), repo.resolve("index-" + (repositoryData.getGenId() + 1))); + Files.move( + repo.resolve(getRepositoryDataBlobName(repositoryData.getGenId())), + repo.resolve(getRepositoryDataBlobName(repositoryData.getGenId() + 1)) + ); logger.info("--> trying to create another snapshot in order for repository to be marked as corrupt"); final SnapshotException snapshotException = expectThrows( @@ -2309,7 +2313,7 @@ private static boolean snapshotHasCompletedShard(String repoName, String snapsho private void corruptIndexN(Path repoPath, long generation) throws IOException { logger.info("--> corrupting [index-{}] in [{}]", generation, repoPath); - Path indexNBlob = repoPath.resolve(BlobStoreRepository.INDEX_FILE_PREFIX + generation); + Path indexNBlob = repoPath.resolve(getRepositoryDataBlobName(generation)); assertFileExists(indexNBlob); Files.write(indexNBlob, randomByteArrayOfLength(1), StandardOpenOption.TRUNCATE_EXISTING); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java index abcac0cade456..5a82b4b1ab99e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java @@ -7,6 +7,7 @@ */ package org.elasticsearch.snapshots; +import org.apache.logging.log4j.Level; import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; @@ -14,8 +15,10 @@ import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.RepositoriesMetadata; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; @@ -23,6 +26,7 @@ import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.Strings; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshotsIntegritySuppressor; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.repositories.IndexMetaDataGenerations; import org.elasticsearch.repositories.Repository; @@ -32,6 +36,8 @@ import org.elasticsearch.repositories.ShardGenerations; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.repositories.fs.FsRepository; +import org.elasticsearch.test.ClusterServiceUtils; +import org.elasticsearch.test.MockLog; import org.elasticsearch.xcontent.XContentFactory; import java.nio.channels.SeekableByteChannel; @@ -45,9 +51,14 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.METADATA_NAME_FORMAT; +import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.SNAPSHOT_INDEX_NAME_FORMAT; +import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.SNAPSHOT_NAME_FORMAT; +import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.getRepositoryDataBlobName; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFileExists; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasSize; @@ -80,7 +91,10 @@ public void testRecreateCorruptedRepositoryUnblocksIt() throws Exception { logger.info("--> move index-N blob to next generation"); final RepositoryData repositoryData = getRepositoryData(repoName); - Files.move(repo.resolve("index-" + repositoryData.getGenId()), repo.resolve("index-" + (repositoryData.getGenId() + 1))); + Files.move( + repo.resolve(getRepositoryDataBlobName(repositoryData.getGenId())), + repo.resolve(getRepositoryDataBlobName(repositoryData.getGenId() + 1)) + ); assertRepositoryBlocked(repoName, snapshot); @@ -133,13 +147,19 @@ public void testConcurrentlyChangeRepositoryContents() throws Exception { logger.info("--> move index-N blob to next generation"); final RepositoryData repositoryData = getRepositoryData(repoName); - Files.move(repo.resolve("index-" + repositoryData.getGenId()), repo.resolve("index-" + (repositoryData.getGenId() + 1))); + Files.move( + repo.resolve(getRepositoryDataBlobName(repositoryData.getGenId())), + repo.resolve(getRepositoryDataBlobName(repositoryData.getGenId() + 1)) + ); assertRepositoryBlocked(repoName, snapshot); if (randomBoolean()) { logger.info("--> move index-N blob back to initial generation"); - Files.move(repo.resolve("index-" + (repositoryData.getGenId() + 1)), repo.resolve("index-" + repositoryData.getGenId())); + Files.move( + repo.resolve(getRepositoryDataBlobName(repositoryData.getGenId() + 1)), + repo.resolve(getRepositoryDataBlobName(repositoryData.getGenId())) + ); logger.info("--> verify repository remains blocked"); assertRepositoryBlocked(repoName, snapshot); @@ -205,7 +225,7 @@ public void testFindDanglingLatestGeneration() throws Exception { logger.info("--> move index-N blob to next generation"); final RepositoryData repositoryData = getRepositoryData(repoName); final long beforeMoveGen = repositoryData.getGenId(); - Files.move(repo.resolve("index-" + beforeMoveGen), repo.resolve("index-" + (beforeMoveGen + 1))); + Files.move(repo.resolve(getRepositoryDataBlobName(beforeMoveGen)), repo.resolve(getRepositoryDataBlobName(beforeMoveGen + 1))); logger.info("--> set next generation as pending in the cluster state"); updateClusterState( @@ -298,7 +318,7 @@ public void testHandlingMissingRootLevelSnapshotMetadata() throws Exception { ); // old-format repository has no cluster UUID Files.write( - repo.resolve(BlobStoreRepository.INDEX_FILE_PREFIX + withoutVersions.getGenId()), + repo.resolve(getRepositoryDataBlobName(withoutVersions.getGenId())), BytesReference.toBytes( BytesReference.bytes(withoutVersions.snapshotsToXContent(XContentFactory.jsonBuilder(), IndexVersion.current(), true)) ), @@ -358,7 +378,7 @@ public void testMountCorruptedRepositoryData() throws Exception { logger.info("--> corrupt index-N blob"); final Repository repository = getRepositoryOnMaster(repoName); final RepositoryData repositoryData = getRepositoryData(repoName); - Files.write(repo.resolve("index-" + repositoryData.getGenId()), randomByteArrayOfLength(randomIntBetween(1, 100))); + Files.write(repo.resolve(getRepositoryDataBlobName(repositoryData.getGenId())), randomByteArrayOfLength(randomIntBetween(1, 100))); logger.info("--> verify loading repository data throws RepositoryException"); asInstanceOf( @@ -398,9 +418,9 @@ public void testHandleSnapshotErrorWithBwCFormat() throws Exception { logger.info("--> move shard level metadata to new generation"); final IndexId indexId = getRepositoryData(repoName).resolveIndexId(indexName); final Path shardPath = repoPath.resolve("indices").resolve(indexId.getId()).resolve("0"); - final Path initialShardMetaPath = shardPath.resolve(BlobStoreRepository.INDEX_FILE_PREFIX + "0"); + final Path initialShardMetaPath = shardPath.resolve(Strings.format(SNAPSHOT_INDEX_NAME_FORMAT, "0")); assertFileExists(initialShardMetaPath); - Files.move(initialShardMetaPath, shardPath.resolve(BlobStoreRepository.INDEX_FILE_PREFIX + "1")); + Files.move(initialShardMetaPath, shardPath.resolve(Strings.format(SNAPSHOT_INDEX_NAME_FORMAT, "1"))); startDeleteSnapshot(repoName, oldVersionSnapshot).get(); @@ -423,9 +443,9 @@ public void testRepairBrokenShardGenerations() throws Exception { logger.info("--> move shard level metadata to new generation and make RepositoryData point at an older generation"); final IndexId indexId = getRepositoryData(repoName).resolveIndexId(indexName); final Path shardPath = repoPath.resolve("indices").resolve(indexId.getId()).resolve("0"); - final Path initialShardMetaPath = shardPath.resolve(BlobStoreRepository.INDEX_FILE_PREFIX + "0"); + final Path initialShardMetaPath = shardPath.resolve(Strings.format(SNAPSHOT_INDEX_NAME_FORMAT, "0")); assertFileExists(initialShardMetaPath); - Files.move(initialShardMetaPath, shardPath.resolve(BlobStoreRepository.INDEX_FILE_PREFIX + randomIntBetween(1, 1000))); + Files.move(initialShardMetaPath, shardPath.resolve(Strings.format(SNAPSHOT_INDEX_NAME_FORMAT, randomIntBetween(1, 1000)))); final RepositoryData repositoryData = getRepositoryData(repoName); final Map snapshotIds = repositoryData.getSnapshotIds() @@ -442,7 +462,7 @@ public void testRepairBrokenShardGenerations() throws Exception { repositoryData.getClusterUUID() ); Files.write( - repoPath.resolve(BlobStoreRepository.INDEX_FILE_PREFIX + repositoryData.getGenId()), + repoPath.resolve(getRepositoryDataBlobName(repositoryData.getGenId())), BytesReference.toBytes( BytesReference.bytes(brokenRepoData.snapshotsToXContent(XContentFactory.jsonBuilder(), IndexVersion.current())) ), @@ -491,7 +511,7 @@ public void testSnapshotWithCorruptedShardIndexFile() throws Exception { final Path shardIndexFile = repo.resolve("indices") .resolve(corruptedIndex.getId()) .resolve("0") - .resolve("index-" + repositoryData.shardGenerations().getShardGen(corruptedIndex, 0)); + .resolve(Strings.format(SNAPSHOT_INDEX_NAME_FORMAT, repositoryData.shardGenerations().getShardGen(corruptedIndex, 0))); logger.info("--> truncating shard index file [{}]", shardIndexFile); try (SeekableByteChannel outChan = Files.newByteChannel(shardIndexFile, StandardOpenOption.WRITE)) { @@ -564,10 +584,15 @@ public void testDeleteSnapshotWithMissingIndexAndShardMetadata() throws Exceptio Path shardZero = indicesPath.resolve(indexIds.get(index).getId()).resolve("0"); if (randomBoolean()) { Files.delete( - shardZero.resolve("index-" + getRepositoryData("test-repo").shardGenerations().getShardGen(indexIds.get(index), 0)) + shardZero.resolve( + Strings.format( + BlobStoreRepository.SNAPSHOT_INDEX_NAME_FORMAT, + getRepositoryData("test-repo").shardGenerations().getShardGen(indexIds.get(index), 0) + ) + ) ); } - Files.delete(shardZero.resolve("snap-" + snapshotInfo.snapshotId().getUUID() + ".dat")); + Files.delete(shardZero.resolve(Strings.format(SNAPSHOT_NAME_FORMAT, snapshotInfo.snapshotId().getUUID()))); } startDeleteSnapshot("test-repo", "test-snap-1").get(); @@ -608,7 +633,7 @@ public void testDeleteSnapshotWithMissingMetadata() throws Exception { ); logger.info("--> delete global state metadata"); - Path metadata = repo.resolve("meta-" + createSnapshotResponse.getSnapshotInfo().snapshotId().getUUID() + ".dat"); + Path metadata = repo.resolve(Strings.format(METADATA_NAME_FORMAT, createSnapshotResponse.getSnapshotInfo().snapshotId().getUUID())); Files.delete(metadata); startDeleteSnapshot("test-repo", "test-snap-1").get(); @@ -651,7 +676,9 @@ public void testDeleteSnapshotWithCorruptedSnapshotFile() throws Exception { ); logger.info("--> truncate snapshot file to make it unreadable"); - Path snapshotPath = repo.resolve("snap-" + createSnapshotResponse.getSnapshotInfo().snapshotId().getUUID() + ".dat"); + Path snapshotPath = repo.resolve( + Strings.format(SNAPSHOT_NAME_FORMAT, createSnapshotResponse.getSnapshotInfo().snapshotId().getUUID()) + ); try (SeekableByteChannel outChan = Files.newByteChannel(snapshotPath, StandardOpenOption.WRITE)) { outChan.truncate(randomInt(10)); } @@ -698,7 +725,7 @@ public void testDeleteSnapshotWithCorruptedGlobalState() throws Exception { SnapshotInfo snapshotInfo = createFullSnapshot("test-repo", "test-snap"); - final Path globalStatePath = repo.resolve("meta-" + snapshotInfo.snapshotId().getUUID() + ".dat"); + final Path globalStatePath = repo.resolve(Strings.format(METADATA_NAME_FORMAT, snapshotInfo.snapshotId().getUUID())); if (randomBoolean()) { // Delete the global state metadata file IOUtils.deleteFilesIgnoringExceptions(globalStatePath); @@ -747,41 +774,131 @@ public void testSnapshotWithMissingShardLevelIndexFile() throws Exception { .setWaitForCompletion(true) .setIndices("test-idx-*") .get(); + final boolean repairWithDelete = randomBoolean(); + if (repairWithDelete || randomBoolean()) { + clusterAdmin().prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, "test-repo", "snap-for-deletion") + .setWaitForCompletion(true) + .setIndices("test-idx-1") + .get(); + } logger.info("--> deleting shard level index file"); final Path indicesPath = repo.resolve("indices"); for (IndexId indexId : getRepositoryData("test-repo").getIndices().values()) { final Path shardGen; try (Stream shardFiles = Files.list(indicesPath.resolve(indexId.getId()).resolve("0"))) { - shardGen = shardFiles.filter(file -> file.getFileName().toString().startsWith(BlobStoreRepository.INDEX_FILE_PREFIX)) + shardGen = shardFiles.filter(file -> file.getFileName().toString().startsWith(BlobStoreRepository.SNAPSHOT_INDEX_PREFIX)) .findFirst() .orElseThrow(() -> new AssertionError("Failed to find shard index blob")); } Files.delete(shardGen); } - logger.info("--> creating another snapshot"); + if (randomBoolean()) { + logger.info(""" + --> restoring the snapshot, the repository should not have lost any shard data despite deleting index-*, \ + because it uses snap-*.dat files and not the index-* to determine what files to restore"""); + indicesAdmin().prepareDelete("test-idx-1", "test-idx-2").get(); + RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot( + TEST_REQUEST_TIMEOUT, + "test-repo", + "test-snap-1" + ).setWaitForCompletion(true).get(); + assertEquals(0, restoreSnapshotResponse.getRestoreInfo().failedShards()); + ensureGreen("test-idx-1", "test-idx-2"); + } + + logger.info("--> creating another snapshot, which should re-create the missing file"); + try ( + var ignored = new BlobStoreIndexShardSnapshotsIntegritySuppressor(); + var mockLog = MockLog.capture(BlobStoreRepository.class) + ) { + mockLog.addExpectation( + new MockLog.SeenEventExpectation( + "fallback message", + "org.elasticsearch.repositories.blobstore.BlobStoreRepository", + Level.ERROR, + "index [test-idx-1/*] shard generation [*] in [test-repo][*] not found - falling back to reading all shard snapshots" + ) + ); + mockLog.addExpectation( + new MockLog.SeenEventExpectation( + "shard blobs list", + "org.elasticsearch.repositories.blobstore.BlobStoreRepository", + Level.ERROR, + "read shard snapshots [*] due to missing shard generation [*] for index [test-idx-1/*] in [test-repo][*]" + ) + ); + if (repairWithDelete) { + clusterAdmin().prepareDeleteSnapshot(TEST_REQUEST_TIMEOUT, "test-repo", "snap-for-deletion").get(); + } else if (randomBoolean()) { + CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot( + TEST_REQUEST_TIMEOUT, + "test-repo", + "test-snap-2" + ).setWaitForCompletion(true).setIndices("test-idx-1").get(); + assertEquals( + createSnapshotResponse.getSnapshotInfo().totalShards(), + createSnapshotResponse.getSnapshotInfo().successfulShards() + ); + } else { + clusterAdmin().prepareCloneSnapshot(TEST_REQUEST_TIMEOUT, "test-repo", "test-snap-1", "test-snap-2") + .setIndices("test-idx-1") + .get(); + safeAwait( + ClusterServiceUtils.addTemporaryStateListener( + internalCluster().getInstance(ClusterService.class), + cs -> SnapshotsInProgress.get(cs).isEmpty() + ) + ); + assertThat( + clusterAdmin().prepareGetSnapshots(TEST_REQUEST_TIMEOUT, "test-repo") + .setSnapshots("test-snap-2") + .get() + .getSnapshots() + .get(0) + .shardFailures(), + empty() + ); + } + mockLog.assertAllExpectationsMatched(); + + try ( + Stream shardFiles = Files.list( + indicesPath.resolve(getRepositoryData("test-repo").resolveIndexId("test-idx-1").getId()).resolve("0") + ) + ) { + assertTrue(shardFiles.anyMatch(file -> file.getFileName().toString().startsWith(BlobStoreRepository.INDEX_FILE_PREFIX))); + } + } + + if (randomBoolean()) { + indicesAdmin().prepareDelete("test-idx-1").get(); + RestoreSnapshotResponse restoreSnapshotResponse2 = clusterAdmin().prepareRestoreSnapshot( + TEST_REQUEST_TIMEOUT, + "test-repo", + repairWithDelete ? "test-snap-1" : randomFrom("test-snap-1", "test-snap-2") + ).setIndices("test-idx-1").setWaitForCompletion(true).get(); + assertEquals(0, restoreSnapshotResponse2.getRestoreInfo().failedShards()); + ensureGreen("test-idx-1", "test-idx-2"); + } + + logger.info("--> creating another snapshot, which should succeed since the shard gen file now exists again"); CreateSnapshotResponse createSnapshotResponse = clusterAdmin().prepareCreateSnapshot( TEST_REQUEST_TIMEOUT, "test-repo", - "test-snap-2" + "test-snap-3" ).setWaitForCompletion(true).setIndices("test-idx-1").get(); - assertEquals( - createSnapshotResponse.getSnapshotInfo().successfulShards(), - createSnapshotResponse.getSnapshotInfo().totalShards() - 1 - ); + assertEquals(createSnapshotResponse.getSnapshotInfo().totalShards(), createSnapshotResponse.getSnapshotInfo().successfulShards()); - logger.info( - "--> restoring the first snapshot, the repository should not have lost any shard data despite deleting index-N, " - + "because it uses snap-*.data files and not the index-N to determine what files to restore" - ); - indicesAdmin().prepareDelete("test-idx-1", "test-idx-2").get(); - RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot( + indicesAdmin().prepareDelete("test-idx-1").get(); + RestoreSnapshotResponse restoreSnapshotResponse3 = clusterAdmin().prepareRestoreSnapshot( TEST_REQUEST_TIMEOUT, "test-repo", - "test-snap-1" - ).setWaitForCompletion(true).get(); - assertEquals(0, restoreSnapshotResponse.getRestoreInfo().failedShards()); + repairWithDelete ? randomFrom("test-snap-1", "test-snap-3") : randomFrom("test-snap-1", "test-snap-2", "test-snap-3") + ).setIndices("test-idx-1").setWaitForCompletion(true).get(); + assertEquals(0, restoreSnapshotResponse3.getRestoreInfo().failedShards()); + ensureGreen("test-idx-1", "test-idx-2"); } public void testDeletesWithUnexpectedIndexBlob() throws Exception { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index b2b3de51dd04b..19f051404bce0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -86,6 +86,7 @@ import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.index.seqno.RetentionLeaseActions.RETAIN_ALL; +import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.METADATA_BLOB_NAME_SUFFIX; import static org.elasticsearch.test.NodeRoles.nonMasterNode; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFutureThrows; @@ -1316,7 +1317,7 @@ private static List findRepoMetaBlobs(Path repoPath) throws IOException { List files = new ArrayList<>(); forEachFileRecursively(repoPath.resolve("indices"), ((file, basicFileAttributes) -> { final String fileName = file.getFileName().toString(); - if (fileName.startsWith(BlobStoreRepository.METADATA_PREFIX) && fileName.endsWith(".dat")) { + if (fileName.startsWith(BlobStoreRepository.METADATA_PREFIX) && fileName.endsWith(METADATA_BLOB_NAME_SUFFIX)) { files.add(file); } })); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java index 477fd9737394e..6f02ac5c983a4 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java @@ -40,7 +40,6 @@ import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.RepositoryData; import org.elasticsearch.repositories.RepositoryMissingException; -import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESTestCase; @@ -63,6 +62,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; +import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.getRepositoryDataBlobName; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.hasSize; @@ -1042,7 +1042,7 @@ private static void removeDetailsForRandomSnapshots(String repositoryName, Actio final var repositoryMetadata = repository.getMetadata(); final var repositorySettings = repositoryMetadata.settings(); final var repositoryDataBlobPath = asInstanceOf(FsBlobStore.class, repository.blobStore()).path() - .resolve(BlobStoreRepository.INDEX_FILE_PREFIX + repositoryMetadata.generation()); + .resolve(getRepositoryDataBlobName(repositoryMetadata.generation())); SubscribableListener diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MultiClusterRepoAccessIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MultiClusterRepoAccessIT.java index fc727007724de..0fd96b96c8756 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MultiClusterRepoAccessIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/MultiClusterRepoAccessIT.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.core.IOUtils; import org.elasticsearch.env.Environment; +import org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshotsIntegritySuppressor; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.repositories.RepositoryException; import org.elasticsearch.test.ESIntegTestCase; @@ -98,7 +99,7 @@ protected Collection> nodePlugins() { return CollectionUtils.appendToCopy(super.nodePlugins(), getTestTransportPlugin()); } - public void testConcurrentDeleteFromOtherCluster() throws InterruptedException { + public void testConcurrentDeleteFromOtherCluster() { internalCluster().startMasterOnlyNode(); internalCluster().startDataOnlyNode(); final String repoNameOnFirstCluster = "test-repo"; @@ -125,10 +126,13 @@ public void testConcurrentDeleteFromOtherCluster() throws InterruptedException { secondCluster.client().admin().cluster().prepareDeleteSnapshot(TEST_REQUEST_TIMEOUT, repoNameOnSecondCluster, "snap-1").get(); secondCluster.client().admin().cluster().prepareDeleteSnapshot(TEST_REQUEST_TIMEOUT, repoNameOnSecondCluster, "snap-2").get(); - final SnapshotException sne = expectThrows( - SnapshotException.class, - clusterAdmin().prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, repoNameOnFirstCluster, "snap-4").setWaitForCompletion(true) - ); + final SnapshotException sne; + try (var ignored = new BlobStoreIndexShardSnapshotsIntegritySuppressor()) { + sne = expectThrows( + SnapshotException.class, + clusterAdmin().prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, repoNameOnFirstCluster, "snap-4").setWaitForCompletion(true) + ); + } assertThat(sne.getMessage(), containsString("failed to update snapshot in repository")); final RepositoryException cause = (RepositoryException) sne.getCause(); assertThat( @@ -147,7 +151,7 @@ public void testConcurrentDeleteFromOtherCluster() throws InterruptedException { createFullSnapshot(repoNameOnFirstCluster, "snap-5"); } - public void testConcurrentWipeAndRecreateFromOtherCluster() throws InterruptedException, IOException { + public void testConcurrentWipeAndRecreateFromOtherCluster() throws IOException { internalCluster().startMasterOnlyNode(); internalCluster().startDataOnlyNode(); final String repoName = "test-repo"; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorServiceIT.java index 1a54df1f85ed6..8a0b74242ba9d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorServiceIT.java @@ -22,6 +22,7 @@ import static org.elasticsearch.health.HealthStatus.GREEN; import static org.elasticsearch.health.HealthStatus.YELLOW; +import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.getRepositoryDataBlobName; import static org.elasticsearch.snapshots.RepositoryIntegrityHealthIndicatorService.NAME; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -73,6 +74,6 @@ private void assertSnapshotRepositoryHealth(String message, Client client, Healt private void corruptRepository(String name, Path location) throws IOException { final RepositoryData repositoryData = getRepositoryData(name); - Files.delete(location.resolve("index-" + repositoryData.getGenId())); + Files.delete(location.resolve(getRepositoryDataBlobName(repositoryData.getGenId()))); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java index 531e9f4f45afa..cd57401550f12 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java @@ -86,7 +86,9 @@ import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY; import static org.elasticsearch.index.shard.IndexShardTests.getEngineFromShard; +import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.METADATA_NAME_FORMAT; import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.READONLY_SETTING_KEY; +import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.SNAPSHOT_NAME_FORMAT; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; @@ -1578,7 +1580,7 @@ public void testListCorruptedSnapshot() throws Exception { final SnapshotInfo snapshotInfo = createSnapshot("test-repo", "test-snap-2", Collections.singletonList("test-idx-*")); logger.info("--> truncate snapshot file to make it unreadable"); - Path snapshotPath = repo.resolve("snap-" + snapshotInfo.snapshotId().getUUID() + ".dat"); + Path snapshotPath = repo.resolve(Strings.format(SNAPSHOT_NAME_FORMAT, snapshotInfo.snapshotId().getUUID())); try (SeekableByteChannel outChan = Files.newByteChannel(snapshotPath, StandardOpenOption.WRITE)) { outChan.truncate(randomInt(10)); } @@ -1621,7 +1623,7 @@ public void testRestoreSnapshotWithCorruptedGlobalState() throws Exception { final String snapshotName = "test-snap"; final SnapshotInfo snapshotInfo = createFullSnapshot(repoName, snapshotName); - final Path globalStatePath = repo.resolve("meta-" + snapshotInfo.snapshotId().getUUID() + ".dat"); + final Path globalStatePath = repo.resolve(Strings.format(METADATA_NAME_FORMAT, snapshotInfo.snapshotId().getUUID())); try (SeekableByteChannel outChan = Files.newByteChannel(globalStatePath, StandardOpenOption.WRITE)) { outChan.truncate(randomInt(10)); } @@ -1701,7 +1703,10 @@ public void testRestoreSnapshotWithCorruptedIndexMetadata() throws Exception { final Path indexMetadataPath = repo.resolve("indices") .resolve(corruptedIndex.getId()) .resolve( - "meta-" + repositoryData.indexMetaDataGenerations().indexMetaBlobId(snapshotInfo.snapshotId(), corruptedIndex) + ".dat" + Strings.format( + METADATA_NAME_FORMAT, + repositoryData.indexMetaDataGenerations().indexMetaBlobId(snapshotInfo.snapshotId(), corruptedIndex) + ) ); // Truncate the index metadata file diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStatusApisIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStatusApisIT.java index b155ef73783eb..fb282b4bf6a48 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStatusApisIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStatusApisIT.java @@ -28,7 +28,6 @@ import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; @@ -45,6 +44,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import static org.elasticsearch.repositories.blobstore.BlobStoreRepository.SNAPSHOT_NAME_FORMAT; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.anEmptyMap; @@ -142,7 +142,7 @@ public void testExceptionOnMissingSnapBlob() throws IOException { final SnapshotInfo snapshotInfo = createFullSnapshot("test-repo", "test-snap"); logger.info("--> delete snap-${uuid}.dat file for this snapshot to simulate concurrent delete"); - IOUtils.rm(repoPath.resolve(BlobStoreRepository.SNAPSHOT_PREFIX + snapshotInfo.snapshotId().getUUID() + ".dat")); + IOUtils.rm(repoPath.resolve(Strings.format(SNAPSHOT_NAME_FORMAT, snapshotInfo.snapshotId().getUUID()))); expectThrows( SnapshotMissingException.class, @@ -173,7 +173,7 @@ public void testExceptionOnMissingShardLevelSnapBlob() throws IOException { repoPath.resolve("indices") .resolve(indexRepoId) .resolve("0") - .resolve(BlobStoreRepository.SNAPSHOT_PREFIX + snapshotInfo.snapshotId().getUUID() + ".dat") + .resolve(Strings.format(SNAPSHOT_NAME_FORMAT, snapshotInfo.snapshotId().getUUID())) ); expectThrows( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java b/server/src/internalClusterTest/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java index 44b6ef1d51ce0..d98b1e7d4e526 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/threadpool/SimpleThreadPoolIT.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.TaskExecutionTimeTrackingEsThreadPoolExecutor; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.PluginsService; @@ -19,6 +20,7 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; +import org.hamcrest.CoreMatchers; import java.lang.management.ManagementFactory; import java.lang.management.ThreadInfo; @@ -36,12 +38,15 @@ import static java.util.function.Function.identity; import static org.elasticsearch.common.util.Maps.toUnmodifiableSortedMap; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.elasticsearch.threadpool.ThreadPool.DEFAULT_INDEX_AUTOSCALING_EWMA_ALPHA; +import static org.elasticsearch.threadpool.ThreadPool.WRITE_THREAD_POOLS_EWMA_ALPHA_SETTING; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.in; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.matchesRegex; @ClusterScope(scope = Scope.TEST, numDataNodes = 0, numClientNodes = 0) @@ -190,4 +195,19 @@ public void testThreadPoolMetrics() throws Exception { }); } + public void testWriteThreadpoolEwmaAlphaSetting() { + Settings settings = Settings.EMPTY; + var ewmaAlpha = DEFAULT_INDEX_AUTOSCALING_EWMA_ALPHA; + if (randomBoolean()) { + ewmaAlpha = randomDoubleBetween(0.0, 1.0, true); + settings = Settings.builder().put(WRITE_THREAD_POOLS_EWMA_ALPHA_SETTING.getKey(), ewmaAlpha).build(); + } + var nodeName = internalCluster().startNode(settings); + var threadPool = internalCluster().getInstance(ThreadPool.class, nodeName); + for (var name : List.of(ThreadPool.Names.WRITE, ThreadPool.Names.SYSTEM_WRITE, ThreadPool.Names.SYSTEM_CRITICAL_WRITE)) { + assertThat(threadPool.executor(name), instanceOf(TaskExecutionTimeTrackingEsThreadPoolExecutor.class)); + final var executor = (TaskExecutionTimeTrackingEsThreadPoolExecutor) threadPool.executor(name); + assertThat(Double.compare(executor.getEwmaAlpha(), ewmaAlpha), CoreMatchers.equalTo(0)); + } + } } diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index d29009cd76b8d..086bfece87172 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -190,6 +190,7 @@ exports org.elasticsearch.common.file; exports org.elasticsearch.common.geo; exports org.elasticsearch.common.hash; + exports org.elasticsearch.injection.api; exports org.elasticsearch.injection.guice; exports org.elasticsearch.injection.guice.binder; exports org.elasticsearch.injection.guice.internal; @@ -364,6 +365,7 @@ exports org.elasticsearch.search.rank.rerank; exports org.elasticsearch.search.rescore; exports org.elasticsearch.search.retriever; + exports org.elasticsearch.search.retriever.rankdoc; exports org.elasticsearch.search.runtime; exports org.elasticsearch.search.searchafter; exports org.elasticsearch.search.slice; @@ -428,6 +430,7 @@ org.elasticsearch.cluster.metadata.MetadataFeatures, org.elasticsearch.rest.RestFeatures, org.elasticsearch.indices.IndicesFeatures, + org.elasticsearch.repositories.RepositoriesFeatures, org.elasticsearch.action.admin.cluster.allocation.AllocationStatsFeatures, org.elasticsearch.index.mapper.MapperFeatures, org.elasticsearch.ingest.IngestGeoIpFeatures, diff --git a/server/src/main/java/org/elasticsearch/ElasticsearchException.java b/server/src/main/java/org/elasticsearch/ElasticsearchException.java index d7db8f4ec09dd..c7f3e25b6a96e 100644 --- a/server/src/main/java/org/elasticsearch/ElasticsearchException.java +++ b/server/src/main/java/org/elasticsearch/ElasticsearchException.java @@ -1646,12 +1646,7 @@ private enum ElasticsearchExceptionHandle { UNKNOWN_VERSION_ADDED ), // 127 used to be org.elasticsearch.search.SearchContextException - SEARCH_SOURCE_BUILDER_EXCEPTION( - org.elasticsearch.search.builder.SearchSourceBuilderException.class, - org.elasticsearch.search.builder.SearchSourceBuilderException::new, - 128, - UNKNOWN_VERSION_ADDED - ), + // 128 used to be org.elasticsearch.search.builder.SearchSourceBuilderException // 129 was EngineClosedException NO_SHARD_AVAILABLE_ACTION_EXCEPTION( org.elasticsearch.action.NoShardAvailableActionException.class, diff --git a/server/src/main/java/org/elasticsearch/ReleaseVersions.java b/server/src/main/java/org/elasticsearch/ReleaseVersions.java index 7b5c8d1d42382..bb90bc79a528a 100644 --- a/server/src/main/java/org/elasticsearch/ReleaseVersions.java +++ b/server/src/main/java/org/elasticsearch/ReleaseVersions.java @@ -41,7 +41,7 @@ public class ReleaseVersions { private static final Pattern VERSION_LINE = Pattern.compile("(\\d+\\.\\d+\\.\\d+),(\\d+)"); - public static IntFunction generateVersionsLookup(Class versionContainer) { + public static IntFunction generateVersionsLookup(Class versionContainer, int current) { if (USES_VERSIONS == false) return Integer::toString; try { @@ -52,6 +52,9 @@ public static IntFunction generateVersionsLookup(Class versionContain } NavigableMap> versions = new TreeMap<>(); + // add the current version id, which won't be in the csv + versions.computeIfAbsent(current, k -> new ArrayList<>()).add(Version.CURRENT); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(versionsFile, StandardCharsets.UTF_8))) { String line; while ((line = reader.readLine()) != null) { @@ -121,8 +124,8 @@ private static IntFunction lookupFunction(NavigableMap getAllVersions() { return VERSION_IDS.values(); } - static final IntFunction VERSION_LOOKUP = ReleaseVersions.generateVersionsLookup(TransportVersions.class); + static final IntFunction VERSION_LOOKUP = ReleaseVersions.generateVersionsLookup(TransportVersions.class, LATEST_DEFINED.id()); // no instance private TransportVersions() {} diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index 333669ca8079c..54b6b1ef9c8c8 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -182,6 +182,7 @@ public class Version implements VersionId, ToXContentFragment { public static final Version V_8_14_3 = new Version(8_14_03_99); public static final Version V_8_15_0 = new Version(8_15_00_99); public static final Version V_8_15_1 = new Version(8_15_01_99); + public static final Version V_8_15_2 = new Version(8_15_02_99); public static final Version V_8_16_0 = new Version(8_16_00_99); public static final Version CURRENT = V_8_16_0; @@ -270,7 +271,9 @@ public static Version max(Version version1, Version version2) { /** * Returns the version given its string representation, current version if the argument is null or empty + * @deprecated Use of semantic release versions should be minimized; please avoid use of this method if possible. */ + @Deprecated public static Version fromString(String version) { if (Strings.hasLength(version) == false) { return Version.CURRENT; diff --git a/server/src/main/java/org/elasticsearch/action/ActionListener.java b/server/src/main/java/org/elasticsearch/action/ActionListener.java index f3fa1dd2e105f..5841648700756 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionListener.java +++ b/server/src/main/java/org/elasticsearch/action/ActionListener.java @@ -388,8 +388,8 @@ static ActionListener assertOnce(ActionListener d private final AtomicReference firstCompletion = new AtomicReference<>(); private void assertFirstRun() { - var previousRun = firstCompletion.compareAndExchange(null, new ElasticsearchException(delegate.toString())); - assert previousRun == null : previousRun; // reports the stack traces of both completions + var previousRun = firstCompletion.compareAndExchange(null, new ElasticsearchException("executed already")); + assert previousRun == null : "[" + delegate + "] " + previousRun; // reports the stack traces of both completions } @Override diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java index bec131341a8f4..2f094b0fc6006 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java @@ -108,6 +108,7 @@ public static class Request extends MasterNodeReadRequest { private final EnumSet metrics; + @SuppressWarnings("this-escape") public Request(TimeValue masterNodeTimeout, TaskId parentTaskId, EnumSet metrics) { super(masterNodeTimeout); setParentTask(parentTaskId); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java index e7e2679e84eb5..8067f92a5a908 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponse.java @@ -45,6 +45,7 @@ public class ClusterHealthResponse extends ActionResponse implements ToXContentO static final String RELOCATING_SHARDS = "relocating_shards"; static final String INITIALIZING_SHARDS = "initializing_shards"; static final String UNASSIGNED_SHARDS = "unassigned_shards"; + static final String UNASSIGNED_PRIMARY_SHARDS = "unassigned_primary_shards"; static final String INDICES = "indices"; private String clusterName; @@ -144,6 +145,10 @@ public int getUnassignedShards() { return clusterStateHealth.getUnassignedShards(); } + public int getUnassignedPrimaryShards() { + return clusterStateHealth.getUnassignedPrimaryShards(); + } + public int getNumberOfNodes() { return clusterStateHealth.getNumberOfNodes(); } @@ -253,6 +258,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(RELOCATING_SHARDS, getRelocatingShards()); builder.field(INITIALIZING_SHARDS, getInitializingShards()); builder.field(UNASSIGNED_SHARDS, getUnassignedShards()); + builder.field(UNASSIGNED_PRIMARY_SHARDS, getUnassignedPrimaryShards()); builder.field(DELAYED_UNASSIGNED_SHARDS, getDelayedUnassignedShards()); builder.field(NUMBER_OF_PENDING_TASKS, getNumberOfPendingTasks()); builder.field(NUMBER_OF_IN_FLIGHT_FETCH, getNumberOfInFlightFetch()); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java index d7a14362026ef..ba34b8cab1021 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java @@ -17,7 +17,6 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.snapshots.RestoreService; import org.elasticsearch.tasks.Task; @@ -49,7 +48,7 @@ public TransportRestoreSnapshotAction( RestoreSnapshotRequest::new, indexNameExpressionResolver, RestoreSnapshotResponse::new, - EsExecutors.DIRECT_EXECUTOR_SERVICE + threadPool.executor(ThreadPool.Names.SNAPSHOT_META) ); this.restoreService = restoreService; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshot.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshot.java new file mode 100644 index 0000000000000..fe1da86dd54c7 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshot.java @@ -0,0 +1,404 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.admin.cluster.stats; + +import org.elasticsearch.action.admin.cluster.stats.LongMetric.LongMetricValue; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.transport.RemoteClusterAware; +import org.elasticsearch.xcontent.ToXContentFragment; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** + * Holds a snapshot of the CCS telemetry statistics from {@link CCSUsageTelemetry}. + * Used to hold the stats for a single node that's part of a {@link ClusterStatsNodeResponse}, as well as to + * accumulate stats for the entire cluster and return them as part of the {@link ClusterStatsResponse}. + *
+ * Theory of operation: + * - The snapshot is created on each particular node with the stats for the node, and is sent to the coordinating node + * - Coordinating node creates an empty snapshot and merges all the node snapshots into it using add() + *
+ * The snapshot contains {@link LongMetricValue}s for latencies, which currently contain full histograms (since you can't + * produce p90 from a set of node p90s, you need the full histogram for that). To avoid excessive copying (histogram weighs several KB), + * the snapshot is designed to be mutable, so that you can add multiple snapshots to it without copying the histograms all the time. + * It is not the intent to mutate the snapshot objects otherwise. + *
+ */ +public final class CCSTelemetrySnapshot implements Writeable, ToXContentFragment { + public static final String CCS_TELEMETRY_FIELD_NAME = "_search"; + private long totalCount; + private long successCount; + private final Map failureReasons; + + /** + * Latency metrics, overall. + */ + private final LongMetricValue took; + /** + * Latency metrics with minimize_roundtrips=true + */ + private final LongMetricValue tookMrtTrue; + /** + * Latency metrics with minimize_roundtrips=false + */ + private final LongMetricValue tookMrtFalse; + private long remotesPerSearchMax; + private double remotesPerSearchAvg; + private long skippedRemotes; + + private final Map featureCounts; + + private final Map clientCounts; + private final Map byRemoteCluster; + + /** + * Creates a new stats instance with the provided info. + */ + public CCSTelemetrySnapshot( + long totalCount, + long successCount, + Map failureReasons, + LongMetricValue took, + LongMetricValue tookMrtTrue, + LongMetricValue tookMrtFalse, + long remotesPerSearchMax, + double remotesPerSearchAvg, + long skippedRemotes, + Map featureCounts, + Map clientCounts, + Map byRemoteCluster + ) { + this.totalCount = totalCount; + this.successCount = successCount; + this.failureReasons = failureReasons; + this.took = took; + this.tookMrtTrue = tookMrtTrue; + this.tookMrtFalse = tookMrtFalse; + this.remotesPerSearchMax = remotesPerSearchMax; + this.remotesPerSearchAvg = remotesPerSearchAvg; + this.skippedRemotes = skippedRemotes; + this.featureCounts = featureCounts; + this.clientCounts = clientCounts; + this.byRemoteCluster = byRemoteCluster; + } + + /** + * Creates a new empty stats instance, that will get additional stats added through {@link #add(CCSTelemetrySnapshot)} + */ + public CCSTelemetrySnapshot() { + // Note this produces modifiable maps, so other snapshots can be merged into it + failureReasons = new HashMap<>(); + featureCounts = new HashMap<>(); + clientCounts = new HashMap<>(); + byRemoteCluster = new HashMap<>(); + took = new LongMetricValue(); + tookMrtTrue = new LongMetricValue(); + tookMrtFalse = new LongMetricValue(); + } + + public CCSTelemetrySnapshot(StreamInput in) throws IOException { + this.totalCount = in.readVLong(); + this.successCount = in.readVLong(); + this.failureReasons = in.readMap(StreamInput::readLong); + this.took = LongMetricValue.fromStream(in); + this.tookMrtTrue = LongMetricValue.fromStream(in); + this.tookMrtFalse = LongMetricValue.fromStream(in); + this.remotesPerSearchMax = in.readVLong(); + this.remotesPerSearchAvg = in.readDouble(); + this.skippedRemotes = in.readVLong(); + this.featureCounts = in.readMap(StreamInput::readLong); + this.clientCounts = in.readMap(StreamInput::readLong); + this.byRemoteCluster = in.readMap(PerClusterCCSTelemetry::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVLong(totalCount); + out.writeVLong(successCount); + out.writeMap(failureReasons, StreamOutput::writeLong); + took.writeTo(out); + tookMrtTrue.writeTo(out); + tookMrtFalse.writeTo(out); + out.writeVLong(remotesPerSearchMax); + out.writeDouble(remotesPerSearchAvg); + out.writeVLong(skippedRemotes); + out.writeMap(featureCounts, StreamOutput::writeLong); + out.writeMap(clientCounts, StreamOutput::writeLong); + out.writeMap(byRemoteCluster, StreamOutput::writeWriteable); + } + + public long getTotalCount() { + return totalCount; + } + + public long getSuccessCount() { + return successCount; + } + + public Map getFailureReasons() { + return Collections.unmodifiableMap(failureReasons); + } + + public LongMetricValue getTook() { + return took; + } + + public LongMetricValue getTookMrtTrue() { + return tookMrtTrue; + } + + public LongMetricValue getTookMrtFalse() { + return tookMrtFalse; + } + + public long getRemotesPerSearchMax() { + return remotesPerSearchMax; + } + + public double getRemotesPerSearchAvg() { + return remotesPerSearchAvg; + } + + public long getSearchCountWithSkippedRemotes() { + return skippedRemotes; + } + + public Map getFeatureCounts() { + return Collections.unmodifiableMap(featureCounts); + } + + public Map getClientCounts() { + return Collections.unmodifiableMap(clientCounts); + } + + public Map getByRemoteCluster() { + return Collections.unmodifiableMap(byRemoteCluster); + } + + public static class PerClusterCCSTelemetry implements Writeable, ToXContentFragment { + private long count; + private long skippedCount; + private final LongMetricValue took; + + public PerClusterCCSTelemetry() { + took = new LongMetricValue(); + } + + public PerClusterCCSTelemetry(long count, long skippedCount, LongMetricValue took) { + this.took = took; + this.skippedCount = skippedCount; + this.count = count; + } + + public PerClusterCCSTelemetry(PerClusterCCSTelemetry other) { + this.count = other.count; + this.skippedCount = other.skippedCount; + this.took = new LongMetricValue(other.took); + } + + public PerClusterCCSTelemetry(StreamInput in) throws IOException { + this.count = in.readVLong(); + this.skippedCount = in.readVLong(); + this.took = LongMetricValue.fromStream(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVLong(count); + out.writeVLong(skippedCount); + took.writeTo(out); + } + + public PerClusterCCSTelemetry add(PerClusterCCSTelemetry v) { + count += v.count; + skippedCount += v.skippedCount; + took.add(v.took); + return this; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("total", count); + builder.field("skipped", skippedCount); + publishLatency(builder, "took", took); + builder.endObject(); + return builder; + } + + public long getCount() { + return count; + } + + public long getSkippedCount() { + return skippedCount; + } + + public LongMetricValue getTook() { + return took; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + PerClusterCCSTelemetry that = (PerClusterCCSTelemetry) o; + return count == that.count && skippedCount == that.skippedCount && Objects.equals(took, that.took); + } + + @Override + public int hashCode() { + return Objects.hash(count, skippedCount, took); + } + } + + /** + * Add the provided stats to the ones held by the current instance, effectively merging the two. + * @param stats the other stats object to add to this one + */ + public void add(CCSTelemetrySnapshot stats) { + // This should be called in ClusterStatsResponse ctor only, so we don't need to worry about concurrency + if (stats.totalCount == 0) { + // Just ignore the empty stats. + // This could happen if the node is brand new or if the stats are not available, e.g. because it runs an old version. + return; + } + long oldCount = totalCount; + totalCount += stats.totalCount; + successCount += stats.successCount; + skippedRemotes += stats.skippedRemotes; + stats.failureReasons.forEach((k, v) -> failureReasons.merge(k, v, Long::sum)); + stats.featureCounts.forEach((k, v) -> featureCounts.merge(k, v, Long::sum)); + stats.clientCounts.forEach((k, v) -> clientCounts.merge(k, v, Long::sum)); + took.add(stats.took); + tookMrtTrue.add(stats.tookMrtTrue); + tookMrtFalse.add(stats.tookMrtFalse); + remotesPerSearchMax = Math.max(remotesPerSearchMax, stats.remotesPerSearchMax); + if (totalCount > 0 && oldCount > 0) { + // Weighted average + remotesPerSearchAvg = (remotesPerSearchAvg * oldCount + stats.remotesPerSearchAvg * stats.totalCount) / totalCount; + } else { + // If we didn't have any old value, we just take the new one + remotesPerSearchAvg = stats.remotesPerSearchAvg; + } + // we copy the object here since we'll be modifying it later on subsequent adds + // TODO: this may be sub-optimal, as we'll be copying histograms when adding first snapshot to an empty container, + // which we could have avoided probably. + stats.byRemoteCluster.forEach((r, v) -> byRemoteCluster.merge(r, new PerClusterCCSTelemetry(v), PerClusterCCSTelemetry::add)); + } + + /** + * Publishes the latency statistics to the provided {@link XContentBuilder}. + * Example: + * "took": { + * "max": 345032, + * "avg": 1620, + * "p90": 2570 + * } + */ + public static void publishLatency(XContentBuilder builder, String name, LongMetricValue took) throws IOException { + builder.startObject(name); + { + builder.field("max", took.max()); + builder.field("avg", took.avg()); + builder.field("p90", took.p90()); + } + builder.endObject(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(CCS_TELEMETRY_FIELD_NAME); + { + builder.field("total", totalCount); + builder.field("success", successCount); + builder.field("skipped", skippedRemotes); + publishLatency(builder, "took", took); + publishLatency(builder, "took_mrt_true", tookMrtTrue); + publishLatency(builder, "took_mrt_false", tookMrtFalse); + builder.field("remotes_per_search_max", remotesPerSearchMax); + builder.field("remotes_per_search_avg", remotesPerSearchAvg); + builder.field("failure_reasons", failureReasons); + builder.field("features", featureCounts); + builder.field("clients", clientCounts); + builder.startObject("clusters"); + { + for (var entry : byRemoteCluster.entrySet()) { + String remoteName = entry.getKey(); + if (RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY.equals(remoteName)) { + remoteName = SearchResponse.LOCAL_CLUSTER_NAME_REPRESENTATION; + } + builder.field(remoteName, entry.getValue()); + } + } + builder.endObject(); + } + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + CCSTelemetrySnapshot that = (CCSTelemetrySnapshot) o; + return totalCount == that.totalCount + && successCount == that.successCount + && skippedRemotes == that.skippedRemotes + && Objects.equals(failureReasons, that.failureReasons) + && Objects.equals(took, that.took) + && Objects.equals(tookMrtTrue, that.tookMrtTrue) + && Objects.equals(tookMrtFalse, that.tookMrtFalse) + && Objects.equals(remotesPerSearchMax, that.remotesPerSearchMax) + && Objects.equals(remotesPerSearchAvg, that.remotesPerSearchAvg) + && Objects.equals(featureCounts, that.featureCounts) + && Objects.equals(clientCounts, that.clientCounts) + && Objects.equals(byRemoteCluster, that.byRemoteCluster); + } + + @Override + public int hashCode() { + return Objects.hash( + totalCount, + successCount, + failureReasons, + took, + tookMrtTrue, + tookMrtFalse, + remotesPerSearchMax, + remotesPerSearchAvg, + skippedRemotes, + featureCounts, + clientCounts, + byRemoteCluster + ); + } + + @Override + public String toString() { + return Strings.toString(this, true, true); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsage.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsage.java new file mode 100644 index 0000000000000..b2d75ac8f61f3 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsage.java @@ -0,0 +1,246 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.admin.cluster.stats; + +import org.elasticsearch.ElasticsearchSecurityException; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ShardOperationFailedException; +import org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry.Result; +import org.elasticsearch.action.search.SearchPhaseExecutionException; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.action.search.TransportSearchAction; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.search.SearchShardTarget; +import org.elasticsearch.search.query.SearchTimeoutException; +import org.elasticsearch.tasks.TaskCancelledException; +import org.elasticsearch.transport.ConnectTransportException; +import org.elasticsearch.transport.NoSeedNodeLeftException; +import org.elasticsearch.transport.NoSuchRemoteClusterException; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.transport.RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; + +/** + * This is a container for telemetry data from an individual cross-cluster search for _search or _async_search (or + * other search endpoints that use the {@link TransportSearchAction} such as _msearch). + */ +public class CCSUsage { + private final long took; + private final Result status; + private final Set features; + private final int remotesCount; + + private final String client; + + private final Set skippedRemotes; + private final Map perClusterUsage; + + public static class Builder { + private long took; + private final Set features; + private Result status = Result.SUCCESS; + private int remotesCount; + private String client; + private final Set skippedRemotes; + private final Map perClusterUsage; + + public Builder() { + features = new HashSet<>(); + skippedRemotes = new HashSet<>(); + perClusterUsage = new HashMap<>(); + } + + public Builder took(long took) { + this.took = took; + return this; + } + + public Builder setFailure(Result failureType) { + this.status = failureType; + return this; + } + + public Builder setFailure(Exception e) { + return setFailure(getFailureType(e)); + } + + public Builder setFeature(String feature) { + this.features.add(feature); + return this; + } + + public Builder setClient(String client) { + this.client = client; + return this; + } + + public Builder skippedRemote(String remote) { + this.skippedRemotes.add(remote); + return this; + } + + public Builder perClusterUsage(String remote, TimeValue took) { + this.perClusterUsage.put(remote, new PerClusterUsage(took)); + return this; + } + + public CCSUsage build() { + return new CCSUsage(took, status, remotesCount, skippedRemotes, features, client, perClusterUsage); + } + + public Builder setRemotesCount(int remotesCount) { + this.remotesCount = remotesCount; + return this; + } + + public int getRemotesCount() { + return remotesCount; + } + + /** + * Get failure type as {@link Result} from the search failure exception. + */ + public static Result getFailureType(Exception e) { + var unwrapped = ExceptionsHelper.unwrapCause(e); + if (unwrapped instanceof Exception) { + e = (Exception) unwrapped; + } + if (isRemoteUnavailable(e)) { + return Result.REMOTES_UNAVAILABLE; + } + if (ExceptionsHelper.unwrap(e, ResourceNotFoundException.class) != null) { + return Result.NOT_FOUND; + } + if (e instanceof TaskCancelledException || (ExceptionsHelper.unwrap(e, TaskCancelledException.class) != null)) { + return Result.CANCELED; + } + if (ExceptionsHelper.unwrap(e, SearchTimeoutException.class) != null) { + return Result.TIMEOUT; + } + if (ExceptionsHelper.unwrap(e, ElasticsearchSecurityException.class) != null) { + return Result.SECURITY; + } + if (ExceptionsHelper.unwrapCorruption(e) != null) { + return Result.CORRUPTION; + } + // This is kind of last resort check - if we still don't know the reason but all shard failures are remote, + // we assume it's remote's fault somehow. + if (e instanceof SearchPhaseExecutionException spe) { + // If this is a failure that happened because of remote failures only + var groupedFails = ExceptionsHelper.groupBy(spe.shardFailures()); + if (Arrays.stream(groupedFails).allMatch(Builder::isRemoteFailure)) { + return Result.REMOTES_UNAVAILABLE; + } + } + // OK we don't know what happened + return Result.UNKNOWN; + } + + /** + * Is this failure exception because remote was unavailable? + * See also: TransportResolveClusterAction#notConnectedError + */ + static boolean isRemoteUnavailable(Exception e) { + if (ExceptionsHelper.unwrap( + e, + ConnectTransportException.class, + NoSuchRemoteClusterException.class, + NoSeedNodeLeftException.class + ) != null) { + return true; + } + Throwable ill = ExceptionsHelper.unwrap(e, IllegalStateException.class, IllegalArgumentException.class); + if (ill != null && (ill.getMessage().contains("Unable to open any connections") || ill.getMessage().contains("unknown host"))) { + return true; + } + // Ok doesn't look like any of the known remote exceptions + return false; + } + + /** + * Is this failure coming from a remote cluster? + */ + static boolean isRemoteFailure(ShardOperationFailedException failure) { + if (failure instanceof ShardSearchFailure shardFailure) { + SearchShardTarget shard = shardFailure.shard(); + return shard != null && shard.getClusterAlias() != null && LOCAL_CLUSTER_GROUP_KEY.equals(shard.getClusterAlias()) == false; + } + return false; + } + } + + private CCSUsage( + long took, + Result status, + int remotesCount, + Set skippedRemotes, + Set features, + String client, + Map perClusterUsage + ) { + this.status = status; + this.remotesCount = remotesCount; + this.features = features; + this.client = client; + this.took = took; + this.skippedRemotes = skippedRemotes; + this.perClusterUsage = perClusterUsage; + } + + public Map getPerClusterUsage() { + return perClusterUsage; + } + + public Result getStatus() { + return status; + } + + public Set getFeatures() { + return features; + } + + public long getRemotesCount() { + return remotesCount; + } + + public String getClient() { + return client; + } + + public long getTook() { + return took; + } + + public Set getSkippedRemotes() { + return skippedRemotes; + } + + public static class PerClusterUsage { + + // if MRT=true, the took time on the remote cluster (if MRT=true), otherwise the overall took time + private long took; + + public PerClusterUsage(TimeValue took) { + if (took != null) { + this.took = took.millis(); + } + } + + public long getTook() { + return took; + } + } + +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetry.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetry.java new file mode 100644 index 0000000000000..60766bd4068e3 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/CCSUsageTelemetry.java @@ -0,0 +1,246 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.admin.cluster.stats; + +import org.elasticsearch.common.util.Maps; + +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.LongAdder; + +/** + * Service holding accumulated CCS search usage statistics. Individual cross-cluster searches will pass + * CCSUsage data here to have it collated and aggregated. Snapshots of the current CCS Telemetry Usage + * can be obtained by getting {@link CCSTelemetrySnapshot} objects. + *
+ * Theory of operation: + * Each search creates a {@link CCSUsage.Builder}, which can be updated during the progress of the search request, + * and then it instantiates a {@link CCSUsage} object when the request is finished. + * That object is passed to {@link #updateUsage(CCSUsage)} on the request processing end (whether successful or not). + * The {@link #updateUsage(CCSUsage)} method will then update the internal counters and metrics. + *
+ * When we need to return the current state of the telemetry, we can call {@link #getCCSTelemetrySnapshot()} which produces + * a snapshot of the current state of the telemetry as {@link CCSTelemetrySnapshot}. These snapshots are additive so + * when collecting the snapshots from multiple nodes, an empty snapshot is created and then all the node's snapshots are added + * to it to obtain the summary telemetry. + */ +public class CCSUsageTelemetry { + + /** + * Result of the request execution. + * Either "success" or a failure reason. + */ + public enum Result { + SUCCESS("success"), + REMOTES_UNAVAILABLE("remotes_unavailable"), + CANCELED("canceled"), + NOT_FOUND("not_found"), + TIMEOUT("timeout"), + CORRUPTION("corruption"), + SECURITY("security"), + // May be helpful if there's a lot of other reasons, and it may be hard to calculate the unknowns for some clients. + UNKNOWN("other"); + + private final String name; + + Result(String name) { + this.name = name; + } + + public String getName() { + return name; + } + } + + // Not enum because we won't mind other places adding their own features + public static final String MRT_FEATURE = "mrt_on"; + public static final String ASYNC_FEATURE = "async"; + public static final String WILDCARD_FEATURE = "wildcards"; + + // The list of known Elastic clients. May be incomplete. + public static final Set KNOWN_CLIENTS = Set.of( + "kibana", + "cloud", + "logstash", + "beats", + "fleet", + "ml", + "security", + "observability", + "enterprise-search", + "elasticsearch", + "connectors", + "connectors-cli" + ); + + private final LongAdder totalCount; + private final LongAdder successCount; + private final Map failureReasons; + + /** + * Latency metrics overall + */ + private final LongMetric took; + /** + * Latency metrics with minimize_roundtrips=true + */ + private final LongMetric tookMrtTrue; + /** + * Latency metrics with minimize_roundtrips=false + */ + private final LongMetric tookMrtFalse; + private final LongMetric remotesPerSearch; + private final LongAdder skippedRemotes; + + private final Map featureCounts; + + private final Map clientCounts; + private final Map byRemoteCluster; + + public CCSUsageTelemetry() { + this.byRemoteCluster = new ConcurrentHashMap<>(); + totalCount = new LongAdder(); + successCount = new LongAdder(); + failureReasons = new ConcurrentHashMap<>(); + took = new LongMetric(); + tookMrtTrue = new LongMetric(); + tookMrtFalse = new LongMetric(); + remotesPerSearch = new LongMetric(); + skippedRemotes = new LongAdder(); + featureCounts = new ConcurrentHashMap<>(); + clientCounts = new ConcurrentHashMap<>(); + } + + public void updateUsage(CCSUsage ccsUsage) { + assert ccsUsage.getRemotesCount() > 0 : "Expected at least one remote cluster in CCSUsage"; + // TODO: fork this to a background thread? + doUpdate(ccsUsage); + } + + // This is not synchronized, instead we ensure that every metric in the class is thread-safe. + private void doUpdate(CCSUsage ccsUsage) { + totalCount.increment(); + long searchTook = ccsUsage.getTook(); + if (isSuccess(ccsUsage)) { + successCount.increment(); + took.record(searchTook); + if (isMRT(ccsUsage)) { + tookMrtTrue.record(searchTook); + } else { + tookMrtFalse.record(searchTook); + } + ccsUsage.getPerClusterUsage().forEach((r, u) -> byRemoteCluster.computeIfAbsent(r, PerClusterCCSTelemetry::new).update(u)); + } else { + failureReasons.computeIfAbsent(ccsUsage.getStatus(), k -> new LongAdder()).increment(); + } + + remotesPerSearch.record(ccsUsage.getRemotesCount()); + if (ccsUsage.getSkippedRemotes().isEmpty() == false) { + skippedRemotes.increment(); + ccsUsage.getSkippedRemotes().forEach(remote -> byRemoteCluster.computeIfAbsent(remote, PerClusterCCSTelemetry::new).skipped()); + } + ccsUsage.getFeatures().forEach(f -> featureCounts.computeIfAbsent(f, k -> new LongAdder()).increment()); + String client = ccsUsage.getClient(); + if (client != null && KNOWN_CLIENTS.contains(client)) { + // We count only known clients for now + clientCounts.computeIfAbsent(ccsUsage.getClient(), k -> new LongAdder()).increment(); + } + } + + private boolean isMRT(CCSUsage ccsUsage) { + return ccsUsage.getFeatures().contains(MRT_FEATURE); + } + + private boolean isSuccess(CCSUsage ccsUsage) { + return ccsUsage.getStatus() == Result.SUCCESS; + } + + public Map getTelemetryByCluster() { + return byRemoteCluster; + } + + /** + * Telemetry of each remote involved in cross cluster searches + */ + public static class PerClusterCCSTelemetry { + private final String clusterAlias; + // The number of successful (not skipped) requests to this cluster. + private final LongAdder count; + private final LongAdder skippedCount; + // This is only over the successful requetss, skipped ones do not count here. + private final LongMetric took; + + PerClusterCCSTelemetry(String clusterAlias) { + this.clusterAlias = clusterAlias; + this.count = new LongAdder(); + took = new LongMetric(); + this.skippedCount = new LongAdder(); + } + + void update(CCSUsage.PerClusterUsage remoteUsage) { + count.increment(); + took.record(remoteUsage.getTook()); + } + + void skipped() { + skippedCount.increment(); + } + + public long getCount() { + return count.longValue(); + } + + @Override + public String toString() { + return "PerClusterCCSTelemetry{" + + "clusterAlias='" + + clusterAlias + + '\'' + + ", count=" + + count + + ", latency=" + + took.toString() + + '}'; + } + + public long getSkippedCount() { + return skippedCount.longValue(); + } + + public CCSTelemetrySnapshot.PerClusterCCSTelemetry getSnapshot() { + return new CCSTelemetrySnapshot.PerClusterCCSTelemetry(count.longValue(), skippedCount.longValue(), took.getValue()); + } + + } + + public CCSTelemetrySnapshot getCCSTelemetrySnapshot() { + Map reasonsMap = Maps.newMapWithExpectedSize(failureReasons.size()); + failureReasons.forEach((k, v) -> reasonsMap.put(k.getName(), v.longValue())); + + LongMetric.LongMetricValue remotes = remotesPerSearch.getValue(); + + // Maps returned here are unmodifiable, but the empty ctor produces modifiable maps + return new CCSTelemetrySnapshot( + totalCount.longValue(), + successCount.longValue(), + Collections.unmodifiableMap(reasonsMap), + took.getValue(), + tookMrtTrue.getValue(), + tookMrtFalse.getValue(), + remotes.max(), + remotes.avg(), + skippedRemotes.longValue(), + Collections.unmodifiableMap(Maps.transformValues(featureCounts, LongAdder::longValue)), + Collections.unmodifiableMap(Maps.transformValues(clientCounts, LongAdder::longValue)), + Collections.unmodifiableMap(Maps.transformValues(byRemoteCluster, PerClusterCCSTelemetry::getSnapshot)) + ); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java index d74889b623589..b48295dc8b3eb 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java @@ -20,29 +20,33 @@ import org.elasticsearch.core.Nullable; import java.io.IOException; +import java.util.Objects; public class ClusterStatsNodeResponse extends BaseNodeResponse { private final NodeInfo nodeInfo; private final NodeStats nodeStats; private final ShardStats[] shardsStats; - private ClusterHealthStatus clusterStatus; + private final ClusterHealthStatus clusterStatus; private final SearchUsageStats searchUsageStats; + private final RepositoryUsageStats repositoryUsageStats; public ClusterStatsNodeResponse(StreamInput in) throws IOException { super(in); - clusterStatus = null; - if (in.readBoolean()) { - clusterStatus = ClusterHealthStatus.readFrom(in); - } + this.clusterStatus = in.readOptionalWriteable(ClusterHealthStatus::readFrom); this.nodeInfo = new NodeInfo(in); this.nodeStats = new NodeStats(in); - shardsStats = in.readArray(ShardStats::new, ShardStats[]::new); + this.shardsStats = in.readArray(ShardStats::new, ShardStats[]::new); if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_6_0)) { searchUsageStats = new SearchUsageStats(in); } else { searchUsageStats = new SearchUsageStats(); } + if (in.getTransportVersion().onOrAfter(TransportVersions.REPOSITORIES_TELEMETRY)) { + repositoryUsageStats = RepositoryUsageStats.readFrom(in); + } else { + repositoryUsageStats = RepositoryUsageStats.EMPTY; + } } public ClusterStatsNodeResponse( @@ -51,14 +55,16 @@ public ClusterStatsNodeResponse( NodeInfo nodeInfo, NodeStats nodeStats, ShardStats[] shardsStats, - SearchUsageStats searchUsageStats + SearchUsageStats searchUsageStats, + RepositoryUsageStats repositoryUsageStats ) { super(node); this.nodeInfo = nodeInfo; this.nodeStats = nodeStats; this.shardsStats = shardsStats; this.clusterStatus = clusterStatus; - this.searchUsageStats = searchUsageStats; + this.searchUsageStats = Objects.requireNonNull(searchUsageStats); + this.repositoryUsageStats = Objects.requireNonNull(repositoryUsageStats); } public NodeInfo nodeInfo() { @@ -85,20 +91,22 @@ public SearchUsageStats searchUsageStats() { return searchUsageStats; } + public RepositoryUsageStats repositoryUsageStats() { + return repositoryUsageStats; + } + @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - if (clusterStatus == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - out.writeByte(clusterStatus.value()); - } + out.writeOptionalWriteable(clusterStatus); nodeInfo.writeTo(out); nodeStats.writeTo(out); out.writeArray(shardsStats); if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_6_0)) { searchUsageStats.writeTo(out); } + if (out.getTransportVersion().onOrAfter(TransportVersions.REPOSITORIES_TELEMETRY)) { + repositoryUsageStats.writeTo(out); + } // else just drop these stats, ok for bwc } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java index 36e7b247befac..b6dd40e8c8b79 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsResponse.java @@ -30,6 +30,7 @@ public class ClusterStatsResponse extends BaseNodesResponse r.isEmpty() == false) + // stats should be the same on every node so just pick one of them + .findAny() + .orElse(RepositoryUsageStats.EMPTY); } public String getClusterUUID() { @@ -113,6 +122,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("snapshots"); clusterSnapshotStats.toXContent(builder, params); + builder.field("repositories"); + repositoryUsageStats.toXContent(builder, params); + return builder; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/LongMetric.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/LongMetric.java new file mode 100644 index 0000000000000..f3bb936b108c0 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/LongMetric.java @@ -0,0 +1,126 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.admin.cluster.stats; + +import org.HdrHistogram.ConcurrentHistogram; +import org.HdrHistogram.Histogram; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; +import java.util.zip.DataFormatException; + +/** + * Metric class that accepts longs and provides count, average, max and percentiles. + * Abstracts out the details of how exactly the values are stored and calculated. + * {@link LongMetricValue} is a snapshot of the current state of the metric. + */ +public class LongMetric { + private final Histogram values; + private static final int SIGNIFICANT_DIGITS = 2; + + LongMetric() { + values = new ConcurrentHistogram(SIGNIFICANT_DIGITS); + } + + void record(long v) { + values.recordValue(v); + } + + LongMetricValue getValue() { + return new LongMetricValue(values); + } + + /** + * Snapshot of {@link LongMetric} value that provides the current state of the metric. + * Can be added with another {@link LongMetricValue} object. + */ + public static final class LongMetricValue implements Writeable { + // We have to carry the full histogram around since we might need to calculate aggregate percentiles + // after collecting individual stats from the nodes, and we can't do that without having the full histogram. + // This costs about 2K per metric, which was deemed acceptable. + private final Histogram values; + + public LongMetricValue(Histogram values) { + // Copy here since we don't want the snapshot value to change if somebody updates the original one + this.values = values.copy(); + } + + public LongMetricValue(LongMetricValue v) { + this.values = v.values.copy(); + } + + LongMetricValue() { + this.values = new Histogram(SIGNIFICANT_DIGITS); + } + + public void add(LongMetricValue v) { + this.values.add(v.values); + } + + public static LongMetricValue fromStream(StreamInput in) throws IOException { + byte[] b = in.readByteArray(); + ByteBuffer bb = ByteBuffer.wrap(b); + try { + // TODO: not sure what is the good value for minBarForHighestToLowestValueRatio here? + Histogram dh = Histogram.decodeFromCompressedByteBuffer(bb, 1); + return new LongMetricValue(dh); + } catch (DataFormatException e) { + throw new IOException(e); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + ByteBuffer b = ByteBuffer.allocate(values.getNeededByteBufferCapacity()); + values.encodeIntoCompressedByteBuffer(b); + int size = b.position(); + out.writeVInt(size); + out.writeBytes(b.array(), 0, size); + } + + public long count() { + return values.getTotalCount(); + } + + public long max() { + return values.getMaxValue(); + } + + public long avg() { + return (long) Math.ceil(values.getMean()); + } + + public long p90() { + return values.getValueAtPercentile(90); + } + + @Override + public boolean equals(Object obj) { + if (obj == this) return true; + if (obj == null || obj.getClass() != this.getClass()) return false; + var that = (LongMetricValue) obj; + return this.values.equals(that.values); + } + + @Override + public int hashCode() { + return Objects.hash(values); + } + + @Override + public String toString() { + return "LongMetricValue[count=" + count() + ", " + "max=" + max() + ", " + "avg=" + avg() + "]"; + } + + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RepositoryUsageStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RepositoryUsageStats.java new file mode 100644 index 0000000000000..771aa0fbef842 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/RepositoryUsageStats.java @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.admin.cluster.stats; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Map; + +/** + * Stats on repository feature usage exposed in cluster stats for telemetry. + * + * @param statsByType a count of the repositories using various named features, keyed by repository type and then by feature name. + */ +public record RepositoryUsageStats(Map> statsByType) implements Writeable, ToXContentObject { + + public static final RepositoryUsageStats EMPTY = new RepositoryUsageStats(Map.of()); + + public static RepositoryUsageStats readFrom(StreamInput in) throws IOException { + final var statsByType = in.readMap(i -> i.readMap(StreamInput::readVLong)); + if (statsByType.isEmpty()) { + return EMPTY; + } else { + return new RepositoryUsageStats(statsByType); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeMap(statsByType, (o, m) -> o.writeMap(m, StreamOutput::writeVLong)); + } + + public boolean isEmpty() { + return statsByType.isEmpty(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + for (Map.Entry> typeAndStats : statsByType.entrySet()) { + builder.startObject(typeAndStats.getKey()); + for (Map.Entry statAndValue : typeAndStats.getValue().entrySet()) { + builder.field(statAndValue.getKey(), statAndValue.getValue()); + } + builder.endObject(); + } + return builder.endObject(); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java index bcf49bca421f6..1912de3cfa4d2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -41,6 +41,7 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.node.NodeService; +import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; @@ -78,6 +79,7 @@ public class TransportClusterStatsAction extends TransportNodesAction< private final NodeService nodeService; private final IndicesService indicesService; + private final RepositoriesService repositoriesService; private final SearchUsageHolder searchUsageHolder; private final MetadataStatsCache mappingStatsCache; @@ -90,6 +92,7 @@ public TransportClusterStatsAction( TransportService transportService, NodeService nodeService, IndicesService indicesService, + RepositoriesService repositoriesService, UsageService usageService, ActionFilters actionFilters ) { @@ -103,6 +106,7 @@ public TransportClusterStatsAction( ); this.nodeService = nodeService; this.indicesService = indicesService; + this.repositoriesService = repositoriesService; this.searchUsageHolder = usageService.getSearchUsageHolder(); this.mappingStatsCache = new MetadataStatsCache<>(threadPool.getThreadContext(), MappingStats::of); this.analysisStatsCache = new MetadataStatsCache<>(threadPool.getThreadContext(), AnalysisStats::of); @@ -237,12 +241,14 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq } } - ClusterHealthStatus clusterStatus = null; - if (clusterService.state().nodes().isLocalNodeElectedMaster()) { - clusterStatus = new ClusterStateHealth(clusterService.state()).getStatus(); - } + final ClusterState clusterState = clusterService.state(); + final ClusterHealthStatus clusterStatus = clusterState.nodes().isLocalNodeElectedMaster() + ? new ClusterStateHealth(clusterState).getStatus() + : null; + + final SearchUsageStats searchUsageStats = searchUsageHolder.getSearchUsageStats(); - SearchUsageStats searchUsageStats = searchUsageHolder.getSearchUsageStats(); + final RepositoryUsageStats repositoryUsageStats = repositoriesService.getUsageStats(); return new ClusterStatsNodeResponse( nodeInfo.getNode(), @@ -250,7 +256,8 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq nodeInfo, nodeStats, shardsStats.toArray(new ShardStats[shardsStats.size()]), - searchUsageStats + searchUsageStats, + repositoryUsageStats ); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java index 8a46daa45e73b..948199fbe74f4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java @@ -12,7 +12,6 @@ import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.cluster.ack.ClusterStateUpdateRequest; -import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; @@ -43,8 +42,6 @@ public class CreateIndexClusterStateUpdateRequest extends ClusterStateUpdateRequ private final Set aliases = new HashSet<>(); - private final Set blocks = new HashSet<>(); - private ActiveShardCount waitForActiveShards = ActiveShardCount.DEFAULT; private boolean performReroute = true; @@ -125,10 +122,6 @@ public Set aliases() { return aliases; } - public Set blocks() { - return blocks; - } - public Index recoverFrom() { return recoverFrom; } @@ -229,8 +222,6 @@ public String toString() { + settings + ", aliases=" + aliases - + ", blocks=" - + blocks + ", waitForActiveShards=" + waitForActiveShards + ", systemDataStreamDescriptor=" diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionRequest.java index 118f139045971..224ea63150420 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionRequest.java @@ -54,12 +54,14 @@ public ResolveClusterActionRequest(String[] names) { this(names, DEFAULT_INDICES_OPTIONS); } + @SuppressWarnings("this-escape") public ResolveClusterActionRequest(String[] names, IndicesOptions indicesOptions) { this.names = names; this.localIndicesRequested = localIndicesPresent(names); this.indicesOptions = indicesOptions; } + @SuppressWarnings("this-escape") public ResolveClusterActionRequest(StreamInput in) throws IOException { super(in); if (in.getTransportVersion().before(TransportVersions.V_8_13_0)) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java index ef72fdd93caeb..65b768a1c629f 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java @@ -21,6 +21,7 @@ import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; import org.elasticsearch.cluster.metadata.MetadataDataStreamsService; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.allocator.AllocationActionMultiListener; @@ -134,7 +135,7 @@ protected void masterOperation( ); final String trialSourceIndexName = trialRolloverNames.sourceName(); final String trialRolloverIndexName = trialRolloverNames.rolloverName(); - MetadataRolloverService.validateIndexName(clusterState, trialRolloverIndexName); + MetadataCreateIndexService.validateIndexName(trialRolloverIndexName, clusterState.metadata(), clusterState.routingTable()); assert metadata.dataStreams().containsKey(rolloverRequest.getRolloverTarget()) : "Auto-rollover applies only to data streams"; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java index 9d34b9ab5f126..b8d975f82980d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java @@ -179,10 +179,6 @@ public RolloverResult rolloverClusterState( }; } - public static void validateIndexName(ClusterState state, String index) { - MetadataCreateIndexService.validateIndexName(index, state); - } - /** * Returns the names that rollover would use, but does not perform the actual rollover */ @@ -252,7 +248,8 @@ private RolloverResult rolloverAlias( final Boolean isHidden = IndexMetadata.INDEX_HIDDEN_SETTING.exists(createIndexRequest.settings()) ? IndexMetadata.INDEX_HIDDEN_SETTING.get(createIndexRequest.settings()) : null; - MetadataCreateIndexService.validateIndexName(rolloverIndexName, currentState); // fails if the index already exists + MetadataCreateIndexService.validateIndexName(rolloverIndexName, metadata, currentState.routingTable()); // fails if the index + // already exists checkNoDuplicatedAliasInIndexTemplate(metadata, rolloverIndexName, aliasName, isHidden); if (onlyValidate) { return new RolloverResult(rolloverIndexName, sourceIndexName, currentState); @@ -328,7 +325,8 @@ private RolloverResult rolloverDataStream( final Tuple nextIndexAndGeneration = dataStream.nextWriteIndexAndGeneration(metadata, dataStreamIndices); final String newWriteIndexName = nextIndexAndGeneration.v1(); final long newGeneration = nextIndexAndGeneration.v2(); - MetadataCreateIndexService.validateIndexName(newWriteIndexName, currentState); // fails if the index already exists + MetadataCreateIndexService.validateIndexName(newWriteIndexName, metadata, currentState.routingTable()); // fails if the index + // already exists if (onlyValidate) { return new RolloverResult(newWriteIndexName, isLazyCreation ? NON_EXISTENT_SOURCE : originalWriteIndex.getName(), currentState); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java index 9df3be1994fdf..c997795bb3b89 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java @@ -36,6 +36,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadataStats; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; import org.elasticsearch.cluster.metadata.MetadataDataStreamsService; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.allocator.AllocationActionMultiListener; @@ -179,7 +180,7 @@ protected void masterOperation( ); final String trialSourceIndexName = trialRolloverNames.sourceName(); final String trialRolloverIndexName = trialRolloverNames.rolloverName(); - MetadataRolloverService.validateIndexName(clusterState, trialRolloverIndexName); + MetadataCreateIndexService.validateIndexName(trialRolloverIndexName, metadata, clusterState.routingTable()); boolean isDataStream = metadata.dataStreams().containsKey(rolloverRequest.getRolloverTarget()); if (rolloverRequest.isLazy()) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index 6fcaad47e0d72..cd8ffea3d3824 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; @@ -249,13 +250,20 @@ public static Template resolveTemplate( .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()); - // empty request mapping as the user can't specify any explicit mappings via the simulate api + /* + * If the index name doesn't look like a data stream backing index, then MetadataCreateIndexService.collectV2Mappings() won't + * include data stream specific mappings in its response. + */ + String simulatedIndexName = template.getDataStreamTemplate() != null + && indexName.startsWith(DataStream.BACKING_INDEX_PREFIX) == false + ? DataStream.getDefaultBackingIndexName(indexName, 1) + : indexName; List mappings = MetadataCreateIndexService.collectV2Mappings( - null, + null, // empty request mapping as the user can't specify any explicit mappings via the simulate api simulatedState, matchingTemplate, xContentRegistry, - indexName + simulatedIndexName ); // First apply settings sourced from index settings providers @@ -303,7 +311,9 @@ public static Template resolveTemplate( ) ); - Map aliasesByName = aliases.stream().collect(Collectors.toMap(AliasMetadata::getAlias, Function.identity())); + Map aliasesByName = aliases == null + ? Map.of() + : aliases.stream().collect(Collectors.toMap(AliasMetadata::getAlias, Function.identity())); CompressedXContent mergedMapping = indicesService.withTempIndexService( indexMetadata, diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java index 99c2d994a8bd0..b8dd0d1fe415e 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java @@ -14,9 +14,10 @@ import java.util.Set; import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_VALIDATION; +import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_VALIDATION_TEMPLATES; public class BulkFeatures implements FeatureSpecification { public Set getFeatures() { - return Set.of(SIMULATE_MAPPING_VALIDATION); + return Set.of(SIMULATE_MAPPING_VALIDATION, SIMULATE_MAPPING_VALIDATION_TEMPLATES); } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkItemRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkItemRequest.java index 425461d1f4ba1..7c1304f92eefd 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkItemRequest.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkItemRequest.java @@ -101,11 +101,11 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(primaryResponse); } - public void writeThin(StreamOutput out) throws IOException { - out.writeVInt(id); - DocWriteRequest.writeDocumentRequestThin(out, request); - out.writeOptionalWriteable(primaryResponse == null ? null : primaryResponse::writeThin); - } + public static final Writer THIN_WRITER = (out, item) -> { + out.writeVInt(item.id); + DocWriteRequest.writeDocumentRequestThin(out, item.request); + out.writeOptional(BulkItemResponse.THIN_WRITER, item.primaryResponse); + }; @Override public long ramBytesUsed() { diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java index 151e8795d0f82..d3e550eaf05b3 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkItemResponse.java @@ -264,7 +264,7 @@ public String toString() { id = in.readVInt(); opType = OpType.fromId(in.readByte()); response = readResponse(shardId, in); - failure = in.readBoolean() ? new Failure(in) : null; + failure = in.readOptionalWriteable(Failure::new); assertConsistent(); } @@ -272,7 +272,7 @@ public String toString() { id = in.readVInt(); opType = OpType.fromId(in.readByte()); response = readResponse(in); - failure = in.readBoolean() ? new Failure(in) : null; + failure = in.readOptionalWriteable(Failure::new); assertConsistent(); } @@ -384,31 +384,21 @@ public void writeTo(StreamOutput out) throws IOException { writeResponseType(out); response.writeTo(out); } - if (failure == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - failure.writeTo(out); - } + out.writeOptionalWriteable(failure); } - public void writeThin(StreamOutput out) throws IOException { - out.writeVInt(id); - out.writeByte(opType.getId()); + public static final Writer THIN_WRITER = (out, item) -> { + out.writeVInt(item.id); + out.writeByte(item.opType.getId()); - if (response == null) { + if (item.response == null) { out.writeByte((byte) 2); } else { - writeResponseType(out); - response.writeThin(out); + item.writeResponseType(out); + item.response.writeThin(out); } - if (failure == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - failure.writeTo(out); - } - } + out.writeOptionalWriteable(item.failure); + }; private void writeResponseType(StreamOutput out) throws IOException { if (response instanceof SimulateIndexResponse) { diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java index 258e5b4c9a58d..64e0b80aca74a 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java @@ -10,7 +10,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRunnable; @@ -42,6 +44,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndexClosedException; import org.elasticsearch.node.NodeClosedException; @@ -91,6 +94,7 @@ final class BulkOperation extends ActionRunnable { private final OriginSettingClient rolloverClient; private final Set failureStoresToBeRolledOver = ConcurrentCollections.newConcurrentSet(); private final Set failedRolloverRequests = ConcurrentCollections.newConcurrentSet(); + private final FailureStoreMetrics failureStoreMetrics; BulkOperation( Task task, @@ -104,7 +108,8 @@ final class BulkOperation extends ActionRunnable { IndexNameExpressionResolver indexNameExpressionResolver, LongSupplier relativeTimeProvider, long startTimeNanos, - ActionListener listener + ActionListener listener, + FailureStoreMetrics failureStoreMetrics ) { this( task, @@ -120,7 +125,8 @@ final class BulkOperation extends ActionRunnable { startTimeNanos, listener, new ClusterStateObserver(clusterService, bulkRequest.timeout(), logger, threadPool.getThreadContext()), - new FailureStoreDocumentConverter() + new FailureStoreDocumentConverter(), + failureStoreMetrics ); } @@ -138,7 +144,8 @@ final class BulkOperation extends ActionRunnable { long startTimeNanos, ActionListener listener, ClusterStateObserver observer, - FailureStoreDocumentConverter failureStoreDocumentConverter + FailureStoreDocumentConverter failureStoreDocumentConverter, + FailureStoreMetrics failureStoreMetrics ) { super(listener); this.task = task; @@ -156,6 +163,7 @@ final class BulkOperation extends ActionRunnable { this.observer = observer; this.failureStoreDocumentConverter = failureStoreDocumentConverter; this.rolloverClient = new OriginSettingClient(client, LAZY_ROLLOVER_ORIGIN); + this.failureStoreMetrics = failureStoreMetrics; } @Override @@ -437,17 +445,11 @@ public void onResponse(BulkShardResponse bulkShardResponse) { for (int idx = 0; idx < bulkShardResponse.getResponses().length; idx++) { // We zip the requests and responses together so that we can identify failed documents and potentially store them BulkItemResponse bulkItemResponse = bulkShardResponse.getResponses()[idx]; + BulkItemRequest bulkItemRequest = bulkShardRequest.items()[idx]; if (bulkItemResponse.isFailed()) { - BulkItemRequest bulkItemRequest = bulkShardRequest.items()[idx]; assert bulkItemRequest.id() == bulkItemResponse.getItemId() : "Bulk items were returned out of order"; - - DataStream failureStoreReference = getRedirectTarget(bulkItemRequest.request(), getClusterState().metadata()); - if (failureStoreReference != null) { - maybeMarkFailureStoreForRollover(failureStoreReference); - var cause = bulkItemResponse.getFailure().getCause(); - addDocumentToRedirectRequests(bulkItemRequest, cause, failureStoreReference.getName()); - } + processFailure(bulkItemRequest, bulkItemResponse.getFailure().getCause()); addFailure(bulkItemResponse); } else { bulkItemResponse.getResponse().setShardInfo(bulkShardResponse.getShardInfo()); @@ -464,11 +466,7 @@ public void onFailure(Exception e) { final String indexName = request.index(); DocWriteRequest docWriteRequest = request.request(); - DataStream failureStoreReference = getRedirectTarget(docWriteRequest, getClusterState().metadata()); - if (failureStoreReference != null) { - maybeMarkFailureStoreForRollover(failureStoreReference); - addDocumentToRedirectRequests(request, e, failureStoreReference.getName()); - } + processFailure(request, e); addFailure(docWriteRequest, request.id(), indexName, e); } completeShardOperation(); @@ -479,45 +477,59 @@ private void completeShardOperation() { clusterState = null; releaseOnFinish.close(); } + + private void processFailure(BulkItemRequest bulkItemRequest, Exception cause) { + var error = ExceptionsHelper.unwrapCause(cause); + var errorType = ElasticsearchException.getExceptionName(error); + DocWriteRequest docWriteRequest = bulkItemRequest.request(); + DataStream failureStoreCandidate = getRedirectTargetCandidate(docWriteRequest, getClusterState().metadata()); + // If the candidate is not null, the BulkItemRequest targets a data stream, but we'll still have to check if + // it has the failure store enabled. + if (failureStoreCandidate != null) { + // Do not redirect documents to a failure store that were already headed to one. + var isFailureStoreDoc = docWriteRequest instanceof IndexRequest indexRequest && indexRequest.isWriteToFailureStore(); + if (isFailureStoreDoc == false + && failureStoreCandidate.isFailureStoreEnabled() + && error instanceof VersionConflictEngineException == false) { + // Redirect to failure store. + maybeMarkFailureStoreForRollover(failureStoreCandidate); + addDocumentToRedirectRequests(bulkItemRequest, cause, failureStoreCandidate.getName()); + failureStoreMetrics.incrementFailureStore( + bulkItemRequest.index(), + errorType, + FailureStoreMetrics.ErrorLocation.SHARD + ); + } else { + // If we can't redirect to a failure store (because either the data stream doesn't have the failure store enabled + // or this request was already targeting a failure store), we increment the rejected counter. + failureStoreMetrics.incrementRejected( + bulkItemRequest.index(), + errorType, + FailureStoreMetrics.ErrorLocation.SHARD, + isFailureStoreDoc + ); + } + } + } }); } /** - * Determines if the write request can be redirected if it fails. Write requests can be redirected IFF they are targeting a data stream - * with a failure store and are not already redirected themselves. If the document can be redirected, the data stream name to use for - * the redirection is returned. + * Tries to find a candidate redirect target for this write request. A candidate redirect target is a data stream that may or + * may not have the failure store enabled. * * @param docWriteRequest the write request to check * @param metadata cluster state metadata for resolving index abstractions - * @return a data stream if the write request points to a data stream that has the failure store enabled, or {@code null} if it does not + * @return a data stream if the write request points to a data stream, or {@code null} if it does not */ - private static DataStream getRedirectTarget(DocWriteRequest docWriteRequest, Metadata metadata) { + private static DataStream getRedirectTargetCandidate(DocWriteRequest docWriteRequest, Metadata metadata) { // Feature flag guard if (DataStream.isFailureStoreFeatureFlagEnabled() == false) { return null; } - // Do not resolve a failure store for documents that were already headed to one - if (docWriteRequest instanceof IndexRequest indexRequest && indexRequest.isWriteToFailureStore()) { - return null; - } // If there is no index abstraction, then the request is using a pattern of some sort, which data streams do not support IndexAbstraction ia = metadata.getIndicesLookup().get(docWriteRequest.index()); - if (ia == null) { - return null; - } - if (ia.isDataStreamRelated()) { - // The index abstraction could be an alias. Alias abstractions (even for data streams) only keep track of which _index_ they - // will write to, not which _data stream_. - // We work backward to find the data stream from the concrete write index to cover this case. - Index concreteIndex = ia.getWriteIndex(); - IndexAbstraction writeIndexAbstraction = metadata.getIndicesLookup().get(concreteIndex.getName()); - DataStream parentDataStream = writeIndexAbstraction.getParentDataStream(); - if (parentDataStream != null && parentDataStream.isFailureStoreEnabled()) { - // Keep the data stream name around to resolve the redirect to failure store if the shard level request fails. - return parentDataStream; - } - } - return null; + return DataStream.resolveDataStream(ia, metadata); } /** diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java index 0d2942e688382..f7860c47d8b73 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java @@ -130,14 +130,7 @@ public void writeTo(StreamOutput out) throws IOException { throw new IllegalStateException("Inference metadata should have been consumed before writing to the stream"); } super.writeTo(out); - out.writeArray((o, item) -> { - if (item != null) { - o.writeBoolean(true); - item.writeThin(o); - } else { - o.writeBoolean(false); - } - }, items); + out.writeArray((o, item) -> o.writeOptional(BulkItemRequest.THIN_WRITER, item), items); if (out.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_VALIDATES_MAPPINGS)) { out.writeBoolean(isSimulated); } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkShardResponse.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkShardResponse.java index 3eeb96546c9b0..eb1bb0468c9bb 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkShardResponse.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkShardResponse.java @@ -56,6 +56,6 @@ public void setForcedRefresh(boolean forcedRefresh) { public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); shardId.writeTo(out); - out.writeArray((o, item) -> item.writeThin(o), responses); + out.writeArray(BulkItemResponse.THIN_WRITER, responses); } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreMetrics.java b/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreMetrics.java new file mode 100644 index 0000000000000..5a36f10785790 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/bulk/FailureStoreMetrics.java @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.bulk; + +import org.elasticsearch.telemetry.metric.LongCounter; +import org.elasticsearch.telemetry.metric.MeterRegistry; + +import java.util.Map; + +/** + * A class containing APM metrics for failure stores. See the JavaDoc on the individual methods for an explanation on what they're tracking. + * General notes: + *

    + *
  • When a document is rerouted in a pipeline, the destination data stream is used for the metric attribute(s).
    • + *
+ */ +public class FailureStoreMetrics { + + public static final FailureStoreMetrics NOOP = new FailureStoreMetrics(MeterRegistry.NOOP); + + public static final String METRIC_TOTAL = "es.data_stream.ingest.documents.total"; + public static final String METRIC_FAILURE_STORE = "es.data_stream.ingest.documents.failure_store.total"; + public static final String METRIC_REJECTED = "es.data_stream.ingest.documents.rejected.total"; + + private final LongCounter totalCounter; + private final LongCounter failureStoreCounter; + private final LongCounter rejectedCounter; + + public FailureStoreMetrics(MeterRegistry meterRegistry) { + totalCounter = meterRegistry.registerLongCounter(METRIC_TOTAL, "total number of documents that were sent to a data stream", "unit"); + failureStoreCounter = meterRegistry.registerLongCounter( + METRIC_FAILURE_STORE, + "number of documents that got redirected to the failure store", + "unit" + ); + rejectedCounter = meterRegistry.registerLongCounter(METRIC_REJECTED, "number of documents that were rejected", "unit"); + } + + /** + * This counter tracks the number of documents that we tried to index into a data stream. This includes documents + * that were dropped by a pipeline. This counter will only be incremented once for every incoming document (even when it gets + * redirected to the failure store and/or gets rejected). + * @param dataStream the name of the data stream + */ + public void incrementTotal(String dataStream) { + totalCounter.incrementBy(1, Map.of("data_stream", dataStream)); + } + + /** + * This counter tracks the number of documents that we tried to store into a failure store. This includes both pipeline and + * shard-level failures. + * @param dataStream the name of the data stream + * @param errorType the error type (i.e. the name of the exception that was thrown) + * @param errorLocation where this failure occurred + */ + public void incrementFailureStore(String dataStream, String errorType, ErrorLocation errorLocation) { + failureStoreCounter.incrementBy( + 1, + Map.of("data_stream", dataStream, "error_type", errorType, "error_location", errorLocation.name()) + ); + } + + /** + * This counter tracks the number of documents that failed to get stored in Elasticsearch. Meaning, any document that did not get + * stored in the data stream or in its failure store. + * @param dataStream the name of the data stream + * @param errorType the error type (i.e. the name of the exception that was thrown) + * @param errorLocation where this failure occurred + * @param failureStore whether this failure occurred while trying to ingest into a failure store (true) or in the data + * stream itself (false) + */ + public void incrementRejected(String dataStream, String errorType, ErrorLocation errorLocation, boolean failureStore) { + rejectedCounter.incrementBy( + 1, + Map.of( + "data_stream", + dataStream, + "error_type", + errorType, + "error_location", + errorLocation.name(), + "failure_store", + failureStore + ) + ); + } + + public enum ErrorLocation { + PIPELINE, + SHARD; + } +} diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java index c44ad505aea84..74864abe3ec50 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java @@ -222,7 +222,7 @@ private void processBulkIndexIngestRequest( original.numberOfActions(), () -> bulkRequestModifier, bulkRequestModifier::markItemAsDropped, - (indexName) -> shouldStoreFailure(indexName, metadata, threadPool.absoluteTimeInMillis()), + (indexName) -> resolveFailureStore(indexName, metadata, threadPool.absoluteTimeInMillis()), bulkRequestModifier::markItemForFailureStore, bulkRequestModifier::markItemAsFailed, (originalThread, exception) -> { @@ -274,13 +274,15 @@ public boolean isForceExecution() { /** * Determines if an index name is associated with either an existing data stream or a template * for one that has the failure store enabled. + * * @param indexName The index name to check. * @param metadata Cluster state metadata. * @param epochMillis A timestamp to use when resolving date math in the index name. * @return true if this is not a simulation, and the given index name corresponds to a data stream with a failure store - * or if it matches a template that has a data stream failure store enabled. + * or if it matches a template that has a data stream failure store enabled. Returns false if the index name corresponds to a + * data stream, but it doesn't have the failure store enabled. Returns null when it doesn't correspond to a data stream. */ - protected abstract boolean shouldStoreFailure(String indexName, Metadata metadata, long epochMillis); + protected abstract Boolean resolveFailureStore(String indexName, Metadata metadata, long epochMillis); /** * Retrieves the {@link IndexRequest} from the provided {@link DocWriteRequest} for index or upsert actions. Upserts are diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index a695e0f5e8ab6..bdda4ff487f6b 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -42,7 +42,6 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.features.FeatureService; -import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.index.VersionType; @@ -57,7 +56,6 @@ import java.util.HashSet; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.SortedMap; import java.util.concurrent.Executor; @@ -82,6 +80,7 @@ public class TransportBulkAction extends TransportAbstractBulkAction { private final NodeClient client; private final IndexNameExpressionResolver indexNameExpressionResolver; private final OriginSettingClient rolloverClient; + private final FailureStoreMetrics failureStoreMetrics; @Inject public TransportBulkAction( @@ -94,7 +93,8 @@ public TransportBulkAction( ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, IndexingPressure indexingPressure, - SystemIndices systemIndices + SystemIndices systemIndices, + FailureStoreMetrics failureStoreMetrics ) { this( threadPool, @@ -107,7 +107,8 @@ public TransportBulkAction( indexNameExpressionResolver, indexingPressure, systemIndices, - threadPool::relativeTimeInNanos + threadPool::relativeTimeInNanos, + failureStoreMetrics ); } @@ -122,7 +123,8 @@ public TransportBulkAction( IndexNameExpressionResolver indexNameExpressionResolver, IndexingPressure indexingPressure, SystemIndices systemIndices, - LongSupplier relativeTimeProvider + LongSupplier relativeTimeProvider, + FailureStoreMetrics failureStoreMetrics ) { this( TYPE, @@ -137,7 +139,8 @@ public TransportBulkAction( indexNameExpressionResolver, indexingPressure, systemIndices, - relativeTimeProvider + relativeTimeProvider, + failureStoreMetrics ); } @@ -154,7 +157,8 @@ public TransportBulkAction( IndexNameExpressionResolver indexNameExpressionResolver, IndexingPressure indexingPressure, SystemIndices systemIndices, - LongSupplier relativeTimeProvider + LongSupplier relativeTimeProvider, + FailureStoreMetrics failureStoreMetrics ) { super( bulkAction, @@ -173,6 +177,7 @@ public TransportBulkAction( this.client = client; this.indexNameExpressionResolver = indexNameExpressionResolver; this.rolloverClient = new OriginSettingClient(client, LAZY_ROLLOVER_ORIGIN); + this.failureStoreMetrics = failureStoreMetrics; } public static ActionListener unwrappingSingleItemBulkResponse( @@ -199,6 +204,8 @@ protected void doInternalExecute( ActionListener listener, long relativeStartTimeNanos ) { + trackIndexRequests(bulkRequest); + Map indicesToAutoCreate = new HashMap<>(); Set dataStreamsToBeRolledOver = new HashSet<>(); Set failureStoresToBeRolledOver = new HashSet<>(); @@ -216,6 +223,27 @@ protected void doInternalExecute( ); } + /** + * Track the number of index requests in our APM metrics. We'll track almost all docs here (pipeline or no pipeline, + * failure store or original), but some docs don't reach this place (dropped and rejected docs), so we increment for those docs in + * different places. + */ + private void trackIndexRequests(BulkRequest bulkRequest) { + final Metadata metadata = clusterService.state().metadata(); + for (DocWriteRequest request : bulkRequest.requests) { + if (request instanceof IndexRequest == false) { + continue; + } + String resolvedIndexName = IndexNameExpressionResolver.resolveDateMathExpression(request.index()); + IndexAbstraction indexAbstraction = metadata.getIndicesLookup().get(resolvedIndexName); + DataStream dataStream = DataStream.resolveDataStream(indexAbstraction, metadata); + // We only track index requests into data streams. + if (dataStream != null) { + failureStoreMetrics.incrementTotal(dataStream.getName()); + } + } + } + /** * Determine all the targets (i.e. indices, data streams, failure stores) that require an action before we can proceed with the bulk * request. Indices might need to be created, and data streams and failure stores might need to be rolled over when they're marked @@ -535,29 +563,29 @@ void executeBulk( indexNameExpressionResolver, relativeTimeNanosProvider, startTimeNanos, - listener + listener, + failureStoreMetrics ).run(); } /** - * Determines if an index name is associated with either an existing data stream or a template - * for one that has the failure store enabled. - * @param indexName The index name to check. - * @param metadata Cluster state metadata. - * @param epochMillis A timestamp to use when resolving date math in the index name. - * @return true if the given index name corresponds to a data stream with a failure store, - * or if it matches a template that has a data stream failure store enabled. + * See {@link #resolveFailureStore(String, Metadata, long)} */ - static boolean shouldStoreFailureInternal(String indexName, Metadata metadata, long epochMillis) { - return DataStream.isFailureStoreFeatureFlagEnabled() - && resolveFailureStoreFromMetadata(indexName, metadata, epochMillis).or( - () -> resolveFailureStoreFromTemplate(indexName, metadata) - ).orElse(false); + // Visibility for testing + static Boolean resolveFailureInternal(String indexName, Metadata metadata, long epochMillis) { + if (DataStream.isFailureStoreFeatureFlagEnabled() == false) { + return null; + } + var resolution = resolveFailureStoreFromMetadata(indexName, metadata, epochMillis); + if (resolution != null) { + return resolution; + } + return resolveFailureStoreFromTemplate(indexName, metadata); } @Override - protected boolean shouldStoreFailure(String indexName, Metadata metadata, long time) { - return shouldStoreFailureInternal(indexName, metadata, time); + protected Boolean resolveFailureStore(String indexName, Metadata metadata, long time) { + return resolveFailureInternal(indexName, metadata, time); } /** @@ -567,30 +595,24 @@ protected boolean shouldStoreFailure(String indexName, Metadata metadata, long t * @param epochMillis A timestamp to use when resolving date math in the index name. * @return true if the given index name corresponds to an existing data stream with a failure store enabled. */ - private static Optional resolveFailureStoreFromMetadata(String indexName, Metadata metadata, long epochMillis) { + private static Boolean resolveFailureStoreFromMetadata(String indexName, Metadata metadata, long epochMillis) { if (indexName == null) { - return Optional.empty(); + return null; } // Get index abstraction, resolving date math if it exists IndexAbstraction indexAbstraction = metadata.getIndicesLookup() .get(IndexNameExpressionResolver.resolveDateMathExpression(indexName, epochMillis)); - - // We only store failures if the failure is being written to a data stream, - // not when directly writing to backing indices/failure stores if (indexAbstraction == null || indexAbstraction.isDataStreamRelated() == false) { - return Optional.empty(); + return null; } - // Locate the write index for the abstraction, and check if it has a data stream associated with it. - // This handles alias resolution as well as data stream resolution. - Index writeIndex = indexAbstraction.getWriteIndex(); - assert writeIndex != null : "Could not resolve write index for resource [" + indexName + "]"; - IndexAbstraction writeAbstraction = metadata.getIndicesLookup().get(writeIndex.getName()); - DataStream targetDataStream = writeAbstraction.getParentDataStream(); + // We only store failures if the failure is being written to a data stream, + // not when directly writing to backing indices/failure stores + DataStream targetDataStream = DataStream.resolveDataStream(indexAbstraction, metadata); // We will store the failure if the write target belongs to a data stream with a failure store. - return Optional.of(targetDataStream != null && targetDataStream.isFailureStoreEnabled()); + return targetDataStream != null && targetDataStream.isFailureStoreEnabled(); } /** @@ -599,9 +621,9 @@ private static Optional resolveFailureStoreFromMetadata(String indexNam * @param metadata Cluster state metadata. * @return true if the given index name corresponds to an index template with a data stream failure store enabled. */ - private static Optional resolveFailureStoreFromTemplate(String indexName, Metadata metadata) { + private static Boolean resolveFailureStoreFromTemplate(String indexName, Metadata metadata) { if (indexName == null) { - return Optional.empty(); + return null; } // Check to see if the index name matches any templates such that an index would have been attributed @@ -612,11 +634,11 @@ private static Optional resolveFailureStoreFromTemplate(String indexNam ComposableIndexTemplate composableIndexTemplate = metadata.templatesV2().get(template); if (composableIndexTemplate.getDataStreamTemplate() != null) { // Check if the data stream has the failure store enabled - return Optional.of(composableIndexTemplate.getDataStreamTemplate().hasFailureStore()); + return composableIndexTemplate.getDataStreamTemplate().hasFailureStore(); } } // Could not locate a failure store via template - return Optional.empty(); + return null; } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java index a4648a7accb5a..8da6fb409cb90 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java @@ -10,16 +10,27 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.admin.indices.template.post.TransportSimulateIndexTemplateAction; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.ingest.SimulateIndexResponse; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; +import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; +import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.index.IndexSettingProvider; +import org.elasticsearch.index.IndexSettingProviders; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.Engine; @@ -35,9 +46,18 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.Executor; +import static java.util.stream.Collectors.toList; +import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.isDataStreamsLifecycleOnlyMode; +import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV1Templates; +import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV2Template; + /** * This action simulates bulk indexing data. Pipelines are executed for all indices that the request routes to, but no data is actually * indexed and no state is changed. Unlike TransportBulkAction, this does not push the work out to the nodes where the shards live (since @@ -45,7 +65,10 @@ */ public class TransportSimulateBulkAction extends TransportAbstractBulkAction { public static final NodeFeature SIMULATE_MAPPING_VALIDATION = new NodeFeature("simulate.mapping.validation"); + public static final NodeFeature SIMULATE_MAPPING_VALIDATION_TEMPLATES = new NodeFeature("simulate.mapping.validation.templates"); private final IndicesService indicesService; + private final NamedXContentRegistry xContentRegistry; + private final Set indexSettingProviders; @Inject public TransportSimulateBulkAction( @@ -56,7 +79,9 @@ public TransportSimulateBulkAction( ActionFilters actionFilters, IndexingPressure indexingPressure, SystemIndices systemIndices, - IndicesService indicesService + IndicesService indicesService, + NamedXContentRegistry xContentRegistry, + IndexSettingProviders indexSettingProviders ) { super( SimulateBulkAction.INSTANCE, @@ -71,6 +96,8 @@ public TransportSimulateBulkAction( threadPool::relativeTimeInNanos ); this.indicesService = indicesService; + this.xContentRegistry = xContentRegistry; + this.indexSettingProviders = indexSettingProviders.getIndexSettingProviders(); } @Override @@ -128,9 +155,9 @@ private Exception validateMappings(IndexRequest request) { ClusterState state = clusterService.state(); Exception mappingValidationException = null; IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(request.index()); - if (indexAbstraction != null) { - IndexMetadata imd = state.metadata().getIndexSafe(indexAbstraction.getWriteIndex(request, state.metadata())); - try { + try { + if (indexAbstraction != null) { + IndexMetadata imd = state.metadata().getIndexSafe(indexAbstraction.getWriteIndex(request, state.metadata())); indicesService.withTempIndexService(imd, indexService -> { indexService.mapperService().updateMapping(null, imd); return IndexShard.prepareIndex( @@ -148,9 +175,102 @@ private Exception validateMappings(IndexRequest request) { 0 ); }); - } catch (Exception e) { - mappingValidationException = e; + } else { + /* + * The index did not exist, so we put together the mappings from existing templates. + * This reproduces a lot of the mapping resolution logic in MetadataCreateIndexService.applyCreateIndexRequest(). However, + * it does not deal with aliases (since an alias cannot be created if an index does not exist, and this is the path for + * when the index does not exist). And it does not deal with system indices since we do not intend for users to simulate + * writing to system indices. + */ + String matchingTemplate = findV2Template(state.metadata(), request.index(), false); + if (matchingTemplate != null) { + final Template template = TransportSimulateIndexTemplateAction.resolveTemplate( + matchingTemplate, + request.index(), + state, + isDataStreamsLifecycleOnlyMode(clusterService.getSettings()), + xContentRegistry, + indicesService, + systemIndices, + indexSettingProviders + ); + CompressedXContent mappings = template.mappings(); + if (mappings != null) { + MappingMetadata mappingMetadata = new MappingMetadata(mappings); + Settings dummySettings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) + .build(); + final IndexMetadata imd = IndexMetadata.builder(request.index()) + .settings(dummySettings) + .putMapping(mappingMetadata) + .build(); + indicesService.withTempIndexService(imd, indexService -> { + indexService.mapperService().updateMapping(null, imd); + return IndexShard.prepareIndex( + indexService.mapperService(), + sourceToParse, + SequenceNumbers.UNASSIGNED_SEQ_NO, + -1, + -1, + VersionType.INTERNAL, + Engine.Operation.Origin.PRIMARY, + Long.MIN_VALUE, + false, + request.ifSeqNo(), + request.ifPrimaryTerm(), + 0 + ); + }); + } + } else { + List matchingTemplates = findV1Templates(state.metadata(), request.index(), false); + final Map mappingsMap = MetadataCreateIndexService.parseV1Mappings( + "{}", + matchingTemplates.stream().map(IndexTemplateMetadata::getMappings).collect(toList()), + xContentRegistry + ); + final CompressedXContent combinedMappings; + if (mappingsMap.isEmpty()) { + combinedMappings = null; + } else { + combinedMappings = new CompressedXContent(mappingsMap); + } + Settings dummySettings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) + .build(); + MappingMetadata mappingMetadata = combinedMappings == null ? null : new MappingMetadata(combinedMappings); + final IndexMetadata imd = IndexMetadata.builder(request.index()) + .putMapping(mappingMetadata) + .settings(dummySettings) + .build(); + indicesService.withTempIndexService(imd, indexService -> { + indexService.mapperService().updateMapping(null, imd); + return IndexShard.prepareIndex( + indexService.mapperService(), + sourceToParse, + SequenceNumbers.UNASSIGNED_SEQ_NO, + -1, + -1, + VersionType.INTERNAL, + Engine.Operation.Origin.PRIMARY, + Long.MIN_VALUE, + false, + request.ifSeqNo(), + request.ifPrimaryTerm(), + 0 + ); + }); + } } + } catch (Exception e) { + mappingValidationException = e; } return mappingValidationException; } @@ -166,8 +286,8 @@ protected IngestService getIngestService(BulkRequest request) { } @Override - protected boolean shouldStoreFailure(String indexName, Metadata metadata, long epochMillis) { + protected Boolean resolveFailureStore(String indexName, Metadata metadata, long epochMillis) { // A simulate bulk request should not change any persistent state in the system, so we never write to the failure store - return false; + return null; } } diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java index 89282b8db3646..2bcd824dfea3c 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java @@ -58,6 +58,7 @@ public static class Request extends MasterNodeReadRequest implements In private String[] names; private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true, false); private boolean includeDefaults = false; + private boolean verbose = false; public Request(TimeValue masterNodeTimeout, String[] names) { super(masterNodeTimeout); @@ -79,6 +80,10 @@ public String[] getNames() { return names; } + public boolean verbose() { + return verbose; + } + @Override public ActionRequestValidationException validate() { return null; @@ -93,6 +98,11 @@ public Request(StreamInput in) throws IOException { } else { this.includeDefaults = false; } + if (in.getTransportVersion().onOrAfter(TransportVersions.GET_DATA_STREAMS_VERBOSE)) { + this.verbose = in.readBoolean(); + } else { + this.verbose = false; + } } @Override @@ -103,6 +113,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { out.writeBoolean(includeDefaults); } + if (out.getTransportVersion().onOrAfter(TransportVersions.GET_DATA_STREAMS_VERBOSE)) { + out.writeBoolean(verbose); + } } @Override @@ -112,12 +125,13 @@ public boolean equals(Object o) { Request request = (Request) o; return Arrays.equals(names, request.names) && indicesOptions.equals(request.indicesOptions) - && includeDefaults == request.includeDefaults; + && includeDefaults == request.includeDefaults + && verbose == request.verbose; } @Override public int hashCode() { - int result = Objects.hash(indicesOptions, includeDefaults); + int result = Objects.hash(indicesOptions, includeDefaults, verbose); result = 31 * result + Arrays.hashCode(names); return result; } @@ -156,6 +170,11 @@ public Request includeDefaults(boolean includeDefaults) { this.includeDefaults = includeDefaults; return this; } + + public Request verbose(boolean verbose) { + this.verbose = verbose; + return this; + } } public static class Response extends ActionResponse implements ToXContentObject { @@ -197,6 +216,7 @@ public static class DataStreamInfo implements SimpleDiffable, To "time_since_last_auto_shard_event_millis" ); public static final ParseField FAILURE_STORE_ENABLED = new ParseField("enabled"); + public static final ParseField MAXIMUM_TIMESTAMP = new ParseField("maximum_timestamp"); private final DataStream dataStream; private final ClusterHealthStatus dataStreamStatus; @@ -208,6 +228,8 @@ public static class DataStreamInfo implements SimpleDiffable, To private final TimeSeries timeSeries; private final Map indexSettingsValues; private final boolean templatePreferIlmValue; + @Nullable + private final Long maximumTimestamp; public DataStreamInfo( DataStream dataStream, @@ -216,7 +238,8 @@ public DataStreamInfo( @Nullable String ilmPolicyName, @Nullable TimeSeries timeSeries, Map indexSettingsValues, - boolean templatePreferIlmValue + boolean templatePreferIlmValue, + @Nullable Long maximumTimestamp ) { this.dataStream = dataStream; this.dataStreamStatus = dataStreamStatus; @@ -225,6 +248,7 @@ public DataStreamInfo( this.timeSeries = timeSeries; this.indexSettingsValues = indexSettingsValues; this.templatePreferIlmValue = templatePreferIlmValue; + this.maximumTimestamp = maximumTimestamp; } @SuppressWarnings("unchecked") @@ -236,7 +260,8 @@ public DataStreamInfo( in.readOptionalString(), in.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0) ? in.readOptionalWriteable(TimeSeries::new) : null, in.getTransportVersion().onOrAfter(V_8_11_X) ? in.readMap(Index::new, IndexProperties::new) : Map.of(), - in.getTransportVersion().onOrAfter(V_8_11_X) ? in.readBoolean() : true + in.getTransportVersion().onOrAfter(V_8_11_X) ? in.readBoolean() : true, + in.getTransportVersion().onOrAfter(TransportVersions.GET_DATA_STREAMS_VERBOSE) ? in.readOptionalVLong() : null ); } @@ -271,6 +296,11 @@ public boolean templatePreferIlmValue() { return templatePreferIlmValue; } + @Nullable + public Long getMaximumTimestamp() { + return maximumTimestamp; + } + @Override public void writeTo(StreamOutput out) throws IOException { dataStream.writeTo(out); @@ -284,6 +314,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeMap(indexSettingsValues); out.writeBoolean(templatePreferIlmValue); } + if (out.getTransportVersion().onOrAfter(TransportVersions.GET_DATA_STREAMS_VERBOSE)) { + out.writeOptionalVLong(maximumTimestamp); + } } @Override @@ -319,8 +352,7 @@ public XContentBuilder toXContent( } if (dataStream.getLifecycle() != null) { builder.field(LIFECYCLE_FIELD.getPreferredName()); - dataStream.getLifecycle() - .toXContent(builder, params, rolloverConfiguration, dataStream.isSystem() ? null : globalRetention); + dataStream.getLifecycle().toXContent(builder, params, rolloverConfiguration, globalRetention, dataStream.isInternal()); } if (ilmPolicyName != null) { builder.field(ILM_POLICY_FIELD.getPreferredName(), ilmPolicyName); @@ -332,6 +364,9 @@ public XContentBuilder toXContent( builder.field(ALLOW_CUSTOM_ROUTING.getPreferredName(), dataStream.isAllowCustomRouting()); builder.field(REPLICATED.getPreferredName(), dataStream.isReplicated()); builder.field(ROLLOVER_ON_WRITE.getPreferredName(), dataStream.rolloverOnWrite()); + if (this.maximumTimestamp != null) { + builder.field(MAXIMUM_TIMESTAMP.getPreferredName(), this.maximumTimestamp); + } addAutoShardingEvent(builder, params, dataStream.getAutoShardingEvent()); if (timeSeries != null) { builder.startObject(TIME_SERIES.getPreferredName()); @@ -432,7 +467,8 @@ public boolean equals(Object o) { && Objects.equals(indexTemplate, that.indexTemplate) && Objects.equals(ilmPolicyName, that.ilmPolicyName) && Objects.equals(timeSeries, that.timeSeries) - && Objects.equals(indexSettingsValues, that.indexSettingsValues); + && Objects.equals(indexSettingsValues, that.indexSettingsValues) + && Objects.equals(maximumTimestamp, that.maximumTimestamp); } @Override @@ -444,7 +480,8 @@ public int hashCode() { ilmPolicyName, timeSeries, indexSettingsValues, - templatePreferIlmValue + templatePreferIlmValue, + maximumTimestamp ); } } @@ -556,7 +593,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws for (DataStreamInfo dataStream : dataStreams) { dataStream.toXContent( builder, - DataStreamLifecycle.maybeAddEffectiveRetentionParams(params), + DataStreamLifecycle.addEffectiveRetentionParams(params), rolloverConfiguration, globalRetention ); diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainDataStreamLifecycleAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainDataStreamLifecycleAction.java index 4dc9ada5dc01f..d51f00681bb5e 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainDataStreamLifecycleAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainDataStreamLifecycleAction.java @@ -217,7 +217,7 @@ public Iterator toXContentChunked(ToXContent.Params outerP builder.field(explainIndexDataLifecycle.getIndex()); explainIndexDataLifecycle.toXContent( builder, - DataStreamLifecycle.maybeAddEffectiveRetentionParams(outerParams), + DataStreamLifecycle.addEffectiveRetentionParams(outerParams), rolloverConfiguration, globalRetention ); diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java index 32be73a7b0960..962c2975f5998 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java @@ -45,7 +45,7 @@ public class ExplainIndexDataStreamLifecycle implements Writeable, ToXContentObj private final String index; private final boolean managedByLifecycle; - private final boolean isSystemDataStream; + private final boolean isInternalDataStream; @Nullable private final Long indexCreationDate; @Nullable @@ -61,7 +61,7 @@ public class ExplainIndexDataStreamLifecycle implements Writeable, ToXContentObj public ExplainIndexDataStreamLifecycle( String index, boolean managedByLifecycle, - boolean isSystemDataStream, + boolean isInternalDataStream, @Nullable Long indexCreationDate, @Nullable Long rolloverDate, @Nullable TimeValue generationDate, @@ -70,7 +70,7 @@ public ExplainIndexDataStreamLifecycle( ) { this.index = index; this.managedByLifecycle = managedByLifecycle; - this.isSystemDataStream = isSystemDataStream; + this.isInternalDataStream = isInternalDataStream; this.indexCreationDate = indexCreationDate; this.rolloverDate = rolloverDate; this.generationDateMillis = generationDate == null ? null : generationDate.millis(); @@ -82,9 +82,9 @@ public ExplainIndexDataStreamLifecycle(StreamInput in) throws IOException { this.index = in.readString(); this.managedByLifecycle = in.readBoolean(); if (in.getTransportVersion().onOrAfter(TransportVersions.NO_GLOBAL_RETENTION_FOR_SYSTEM_DATA_STREAMS)) { - this.isSystemDataStream = in.readBoolean(); + this.isInternalDataStream = in.readBoolean(); } else { - this.isSystemDataStream = false; + this.isInternalDataStream = false; } if (managedByLifecycle) { this.indexCreationDate = in.readOptionalLong(); @@ -141,7 +141,7 @@ public XContentBuilder toXContent( } if (this.lifecycle != null) { builder.field(LIFECYCLE_FIELD.getPreferredName()); - lifecycle.toXContent(builder, params, rolloverConfiguration, isSystemDataStream ? null : globalRetention); + lifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention, isInternalDataStream); } if (this.error != null) { if (error.firstOccurrenceTimestamp() != -1L && error.recordedTimestamp() != -1L && error.retryCount() != -1) { @@ -161,7 +161,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(index); out.writeBoolean(managedByLifecycle); if (out.getTransportVersion().onOrAfter(TransportVersions.NO_GLOBAL_RETENTION_FOR_SYSTEM_DATA_STREAMS)) { - out.writeBoolean(isSystemDataStream); + out.writeBoolean(isInternalDataStream); } if (managedByLifecycle) { out.writeOptionalLong(indexCreationDate); diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java index e038763169ef8..2ae6e544b3f53 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/GetDataStreamLifecycleAction.java @@ -143,7 +143,7 @@ public static class Response extends ActionResponse implements ChunkedToXContent public record DataStreamLifecycle( String dataStreamName, @Nullable org.elasticsearch.cluster.metadata.DataStreamLifecycle lifecycle, - boolean isSystemDataStream + boolean isInternalDataStream ) implements Writeable, ToXContentObject { public static final ParseField NAME_FIELD = new ParseField("name"); @@ -162,7 +162,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(dataStreamName); out.writeOptionalWriteable(lifecycle); if (out.getTransportVersion().onOrAfter(TransportVersions.NO_GLOBAL_RETENTION_FOR_SYSTEM_DATA_STREAMS)) { - out.writeBoolean(isSystemDataStream); + out.writeBoolean(isInternalDataStream); } } @@ -187,9 +187,10 @@ public XContentBuilder toXContent( builder.field(LIFECYCLE_FIELD.getPreferredName()); lifecycle.toXContent( builder, - org.elasticsearch.cluster.metadata.DataStreamLifecycle.maybeAddEffectiveRetentionParams(params), + org.elasticsearch.cluster.metadata.DataStreamLifecycle.addEffectiveRetentionParams(params), rolloverConfiguration, - isSystemDataStream ? null : globalRetention + globalRetention, + isInternalDataStream ); } builder.endObject(); @@ -253,6 +254,16 @@ public void writeTo(StreamOutput out) throws IOException { public Iterator toXContentChunked(ToXContent.Params outerParams) { return Iterators.concat(Iterators.single((builder, params) -> { builder.startObject(); + builder.startObject("global_retention"); + if (globalRetention != null) { + if (globalRetention.maxRetention() != null) { + builder.field("max_retention", globalRetention.maxRetention().getStringRep()); + } + if (globalRetention.defaultRetention() != null) { + builder.field("default_retention", globalRetention.defaultRetention().getStringRep()); + } + } + builder.endObject(); builder.startArray(DATA_STREAMS_FIELD.getPreferredName()); return builder; }), diff --git a/server/src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java index f2b1dc7cd556c..40810f004b0de 100644 --- a/server/src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java @@ -30,6 +30,7 @@ public class DeleteRequestBuilder extends ReplicationRequestBuilder existing = indexMappingHashToResponses.get(indexMappingHash); if (existing != null) { - return new FieldCapabilitiesIndexResponse(shardId.getIndexName(), indexMappingHash, existing, true); + return new FieldCapabilitiesIndexResponse(shardId.getIndexName(), indexMappingHash, existing, true, indexMode); } } task.ensureNotCancelled(); @@ -145,7 +145,7 @@ private FieldCapabilitiesIndexResponse doFetch( if (indexMappingHash != null) { indexMappingHashToResponses.put(indexMappingHash, responseMap); } - return new FieldCapabilitiesIndexResponse(shardId.getIndexName(), indexMappingHash, responseMap, true); + return new FieldCapabilitiesIndexResponse(shardId.getIndexName(), indexMappingHash, responseMap, true, indexMode); } static Map retrieveFieldCaps( diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java index cc72dd80dceac..5a50ed4c9f573 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexResponse.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.Maps; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IndexMode; import java.io.IOException; import java.util.ArrayList; @@ -33,18 +34,21 @@ public final class FieldCapabilitiesIndexResponse implements Writeable { private final Map responseMap; private final boolean canMatch; private final transient TransportVersion originVersion; + private final IndexMode indexMode; public FieldCapabilitiesIndexResponse( String indexName, @Nullable String indexMappingHash, Map responseMap, - boolean canMatch + boolean canMatch, + IndexMode indexMode ) { this.indexName = indexName; this.indexMappingHash = indexMappingHash; this.responseMap = responseMap; this.canMatch = canMatch; this.originVersion = TransportVersion.current(); + this.indexMode = indexMode; } FieldCapabilitiesIndexResponse(StreamInput in) throws IOException { @@ -57,6 +61,11 @@ public FieldCapabilitiesIndexResponse( } else { this.indexMappingHash = null; } + if (in.getTransportVersion().onOrAfter(TransportVersions.FIELD_CAPS_RESPONSE_INDEX_MODE)) { + this.indexMode = IndexMode.readFrom(in); + } else { + this.indexMode = IndexMode.STANDARD; + } } @Override @@ -67,9 +76,12 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(MAPPING_HASH_VERSION)) { out.writeOptionalString(indexMappingHash); } + if (out.getTransportVersion().onOrAfter(TransportVersions.FIELD_CAPS_RESPONSE_INDEX_MODE)) { + IndexMode.writeTo(indexMode, out); + } } - private record CompressedGroup(String[] indices, String mappingHash, int[] fields) {} + private record CompressedGroup(String[] indices, IndexMode indexMode, String mappingHash, int[] fields) {} static List readList(StreamInput input) throws IOException { if (input.getTransportVersion().before(MAPPING_HASH_VERSION)) { @@ -92,10 +104,12 @@ static List readList(StreamInput input) throws I private static void collectCompressedResponses(StreamInput input, int groups, ArrayList responses) throws IOException { final CompressedGroup[] compressedGroups = new CompressedGroup[groups]; + final boolean readIndexMode = input.getTransportVersion().onOrAfter(TransportVersions.FIELD_CAPS_RESPONSE_INDEX_MODE); for (int i = 0; i < groups; i++) { final String[] indices = input.readStringArray(); + final IndexMode indexMode = readIndexMode ? IndexMode.readFrom(input) : IndexMode.STANDARD; final String mappingHash = input.readString(); - compressedGroups[i] = new CompressedGroup(indices, mappingHash, input.readIntArray()); + compressedGroups[i] = new CompressedGroup(indices, indexMode, mappingHash, input.readIntArray()); } final IndexFieldCapabilities[] ifcLookup = input.readArray(IndexFieldCapabilities::readFrom, IndexFieldCapabilities[]::new); for (CompressedGroup compressedGroup : compressedGroups) { @@ -105,7 +119,7 @@ private static void collectCompressedResponses(StreamInput input, int groups, Ar ifc.put(val.name(), val); } for (String index : compressedGroup.indices) { - responses.add(new FieldCapabilitiesIndexResponse(index, compressedGroup.mappingHash, ifc, true)); + responses.add(new FieldCapabilitiesIndexResponse(index, compressedGroup.mappingHash, ifc, true, compressedGroup.indexMode)); } } } @@ -117,7 +131,7 @@ private static void collectResponsesLegacyFormat(StreamInput input, int groups, final String mappingHash = input.readString(); final Map ifc = input.readMap(IndexFieldCapabilities::readFrom); for (String index : indices) { - responses.add(new FieldCapabilitiesIndexResponse(index, mappingHash, ifc, true)); + responses.add(new FieldCapabilitiesIndexResponse(index, mappingHash, ifc, true, IndexMode.STANDARD)); } } } @@ -164,6 +178,9 @@ private static void writeCompressedResponses(StreamOutput output, Map { o.writeCollection(fieldCapabilitiesIndexResponses, (oo, r) -> oo.writeString(r.indexName)); var first = fieldCapabilitiesIndexResponses.get(0); + if (output.getTransportVersion().onOrAfter(TransportVersions.FIELD_CAPS_RESPONSE_INDEX_MODE)) { + IndexMode.writeTo(first.indexMode, o); + } o.writeString(first.indexMappingHash); o.writeVInt(first.responseMap.size()); for (IndexFieldCapabilities ifc : first.responseMap.values()) { @@ -192,6 +209,10 @@ public String getIndexMappingHash() { return indexMappingHash; } + public IndexMode getIndexMode() { + return indexMode; + } + public boolean canMatch() { return canMatch; } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java index b9bf3bb37c7b4..bb97b0dc48c42 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java @@ -174,7 +174,13 @@ private void doExecuteForked( if (resp.canMatch() && resp.getIndexMappingHash() != null) { FieldCapabilitiesIndexResponse curr = indexMappingHashToResponses.putIfAbsent(resp.getIndexMappingHash(), resp); if (curr != null) { - resp = new FieldCapabilitiesIndexResponse(resp.getIndexName(), curr.getIndexMappingHash(), curr.get(), true); + resp = new FieldCapabilitiesIndexResponse( + resp.getIndexName(), + curr.getIndexMappingHash(), + curr.get(), + true, + curr.getIndexMode() + ); } } if (request.includeEmptyFields()) { @@ -186,7 +192,13 @@ private void doExecuteForked( } Map mergedCaps = new HashMap<>(a.get()); mergedCaps.putAll(b.get()); - return new FieldCapabilitiesIndexResponse(a.getIndexName(), a.getIndexMappingHash(), mergedCaps, true); + return new FieldCapabilitiesIndexResponse( + a.getIndexName(), + a.getIndexMappingHash(), + mergedCaps, + true, + a.getIndexMode() + ); }); } if (fieldCapTask.isCancelled()) { @@ -249,7 +261,13 @@ private void doExecuteForked( for (FieldCapabilitiesIndexResponse resp : response.getIndexResponses()) { String indexName = RemoteClusterAware.buildRemoteIndexName(clusterAlias, resp.getIndexName()); handleIndexResponse.accept( - new FieldCapabilitiesIndexResponse(indexName, resp.getIndexMappingHash(), resp.get(), resp.canMatch()) + new FieldCapabilitiesIndexResponse( + indexName, + resp.getIndexMappingHash(), + resp.get(), + resp.canMatch(), + resp.getIndexMode() + ) ); } for (FieldCapabilitiesFailure failure : response.getFailures()) { diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java index 0cb04fbdba1a6..3f3d97e2115cb 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java @@ -54,6 +54,7 @@ public IndexRequestBuilder(ElasticsearchClient client) { this(client, null); } + @SuppressWarnings("this-escape") public IndexRequestBuilder(ElasticsearchClient client, @Nullable String index) { super(client, TransportIndexAction.TYPE); setIndex(index); diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulateIndexResponse.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulateIndexResponse.java index 445492f037926..258cd5ceaa8e7 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulateIndexResponse.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulateIndexResponse.java @@ -96,6 +96,10 @@ public void writeTo(StreamOutput out) throws IOException { } } + public Exception getException() { + return this.exception; + } + @Override public String toString() { StringBuilder builder = new StringBuilder(); diff --git a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java index 4fd551994e2a0..1e5b5ebbefe48 100644 --- a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java @@ -707,7 +707,7 @@ public void sendSearchResponse(SearchResponseSections internalSearchResponse, At final String scrollId = request.scroll() != null ? TransportSearchHelper.buildScrollId(queryResults) : null; final BytesReference searchContextId; if (buildPointInTimeFromSearchResults()) { - searchContextId = SearchContextId.encode(queryResults.asList(), aliasFilter, minTransportVersion); + searchContextId = SearchContextId.encode(queryResults.asList(), aliasFilter, minTransportVersion, failures); } else { if (request.source() != null && request.source().pointInTimeBuilder() != null diff --git a/server/src/main/java/org/elasticsearch/action/search/CanMatchNodeRequest.java b/server/src/main/java/org/elasticsearch/action/search/CanMatchNodeRequest.java index bc50a9f8f0c2c..30fa4caa79a15 100644 --- a/server/src/main/java/org/elasticsearch/action/search/CanMatchNodeRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/CanMatchNodeRequest.java @@ -229,10 +229,6 @@ public List getShardLevelRequests() { return shards; } - public List createShardSearchRequests() { - return shards.stream().map(this::createShardSearchRequest).toList(); - } - public ShardSearchRequest createShardSearchRequest(Shard r) { ShardSearchRequest shardSearchRequest = new ShardSearchRequest( new OriginalIndices(r.indices, indicesOptions), diff --git a/server/src/main/java/org/elasticsearch/action/search/ClearScrollController.java b/server/src/main/java/org/elasticsearch/action/search/ClearScrollController.java index 04573f72068f3..965b19a69b858 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ClearScrollController.java +++ b/server/src/main/java/org/elasticsearch/action/search/ClearScrollController.java @@ -166,6 +166,10 @@ public static void closeContexts( final var successes = new AtomicInteger(); try (RefCountingRunnable refs = new RefCountingRunnable(() -> l.onResponse(successes.get()))) { for (SearchContextIdForNode contextId : contextIds) { + if (contextId.getNode() == null) { + // the shard was missing when creating the PIT, ignore. + continue; + } final DiscoveryNode node = nodeLookup.apply(contextId.getClusterAlias(), contextId.getNode()); if (node != null) { try { diff --git a/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeRequest.java b/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeRequest.java index a1cd4df25a25c..146418839f063 100644 --- a/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeRequest.java @@ -41,6 +41,8 @@ public final class OpenPointInTimeRequest extends ActionRequest implements Indic private QueryBuilder indexFilter; + private boolean allowPartialSearchResults = false; + public static final IndicesOptions DEFAULT_INDICES_OPTIONS = SearchRequest.DEFAULT_INDICES_OPTIONS; public OpenPointInTimeRequest(String... indices) { @@ -60,6 +62,9 @@ public OpenPointInTimeRequest(StreamInput in) throws IOException { if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) { this.indexFilter = in.readOptionalNamedWriteable(QueryBuilder.class); } + if (in.getTransportVersion().onOrAfter(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT)) { + this.allowPartialSearchResults = in.readBoolean(); + } } @Override @@ -76,6 +81,11 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) { out.writeOptionalWriteable(indexFilter); } + if (out.getTransportVersion().onOrAfter(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT)) { + out.writeBoolean(allowPartialSearchResults); + } else if (allowPartialSearchResults) { + throw new IOException("[allow_partial_search_results] is not supported on nodes with version " + out.getTransportVersion()); + } } @Override @@ -180,6 +190,15 @@ public boolean includeDataStreams() { return true; } + public boolean allowPartialSearchResults() { + return allowPartialSearchResults; + } + + public OpenPointInTimeRequest allowPartialSearchResults(boolean allowPartialSearchResults) { + this.allowPartialSearchResults = allowPartialSearchResults; + return this; + } + @Override public String getDescription() { return "open search context: indices [" + String.join(",", indices) + "] keep_alive [" + keepAlive + "]"; @@ -200,6 +219,8 @@ public String toString() { + ", preference='" + preference + '\'' + + ", allowPartialSearchResults=" + + allowPartialSearchResults + '}'; } @@ -218,12 +239,13 @@ public boolean equals(Object o) { && indicesOptions.equals(that.indicesOptions) && keepAlive.equals(that.keepAlive) && Objects.equals(routing, that.routing) - && Objects.equals(preference, that.preference); + && Objects.equals(preference, that.preference) + && Objects.equals(allowPartialSearchResults, that.allowPartialSearchResults); } @Override public int hashCode() { - int result = Objects.hash(indicesOptions, keepAlive, maxConcurrentShardRequests, routing, preference); + int result = Objects.hash(indicesOptions, keepAlive, maxConcurrentShardRequests, routing, preference, allowPartialSearchResults); result = 31 * result + Arrays.hashCode(indices); return result; } diff --git a/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeResponse.java b/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeResponse.java index dafcee894c9a6..4a4c0252fb109 100644 --- a/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeResponse.java +++ b/server/src/main/java/org/elasticsearch/action/search/OpenPointInTimeResponse.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.search; +import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamOutput; @@ -18,22 +19,46 @@ import java.util.Base64; import java.util.Objects; +import static org.elasticsearch.rest.action.RestActions.buildBroadcastShardsHeader; + public final class OpenPointInTimeResponse extends ActionResponse implements ToXContentObject { private final BytesReference pointInTimeId; - public OpenPointInTimeResponse(BytesReference pointInTimeId) { + private final int totalShards; + private final int successfulShards; + private final int failedShards; + private final int skippedShards; + + public OpenPointInTimeResponse( + BytesReference pointInTimeId, + int totalShards, + int successfulShards, + int failedShards, + int skippedShards + ) { this.pointInTimeId = Objects.requireNonNull(pointInTimeId, "Point in time parameter must be not null"); + this.totalShards = totalShards; + this.successfulShards = successfulShards; + this.failedShards = failedShards; + this.skippedShards = skippedShards; } @Override public void writeTo(StreamOutput out) throws IOException { out.writeBytesReference(pointInTimeId); + if (out.getTransportVersion().onOrAfter(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT)) { + out.writeVInt(totalShards); + out.writeVInt(successfulShards); + out.writeVInt(failedShards); + out.writeVInt(skippedShards); + } } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field("id", Base64.getUrlEncoder().encodeToString(BytesReference.toBytes(pointInTimeId))); + buildBroadcastShardsHeader(builder, params, totalShards, successfulShards, failedShards, skippedShards, null); builder.endObject(); return builder; } @@ -42,4 +67,19 @@ public BytesReference getPointInTimeId() { return pointInTimeId; } + public int getTotalShards() { + return totalShards; + } + + public int getSuccessfulShards() { + return successfulShards; + } + + public int getFailedShards() { + return failedShards; + } + + public int getSkippedShards() { + return skippedShards; + } } diff --git a/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java b/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java index 5b42afcb86928..0f7cbd65a63c2 100644 --- a/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/RankFeaturePhase.java @@ -28,8 +28,8 @@ /** * This search phase is responsible for executing any re-ranking needed for the given search request, iff that is applicable. - * It starts by retrieving {@code num_shards * window_size} results from the query phase and reduces them to a global list of - * the top {@code window_size} results. It then reaches out to the shards to extract the needed feature data, + * It starts by retrieving {@code num_shards * rank_window_size} results from the query phase and reduces them to a global list of + * the top {@code rank_window_size} results. It then reaches out to the shards to extract the needed feature data, * and finally passes all this information to the appropriate {@code RankFeatureRankCoordinatorContext} which is responsible for reranking * the results. If no rank query is specified, it proceeds directly to the next phase (FetchSearchPhase) by first reducing the results. */ @@ -88,7 +88,7 @@ public void onFailure(Exception e) { void innerRun() throws Exception { // if the RankBuilder specifies a QueryPhaseCoordinatorContext, it will be called as part of the reduce call - // to operate on the first `window_size * num_shards` results and merge them appropriately. + // to operate on the first `rank_window_size * num_shards` results and merge them appropriately. SearchPhaseController.ReducedQueryPhase reducedQueryPhase = queryPhaseResults.reduce(); RankFeaturePhaseRankCoordinatorContext rankFeaturePhaseRankCoordinatorContext = coordinatorContext(context.getRequest().source()); if (rankFeaturePhaseRankCoordinatorContext != null) { diff --git a/server/src/main/java/org/elasticsearch/action/search/RestOpenPointInTimeAction.java b/server/src/main/java/org/elasticsearch/action/search/RestOpenPointInTimeAction.java index 0e7f3f9111842..5966a1c924745 100644 --- a/server/src/main/java/org/elasticsearch/action/search/RestOpenPointInTimeAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/RestOpenPointInTimeAction.java @@ -47,6 +47,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC openRequest.routing(request.param("routing")); openRequest.preference(request.param("preference")); openRequest.keepAlive(TimeValue.parseTimeValue(request.param("keep_alive"), null, "keep_alive")); + openRequest.allowPartialSearchResults(request.paramAsBoolean("allow_partial_search_results", false)); if (request.hasParam("max_concurrent_shard_requests")) { final int maxConcurrentShardRequests = request.paramAsInt( "max_concurrent_shard_requests", diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchContextId.java b/server/src/main/java/org/elasticsearch/action/search/SearchContextId.java index 95d22e8a9034e..2e4dc724413ea 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchContextId.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchContextId.java @@ -9,6 +9,7 @@ package org.elasticsearch.action.search; import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -58,12 +59,30 @@ public boolean contains(ShardSearchContextId contextId) { public static BytesReference encode( List searchPhaseResults, Map aliasFilter, - TransportVersion version + TransportVersion version, + ShardSearchFailure[] shardFailures ) { + assert shardFailures.length == 0 || version.onOrAfter(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT) + : "[allow_partial_search_results] cannot be enabled on a cluster that has not been fully upgraded to version [" + + TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT + + "] or higher."; try (var out = new BytesStreamOutput()) { out.setTransportVersion(version); TransportVersion.writeVersion(version, out); - out.writeCollection(searchPhaseResults, SearchContextId::writeSearchPhaseResult); + boolean allowNullContextId = out.getTransportVersion().onOrAfter(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT); + int shardSize = searchPhaseResults.size() + (allowNullContextId ? shardFailures.length : 0); + out.writeVInt(shardSize); + for (var searchResult : searchPhaseResults) { + final SearchShardTarget target = searchResult.getSearchShardTarget(); + target.getShardId().writeTo(out); + new SearchContextIdForNode(target.getClusterAlias(), target.getNodeId(), searchResult.getContextId()).writeTo(out); + } + if (allowNullContextId) { + for (var failure : shardFailures) { + failure.shard().getShardId().writeTo(out); + new SearchContextIdForNode(failure.shard().getClusterAlias(), null, null).writeTo(out); + } + } out.writeMap(aliasFilter, StreamOutput::writeWriteable); return out.bytes(); } catch (IOException e) { @@ -72,12 +91,6 @@ public static BytesReference encode( } } - private static void writeSearchPhaseResult(StreamOutput out, SearchPhaseResult searchPhaseResult) throws IOException { - final SearchShardTarget target = searchPhaseResult.getSearchShardTarget(); - target.getShardId().writeTo(out); - new SearchContextIdForNode(target.getClusterAlias(), target.getNodeId(), searchPhaseResult.getContextId()).writeTo(out); - } - public static SearchContextId decode(NamedWriteableRegistry namedWriteableRegistry, BytesReference id) { try (var in = new NamedWriteableAwareStreamInput(id.streamInput(), namedWriteableRegistry)) { final TransportVersion version = TransportVersion.readVersion(in); diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchContextIdForNode.java b/server/src/main/java/org/elasticsearch/action/search/SearchContextIdForNode.java index 3071362f552ea..a70ddf6ee14b9 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchContextIdForNode.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchContextIdForNode.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.search; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -21,25 +22,59 @@ public final class SearchContextIdForNode implements Writeable { private final ShardSearchContextId searchContextId; private final String clusterAlias; - SearchContextIdForNode(@Nullable String clusterAlias, String node, ShardSearchContextId searchContextId) { + /** + * Contains the details required to retrieve a {@link ShardSearchContextId} for a shard on a specific node. + * + * @param clusterAlias The alias of the cluster, or {@code null} if the shard is local. + * @param node The target node where the search context ID is defined, or {@code null} if the shard is missing or unavailable. + * @param searchContextId The {@link ShardSearchContextId}, or {@code null} if the shard is missing or unavailable. + */ + SearchContextIdForNode(@Nullable String clusterAlias, @Nullable String node, @Nullable ShardSearchContextId searchContextId) { this.node = node; this.clusterAlias = clusterAlias; this.searchContextId = searchContextId; } SearchContextIdForNode(StreamInput in) throws IOException { - this.node = in.readString(); + boolean allowNull = in.getTransportVersion().onOrAfter(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT); + this.node = allowNull ? in.readOptionalString() : in.readString(); this.clusterAlias = in.readOptionalString(); - this.searchContextId = new ShardSearchContextId(in); + this.searchContextId = allowNull ? in.readOptionalWriteable(ShardSearchContextId::new) : new ShardSearchContextId(in); } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeString(node); + boolean allowNull = out.getTransportVersion().onOrAfter(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT); + if (allowNull) { + out.writeOptionalString(node); + } else { + if (node == null) { + // We should never set a null node if the cluster is not fully upgraded to a version that can handle it. + throw new IOException( + "Cannot write null node value to a node in version " + + out.getTransportVersion() + + ". The target node must be specified to retrieve the ShardSearchContextId." + ); + } + out.writeString(node); + } out.writeOptionalString(clusterAlias); - searchContextId.writeTo(out); + if (allowNull) { + out.writeOptionalWriteable(searchContextId); + } else { + if (searchContextId == null) { + // We should never set a null search context id if the cluster is not fully upgraded to a version that can handle it. + throw new IOException( + "Cannot write null search context ID to a node in version " + + out.getTransportVersion() + + ". A valid search context ID is required to identify the shard's search context in this version." + ); + } + searchContextId.writeTo(out); + } } + @Nullable public String getNode() { return node; } @@ -49,6 +84,7 @@ public String getClusterAlias() { return clusterAlias; } + @Nullable public ShardSearchContextId getSearchContextId() { return searchContextId; } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java index 45cb118691082..8d70e2dd6bb66 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java @@ -47,6 +47,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.function.BiFunction; import java.util.function.Predicate; import java.util.function.Supplier; @@ -701,6 +702,13 @@ public Cluster getCluster(String clusterAlias) { return clusterInfo.get(clusterAlias); } + /** + * @return collection of cluster aliases in the search response (including "(local)" if was searched). + */ + public Set getClusterAliases() { + return clusterInfo.keySet(); + } + /** * Utility to swap a Cluster object. Guidelines for the remapping function: *
    @@ -803,6 +811,7 @@ public boolean hasClusterObjects() { public boolean hasRemoteClusters() { return total > 1 || clusterInfo.keySet().stream().anyMatch(alias -> alias != RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY); } + } /** diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTask.java b/server/src/main/java/org/elasticsearch/action/search/SearchTask.java index 3bf72313c4c21..cc5d60ad0b0c0 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchTask.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchTask.java @@ -69,4 +69,11 @@ public Supplier getSearchResponseMergerSupplier() { public void setSearchResponseMergerSupplier(Supplier supplier) { this.searchResponseMergerSupplier = supplier; } + + /** + * Is this async search? + */ + public boolean isAsync() { + return false; + } } diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java index a929b774edf5e..717b1805547be 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java @@ -10,6 +10,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListenerResponseHandler; import org.elasticsearch.action.ActionType; @@ -21,6 +23,7 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.routing.GroupShardsIterator; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -29,6 +32,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchService; import org.elasticsearch.search.SearchShardTarget; @@ -50,6 +54,8 @@ import java.util.concurrent.Executor; import java.util.function.BiFunction; +import static org.elasticsearch.core.Strings.format; + public class TransportOpenPointInTimeAction extends HandledTransportAction { private static final Logger logger = LogManager.getLogger(TransportOpenPointInTimeAction.class); @@ -62,6 +68,7 @@ public class TransportOpenPointInTimeAction extends HandledTransportAction listener) { + final ClusterState clusterState = clusterService.state(); + // Check if all the nodes in this cluster know about the service + if (request.allowPartialSearchResults() + && clusterState.getMinTransportVersion().before(TransportVersions.ALLOW_PARTIAL_SEARCH_RESULTS_IN_PIT)) { + listener.onFailure( + new ElasticsearchStatusException( + format( + "The [allow_partial_search_results] parameter cannot be used while the cluster is still upgrading. " + + "Please wait until the upgrade is fully completed and try again." + ), + RestStatus.BAD_REQUEST + ) + ); + return; + } final SearchRequest searchRequest = new SearchRequest().indices(request.indices()) .indicesOptions(request.indicesOptions()) .preference(request.preference()) .routing(request.routing()) - .allowPartialSearchResults(false) + .allowPartialSearchResults(request.allowPartialSearchResults()) .source(new SearchSourceBuilder().query(request.indexFilter())); searchRequest.setMaxConcurrentShardRequests(request.maxConcurrentShardRequests()); searchRequest.setCcsMinimizeRoundtrips(false); transportSearchAction.executeRequest((SearchTask) task, searchRequest, listener.map(r -> { assert r.pointInTimeId() != null : r; - return new OpenPointInTimeResponse(r.pointInTimeId()); + return new OpenPointInTimeResponse( + r.pointInTimeId(), + r.getTotalShards(), + r.getSuccessfulShards(), + r.getFailedShards(), + r.getSkippedShards() + ); }), searchListener -> new OpenPointInTimePhase(request, searchListener)); } @@ -215,7 +245,9 @@ SearchPhase openPointInTimePhase( ) { @Override protected String missingShardsErrorMessage(StringBuilder missingShards) { - return "[open_point_in_time] action requires all shards to be available. Missing shards: [" + missingShards + "]"; + return "[open_point_in_time] action requires all shards to be available. Missing shards: [" + + missingShards + + "]. Consider using `allow_partial_search_results` setting to bypass this error."; } @Override diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index 75668f5ebce51..23ff692da4887 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -23,6 +23,8 @@ import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsRequest; import org.elasticsearch.action.admin.cluster.shards.ClusterSearchShardsResponse; import org.elasticsearch.action.admin.cluster.shards.TransportClusterSearchShardsAction; +import org.elasticsearch.action.admin.cluster.stats.CCSUsage; +import org.elasticsearch.action.admin.cluster.stats.CCSUsageTelemetry; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.action.support.IndicesOptions; @@ -46,6 +48,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.util.ArrayUtils; @@ -84,6 +87,7 @@ import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.usage.UsageService; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentFactory; @@ -156,6 +160,7 @@ public class TransportSearchAction extends HandledTransportAction buildPerIndexOriginalIndices( @@ -305,44 +312,12 @@ public long buildTookInMillis() { @Override protected void doExecute(Task task, SearchRequest searchRequest, ActionListener listener) { - ActionListener loggingAndMetrics = new ActionListener<>() { - @Override - public void onResponse(SearchResponse searchResponse) { - try { - searchResponseMetrics.recordTookTime(searchResponse.getTookInMillis()); - SearchResponseMetrics.ResponseCountTotalStatus responseCountTotalStatus = - SearchResponseMetrics.ResponseCountTotalStatus.SUCCESS; - if (searchResponse.getShardFailures() != null && searchResponse.getShardFailures().length > 0) { - // Deduplicate failures by exception message and index - ShardOperationFailedException[] groupedFailures = ExceptionsHelper.groupBy(searchResponse.getShardFailures()); - for (ShardOperationFailedException f : groupedFailures) { - boolean causeHas500Status = false; - if (f.getCause() != null) { - causeHas500Status = ExceptionsHelper.status(f.getCause()).getStatus() >= 500; - } - if ((f.status().getStatus() >= 500 || causeHas500Status) - && ExceptionsHelper.isNodeOrShardUnavailableTypeException(f.getCause()) == false) { - logger.warn("TransportSearchAction shard failure (partial results response)", f); - responseCountTotalStatus = SearchResponseMetrics.ResponseCountTotalStatus.PARTIAL_FAILURE; - } - } - } - listener.onResponse(searchResponse); - // increment after the delegated onResponse to ensure we don't - // record both a success and a failure if there is an exception - searchResponseMetrics.incrementResponseCount(responseCountTotalStatus); - } catch (Exception e) { - onFailure(e); - } - } - - @Override - public void onFailure(Exception e) { - searchResponseMetrics.incrementResponseCount(SearchResponseMetrics.ResponseCountTotalStatus.FAILURE); - listener.onFailure(e); - } - }; - executeRequest((SearchTask) task, searchRequest, loggingAndMetrics, AsyncSearchActionProvider::new); + executeRequest( + (SearchTask) task, + searchRequest, + new SearchResponseActionListener((SearchTask) task, listener), + AsyncSearchActionProvider::new + ); } void executeRequest( @@ -396,8 +371,32 @@ void executeRequest( searchPhaseProvider.apply(delegate) ); } else { + if ((listener instanceof TelemetryListener tl) && CCS_TELEMETRY_FEATURE_FLAG.isEnabled()) { + tl.setRemotes(resolvedIndices.getRemoteClusterIndices().size()); + if (task.isAsync()) { + tl.setFeature(CCSUsageTelemetry.ASYNC_FEATURE); + } + String client = task.getHeader(Task.X_ELASTIC_PRODUCT_ORIGIN_HTTP_HEADER); + if (client != null) { + tl.setClient(client); + } + // Check if any of the index patterns are wildcard patterns + var localIndices = resolvedIndices.getLocalIndices(); + if (localIndices != null && Arrays.stream(localIndices.indices()).anyMatch(Regex::isSimpleMatchPattern)) { + tl.setFeature(CCSUsageTelemetry.WILDCARD_FEATURE); + } + if (resolvedIndices.getRemoteClusterIndices() + .values() + .stream() + .anyMatch(indices -> Arrays.stream(indices.indices()).anyMatch(Regex::isSimpleMatchPattern))) { + tl.setFeature(CCSUsageTelemetry.WILDCARD_FEATURE); + } + } final TaskId parentTaskId = task.taskInfo(clusterService.localNode().getId(), false).taskId(); if (shouldMinimizeRoundtrips(rewritten)) { + if ((listener instanceof TelemetryListener tl) && CCS_TELEMETRY_FEATURE_FLAG.isEnabled()) { + tl.setFeature(CCSUsageTelemetry.MRT_FEATURE); + } final AggregationReduceContext.Builder aggregationReduceContextBuilder = rewritten.source() != null && rewritten.source().aggregations() != null ? searchService.aggReduceContextBuilder(task::isCancelled, rewritten.source().aggregations()) @@ -503,7 +502,7 @@ void executeRequest( // We set the keep alive to -1 to indicate that we don't need the pit id in the response. // This is needed since we delete the pit prior to sending the response so the id doesn't exist anymore. source.pointInTimeBuilder(new PointInTimeBuilder(resp.getPointInTimeId()).setKeepAlive(TimeValue.MINUS_ONE)); - executeRequest(task, original, new ActionListener<>() { + var pitListener = new SearchResponseActionListener(task, listener) { @Override public void onResponse(SearchResponse response) { // we need to close the PIT first so we delay the release of the response to after the closing @@ -519,7 +518,8 @@ public void onResponse(SearchResponse response) { public void onFailure(Exception e) { closePIT(client, original.source().pointInTimeBuilder(), () -> listener.onFailure(e)); } - }, searchPhaseProvider); + }; + executeRequest(task, original, pitListener, searchPhaseProvider); })); } else { Rewriteable.rewriteAndFetch( @@ -805,27 +805,26 @@ static void collectSearchShards( for (Map.Entry entry : remoteIndicesByCluster.entrySet()) { final String clusterAlias = entry.getKey(); boolean skipUnavailable = remoteClusterService.isSkipUnavailable(clusterAlias); - TransportSearchAction.CCSActionListener> singleListener = - new TransportSearchAction.CCSActionListener<>( - clusterAlias, - skipUnavailable, - responsesCountDown, - exceptions, - clusters, - listener - ) { - @Override - void innerOnResponse(SearchShardsResponse searchShardsResponse) { - assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SEARCH_COORDINATION); - ccsClusterInfoUpdate(searchShardsResponse, clusters, clusterAlias, timeProvider); - searchShardsResponses.put(clusterAlias, searchShardsResponse); - } + CCSActionListener> singleListener = new CCSActionListener<>( + clusterAlias, + skipUnavailable, + responsesCountDown, + exceptions, + clusters, + listener + ) { + @Override + void innerOnResponse(SearchShardsResponse searchShardsResponse) { + assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SEARCH_COORDINATION); + ccsClusterInfoUpdate(searchShardsResponse, clusters, clusterAlias, timeProvider); + searchShardsResponses.put(clusterAlias, searchShardsResponse); + } - @Override - Map createFinalResponse() { - return searchShardsResponses; - } - }; + @Override + Map createFinalResponse() { + return searchShardsResponses; + } + }; remoteClusterService.maybeEnsureConnectedAndGetConnection( clusterAlias, skipUnavailable == false, @@ -1116,11 +1115,16 @@ static List getRemoteShardsIteratorFromPointInTime( final String clusterAlias = entry.getKey(); assert clusterAlias.equals(perNode.getClusterAlias()) : clusterAlias + " != " + perNode.getClusterAlias(); final List targetNodes = new ArrayList<>(group.allocatedNodes().size()); - targetNodes.add(perNode.getNode()); - if (perNode.getSearchContextId().getSearcherId() != null) { - for (String node : group.allocatedNodes()) { - if (node.equals(perNode.getNode()) == false) { - targetNodes.add(node); + if (perNode.getNode() != null) { + // If the shard was available when the PIT was created, it's included. + // Otherwise, we add the shard iterator without a target node, allowing a partial search failure to + // be thrown when a search phase attempts to access it. + targetNodes.add(perNode.getNode()); + if (perNode.getSearchContextId().getSearcherId() != null) { + for (String node : group.allocatedNodes()) { + if (node.equals(perNode.getNode()) == false) { + targetNodes.add(node); + } } } } @@ -1216,7 +1220,7 @@ private void executeSearch( assert searchRequest.pointInTimeBuilder() != null; aliasFilter = resolvedIndices.getSearchContextId().aliasFilter(); concreteLocalIndices = resolvedIndices.getLocalIndices() == null ? new String[0] : resolvedIndices.getLocalIndices().indices(); - localShardIterators = getLocalLocalShardsIteratorFromPointInTime( + localShardIterators = getLocalShardsIteratorFromPointInTime( clusterState, searchRequest.indicesOptions(), searchRequest.getLocalClusterAlias(), @@ -1286,16 +1290,27 @@ private void executeSearch( } Executor asyncSearchExecutor(final String[] indices) { - final List executorsForIndices = Arrays.stream(indices).map(executorSelector::executorForSearch).toList(); - if (executorsForIndices.size() == 1) { // all indices have same executor - return threadPool.executor(executorsForIndices.get(0)); + boolean seenSystem = false; + boolean seenCritical = false; + for (String index : indices) { + final String executorName = executorSelector.executorForSearch(index); + switch (executorName) { + case SYSTEM_READ -> seenSystem = true; + case SYSTEM_CRITICAL_READ -> seenCritical = true; + default -> { + return threadPool.executor(executorName); + } + } } - if (executorsForIndices.size() == 2 - && executorsForIndices.contains(SYSTEM_READ) - && executorsForIndices.contains(SYSTEM_CRITICAL_READ)) { // mix of critical and non critical system indices - return threadPool.executor(SYSTEM_READ); + final String executor; + if (seenSystem == false && seenCritical) { + executor = SYSTEM_CRITICAL_READ; + } else if (seenSystem) { + executor = SYSTEM_READ; + } else { + executor = ThreadPool.Names.SEARCH; } - return threadPool.executor(ThreadPool.Names.SEARCH); + return threadPool.executor(executor); } static BiFunction buildConnectionLookup( @@ -1723,7 +1738,7 @@ private static RemoteTransportException wrapRemoteClusterFailure(String clusterA return new RemoteTransportException("error while communicating with remote cluster [" + clusterAlias + "]", e); } - static List getLocalLocalShardsIteratorFromPointInTime( + static List getLocalShardsIteratorFromPointInTime( ClusterState clusterState, IndicesOptions indicesOptions, String localClusterAlias, @@ -1737,25 +1752,30 @@ static List getLocalLocalShardsIteratorFromPointInTime( if (Strings.isEmpty(perNode.getClusterAlias())) { final ShardId shardId = entry.getKey(); final List targetNodes = new ArrayList<>(2); - try { - final ShardIterator shards = OperationRouting.getShards(clusterState, shardId); - // Prefer executing shard requests on nodes that are part of PIT first. - if (clusterState.nodes().nodeExists(perNode.getNode())) { - targetNodes.add(perNode.getNode()); - } - if (perNode.getSearchContextId().getSearcherId() != null) { - for (ShardRouting shard : shards) { - if (shard.currentNodeId().equals(perNode.getNode()) == false) { - targetNodes.add(shard.currentNodeId()); + if (perNode.getNode() != null) { + // If the shard was available when the PIT was created, it's included. + // Otherwise, we add the shard iterator without a target node, allowing a partial search failure to + // be thrown when a search phase attempts to access it. + try { + final ShardIterator shards = OperationRouting.getShards(clusterState, shardId); + // Prefer executing shard requests on nodes that are part of PIT first. + if (clusterState.nodes().nodeExists(perNode.getNode())) { + targetNodes.add(perNode.getNode()); + } + if (perNode.getSearchContextId().getSearcherId() != null) { + for (ShardRouting shard : shards) { + if (shard.currentNodeId().equals(perNode.getNode()) == false) { + targetNodes.add(shard.currentNodeId()); + } } } - } - } catch (IndexNotFoundException | ShardNotFoundException e) { - // We can hit these exceptions if the index was deleted after creating PIT or the cluster state on - // this coordinating node is outdated. It's fine to ignore these extra "retry-able" target shards - // when allowPartialSearchResults is false - if (allowPartialSearchResults == false) { - throw e; + } catch (IndexNotFoundException | ShardNotFoundException e) { + // We can hit these exceptions if the index was deleted after creating PIT or the cluster state on + // this coordinating node is outdated. It's fine to ignore these extra "retry-able" target shards + // when allowPartialSearchResults is false + if (allowPartialSearchResults == false) { + throw e; + } } } OriginalIndices finalIndices = new OriginalIndices(new String[] { shardId.getIndexName() }, indicesOptions); @@ -1814,4 +1834,116 @@ List getLocalShardsIterator( // the returned list must support in-place sorting, so this is the most memory efficient we can do here return Arrays.asList(list); } + + private interface TelemetryListener { + void setRemotes(int count); + + void setFeature(String feature); + + void setClient(String client); + } + + private class SearchResponseActionListener implements ActionListener, TelemetryListener { + private final SearchTask task; + private final ActionListener listener; + private final CCSUsage.Builder usageBuilder; + + SearchResponseActionListener(SearchTask task, ActionListener listener) { + this.task = task; + this.listener = listener; + if (listener instanceof SearchResponseActionListener srListener) { + usageBuilder = srListener.usageBuilder; + } else { + usageBuilder = new CCSUsage.Builder(); + } + } + + /** + * Should we collect telemetry for this search? + */ + private boolean collectTelemetry() { + return CCS_TELEMETRY_FEATURE_FLAG.isEnabled() && usageBuilder.getRemotesCount() > 0; + } + + public void setRemotes(int count) { + usageBuilder.setRemotesCount(count); + } + + @Override + public void setFeature(String feature) { + usageBuilder.setFeature(feature); + } + + @Override + public void setClient(String client) { + usageBuilder.setClient(client); + } + + @Override + public void onResponse(SearchResponse searchResponse) { + try { + searchResponseMetrics.recordTookTime(searchResponse.getTookInMillis()); + SearchResponseMetrics.ResponseCountTotalStatus responseCountTotalStatus = + SearchResponseMetrics.ResponseCountTotalStatus.SUCCESS; + if (searchResponse.getShardFailures() != null && searchResponse.getShardFailures().length > 0) { + // Deduplicate failures by exception message and index + ShardOperationFailedException[] groupedFailures = ExceptionsHelper.groupBy(searchResponse.getShardFailures()); + for (ShardOperationFailedException f : groupedFailures) { + boolean causeHas500Status = false; + if (f.getCause() != null) { + causeHas500Status = ExceptionsHelper.status(f.getCause()).getStatus() >= 500; + } + if ((f.status().getStatus() >= 500 || causeHas500Status) + && ExceptionsHelper.isNodeOrShardUnavailableTypeException(f.getCause()) == false) { + logger.warn("TransportSearchAction shard failure (partial results response)", f); + responseCountTotalStatus = SearchResponseMetrics.ResponseCountTotalStatus.PARTIAL_FAILURE; + } + } + } + searchResponseMetrics.incrementResponseCount(responseCountTotalStatus); + + if (collectTelemetry()) { + extractCCSTelemetry(searchResponse); + recordTelemetry(); + } + } catch (Exception e) { + onFailure(e); + return; + } + // This is last because we want to collect telemetry before returning the response. + listener.onResponse(searchResponse); + } + + @Override + public void onFailure(Exception e) { + searchResponseMetrics.incrementResponseCount(SearchResponseMetrics.ResponseCountTotalStatus.FAILURE); + if (collectTelemetry()) { + usageBuilder.setFailure(e); + recordTelemetry(); + } + listener.onFailure(e); + } + + private void recordTelemetry() { + usageService.getCcsUsageHolder().updateUsage(usageBuilder.build()); + } + + /** + * Extract telemetry data from the search response. + * @param searchResponse The final response from the search. + */ + private void extractCCSTelemetry(SearchResponse searchResponse) { + usageBuilder.took(searchResponse.getTookInMillis()); + for (String clusterAlias : searchResponse.getClusters().getClusterAliases()) { + SearchResponse.Cluster cluster = searchResponse.getClusters().getCluster(clusterAlias); + if (cluster.getStatus() == SearchResponse.Cluster.Status.SKIPPED) { + usageBuilder.skippedRemote(clusterAlias); + } else { + usageBuilder.perClusterUsage(clusterAlias, cluster.getTook()); + } + } + + } + + } } diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java index c1ee0f7b8af37..587ed2ef75eba 100644 --- a/server/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java @@ -68,6 +68,7 @@ public UpdateRequestBuilder(ElasticsearchClient client) { this(client, null, null); } + @SuppressWarnings("this-escape") public UpdateRequestBuilder(ElasticsearchClient client, String index, String id) { super(client, TransportUpdateAction.TYPE); setIndex(index); diff --git a/server/src/main/java/org/elasticsearch/client/internal/support/AbstractClient.java b/server/src/main/java/org/elasticsearch/client/internal/support/AbstractClient.java index f4e86c8a4eca6..c2268bfd9dc62 100644 --- a/server/src/main/java/org/elasticsearch/client/internal/support/AbstractClient.java +++ b/server/src/main/java/org/elasticsearch/client/internal/support/AbstractClient.java @@ -93,6 +93,7 @@ public abstract class AbstractClient implements Client { private final ThreadPool threadPool; private final AdminClient admin; + @SuppressWarnings("this-escape") public AbstractClient(Settings settings, ThreadPool threadPool) { this.settings = settings; this.threadPool = threadPool; diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterFeatures.java b/server/src/main/java/org/elasticsearch/cluster/ClusterFeatures.java index 95cc53376af59..bab68303e8de5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterFeatures.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterFeatures.java @@ -140,11 +140,10 @@ private static void writeCanonicalSets(StreamOutput out, Map out.writeMap(nodeFeatureSetIndexes, StreamOutput::writeVInt); } + @SuppressWarnings("unchecked") private static Map> readCanonicalSets(StreamInput in) throws IOException { - List> featureSets = in.readCollectionAsList(i -> i.readCollectionAsImmutableSet(StreamInput::readString)); - Map nodeIndexes = in.readMap(StreamInput::readVInt); - - return nodeIndexes.entrySet().stream().collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, e -> featureSets.get(e.getValue()))); + Set[] featureSets = in.readArray(i -> i.readCollectionAsImmutableSet(StreamInput::readString), Set[]::new); + return in.readImmutableMap(streamInput -> featureSets[streamInput.readVInt()]); } public static ClusterFeatures readFrom(StreamInput in) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java index 3fba3a7bdbe13..e399e739da047 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java @@ -45,6 +45,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.DiskThresholdDecider; import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider; +import org.elasticsearch.cluster.routing.allocation.decider.IndexVersionAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.NodeReplacementAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.NodeShutdownAllocationDecider; @@ -364,6 +365,7 @@ public static Collection createAllocationDeciders( addAllocationDecider(deciders, new ClusterRebalanceAllocationDecider(clusterSettings)); addAllocationDecider(deciders, new ConcurrentRebalanceAllocationDecider(clusterSettings)); addAllocationDecider(deciders, new EnableAllocationDecider(clusterSettings)); + addAllocationDecider(deciders, new IndexVersionAllocationDecider()); addAllocationDecider(deciders, new NodeVersionAllocationDecider()); addAllocationDecider(deciders, new SnapshotInProgressAllocationDecider()); addAllocationDecider(deciders, new RestoreInProgressAllocationDecider()); diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterName.java b/server/src/main/java/org/elasticsearch/cluster/ClusterName.java index 711c2a7fee8e0..dd4194b60e6ac 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterName.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterName.java @@ -39,7 +39,8 @@ public ClusterName(StreamInput input) throws IOException { } public ClusterName(String value) { - this.value = value.intern(); + // cluster name string is most likely part of a setting so we can speed things up over outright interning here + this.value = Settings.internKeyOrValue(value); } public String value() { diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java index c54269da68507..02d5bdfdbebc0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -984,7 +984,9 @@ public ClusterState build() { routingTable, nodes, compatibilityVersions, - new ClusterFeatures(nodeFeatures), + previous != null && getNodeFeatures(previous.clusterFeatures).equals(nodeFeatures) + ? previous.clusterFeatures + : new ClusterFeatures(nodeFeatures), blocks, customs.build(), fromDiff, @@ -1081,7 +1083,7 @@ public void writeTo(StreamOutput out) throws IOException { routingTable.writeTo(out); nodes.writeTo(out); if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_8_0)) { - out.writeMap(compatibilityVersions, (streamOutput, versions) -> versions.writeTo(streamOutput)); + out.writeMap(compatibilityVersions, StreamOutput::writeWriteable); } if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_12_0)) { clusterFeatures.writeTo(out); diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java index 437219b312045..e922d130d7f83 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java @@ -41,6 +41,7 @@ import org.elasticsearch.cluster.service.MasterServiceTaskQueue; import org.elasticsearch.cluster.version.CompatibilityVersions; import org.elasticsearch.common.Priority; +import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.Strings; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -831,10 +832,12 @@ public void run() { discover other nodes and form a multi-node cluster via the [{}={}] setting. Fully-formed clusters do \ not attempt to discover other nodes, and nodes with different cluster UUIDs cannot belong to the same \ cluster. The cluster UUID persists across restarts and can only be changed by deleting the contents of \ - the node's data path(s). Remove the discovery configuration to suppress this message.""", + the node's data path(s). Remove the discovery configuration to suppress this message. See [{}] for \ + more information.""", applierState.metadata().clusterUUID(), DISCOVERY_SEED_HOSTS_SETTING.getKey(), - DISCOVERY_SEED_HOSTS_SETTING.get(settings) + DISCOVERY_SEED_HOSTS_SETTING.get(settings), + ReferenceDocs.FORMING_SINGLE_NODE_CLUSTERS ); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/health/ClusterHealthStatus.java b/server/src/main/java/org/elasticsearch/cluster/health/ClusterHealthStatus.java index d025ddab26af6..c53395b5d76c1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/health/ClusterHealthStatus.java +++ b/server/src/main/java/org/elasticsearch/cluster/health/ClusterHealthStatus.java @@ -19,7 +19,7 @@ public enum ClusterHealthStatus implements Writeable { YELLOW((byte) 1), RED((byte) 2); - private byte value; + private final byte value; ClusterHealthStatus(byte value) { this.value = value; diff --git a/server/src/main/java/org/elasticsearch/cluster/health/ClusterIndexHealth.java b/server/src/main/java/org/elasticsearch/cluster/health/ClusterIndexHealth.java index ad957f7a8f37f..46273e92b84b2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/health/ClusterIndexHealth.java +++ b/server/src/main/java/org/elasticsearch/cluster/health/ClusterIndexHealth.java @@ -8,6 +8,7 @@ package org.elasticsearch.cluster.health; +import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ClusterStatsLevel; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.routing.IndexRoutingTable; @@ -33,6 +34,7 @@ public final class ClusterIndexHealth implements Writeable, ToXContentFragment { static final String RELOCATING_SHARDS = "relocating_shards"; static final String INITIALIZING_SHARDS = "initializing_shards"; static final String UNASSIGNED_SHARDS = "unassigned_shards"; + static final String UNASSIGNED_PRIMARY_SHARDS = "unassigned_primary_shards"; static final String SHARDS = "shards"; private final String index; @@ -42,6 +44,7 @@ public final class ClusterIndexHealth implements Writeable, ToXContentFragment { private final int relocatingShards; private final int initializingShards; private final int unassignedShards; + private final int unassignedPrimaryShards; private final int activePrimaryShards; private final ClusterHealthStatus status; private final Map shards; @@ -64,6 +67,7 @@ public ClusterIndexHealth(final IndexMetadata indexMetadata, final IndexRoutingT int computeActiveShards = 0; int computeRelocatingShards = 0; int computeInitializingShards = 0; + int computeUnassignedPrimaryShards = 0; int computeUnassignedShards = 0; for (ClusterShardHealth shardHealth : shards.values()) { if (shardHealth.isPrimaryActive()) { @@ -73,6 +77,7 @@ public ClusterIndexHealth(final IndexMetadata indexMetadata, final IndexRoutingT computeRelocatingShards += shardHealth.getRelocatingShards(); computeInitializingShards += shardHealth.getInitializingShards(); computeUnassignedShards += shardHealth.getUnassignedShards(); + computeUnassignedPrimaryShards += shardHealth.getUnassignedPrimaryShards(); if (shardHealth.getStatus() == ClusterHealthStatus.RED) { computeStatus = ClusterHealthStatus.RED; @@ -91,6 +96,7 @@ public ClusterIndexHealth(final IndexMetadata indexMetadata, final IndexRoutingT this.relocatingShards = computeRelocatingShards; this.initializingShards = computeInitializingShards; this.unassignedShards = computeUnassignedShards; + this.unassignedPrimaryShards = computeUnassignedPrimaryShards; } public ClusterIndexHealth(final StreamInput in) throws IOException { @@ -104,6 +110,11 @@ public ClusterIndexHealth(final StreamInput in) throws IOException { unassignedShards = in.readVInt(); status = ClusterHealthStatus.readFrom(in); shards = in.readMapValues(ClusterShardHealth::new, ClusterShardHealth::getShardId); + if (in.getTransportVersion().onOrAfter(TransportVersions.UNASSIGNED_PRIMARY_COUNT_ON_CLUSTER_HEALTH)) { + unassignedPrimaryShards = in.readVInt(); + } else { + unassignedPrimaryShards = 0; + } } /** @@ -117,6 +128,7 @@ public ClusterIndexHealth(final StreamInput in) throws IOException { int relocatingShards, int initializingShards, int unassignedShards, + int unassignedPrimaryShards, int activePrimaryShards, ClusterHealthStatus status, Map shards @@ -128,6 +140,7 @@ public ClusterIndexHealth(final StreamInput in) throws IOException { this.relocatingShards = relocatingShards; this.initializingShards = initializingShards; this.unassignedShards = unassignedShards; + this.unassignedPrimaryShards = unassignedPrimaryShards; this.activePrimaryShards = activePrimaryShards; this.status = status; this.shards = shards; @@ -165,6 +178,10 @@ public int getUnassignedShards() { return unassignedShards; } + public int getUnassignedPrimaryShards() { + return unassignedPrimaryShards; + } + public ClusterHealthStatus getStatus() { return status; } @@ -185,6 +202,9 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeVInt(unassignedShards); out.writeByte(status.value()); out.writeMapValues(shards); + if (out.getTransportVersion().onOrAfter(TransportVersions.UNASSIGNED_PRIMARY_COUNT_ON_CLUSTER_HEALTH)) { + out.writeVInt(unassignedPrimaryShards); + } } @Override @@ -198,6 +218,7 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.field(RELOCATING_SHARDS, getRelocatingShards()); builder.field(INITIALIZING_SHARDS, getInitializingShards()); builder.field(UNASSIGNED_SHARDS, getUnassignedShards()); + builder.field(UNASSIGNED_PRIMARY_SHARDS, getUnassignedPrimaryShards()); ClusterStatsLevel level = ClusterStatsLevel.of(params, ClusterStatsLevel.INDICES); if (level == ClusterStatsLevel.SHARDS) { @@ -229,6 +250,8 @@ public String toString() { + initializingShards + ", unassignedShards=" + unassignedShards + + ", unassignedPrimaryShards=" + + unassignedPrimaryShards + ", activePrimaryShards=" + activePrimaryShards + ", status=" @@ -250,6 +273,7 @@ public boolean equals(Object o) { && relocatingShards == that.relocatingShards && initializingShards == that.initializingShards && unassignedShards == that.unassignedShards + && unassignedPrimaryShards == that.unassignedPrimaryShards && activePrimaryShards == that.activePrimaryShards && status == that.status && Objects.equals(shards, that.shards); @@ -265,6 +289,7 @@ public int hashCode() { relocatingShards, initializingShards, unassignedShards, + unassignedPrimaryShards, activePrimaryShards, status, shards diff --git a/server/src/main/java/org/elasticsearch/cluster/health/ClusterShardHealth.java b/server/src/main/java/org/elasticsearch/cluster/health/ClusterShardHealth.java index adb5a7caf2f45..7ebd90050bff3 100644 --- a/server/src/main/java/org/elasticsearch/cluster/health/ClusterShardHealth.java +++ b/server/src/main/java/org/elasticsearch/cluster/health/ClusterShardHealth.java @@ -8,6 +8,7 @@ package org.elasticsearch.cluster.health; +import org.elasticsearch.TransportVersions; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.cluster.routing.ShardRouting; @@ -30,6 +31,7 @@ public final class ClusterShardHealth implements Writeable, ToXContentFragment { static final String RELOCATING_SHARDS = "relocating_shards"; static final String INITIALIZING_SHARDS = "initializing_shards"; static final String UNASSIGNED_SHARDS = "unassigned_shards"; + static final String UNASSIGNED_PRIMARY_SHARDS = "unassigned_primary_shards"; static final String PRIMARY_ACTIVE = "primary_active"; private final int shardId; @@ -37,6 +39,7 @@ public final class ClusterShardHealth implements Writeable, ToXContentFragment { private final int activeShards; private final int relocatingShards; private final int initializingShards; + private final int unassignedPrimaryShards; private final int unassignedShards; private final boolean primaryActive; @@ -45,6 +48,7 @@ public ClusterShardHealth(final int shardId, final IndexShardRoutingTable shardR int computeActiveShards = 0; int computeRelocatingShards = 0; int computeInitializingShards = 0; + int computeUnassignedPrimaryShards = 0; int computeUnassignedShards = 0; for (int j = 0; j < shardRoutingTable.size(); j++) { ShardRouting shardRouting = shardRoutingTable.shard(j); @@ -57,6 +61,9 @@ public ClusterShardHealth(final int shardId, final IndexShardRoutingTable shardR } else if (shardRouting.initializing()) { computeInitializingShards++; } else if (shardRouting.unassigned()) { + if (shardRouting.primary()) { + computeUnassignedPrimaryShards++; + } computeUnassignedShards++; } } @@ -76,6 +83,7 @@ public ClusterShardHealth(final int shardId, final IndexShardRoutingTable shardR this.relocatingShards = computeRelocatingShards; this.initializingShards = computeInitializingShards; this.unassignedShards = computeUnassignedShards; + this.unassignedPrimaryShards = computeUnassignedPrimaryShards; this.primaryActive = primaryRouting.active(); } @@ -87,6 +95,11 @@ public ClusterShardHealth(final StreamInput in) throws IOException { initializingShards = in.readVInt(); unassignedShards = in.readVInt(); primaryActive = in.readBoolean(); + if (in.getTransportVersion().onOrAfter(TransportVersions.UNASSIGNED_PRIMARY_COUNT_ON_CLUSTER_HEALTH)) { + unassignedPrimaryShards = in.readVInt(); + } else { + unassignedPrimaryShards = 0; + } } /** @@ -99,6 +112,7 @@ public ClusterShardHealth(final StreamInput in) throws IOException { int relocatingShards, int initializingShards, int unassignedShards, + int unassignedPrimaryShards, boolean primaryActive ) { this.shardId = shardId; @@ -107,6 +121,7 @@ public ClusterShardHealth(final StreamInput in) throws IOException { this.relocatingShards = relocatingShards; this.initializingShards = initializingShards; this.unassignedShards = unassignedShards; + this.unassignedPrimaryShards = unassignedPrimaryShards; this.primaryActive = primaryActive; } @@ -138,6 +153,10 @@ public int getUnassignedShards() { return unassignedShards; } + public int getUnassignedPrimaryShards() { + return unassignedPrimaryShards; + } + @Override public void writeTo(final StreamOutput out) throws IOException { out.writeVInt(shardId); @@ -147,6 +166,9 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeVInt(initializingShards); out.writeVInt(unassignedShards); out.writeBoolean(primaryActive); + if (out.getTransportVersion().onOrAfter(TransportVersions.UNASSIGNED_PRIMARY_COUNT_ON_CLUSTER_HEALTH)) { + out.writeVInt(unassignedPrimaryShards); + } } /** @@ -187,6 +209,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(RELOCATING_SHARDS, getRelocatingShards()); builder.field(INITIALIZING_SHARDS, getInitializingShards()); builder.field(UNASSIGNED_SHARDS, getUnassignedShards()); + builder.field(UNASSIGNED_PRIMARY_SHARDS, getUnassignedPrimaryShards()); builder.endObject(); return builder; } @@ -206,12 +229,22 @@ public boolean equals(Object o) { && relocatingShards == that.relocatingShards && initializingShards == that.initializingShards && unassignedShards == that.unassignedShards + && unassignedPrimaryShards == that.unassignedPrimaryShards && primaryActive == that.primaryActive && status == that.status; } @Override public int hashCode() { - return Objects.hash(shardId, status, activeShards, relocatingShards, initializingShards, unassignedShards, primaryActive); + return Objects.hash( + shardId, + status, + activeShards, + relocatingShards, + initializingShards, + unassignedShards, + unassignedPrimaryShards, + primaryActive + ); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java b/server/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java index 63f1c44b3d1e4..3f27d8d7c2c63 100644 --- a/server/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java +++ b/server/src/main/java/org/elasticsearch/cluster/health/ClusterStateHealth.java @@ -7,6 +7,7 @@ */ package org.elasticsearch.cluster.health; +import org.elasticsearch.TransportVersions; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.routing.IndexRoutingTable; @@ -30,6 +31,7 @@ public final class ClusterStateHealth implements Writeable { private final int activePrimaryShards; private final int initializingShards; private final int unassignedShards; + private final int unassignedPrimaryShards; private final double activeShardsPercent; private final ClusterHealthStatus status; private final Map indices; @@ -58,6 +60,7 @@ public ClusterStateHealth(final ClusterState clusterState, final String[] concre int computeActiveShards = 0; int computeRelocatingShards = 0; int computeInitializingShards = 0; + int computeUnassignedPrimaryShards = 0; int computeUnassignedShards = 0; int totalShardCount = 0; @@ -77,6 +80,7 @@ public ClusterStateHealth(final ClusterState clusterState, final String[] concre computeRelocatingShards += indexHealth.getRelocatingShards(); computeInitializingShards += indexHealth.getInitializingShards(); computeUnassignedShards += indexHealth.getUnassignedShards(); + computeUnassignedPrimaryShards += indexHealth.getUnassignedPrimaryShards(); if (indexHealth.getStatus() == ClusterHealthStatus.RED) { computeStatus = ClusterHealthStatus.RED; } else if (indexHealth.getStatus() == ClusterHealthStatus.YELLOW && computeStatus != ClusterHealthStatus.RED) { @@ -94,6 +98,7 @@ public ClusterStateHealth(final ClusterState clusterState, final String[] concre this.relocatingShards = computeRelocatingShards; this.initializingShards = computeInitializingShards; this.unassignedShards = computeUnassignedShards; + this.unassignedPrimaryShards = computeUnassignedPrimaryShards; // shortcut on green if (computeStatus.equals(ClusterHealthStatus.GREEN)) { @@ -114,6 +119,11 @@ public ClusterStateHealth(final StreamInput in) throws IOException { status = ClusterHealthStatus.readFrom(in); indices = in.readMapValues(ClusterIndexHealth::new, ClusterIndexHealth::getIndex); activeShardsPercent = in.readDouble(); + if (in.getTransportVersion().onOrAfter(TransportVersions.UNASSIGNED_PRIMARY_COUNT_ON_CLUSTER_HEALTH)) { + unassignedPrimaryShards = in.readVInt(); + } else { + unassignedPrimaryShards = 0; + } } /** @@ -125,6 +135,7 @@ public ClusterStateHealth( int relocatingShards, int initializingShards, int unassignedShards, + int unassignedPrimaryShards, int numberOfNodes, int numberOfDataNodes, double activeShardsPercent, @@ -136,6 +147,7 @@ public ClusterStateHealth( this.relocatingShards = relocatingShards; this.initializingShards = initializingShards; this.unassignedShards = unassignedShards; + this.unassignedPrimaryShards = unassignedPrimaryShards; this.numberOfNodes = numberOfNodes; this.numberOfDataNodes = numberOfDataNodes; this.activeShardsPercent = activeShardsPercent; @@ -159,6 +171,10 @@ public int getInitializingShards() { return initializingShards; } + public int getUnassignedPrimaryShards() { + return unassignedPrimaryShards; + } + public int getUnassignedShards() { return unassignedShards; } @@ -195,6 +211,9 @@ public void writeTo(final StreamOutput out) throws IOException { out.writeByte(status.value()); out.writeMapValues(indices); out.writeDouble(activeShardsPercent); + if (out.getTransportVersion().onOrAfter(TransportVersions.UNASSIGNED_PRIMARY_COUNT_ON_CLUSTER_HEALTH)) { + out.writeVInt(unassignedPrimaryShards); + } } @Override @@ -214,6 +233,8 @@ public String toString() { + initializingShards + ", unassignedShards=" + unassignedShards + + ", unassignedPrimaryShards=" + + unassignedPrimaryShards + ", activeShardsPercent=" + activeShardsPercent + ", status=" @@ -235,6 +256,7 @@ public boolean equals(Object o) { && activePrimaryShards == that.activePrimaryShards && initializingShards == that.initializingShards && unassignedShards == that.unassignedShards + && unassignedPrimaryShards == that.unassignedPrimaryShards && Double.compare(that.activeShardsPercent, activeShardsPercent) == 0 && status == that.status && Objects.equals(indices, that.indices); @@ -250,6 +272,7 @@ public int hashCode() { activePrimaryShards, initializingShards, unassignedShards, + unassignedPrimaryShards, activeShardsPercent, status, indices diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java index a0f4a929dafdb..ff412d629b3b1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/AliasMetadata.java @@ -396,6 +396,8 @@ public static AliasMetadata fromXContent(XContentParser parser) throws IOExcepti } else if ("is_hidden".equals(currentFieldName)) { builder.isHidden(parser.booleanValue()); } + } else if (token == null) { + throw new IllegalArgumentException("unexpected null token while parsing alias"); } } return builder.build(); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index 6b20399a1bc59..8acee4f6be821 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -277,6 +277,18 @@ public boolean rolloverOnWrite() { return backingIndices.rolloverOnWrite; } + /** + * We define that a data stream is considered internal either if it is a system index or if + * its name starts with a dot. + * + * Note: Dot-prefixed internal data streams is a naming convention for internal data streams, + * but it's not yet enforced. + * @return true if it's a system index or has a dot-prefixed name. + */ + public boolean isInternal() { + return isSystem() || name.charAt(0) == '.'; + } + /** * @param timestamp The timestamp used to select a backing index based on its start and end time. * @param metadata The metadata that is used to fetch the start and end times for backing indices of this data stream. @@ -796,12 +808,12 @@ public List getIndicesPastRetention( ) { if (lifecycle == null || lifecycle.isEnabled() == false - || lifecycle.getEffectiveDataRetention(isSystem() ? null : globalRetention) == null) { + || lifecycle.getEffectiveDataRetention(globalRetention, isInternal()) == null) { return List.of(); } List indicesPastRetention = getNonWriteIndicesOlderThan( - lifecycle.getEffectiveDataRetention(isSystem() ? null : globalRetention), + lifecycle.getEffectiveDataRetention(globalRetention, isInternal()), indexMetadataSupplier, this::isIndexManagedByDataStreamLifecycle, nowSupplier @@ -1202,7 +1214,7 @@ public XContentBuilder toXContent( } if (lifecycle != null) { builder.field(LIFECYCLE.getPreferredName()); - lifecycle.toXContent(builder, params, rolloverConfiguration, isSystem() ? null : globalRetention); + lifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention, isInternal()); } builder.field(ROLLOVER_ON_WRITE_FIELD.getPreferredName(), backingIndices.rolloverOnWrite); if (backingIndices.autoShardingEvent != null) { @@ -1376,6 +1388,25 @@ private static Instant getTimestampFromParser(BytesReference source, XContentTyp } } + /** + * Resolve the index abstraction to a data stream. This handles alias resolution as well as data stream resolution. This does NOT + * resolve a data stream by providing a concrete backing index. + */ + public static DataStream resolveDataStream(IndexAbstraction indexAbstraction, Metadata metadata) { + // We do not consider concrete indices - only data streams and data stream aliases. + if (indexAbstraction == null || indexAbstraction.isDataStreamRelated() == false) { + return null; + } + + // Locate the write index for the abstraction, and check if it has a data stream associated with it. + Index writeIndex = indexAbstraction.getWriteIndex(); + if (writeIndex == null) { + return null; + } + IndexAbstraction writeAbstraction = metadata.getIndicesLookup().get(writeIndex.getName()); + return writeAbstraction.getParentDataStream(); + } + /** * Modifies the passed Instant object to be used as a bound for a timestamp field in TimeSeries. It needs to be called in both backing * index construction (rollover) and index selection for doc insertion. Failure to do so may lead to errors due to document timestamps diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java index de9d615022975..bd9a65735be05 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamLifecycle.java @@ -24,7 +24,6 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; -import org.elasticsearch.rest.RestRequest; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.xcontent.AbstractObjectParser; import org.elasticsearch.xcontent.ConstructingObjectParser; @@ -55,6 +54,7 @@ public class DataStreamLifecycle implements SimpleDiffable, // Versions over the wire public static final TransportVersion ADDED_ENABLED_FLAG_VERSION = TransportVersions.V_8_10_X; + public static final String EFFECTIVE_RETENTION_REST_API_CAPABILITY = "data_stream_lifecycle_effective_retention"; public static final String DATA_STREAMS_LIFECYCLE_ONLY_SETTING_NAME = "data_streams.lifecycle_only.mode"; // The following XContent params are used to enrich the DataStreamLifecycle json with effective retention information @@ -65,6 +65,7 @@ public class DataStreamLifecycle implements SimpleDiffable, DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAM_NAME, "true" ); + public static final Tuple INFINITE_RETENTION = Tuple.tuple(null, RetentionSource.DATA_STREAM_CONFIGURATION); /** * Check if {@link #DATA_STREAMS_LIFECYCLE_ONLY_SETTING_NAME} is present and set to {@code true}, indicating that @@ -145,32 +146,37 @@ public boolean isEnabled() { } /** - * The least amount of time data should be kept by elasticsearch. If a caller does not want the global retention considered (for - * example, when evaluating the effective retention for a system data stream or a template) then null should be given for - * globalRetention. - * @param globalRetention The global retention, or null if global retention does not exist or should not be applied + * The least amount of time data should be kept by elasticsearch. The effective retention is a function with three parameters, + * the {@link DataStreamLifecycle#dataRetention}, the global retention and whether this lifecycle is associated with an internal + * data stream. + * @param globalRetention The global retention, or null if global retention does not exist. + * @param isInternalDataStream A flag denoting if this lifecycle is associated with an internal data stream or not * @return the time period or null, null represents that data should never be deleted. */ @Nullable - public TimeValue getEffectiveDataRetention(@Nullable DataStreamGlobalRetention globalRetention) { - return getEffectiveDataRetentionWithSource(globalRetention).v1(); + public TimeValue getEffectiveDataRetention(@Nullable DataStreamGlobalRetention globalRetention, boolean isInternalDataStream) { + return getEffectiveDataRetentionWithSource(globalRetention, isInternalDataStream).v1(); } /** - * The least amount of time data should be kept by elasticsearch. If a caller does not want the global retention considered (for - * example, when evaluating the effective retention for a system data stream or a template) then null should be given for - * globalRetention. - * @param globalRetention The global retention, or null if global retention does not exist or should not be applied + * The least amount of time data should be kept by elasticsearch.. The effective retention is a function with three parameters, + * the {@link DataStreamLifecycle#dataRetention}, the global retention and whether this lifecycle is associated with an internal + * data stream. + * @param globalRetention The global retention, or null if global retention does not exist. + * @param isInternalDataStream A flag denoting if this lifecycle is associated with an internal data stream or not * @return A tuple containing the time period or null as v1 (where null represents that data should never be deleted), and the non-null * retention source as v2. */ - public Tuple getEffectiveDataRetentionWithSource(@Nullable DataStreamGlobalRetention globalRetention) { + public Tuple getEffectiveDataRetentionWithSource( + @Nullable DataStreamGlobalRetention globalRetention, + boolean isInternalDataStream + ) { // If lifecycle is disabled there is no effective retention if (enabled == false) { - return Tuple.tuple(null, RetentionSource.DATA_STREAM_CONFIGURATION); + return INFINITE_RETENTION; } var dataStreamRetention = getDataStreamRetention(); - if (globalRetention == null) { + if (globalRetention == null || isInternalDataStream) { return Tuple.tuple(dataStreamRetention, RetentionSource.DATA_STREAM_CONFIGURATION); } if (dataStreamRetention == null) { @@ -187,7 +193,7 @@ public Tuple getEffectiveDataRetentionWithSource(@Nu /** * The least amount of time data the data stream is requesting es to keep the data. - * NOTE: this can be overridden by the {@link DataStreamLifecycle#getEffectiveDataRetention(DataStreamGlobalRetention)}. + * NOTE: this can be overridden by the {@link DataStreamLifecycle#getEffectiveDataRetention(DataStreamGlobalRetention,boolean)}. * @return the time period or null, null represents that data should never be deleted. */ @Nullable @@ -199,12 +205,16 @@ public TimeValue getDataStreamRetention() { * This method checks if the effective retention is matching what the user has configured; if the effective retention * does not match then it adds a warning informing the user about the effective retention and the source. */ - public void addWarningHeaderIfDataRetentionNotEffective(@Nullable DataStreamGlobalRetention globalRetention) { - if (globalRetention == null) { + public void addWarningHeaderIfDataRetentionNotEffective( + @Nullable DataStreamGlobalRetention globalRetention, + boolean isInternalDataStream + ) { + if (globalRetention == null || isInternalDataStream) { return; } Tuple effectiveDataRetentionWithSource = getEffectiveDataRetentionWithSource( - globalRetention + globalRetention, + isInternalDataStream ); if (effectiveDataRetentionWithSource.v1() == null) { return; @@ -318,7 +328,7 @@ public String toString() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return toXContent(builder, params, null, null); + return toXContent(builder, params, null, null, false); } /** @@ -326,12 +336,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws * and injects the RolloverConditions if they exist. * In order to request the effective retention you need to set {@link #INCLUDE_EFFECTIVE_RETENTION_PARAM_NAME} to true * in the XContent params. + * NOTE: this is used for serialising user output and the result is never deserialised in elasticsearch. */ public XContentBuilder toXContent( XContentBuilder builder, Params params, @Nullable RolloverConfiguration rolloverConfiguration, - @Nullable DataStreamGlobalRetention globalRetention + @Nullable DataStreamGlobalRetention globalRetention, + boolean isInternalDataStream ) throws IOException { builder.startObject(); builder.field(ENABLED_FIELD.getPreferredName(), enabled); @@ -342,11 +354,14 @@ public XContentBuilder toXContent( builder.field(DATA_RETENTION_FIELD.getPreferredName(), dataRetention.value().getStringRep()); } } + Tuple effectiveDataRetentionWithSource = getEffectiveDataRetentionWithSource( + globalRetention, + isInternalDataStream + ); if (params.paramAsBoolean(INCLUDE_EFFECTIVE_RETENTION_PARAM_NAME, false)) { - Tuple effectiveRetention = getEffectiveDataRetentionWithSource(globalRetention); - if (effectiveRetention.v1() != null) { - builder.field(EFFECTIVE_RETENTION_FIELD.getPreferredName(), effectiveRetention.v1().getStringRep()); - builder.field(RETENTION_SOURCE_FIELD.getPreferredName(), effectiveRetention.v2().displayName()); + if (effectiveDataRetentionWithSource.v1() != null) { + builder.field(EFFECTIVE_RETENTION_FIELD.getPreferredName(), effectiveDataRetentionWithSource.v1().getStringRep()); + builder.field(RETENTION_SOURCE_FIELD.getPreferredName(), effectiveDataRetentionWithSource.v2().displayName()); } } @@ -356,25 +371,29 @@ public XContentBuilder toXContent( } if (rolloverConfiguration != null) { builder.field(ROLLOVER_FIELD.getPreferredName()); - rolloverConfiguration.evaluateAndConvertToXContent(builder, params, getEffectiveDataRetention(globalRetention)); + rolloverConfiguration.evaluateAndConvertToXContent(builder, params, effectiveDataRetentionWithSource.v1()); } builder.endObject(); return builder; } + /** + * This method deserialises XContent format as it was generated ONLY by {@link DataStreamLifecycle#toXContent(XContentBuilder, Params)}. + * It does not support the output of + * {@link DataStreamLifecycle#toXContent(XContentBuilder, Params, RolloverConfiguration, DataStreamGlobalRetention, boolean)} because + * this output is enriched with derived fields we do not handle in this deserialisation. + */ public static DataStreamLifecycle fromXContent(XContentParser parser) throws IOException { return PARSER.parse(parser, null); } /** - * Adds a retention param to signal that this serialisation should include the effective retention metadata + * Adds a retention param to signal that this serialisation should include the effective retention metadata. + * @param params the XContent params to be extended with the new flag + * @return XContent params with `include_effective_retention` set to true. If the flag exists it will override it. */ - public static ToXContent.Params maybeAddEffectiveRetentionParams(ToXContent.Params params) { - boolean shouldAddEffectiveRetention = Objects.equals(params.param(RestRequest.PATH_RESTRICTED), "serverless"); - return new DelegatingMapParams( - Map.of(INCLUDE_EFFECTIVE_RETENTION_PARAM_NAME, Boolean.toString(shouldAddEffectiveRetention)), - params - ); + public static ToXContent.Params addEffectiveRetentionParams(ToXContent.Params params) { + return new DelegatingMapParams(INCLUDE_EFFECTIVE_RETENTION_PARAMS, params); } public static Builder newBuilder(DataStreamLifecycle lifecycle) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 742439c9a2484..611640f4a3b0f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -456,7 +456,7 @@ public Iterator> settings() { ); public static final Setting.AffixSetting> INDEX_ROUTING_INITIAL_RECOVERY_GROUP_SETTING = Setting.prefixKeySetting( "index.routing.allocation.initial_recovery.", - key -> Setting.stringListSetting(key) + Setting::stringListSetting ); /** diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index b5ee0ebd7e387..17db4f9253824 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -25,7 +25,6 @@ import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; -import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.node.DiscoveryNodes; @@ -168,7 +167,7 @@ public MetadataCreateIndexService( /** * Validate the name for an index against some static rules and a cluster state. */ - public static void validateIndexName(String index, ClusterState state) { + public static void validateIndexName(String index, Metadata metadata, RoutingTable routingTable) { validateIndexOrAliasName(index, InvalidIndexNameException::new); if (index.toLowerCase(Locale.ROOT).equals(index) == false) { throw new InvalidIndexNameException(index, "must be lowercase"); @@ -176,13 +175,13 @@ public static void validateIndexName(String index, ClusterState state) { // NOTE: dot-prefixed index names are validated after template application, not here - if (state.routingTable().hasIndex(index)) { - throw new ResourceAlreadyExistsException(state.routingTable().index(index).getIndex()); + if (routingTable.hasIndex(index)) { + throw new ResourceAlreadyExistsException(routingTable.index(index).getIndex()); } - if (state.metadata().hasIndex(index)) { - throw new ResourceAlreadyExistsException(state.metadata().index(index).getIndex()); + if (metadata.hasIndex(index)) { + throw new ResourceAlreadyExistsException(metadata.index(index).getIndex()); } - if (state.metadata().hasAlias(index)) { + if (metadata.hasAlias(index)) { throw new InvalidIndexNameException(index, "already exists as alias"); } } @@ -345,7 +344,7 @@ public ClusterState applyCreateIndexRequest( normalizeRequestSetting(request); logger.trace("executing IndexCreationTask for [{}] against cluster state version [{}]", request, currentState.version()); - validate(request, currentState); + validate(request, currentState.metadata(), currentState.routingTable()); final Index recoverFromIndex = request.recoverFrom(); final IndexMetadata sourceMetadata = recoverFromIndex == null ? null : currentState.metadata().getIndexSafe(recoverFromIndex); @@ -514,7 +513,6 @@ private ClusterState applyCreateIndexWithTemporaryService( ClusterState updated = clusterStateCreateIndex( currentState, - request.blocks(), indexMetadata, metadataTransformer, allocationService.getShardRoutingRoleStrategy() @@ -904,7 +902,7 @@ private ClusterState applyCreateIndexRequestWithExistingMetadata( * {@link IndexTemplateMetadata#order()}). This merging makes no distinction between field * definitions, as may result in an invalid field definition */ - static Map parseV1Mappings( + public static Map parseV1Mappings( String mappingsJson, List templateMappings, NamedXContentRegistry xContentRegistry @@ -1071,7 +1069,9 @@ static Settings aggregateIndexSettings( if (sourceMetadata != null) { assert request.resizeType() != null; prepareResizeIndexSettings( - currentState, + currentState.metadata(), + currentState.blocks(), + currentState.routingTable(), indexSettingsBuilder, request.recoverFrom(), request.index(), @@ -1086,7 +1086,7 @@ static Settings aggregateIndexSettings( * We can not validate settings until we have applied templates, otherwise we do not know the actual settings * that will be used to create this index. */ - shardLimitValidator.validateShardLimit(indexSettings, currentState); + shardLimitValidator.validateShardLimit(indexSettings, currentState.nodes(), currentState.metadata()); validateSoftDeleteSettings(indexSettings); validateTranslogRetentionSettings(indexSettings); validateStoreTypeSetting(indexSettings); @@ -1231,7 +1231,6 @@ public static List resolveAndValidateAliases( */ static ClusterState clusterStateCreateIndex( ClusterState currentState, - Set clusterBlocks, IndexMetadata indexMetadata, BiConsumer metadataTransformer, ShardRoutingRoleStrategy shardRoutingRoleStrategy @@ -1245,15 +1244,13 @@ static ClusterState clusterStateCreateIndex( newMetadata = currentState.metadata().withAddedIndex(indexMetadata); } - String indexName = indexMetadata.getIndex().getName(); - ClusterBlocks.Builder blocks = createClusterBlocksBuilder(currentState, indexName, clusterBlocks); - blocks.updateBlocks(indexMetadata); + var blocksBuilder = ClusterBlocks.builder().blocks(currentState.blocks()); + blocksBuilder.updateBlocks(indexMetadata); - ClusterState updatedState = ClusterState.builder(currentState).blocks(blocks).metadata(newMetadata).build(); + var routingTableBuilder = RoutingTable.builder(shardRoutingRoleStrategy, currentState.routingTable()) + .addAsNew(newMetadata.index(indexMetadata.getIndex().getName())); - RoutingTable.Builder routingTableBuilder = RoutingTable.builder(shardRoutingRoleStrategy, updatedState.routingTable()) - .addAsNew(updatedState.metadata().index(indexName)); - return ClusterState.builder(updatedState).routingTable(routingTableBuilder.build()).build(); + return ClusterState.builder(currentState).blocks(blocksBuilder).metadata(newMetadata).routingTable(routingTableBuilder).build(); } static IndexMetadata buildIndexMetadata( @@ -1326,16 +1323,6 @@ private static IndexMetadata.Builder createIndexMetadataBuilder( return builder; } - private static ClusterBlocks.Builder createClusterBlocksBuilder(ClusterState currentState, String index, Set blocks) { - ClusterBlocks.Builder blocksBuilder = ClusterBlocks.builder().blocks(currentState.blocks()); - if (blocks.isEmpty() == false) { - for (ClusterBlock block : blocks) { - blocksBuilder.addIndexBlock(index, block); - } - } - return blocksBuilder; - } - private static void updateIndexMappingsAndBuildSortOrder( IndexService indexService, CreateIndexClusterStateUpdateRequest request, @@ -1378,8 +1365,8 @@ private static void validateActiveShardCount(ActiveShardCount waitForActiveShard } } - private void validate(CreateIndexClusterStateUpdateRequest request, ClusterState state) { - validateIndexName(request.index(), state); + private void validate(CreateIndexClusterStateUpdateRequest request, Metadata metadata, RoutingTable routingTable) { + validateIndexName(request.index(), metadata, routingTable); validateIndexSettings(request.index(), request.settings(), forbidPrivateIndexSettings); } @@ -1443,8 +1430,15 @@ private static List validateIndexCustomPath(Settings settings, @Nullable * * @return the list of nodes at least one instance of the source index shards are allocated */ - static List validateShrinkIndex(ClusterState state, String sourceIndex, String targetIndexName, Settings targetIndexSettings) { - IndexMetadata sourceMetadata = validateResize(state, sourceIndex, targetIndexName, targetIndexSettings); + static List validateShrinkIndex( + Metadata metadata, + ClusterBlocks clusterBlocks, + RoutingTable routingTable, + String sourceIndex, + String targetIndexName, + Settings targetIndexSettings + ) { + IndexMetadata sourceMetadata = validateResize(metadata, clusterBlocks, sourceIndex, targetIndexName, targetIndexSettings); if (sourceMetadata.isSearchableSnapshot()) { throw new IllegalArgumentException("can't shrink searchable snapshot index [" + sourceIndex + ']'); } @@ -1456,7 +1450,7 @@ static List validateShrinkIndex(ClusterState state, String sourceIndex, } // now check that index is all on one node - final IndexRoutingTable table = state.routingTable().index(sourceIndex); + final IndexRoutingTable table = routingTable.index(sourceIndex); Map nodesToNumRouting = new HashMap<>(); int numShards = sourceMetadata.getNumberOfShards(); for (ShardRouting routing : table.shardsWithState(ShardRoutingState.STARTED)) { @@ -1476,16 +1470,28 @@ static List validateShrinkIndex(ClusterState state, String sourceIndex, return nodesToAllocateOn; } - static void validateSplitIndex(ClusterState state, String sourceIndex, String targetIndexName, Settings targetIndexSettings) { - IndexMetadata sourceMetadata = validateResize(state, sourceIndex, targetIndexName, targetIndexSettings); + static void validateSplitIndex( + Metadata metadata, + ClusterBlocks clusterBlocks, + String sourceIndex, + String targetIndexName, + Settings targetIndexSettings + ) { + IndexMetadata sourceMetadata = validateResize(metadata, clusterBlocks, sourceIndex, targetIndexName, targetIndexSettings); if (sourceMetadata.isSearchableSnapshot()) { throw new IllegalArgumentException("can't split searchable snapshot index [" + sourceIndex + ']'); } IndexMetadata.selectSplitShard(0, sourceMetadata, INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings)); } - static void validateCloneIndex(ClusterState state, String sourceIndex, String targetIndexName, Settings targetIndexSettings) { - IndexMetadata sourceMetadata = validateResize(state, sourceIndex, targetIndexName, targetIndexSettings); + static void validateCloneIndex( + Metadata metadata, + ClusterBlocks clusterBlocks, + String sourceIndex, + String targetIndexName, + Settings targetIndexSettings + ) { + IndexMetadata sourceMetadata = validateResize(metadata, clusterBlocks, sourceIndex, targetIndexName, targetIndexSettings); if (sourceMetadata.isSearchableSnapshot()) { for (Setting nonCloneableSetting : Arrays.asList(INDEX_STORE_TYPE_SETTING, INDEX_RECOVERY_TYPE_SETTING)) { if (nonCloneableSetting.exists(targetIndexSettings) == false) { @@ -1502,16 +1508,22 @@ static void validateCloneIndex(ClusterState state, String sourceIndex, String ta IndexMetadata.selectCloneShard(0, sourceMetadata, INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings)); } - static IndexMetadata validateResize(ClusterState state, String sourceIndex, String targetIndexName, Settings targetIndexSettings) { - if (state.metadata().hasIndex(targetIndexName)) { - throw new ResourceAlreadyExistsException(state.metadata().index(targetIndexName).getIndex()); + static IndexMetadata validateResize( + Metadata metadata, + ClusterBlocks clusterBlocks, + String sourceIndex, + String targetIndexName, + Settings targetIndexSettings + ) { + if (metadata.hasIndex(targetIndexName)) { + throw new ResourceAlreadyExistsException(metadata.index(targetIndexName).getIndex()); } - final IndexMetadata sourceMetadata = state.metadata().index(sourceIndex); + final IndexMetadata sourceMetadata = metadata.index(sourceIndex); if (sourceMetadata == null) { throw new IndexNotFoundException(sourceIndex); } - IndexAbstraction source = state.metadata().getIndicesLookup().get(sourceIndex); + IndexAbstraction source = metadata.getIndicesLookup().get(sourceIndex); assert source != null; if (source.getParentDataStream() != null && source.getParentDataStream().getWriteIndex().equals(sourceMetadata.getIndex())) { throw new IllegalArgumentException( @@ -1524,7 +1536,7 @@ static IndexMetadata validateResize(ClusterState state, String sourceIndex, Stri ); } // ensure index is read-only - if (state.blocks().indexBlocked(ClusterBlockLevel.WRITE, sourceIndex) == false) { + if (clusterBlocks.indexBlocked(ClusterBlockLevel.WRITE, sourceIndex) == false) { throw new IllegalStateException("index " + sourceIndex + " must be read-only to resize index. use \"index.blocks.write=true\""); } @@ -1537,7 +1549,9 @@ static IndexMetadata validateResize(ClusterState state, String sourceIndex, Stri } static void prepareResizeIndexSettings( - final ClusterState currentState, + final Metadata metadata, + final ClusterBlocks clusterBlocks, + final RoutingTable routingTable, final Settings.Builder indexSettingsBuilder, final Index resizeSourceIndex, final String resizeIntoName, @@ -1545,20 +1559,22 @@ static void prepareResizeIndexSettings( final boolean copySettings, final IndexScopedSettings indexScopedSettings ) { - final IndexMetadata sourceMetadata = currentState.metadata().index(resizeSourceIndex.getName()); + final IndexMetadata sourceMetadata = metadata.index(resizeSourceIndex.getName()); if (type == ResizeType.SHRINK) { final List nodesToAllocateOn = validateShrinkIndex( - currentState, + metadata, + clusterBlocks, + routingTable, resizeSourceIndex.getName(), resizeIntoName, indexSettingsBuilder.build() ); indexSettingsBuilder.put(INDEX_SHRINK_INITIAL_RECOVERY_KEY, Strings.arrayToCommaDelimitedString(nodesToAllocateOn.toArray())); } else if (type == ResizeType.SPLIT) { - validateSplitIndex(currentState, resizeSourceIndex.getName(), resizeIntoName, indexSettingsBuilder.build()); + validateSplitIndex(metadata, clusterBlocks, resizeSourceIndex.getName(), resizeIntoName, indexSettingsBuilder.build()); indexSettingsBuilder.putNull(INDEX_SHRINK_INITIAL_RECOVERY_KEY); } else if (type == ResizeType.CLONE) { - validateCloneIndex(currentState, resizeSourceIndex.getName(), resizeIntoName, indexSettingsBuilder.build()); + validateCloneIndex(metadata, clusterBlocks, resizeSourceIndex.getName(), resizeIntoName, indexSettingsBuilder.build()); indexSettingsBuilder.putNull(INDEX_SHRINK_INITIAL_RECOVERY_KEY); } else { throw new IllegalStateException("unknown resize type is " + type); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java index 9cac6fa3e8796..9e8a99351d84a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java @@ -214,17 +214,15 @@ static ClusterState modifyDataStream( ClusterState updateDataLifecycle(ClusterState currentState, List dataStreamNames, @Nullable DataStreamLifecycle lifecycle) { Metadata metadata = currentState.metadata(); Metadata.Builder builder = Metadata.builder(metadata); - boolean atLeastOneDataStreamIsNotSystem = false; + boolean onlyInternalDataStreams = true; for (var dataStreamName : dataStreamNames) { var dataStream = validateDataStream(metadata, dataStreamName); builder.put(dataStream.copy().setLifecycle(lifecycle).build()); - atLeastOneDataStreamIsNotSystem = atLeastOneDataStreamIsNotSystem || dataStream.isSystem() == false; + onlyInternalDataStreams = onlyInternalDataStreams && dataStream.isInternal(); } if (lifecycle != null) { - if (atLeastOneDataStreamIsNotSystem) { - // We don't issue any warnings if all data streams are system data streams - lifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetentionSettings.get()); - } + // We don't issue any warnings if all data streams are internal data streams + lifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetentionSettings.get(), onlyInternalDataStreams); } return ClusterState.builder(currentState).metadata(builder.build()).build(); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java index be12198cbaaaa..272c107883043 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java @@ -1100,7 +1100,7 @@ private ClusterState openIndices(final Index[] indices, final ClusterState curre } } - shardLimitValidator.validateShardLimit(currentState, indices); + shardLimitValidator.validateShardLimit(currentState.nodes(), currentState.metadata(), indices); if (indicesToOpen.isEmpty()) { return currentState; } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index ac56f3f670f43..ff23f50ef7afe 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -369,7 +369,8 @@ public ClusterState addComponentTemplate( } if (finalComponentTemplate.template().lifecycle() != null) { - finalComponentTemplate.template().lifecycle().addWarningHeaderIfDataRetentionNotEffective(globalRetentionSettings.get()); + // We do not know if this lifecycle will belong to an internal data stream, so we fall back to a non internal. + finalComponentTemplate.template().lifecycle().addWarningHeaderIfDataRetentionNotEffective(globalRetentionSettings.get(), false); } logger.info("{} component template [{}]", existing == null ? "adding" : "updating", name); @@ -815,7 +816,12 @@ static void validateLifecycle( + "] specifies lifecycle configuration that can only be used in combination with a data stream" ); } - lifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetention); + if (globalRetention != null) { + // We cannot know for sure if the template will apply to internal data streams, so we use a simpler heuristic: + // If all the index patterns start with a dot, we consider that all the connected data streams are internal. + boolean isInternalDataStream = template.indexPatterns().stream().allMatch(indexPattern -> indexPattern.charAt(0) == '.'); + lifecycle.addWarningHeaderIfDataRetentionNotEffective(globalRetention, isInternalDataStream); + } } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataUpdateSettingsService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataUpdateSettingsService.java index 5891b953acfca..3272462dd3725 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataUpdateSettingsService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataUpdateSettingsService.java @@ -255,7 +255,12 @@ ClusterState execute(ClusterState currentState) { final int updatedNumberOfReplicas = IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.get(openSettings); if (preserveExisting == false) { // Verify that this won't take us over the cluster shard limit. - shardLimitValidator.validateShardLimitOnReplicaUpdate(currentState, request.indices(), updatedNumberOfReplicas); + shardLimitValidator.validateShardLimitOnReplicaUpdate( + currentState.nodes(), + currentState.metadata(), + request.indices(), + updatedNumberOfReplicas + ); /* * We do not update the in-sync allocation IDs as they will be removed upon the first index operation diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java index 70440adc4ebbe..0a045261e07b8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Template.java @@ -70,7 +70,11 @@ public class Template implements SimpleDiffable