From af8e4bf26d15af49824cdb94fcd149f8a984b48e Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Tue, 16 Apr 2024 09:47:25 -0400 Subject: [PATCH 01/43] [ES|QL] Moving argument compatibility checking for Equals (#105217) Continuing my work from #104490, this PR moves the parameter compatibility checking for Equals into the type resolution check. This is a somewhat bigger change than for Add, as there was no ES|QL base class for binary comparison operators before this. I've added EsqlBinaryComparison as that base class, and migrated all of the binary comparisons to be based off of that (except for NullEquals, see note below). In order to maintain compatibility with the current behavior, I've kept it so that unsigned longs are only inter-operable with other unsigned longs. We've talked a lot about changing that, and I consider this work a prerequisite for that. I've also added a bunch of test cases to Equals and NotEquals, which should have the side effect of filling out the type support table in the equals docs. As noted in the comments, I'll have follow up PRs for the other binary comparisons to add tests, but this PR is already too long. Note about NullEquals: There is an ES|QL NullEquals class, which inherits from the QL version, but I don't think it works. I didn't see any tests or docs for it, and trying it out in the demo instance gave me a syntax error. I think we need to delve into what's going on there, but this PR isn't the right place for it. --- .../src/main/resources/conditional.csv-spec | 7 + .../predicate/operator/comparison/Equals.java | 47 +++-- .../comparison/EsqlBinaryComparison.java | 164 +++++++++++++++ .../operator/comparison/GreaterThan.java | 33 ++- .../comparison/GreaterThanOrEqual.java | 34 +++- .../operator/comparison/LessThan.java | 36 ++-- .../operator/comparison/LessThanOrEqual.java | 31 ++- .../operator/comparison/NotEquals.java | 82 +++++--- .../DateTimeArithmeticOperation.java | 8 +- .../arithmetic/EsqlArithmeticOperation.java | 23 ++- .../function/AbstractFunctionTestCase.java | 49 +++++ .../expression/function/TestCaseSupplier.java | 84 ++++++-- .../operator/arithmetic/AddTests.java | 8 +- .../operator/comparison/EqualsTests.java | 188 +++++++++++++++--- .../comparison/GreaterThanOrEqualTests.java | 21 +- .../operator/comparison/GreaterThanTests.java | 21 +- .../comparison/LessThanOrEqualTests.java | 20 +- .../operator/comparison/LessThanTests.java | 20 +- .../operator/comparison/NotEqualsTests.java | 187 ++++++++++++++--- .../esql/optimizer/OptimizerRulesTests.java | 2 +- 20 files changed, 825 insertions(+), 240 deletions(-) create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EsqlBinaryComparison.java diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/conditional.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/conditional.csv-spec index f574722f691e..64a8c1d9da31 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/conditional.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/conditional.csv-spec @@ -156,6 +156,9 @@ nullOnMultivaluesComparisonOperation required_feature: esql.disable_nullable_opts ROW a = 5, b = [ 1, 2 ]| EVAL same = a == b| LIMIT 1 | WHERE same IS NULL; +warning:Line 1:38: evaluation of [a == b] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:38: java.lang.IllegalArgumentException: single-value function encountered multi-value + a:integer | b:integer | same:boolean 5 | [1, 2] | null @@ -166,6 +169,8 @@ notNullOnMultivaluesComparisonOperation required_feature: esql.disable_nullable_opts ROW a = 5, b = [ 1, 2 ]| EVAL same = a == b| LIMIT 1 | WHERE same IS NOT NULL; +warning:Line 1:38: evaluation of [a == b] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:38: java.lang.IllegalArgumentException: single-value function encountered multi-value a:integer | b:integer | same:boolean ; @@ -175,6 +180,8 @@ notNullOnMultivaluesComparisonOperationWithPartialMatch required_feature: esql.disable_nullable_opts ROW a = 5, b = [ 5, 2 ]| EVAL same = a == b| LIMIT 1 | WHERE same IS NOT NULL; +warning:Line 1:38: evaluation of [a == b] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:38: java.lang.IllegalArgumentException: single-value function encountered multi-value a:integer | b:integer | same:boolean ; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/Equals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/Equals.java index 9fb899b8e36d..62eec13af008 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/Equals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/Equals.java @@ -8,33 +8,48 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Evaluator; -import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; +import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; +import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.TypeResolutions; +import org.elasticsearch.xpack.ql.expression.predicate.Negatable; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; import java.time.ZoneId; +import java.util.Map; + +public class Equals extends EsqlBinaryComparison implements Negatable { + private static final Map evaluatorMap = Map.ofEntries( + Map.entry(DataTypes.BOOLEAN, EqualsBoolsEvaluator.Factory::new), + Map.entry(DataTypes.INTEGER, EqualsIntsEvaluator.Factory::new), + Map.entry(DataTypes.DOUBLE, EqualsDoublesEvaluator.Factory::new), + Map.entry(DataTypes.LONG, EqualsLongsEvaluator.Factory::new), + Map.entry(DataTypes.UNSIGNED_LONG, EqualsLongsEvaluator.Factory::new), + Map.entry(DataTypes.DATETIME, EqualsLongsEvaluator.Factory::new), + Map.entry(EsqlDataTypes.GEO_POINT, EqualsGeometriesEvaluator.Factory::new), + Map.entry(EsqlDataTypes.CARTESIAN_POINT, EqualsGeometriesEvaluator.Factory::new), + Map.entry(EsqlDataTypes.GEO_SHAPE, EqualsGeometriesEvaluator.Factory::new), + Map.entry(EsqlDataTypes.CARTESIAN_SHAPE, EqualsGeometriesEvaluator.Factory::new), + Map.entry(DataTypes.KEYWORD, EqualsKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.TEXT, EqualsKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.VERSION, EqualsKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.IP, EqualsKeywordsEvaluator.Factory::new) + ); -import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; - -public class Equals extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals { public Equals(Source source, Expression left, Expression right) { - super(source, left, right); + super(source, left, right, BinaryComparisonProcessor.BinaryComparisonOperation.EQ, evaluatorMap); } public Equals(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, zoneId); - } - - @Override - protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { - return EsqlTypeResolutions.isExact(e, sourceText(), DEFAULT); + super(source, left, right, BinaryComparisonProcessor.BinaryComparisonOperation.EQ, zoneId, evaluatorMap); } @Override - protected NodeInfo info() { + protected NodeInfo info() { return NodeInfo.create(this, Equals::new, left(), right(), zoneId()); } @@ -48,6 +63,11 @@ public Equals swapLeftAndRight() { return new Equals(source(), right(), left(), zoneId()); } + @Override + public BinaryComparison reverse() { + return this; + } + @Override public BinaryComparison negate() { return new NotEquals(source(), left(), right(), zoneId()); @@ -82,4 +102,5 @@ static boolean processBools(boolean lhs, boolean rhs) { static boolean processGeometries(BytesRef lhs, BytesRef rhs) { return lhs.equals(rhs); } + } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EsqlBinaryComparison.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EsqlBinaryComparison.java new file mode 100644 index 000000000000..58a808893c4c --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EsqlBinaryComparison.java @@ -0,0 +1,164 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison; + +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; +import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; +import org.elasticsearch.xpack.esql.expression.function.scalar.math.Cast; +import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; +import org.elasticsearch.xpack.esql.type.EsqlDataTypeRegistry; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.TypeResolutions; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; + +import java.time.ZoneId; +import java.util.Map; +import java.util.function.Function; + +import static org.elasticsearch.common.logging.LoggerMessageFormat.format; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; + +public abstract class EsqlBinaryComparison extends BinaryComparison implements EvaluatorMapper { + + private final Map evaluatorMap; + + protected EsqlBinaryComparison( + Source source, + Expression left, + Expression right, + /* TODO: BinaryComparisonOperator is an enum with a bunch of functionality we don't really want. We should extract an interface and + create a symbol only version like we did for BinaryArithmeticOperation. Ideally, they could be the same class. + */ + BinaryComparisonProcessor.BinaryComparisonOperation operation, + Map evaluatorMap + ) { + this(source, left, right, operation, null, evaluatorMap); + } + + protected EsqlBinaryComparison( + Source source, + Expression left, + Expression right, + BinaryComparisonProcessor.BinaryComparisonOperation operation, + // TODO: We are definitely not doing the right thing with this zoneId + ZoneId zoneId, + Map evaluatorMap + ) { + super(source, left, right, operation, zoneId); + this.evaluatorMap = evaluatorMap; + } + + @Override + public EvalOperator.ExpressionEvaluator.Factory toEvaluator( + Function toEvaluator + ) { + // Our type is always boolean, so figure out the evaluator type from the inputs + DataType commonType = EsqlDataTypeRegistry.INSTANCE.commonType(left().dataType(), right().dataType()); + EvalOperator.ExpressionEvaluator.Factory lhs; + EvalOperator.ExpressionEvaluator.Factory rhs; + + if (commonType.isNumeric()) { + lhs = Cast.cast(source(), left().dataType(), commonType, toEvaluator.apply(left())); + rhs = Cast.cast(source(), right().dataType(), commonType, toEvaluator.apply(right())); + } else { + lhs = toEvaluator.apply(left()); + rhs = toEvaluator.apply(right()); + } + + if (evaluatorMap.containsKey(commonType) == false) { + throw new EsqlIllegalArgumentException("Unsupported type " + left().dataType()); + } + return evaluatorMap.get(commonType).apply(source(), lhs, rhs); + } + + @Override + public Boolean fold() { + return (Boolean) EvaluatorMapper.super.fold(); + } + + @Override + protected TypeResolution resolveType() { + TypeResolution typeResolution = super.resolveType(); + if (typeResolution.unresolved()) { + return typeResolution; + } + + return checkCompatibility(); + } + + @Override + protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { + return TypeResolutions.isType( + e, + evaluatorMap::containsKey, + sourceText(), + paramOrdinal, + evaluatorMap.keySet().stream().map(DataType::typeName).toArray(String[]::new) + ); + } + + /** + * Check if the two input types are compatible for this operation + * + * @return TypeResolution.TYPE_RESOLVED iff the types are compatible. Otherwise, an appropriate type resolution error. + */ + protected TypeResolution checkCompatibility() { + DataType leftType = left().dataType(); + DataType rightType = right().dataType(); + + // Unsigned long is only interoperable with other unsigned longs + if ((rightType == UNSIGNED_LONG && (false == (leftType == UNSIGNED_LONG || leftType == DataTypes.NULL))) + || (leftType == UNSIGNED_LONG && (false == (rightType == UNSIGNED_LONG || rightType == DataTypes.NULL)))) { + return new TypeResolution(formatIncompatibleTypesMessage()); + } + + if ((leftType.isNumeric() && rightType.isNumeric()) + || (DataTypes.isString(leftType) && DataTypes.isString(rightType)) + || leftType.equals(rightType) + || DataTypes.isNull(leftType) + || DataTypes.isNull(rightType)) { + return TypeResolution.TYPE_RESOLVED; + } + return new TypeResolution(formatIncompatibleTypesMessage()); + } + + public String formatIncompatibleTypesMessage() { + if (left().dataType().equals(UNSIGNED_LONG)) { + return format( + null, + "first argument of [{}] is [unsigned_long] and second is [{}]. " + + "[unsigned_long] can only be operated on together with another [unsigned_long]", + sourceText(), + right().dataType().typeName() + ); + } + if (right().dataType().equals(UNSIGNED_LONG)) { + return format( + null, + "first argument of [{}] is [{}] and second is [unsigned_long]. " + + "[unsigned_long] can only be operated on together with another [unsigned_long]", + sourceText(), + left().dataType().typeName() + ); + } + return format( + null, + "first argument of [{}] is [{}] so second argument must also be [{}] but was [{}]", + sourceText(), + left().dataType().isNumeric() ? "numeric" : left().dataType().typeName(), + left().dataType().isNumeric() ? "numeric" : left().dataType().typeName(), + right().dataType().typeName() + ); + } + +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThan.java index 5683a9d0d7e8..3eca0e858acb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThan.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThan.java @@ -8,29 +8,42 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Evaluator; -import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; +import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.TypeResolutions; +import org.elasticsearch.xpack.ql.expression.predicate.Negatable; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; import java.time.ZoneId; +import java.util.Map; -import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; +public class GreaterThan extends EsqlBinaryComparison implements Negatable { + private static final Map evaluatorMap = Map.ofEntries( + Map.entry(DataTypes.INTEGER, GreaterThanIntsEvaluator.Factory::new), + Map.entry(DataTypes.DOUBLE, GreaterThanDoublesEvaluator.Factory::new), + Map.entry(DataTypes.LONG, GreaterThanLongsEvaluator.Factory::new), + Map.entry(DataTypes.UNSIGNED_LONG, GreaterThanLongsEvaluator.Factory::new), + Map.entry(DataTypes.DATETIME, GreaterThanLongsEvaluator.Factory::new), + Map.entry(DataTypes.KEYWORD, GreaterThanKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.TEXT, GreaterThanKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.VERSION, GreaterThanKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.IP, GreaterThanKeywordsEvaluator.Factory::new) + ); -public class GreaterThan extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan { - public GreaterThan(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, zoneId); + public GreaterThan(Source source, Expression left, Expression right) { + super(source, left, right, BinaryComparisonProcessor.BinaryComparisonOperation.GT, evaluatorMap); } - @Override - protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { - return EsqlTypeResolutions.isExact(e, sourceText(), DEFAULT); + public GreaterThan(Source source, Expression left, Expression right, ZoneId zoneId) { + super(source, left, right, BinaryComparisonProcessor.BinaryComparisonOperation.GT, zoneId, evaluatorMap); } @Override - protected NodeInfo info() { + protected NodeInfo info() { return NodeInfo.create(this, GreaterThan::new, left(), right(), zoneId()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqual.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqual.java index ebb29998fb99..f99a85420870 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqual.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqual.java @@ -8,30 +8,42 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Evaluator; -import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; +import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.TypeResolutions; +import org.elasticsearch.xpack.ql.expression.predicate.Negatable; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; import java.time.ZoneId; +import java.util.Map; -import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; +public class GreaterThanOrEqual extends EsqlBinaryComparison implements Negatable { + private static final Map evaluatorMap = Map.ofEntries( + Map.entry(DataTypes.INTEGER, GreaterThanOrEqualIntsEvaluator.Factory::new), + Map.entry(DataTypes.DOUBLE, GreaterThanOrEqualDoublesEvaluator.Factory::new), + Map.entry(DataTypes.LONG, GreaterThanOrEqualLongsEvaluator.Factory::new), + Map.entry(DataTypes.UNSIGNED_LONG, GreaterThanOrEqualLongsEvaluator.Factory::new), + Map.entry(DataTypes.DATETIME, GreaterThanOrEqualLongsEvaluator.Factory::new), + Map.entry(DataTypes.KEYWORD, GreaterThanOrEqualKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.TEXT, GreaterThanOrEqualKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.VERSION, GreaterThanOrEqualKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.IP, GreaterThanOrEqualKeywordsEvaluator.Factory::new) + ); -public class GreaterThanOrEqual extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThanOrEqual { - - public GreaterThanOrEqual(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, zoneId); + public GreaterThanOrEqual(Source source, Expression left, Expression right) { + super(source, left, right, BinaryComparisonProcessor.BinaryComparisonOperation.GTE, evaluatorMap); } - @Override - protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { - return EsqlTypeResolutions.isExact(e, sourceText(), DEFAULT); + public GreaterThanOrEqual(Source source, Expression left, Expression right, ZoneId zoneId) { + super(source, left, right, BinaryComparisonProcessor.BinaryComparisonOperation.GTE, zoneId, evaluatorMap); } @Override - protected NodeInfo info() { + protected NodeInfo info() { return NodeInfo.create(this, GreaterThanOrEqual::new, left(), right(), zoneId()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThan.java index 12f54270b65d..6b82df1d67da 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThan.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThan.java @@ -8,38 +8,44 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Evaluator; -import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; +import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.TypeResolutions; +import org.elasticsearch.xpack.ql.expression.predicate.Negatable; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; import java.time.ZoneId; +import java.util.Map; -import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; +public class LessThan extends EsqlBinaryComparison implements Negatable { -public class LessThan extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan { + private static final Map evaluatorMap = Map.ofEntries( + Map.entry(DataTypes.INTEGER, LessThanIntsEvaluator.Factory::new), + Map.entry(DataTypes.DOUBLE, LessThanDoublesEvaluator.Factory::new), + Map.entry(DataTypes.LONG, LessThanLongsEvaluator.Factory::new), + Map.entry(DataTypes.UNSIGNED_LONG, LessThanLongsEvaluator.Factory::new), + Map.entry(DataTypes.DATETIME, LessThanLongsEvaluator.Factory::new), + Map.entry(DataTypes.KEYWORD, LessThanKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.TEXT, LessThanKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.VERSION, LessThanKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.IP, LessThanKeywordsEvaluator.Factory::new) + ); public LessThan(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, zoneId); + super(source, left, right, BinaryComparisonProcessor.BinaryComparisonOperation.LT, zoneId, evaluatorMap); } @Override - protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { - return EsqlTypeResolutions.isExact(e, sourceText(), DEFAULT); - } - - @Override - protected NodeInfo info() { + protected NodeInfo info() { return NodeInfo.create(this, LessThan::new, left(), right(), zoneId()); } @Override - protected org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan replaceChildren( - Expression newLeft, - Expression newRight - ) { + protected LessThan replaceChildren(Expression newLeft, Expression newRight) { return new LessThan(source(), newLeft, newRight, zoneId()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqual.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqual.java index e75733a9e234..ac6a92aaf097 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqual.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqual.java @@ -8,29 +8,38 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Evaluator; -import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; +import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.TypeResolutions; +import org.elasticsearch.xpack.ql.expression.predicate.Negatable; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; import java.time.ZoneId; +import java.util.Map; -import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; +public class LessThanOrEqual extends EsqlBinaryComparison implements Negatable { + private static final Map evaluatorMap = Map.ofEntries( + Map.entry(DataTypes.INTEGER, LessThanOrEqualIntsEvaluator.Factory::new), + Map.entry(DataTypes.DOUBLE, LessThanOrEqualDoublesEvaluator.Factory::new), + Map.entry(DataTypes.LONG, LessThanOrEqualLongsEvaluator.Factory::new), + Map.entry(DataTypes.UNSIGNED_LONG, LessThanOrEqualLongsEvaluator.Factory::new), + Map.entry(DataTypes.DATETIME, LessThanOrEqualLongsEvaluator.Factory::new), + Map.entry(DataTypes.KEYWORD, LessThanOrEqualKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.TEXT, LessThanOrEqualKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.VERSION, LessThanOrEqualKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.IP, LessThanOrEqualKeywordsEvaluator.Factory::new) + ); -public class LessThanOrEqual extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThanOrEqual { public LessThanOrEqual(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, zoneId); + super(source, left, right, BinaryComparisonProcessor.BinaryComparisonOperation.LTE, zoneId, evaluatorMap); } @Override - protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { - return EsqlTypeResolutions.isExact(e, sourceText(), DEFAULT); - } - - @Override - protected NodeInfo info() { + protected NodeInfo info() { return NodeInfo.create(this, LessThanOrEqual::new, left(), right(), zoneId()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEquals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEquals.java index 6fbed572cdc0..9c931ec7433e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEquals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEquals.java @@ -8,45 +8,44 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Evaluator; -import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; +import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; +import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.TypeResolutions; +import org.elasticsearch.xpack.ql.expression.predicate.Negatable; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; import java.time.ZoneId; +import java.util.Map; -import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; +public class NotEquals extends EsqlBinaryComparison implements Negatable { + private static final Map evaluatorMap = Map.ofEntries( + Map.entry(DataTypes.BOOLEAN, NotEqualsBoolsEvaluator.Factory::new), + Map.entry(DataTypes.INTEGER, NotEqualsIntsEvaluator.Factory::new), + Map.entry(DataTypes.DOUBLE, NotEqualsDoublesEvaluator.Factory::new), + Map.entry(DataTypes.LONG, NotEqualsLongsEvaluator.Factory::new), + Map.entry(DataTypes.UNSIGNED_LONG, NotEqualsLongsEvaluator.Factory::new), + Map.entry(DataTypes.DATETIME, NotEqualsLongsEvaluator.Factory::new), + Map.entry(EsqlDataTypes.GEO_POINT, NotEqualsGeometriesEvaluator.Factory::new), + Map.entry(EsqlDataTypes.CARTESIAN_POINT, NotEqualsGeometriesEvaluator.Factory::new), + Map.entry(EsqlDataTypes.GEO_SHAPE, NotEqualsGeometriesEvaluator.Factory::new), + Map.entry(EsqlDataTypes.CARTESIAN_SHAPE, NotEqualsGeometriesEvaluator.Factory::new), + Map.entry(DataTypes.KEYWORD, NotEqualsKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.TEXT, NotEqualsKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.VERSION, NotEqualsKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.IP, NotEqualsKeywordsEvaluator.Factory::new) + ); -public class NotEquals extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals { - public NotEquals(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, zoneId); - } - - @Override - protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { - return EsqlTypeResolutions.isExact(e, sourceText(), DEFAULT); - } - - @Override - protected NodeInfo info() { - return NodeInfo.create(this, NotEquals::new, left(), right(), zoneId()); - } - - @Override - protected NotEquals replaceChildren(Expression newLeft, Expression newRight) { - return new NotEquals(source(), newLeft, newRight, zoneId()); + public NotEquals(Source source, Expression left, Expression right) { + super(source, left, right, BinaryComparisonProcessor.BinaryComparisonOperation.NEQ, evaluatorMap); } - @Override - public NotEquals swapLeftAndRight() { - return new NotEquals(source(), right(), left(), zoneId()); - } - - @Override - public BinaryComparison negate() { - return new Equals(source(), left(), right(), zoneId()); + public NotEquals(Source source, Expression left, Expression right, ZoneId zoneId) { + super(source, left, right, BinaryComparisonProcessor.BinaryComparisonOperation.NEQ, zoneId, evaluatorMap); } @Evaluator(extraName = "Ints") @@ -78,4 +77,29 @@ static boolean processBools(boolean lhs, boolean rhs) { static boolean processGeometries(BytesRef lhs, BytesRef rhs) { return false == lhs.equals(rhs); } + + @Override + public BinaryComparison reverse() { + return this; + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, NotEquals::new, left(), right(), zoneId()); + } + + @Override + protected NotEquals replaceChildren(Expression newLeft, Expression newRight) { + return new NotEquals(source(), newLeft, newRight, zoneId()); + } + + @Override + public NotEquals swapLeftAndRight() { + return new NotEquals(source(), right(), left(), zoneId()); + } + + @Override + public BinaryComparison negate() { + return new Equals(source(), left(), right(), zoneId()); + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java index 0f550862ed1f..a45707a0197d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java @@ -43,10 +43,10 @@ interface DatetimeArithmeticEvaluator { Expression left, Expression right, OperationSymbol op, - ArithmeticEvaluator ints, - ArithmeticEvaluator longs, - ArithmeticEvaluator ulongs, - ArithmeticEvaluator doubles, + BinaryEvaluator ints, + BinaryEvaluator longs, + BinaryEvaluator ulongs, + BinaryEvaluator doubles, DatetimeArithmeticEvaluator datetimes ) { super(source, left, right, op, ints, longs, ulongs, doubles); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java index 22f5798e5b1c..ba283bc4d877 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java @@ -71,14 +71,15 @@ public String symbol() { } /** Arithmetic (quad) function. */ - interface ArithmeticEvaluator { + @FunctionalInterface + public interface BinaryEvaluator { ExpressionEvaluator.Factory apply(Source source, ExpressionEvaluator.Factory lhs, ExpressionEvaluator.Factory rhs); } - private final ArithmeticEvaluator ints; - private final ArithmeticEvaluator longs; - private final ArithmeticEvaluator ulongs; - private final ArithmeticEvaluator doubles; + private final BinaryEvaluator ints; + private final BinaryEvaluator longs; + private final BinaryEvaluator ulongs; + private final BinaryEvaluator doubles; private DataType dataType; @@ -87,10 +88,10 @@ interface ArithmeticEvaluator { Expression left, Expression right, OperationSymbol op, - ArithmeticEvaluator ints, - ArithmeticEvaluator longs, - ArithmeticEvaluator ulongs, - ArithmeticEvaluator doubles + BinaryEvaluator ints, + BinaryEvaluator longs, + BinaryEvaluator ulongs, + BinaryEvaluator doubles ) { super(source, left, right, op); this.ints = ints; @@ -139,7 +140,7 @@ protected TypeResolution checkCompatibility() { return TypeResolution.TYPE_RESOLVED; } - static String formatIncompatibleTypesMessage(String symbol, DataType leftType, DataType rightType) { + public static String formatIncompatibleTypesMessage(String symbol, DataType leftType, DataType rightType) { return format(null, "[{}] has arguments with incompatible types [{}] and [{}]", symbol, leftType.typeName(), rightType.typeName()); } @@ -152,7 +153,7 @@ public ExpressionEvaluator.Factory toEvaluator(Function errorsForCasesWithoutExamples( return suppliers; } + public static String errorMessageStringForBinaryOperators( + boolean includeOrdinal, + List> validPerPosition, + List types + ) { + try { + return typeErrorMessage(includeOrdinal, validPerPosition, types); + } catch (IllegalStateException e) { + // This means all the positional args were okay, so the expected error is from the combination + if (types.get(0).equals(DataTypes.UNSIGNED_LONG)) { + return "first argument of [] is [unsigned_long] and second is [" + + types.get(1).typeName() + + "]. [unsigned_long] can only be operated on together with another [unsigned_long]"; + + } + if (types.get(1).equals(DataTypes.UNSIGNED_LONG)) { + return "first argument of [] is [" + + types.get(0).typeName() + + "] and second is [unsigned_long]. [unsigned_long] can only be operated on together with another [unsigned_long]"; + } + return "first argument of [] is [" + + (types.get(0).isNumeric() ? "numeric" : types.get(0).typeName()) + + "] so second argument must also be [" + + (types.get(0).isNumeric() ? "numeric" : types.get(0).typeName()) + + "] but was [" + + types.get(1).typeName() + + "]"; + + } + } + /** * Adds test cases containing unsupported parameter types that immediately fail. */ @@ -931,6 +962,24 @@ protected static String typeErrorMessage(boolean includeOrdinal, List types) { return types.stream().map(t -> "<" + t.typeName() + ">").collect(Collectors.joining(", ")); } + public static List stringCases( + BinaryOperator expected, + BiFunction evaluatorToString, + List warnings, + DataType expectedType + ) { + List lhsSuppliers = new ArrayList<>(); + List rhsSuppliers = new ArrayList<>(); + List suppliers = new ArrayList<>(); + for (DataType type : AbstractConvertFunction.STRING_TYPES) { + lhsSuppliers.addAll(stringCases(type)); + rhsSuppliers.addAll(stringCases(type)); + casesCrossProduct(expected, lhsSuppliers, rhsSuppliers, evaluatorToString, warnings, suppliers, expectedType, true); + } + return suppliers; + } + @Override public TestCase get() { TestCase supplied = supplier.get(); @@ -258,14 +275,14 @@ public static List castToDoubleSuppliersFromRange(Double Min, return suppliers; } - public record NumericTypeTestConfig(Number min, Number max, BinaryOperator expected, String evaluatorName) {} + public record NumericTypeTestConfig(Number min, Number max, BiFunction expected, String evaluatorName) {} - public record NumericTypeTestConfigs( - NumericTypeTestConfig intStuff, - NumericTypeTestConfig longStuff, - NumericTypeTestConfig doubleStuff + public record NumericTypeTestConfigs( + NumericTypeTestConfig intStuff, + NumericTypeTestConfig longStuff, + NumericTypeTestConfig doubleStuff ) { - public NumericTypeTestConfig get(DataType type) { + public NumericTypeTestConfig get(DataType type) { if (type == DataTypes.INTEGER) { return intStuff; } @@ -312,8 +329,8 @@ public static List getSuppliersForNumericType(DataType type, throw new IllegalArgumentException("bogus numeric type [" + type + "]"); } - public static List forBinaryWithWidening( - NumericTypeTestConfigs typeStuff, + public static List forBinaryComparisonWithWidening( + NumericTypeTestConfigs typeStuff, String lhsName, String rhsName, BiFunction> warnings, @@ -325,7 +342,45 @@ public static List forBinaryWithWidening( for (DataType lhsType : numericTypes) { for (DataType rhsType : numericTypes) { DataType expected = widen(lhsType, rhsType); - NumericTypeTestConfig expectedTypeStuff = typeStuff.get(expected); + NumericTypeTestConfig expectedTypeStuff = typeStuff.get(expected); + BiFunction evaluatorToString = (lhs, rhs) -> expectedTypeStuff.evaluatorName() + + "[" + + lhsName + + "=" + + getCastEvaluator("Attribute[channel=0]", lhs, expected) + + ", " + + rhsName + + "=" + + getCastEvaluator("Attribute[channel=1]", rhs, expected) + + "]"; + casesCrossProduct( + (l, r) -> expectedTypeStuff.expected().apply((Number) l, (Number) r), + getSuppliersForNumericType(lhsType, expectedTypeStuff.min(), expectedTypeStuff.max()), + getSuppliersForNumericType(rhsType, expectedTypeStuff.min(), expectedTypeStuff.max()), + evaluatorToString, + warnings, + suppliers, + DataTypes.BOOLEAN, + true + ); + } + } + return suppliers; + } + + public static List forBinaryWithWidening( + NumericTypeTestConfigs typeStuff, + String lhsName, + String rhsName, + List warnings + ) { + List suppliers = new ArrayList<>(); + List numericTypes = List.of(DataTypes.INTEGER, DataTypes.LONG, DataTypes.DOUBLE); + + for (DataType lhsType : numericTypes) { + for (DataType rhsType : numericTypes) { + DataType expected = widen(lhsType, rhsType); + NumericTypeTestConfig expectedTypeStuff = typeStuff.get(expected); BiFunction evaluatorToString = (lhs, rhs) -> expectedTypeStuff.evaluatorName() + "[" + lhsName @@ -885,7 +940,7 @@ public static List doubleCases(double min, double max, boolea return cases; } - private static List booleanCases() { + public static List booleanCases() { return List.of( new TypedDataSupplier("", () -> true, DataTypes.BOOLEAN), new TypedDataSupplier("", () -> false, DataTypes.BOOLEAN) @@ -1267,9 +1322,14 @@ public Matcher evaluatorToString() { * exists because we can't generate random values from the test parameter generation functions, and instead need to return * suppliers which generate the random values at test execution time. */ - public record TypedDataSupplier(String name, Supplier supplier, DataType type) { + public record TypedDataSupplier(String name, Supplier supplier, DataType type, boolean forceLiteral) { + + public TypedDataSupplier(String name, Supplier supplier, DataType type) { + this(name, supplier, type, false); + } + public TypedData get() { - return new TypedData(supplier.get(), type, name); + return new TypedData(supplier.get(), type, name, forceLiteral); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java index c40d037890d5..2596959c449d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java @@ -43,20 +43,20 @@ public static Iterable parameters() { List suppliers = new ArrayList<>(); suppliers.addAll( TestCaseSupplier.forBinaryWithWidening( - new TestCaseSupplier.NumericTypeTestConfigs( - new TestCaseSupplier.NumericTypeTestConfig( + new TestCaseSupplier.NumericTypeTestConfigs( + new TestCaseSupplier.NumericTypeTestConfig<>( (Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1, (l, r) -> l.intValue() + r.intValue(), "AddIntsEvaluator" ), - new TestCaseSupplier.NumericTypeTestConfig( + new TestCaseSupplier.NumericTypeTestConfig<>( (Long.MIN_VALUE >> 1) - 1, (Long.MAX_VALUE >> 1) - 1, (l, r) -> l.longValue() + r.longValue(), "AddLongsEvaluator" ), - new TestCaseSupplier.NumericTypeTestConfig( + new TestCaseSupplier.NumericTypeTestConfig<>( Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, (l, r) -> l.doubleValue() + r.doubleValue(), diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java index 0a1e9bdfaf34..6fcc4235f5b7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java @@ -11,52 +11,188 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.Equals; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; -import org.hamcrest.Matcher; +import org.elasticsearch.xpack.ql.util.NumericUtils; +import java.math.BigInteger; +import java.util.ArrayList; import java.util.List; import java.util.function.Supplier; -import static org.hamcrest.Matchers.equalTo; - -public class EqualsTests extends AbstractBinaryComparisonTestCase { +public class EqualsTests extends AbstractFunctionTestCase { public EqualsTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); } @ParametersFactory public static Iterable parameters() { - return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("Int == Int", () -> { - int rhs = randomInt(); - int lhs = randomInt(); - return new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(lhs, DataTypes.INTEGER, "lhs"), - new TestCaseSupplier.TypedData(rhs, DataTypes.INTEGER, "rhs") + List suppliers = new ArrayList<>(); + suppliers.addAll( + TestCaseSupplier.forBinaryComparisonWithWidening( + new TestCaseSupplier.NumericTypeTestConfigs<>( + new TestCaseSupplier.NumericTypeTestConfig<>( + (Integer.MIN_VALUE >> 1) - 1, + (Integer.MAX_VALUE >> 1) - 1, + (l, r) -> l.intValue() == r.intValue(), + "EqualsIntsEvaluator" + ), + new TestCaseSupplier.NumericTypeTestConfig<>( + (Long.MIN_VALUE >> 1) - 1, + (Long.MAX_VALUE >> 1) - 1, + (l, r) -> l.longValue() == r.longValue(), + "EqualsLongsEvaluator" + ), + new TestCaseSupplier.NumericTypeTestConfig<>( + Double.NEGATIVE_INFINITY, + Double.POSITIVE_INFINITY, + // NB: this has different behavior than Double::equals + (l, r) -> l.doubleValue() == r.doubleValue(), + "EqualsDoublesEvaluator" + ) ), - "EqualsIntsEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]", + "lhs", + "rhs", + List.of() + ) + ); + + // Unsigned Long cases + // TODO: These should be integrated into the type cross product above, but are currently broken + // see https://github.com/elastic/elasticsearch/issues/102935 + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "EqualsLongsEvaluator", + "lhs", + "rhs", + Object::equals, DataTypes.BOOLEAN, - equalTo(lhs == rhs) - ); - }))); - } + TestCaseSupplier.ulongCases(BigInteger.ZERO, NumericUtils.UNSIGNED_LONG_MAX), + TestCaseSupplier.ulongCases(BigInteger.ZERO, NumericUtils.UNSIGNED_LONG_MAX), + List.of() + ) + ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "EqualsBoolsEvaluator", + "lhs", + "rhs", + Object::equals, + DataTypes.BOOLEAN, + TestCaseSupplier.booleanCases(), + TestCaseSupplier.booleanCases(), + List.of() + ) + ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "EqualsKeywordsEvaluator", + "lhs", + "rhs", + Object::equals, + DataTypes.BOOLEAN, + TestCaseSupplier.ipCases(), + TestCaseSupplier.ipCases(), + List.of() + ) + ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "EqualsKeywordsEvaluator", + "lhs", + "rhs", + Object::equals, + DataTypes.BOOLEAN, + TestCaseSupplier.versionCases(""), + TestCaseSupplier.versionCases(""), + List.of() + ) + ); + // Datetime + // TODO: I'm surprised this passes. Shouldn't there be a cast from DateTime to Long? + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "EqualsLongsEvaluator", + "lhs", + "rhs", + Object::equals, + DataTypes.BOOLEAN, + TestCaseSupplier.dateCases(), + TestCaseSupplier.dateCases(), + List.of() + ) + ); - @Override - protected > Matcher resultMatcher(T lhs, T rhs) { - return equalTo(lhs.equals(rhs)); - } + suppliers.addAll( + TestCaseSupplier.stringCases( + Object::equals, + (lhsType, rhsType) -> "EqualsKeywordsEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]", + List.of(), + DataTypes.BOOLEAN + ) + ); - @Override - protected BinaryComparison build(Source source, Expression lhs, Expression rhs) { - return new Equals(source, lhs, rhs); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "EqualsGeometriesEvaluator", + "lhs", + "rhs", + Object::equals, + DataTypes.BOOLEAN, + TestCaseSupplier.geoPointCases(), + TestCaseSupplier.geoPointCases(), + List.of() + ) + ); + + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "EqualsGeometriesEvaluator", + "lhs", + "rhs", + Object::equals, + DataTypes.BOOLEAN, + TestCaseSupplier.geoShapeCases(), + TestCaseSupplier.geoShapeCases(), + List.of() + ) + ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "EqualsGeometriesEvaluator", + "lhs", + "rhs", + Object::equals, + DataTypes.BOOLEAN, + TestCaseSupplier.cartesianPointCases(), + TestCaseSupplier.cartesianPointCases(), + List.of() + ) + ); + + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "EqualsGeometriesEvaluator", + "lhs", + "rhs", + Object::equals, + DataTypes.BOOLEAN, + TestCaseSupplier.cartesianShapeCases(), + TestCaseSupplier.cartesianShapeCases(), + List.of() + ) + ); + + return parameterSuppliersFromTypedData( + errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), AbstractFunctionTestCase::errorMessageStringForBinaryOperators) + ); } @Override - protected boolean isEquality() { - return true; + protected Expression build(Source source, List args) { + return new Equals(source, args.get(0), args.get(1)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java index ad8dba7d6306..f45dedff837c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java @@ -11,26 +11,25 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.GreaterThanOrEqual; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; -import org.hamcrest.Matcher; -import java.time.ZoneOffset; import java.util.List; import java.util.function.Supplier; import static org.hamcrest.Matchers.equalTo; -public class GreaterThanOrEqualTests extends AbstractBinaryComparisonTestCase { +public class GreaterThanOrEqualTests extends AbstractFunctionTestCase { public GreaterThanOrEqualTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); } @ParametersFactory public static Iterable parameters() { + // ToDo: Add the full set of typed test cases here return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("Int >= Int", () -> { int rhs = randomInt(); int lhs = randomInt(); @@ -47,17 +46,7 @@ public static Iterable parameters() { } @Override - protected > Matcher resultMatcher(T lhs, T rhs) { - return equalTo(lhs.compareTo(rhs) >= 0); - } - - @Override - protected BinaryComparison build(Source source, Expression lhs, Expression rhs) { - return new GreaterThanOrEqual(source, lhs, rhs, ZoneOffset.UTC); - } - - @Override - protected boolean isEquality() { - return false; + protected Expression build(Source source, List args) { + return new GreaterThanOrEqual(source, args.get(0), args.get(1)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java index b631a742f788..e872af5b7c77 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java @@ -11,26 +11,25 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.GreaterThan; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; -import org.hamcrest.Matcher; -import java.time.ZoneOffset; import java.util.List; import java.util.function.Supplier; import static org.hamcrest.Matchers.equalTo; -public class GreaterThanTests extends AbstractBinaryComparisonTestCase { +public class GreaterThanTests extends AbstractFunctionTestCase { public GreaterThanTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); } @ParametersFactory public static Iterable parameters() { + // ToDo: Add the full set of typed test cases here return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("Int > Int", () -> { int rhs = randomInt(); int lhs = randomInt(); @@ -47,17 +46,7 @@ public static Iterable parameters() { } @Override - protected > Matcher resultMatcher(T lhs, T rhs) { - return equalTo(lhs.compareTo(rhs) > 0); - } - - @Override - protected BinaryComparison build(Source source, Expression lhs, Expression rhs) { - return new GreaterThan(source, lhs, rhs, ZoneOffset.UTC); - } - - @Override - protected boolean isEquality() { - return false; + protected Expression build(Source source, List args) { + return new GreaterThan(source, args.get(0), args.get(1)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java index 7864a0dda9fe..8bba0c4a5afb 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java @@ -11,20 +11,18 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.LessThanOrEqual; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; -import org.hamcrest.Matcher; -import java.time.ZoneOffset; import java.util.List; import java.util.function.Supplier; import static org.hamcrest.Matchers.equalTo; -public class LessThanOrEqualTests extends AbstractBinaryComparisonTestCase { +public class LessThanOrEqualTests extends AbstractFunctionTestCase { public LessThanOrEqualTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); } @@ -47,17 +45,7 @@ public static Iterable parameters() { } @Override - protected > Matcher resultMatcher(T lhs, T rhs) { - return equalTo(lhs.compareTo(rhs) <= 0); - } - - @Override - protected BinaryComparison build(Source source, Expression lhs, Expression rhs) { - return new LessThanOrEqual(source, lhs, rhs, ZoneOffset.UTC); - } - - @Override - protected boolean isEquality() { - return false; + protected Expression build(Source source, List args) { + return new LessThanOrEqual(source, args.get(0), args.get(1), null); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java index 826e88551077..ab726dc51fbe 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java @@ -11,20 +11,18 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.LessThan; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; -import org.hamcrest.Matcher; -import java.time.ZoneOffset; import java.util.List; import java.util.function.Supplier; import static org.hamcrest.Matchers.equalTo; -public class LessThanTests extends AbstractBinaryComparisonTestCase { +public class LessThanTests extends AbstractFunctionTestCase { public LessThanTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); } @@ -47,17 +45,7 @@ public static Iterable parameters() { } @Override - protected > Matcher resultMatcher(T lhs, T rhs) { - return equalTo(lhs.compareTo(rhs) < 0); - } - - @Override - protected BinaryComparison build(Source source, Expression lhs, Expression rhs) { - return new LessThan(source, lhs, rhs, ZoneOffset.UTC); - } - - @Override - protected boolean isEquality() { - return false; + protected Expression build(Source source, List args) { + return new LessThan(source, args.get(0), args.get(1), null); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java index 0d6bb32fe248..d6ee5806e0c9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java @@ -11,53 +11,182 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.NotEquals; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; -import org.hamcrest.Matcher; -import java.time.ZoneOffset; +import java.math.BigInteger; +import java.util.ArrayList; import java.util.List; import java.util.function.Supplier; -import static org.hamcrest.Matchers.equalTo; - -public class NotEqualsTests extends AbstractBinaryComparisonTestCase { +public class NotEqualsTests extends AbstractFunctionTestCase { public NotEqualsTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); } @ParametersFactory public static Iterable parameters() { - return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("Int != Int", () -> { - int rhs = randomInt(); - int lhs = randomInt(); - return new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(lhs, DataTypes.INTEGER, "lhs"), - new TestCaseSupplier.TypedData(rhs, DataTypes.INTEGER, "rhs") + List suppliers = new ArrayList<>(); + suppliers.addAll( + + TestCaseSupplier.forBinaryComparisonWithWidening( + new TestCaseSupplier.NumericTypeTestConfigs<>( + new TestCaseSupplier.NumericTypeTestConfig<>( + (Integer.MIN_VALUE >> 1) - 1, + (Integer.MAX_VALUE >> 1) - 1, + (l, r) -> l.intValue() != r.intValue(), + "NotEqualsIntsEvaluator" + ), + new TestCaseSupplier.NumericTypeTestConfig<>( + (Long.MIN_VALUE >> 1) - 1, + (Long.MAX_VALUE >> 1) - 1, + (l, r) -> l.longValue() != r.longValue(), + "NotEqualsLongsEvaluator" + ), + new TestCaseSupplier.NumericTypeTestConfig<>( + Double.NEGATIVE_INFINITY, + Double.POSITIVE_INFINITY, + // NB: this has different behavior than Double::equals + (l, r) -> l.doubleValue() != r.doubleValue(), + "NotEqualsDoublesEvaluator" + ) ), - "NotEqualsIntsEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]", + "lhs", + "rhs", + List.of() + ) + ); + // Unsigned Long cases + // TODO: These should be integrated into the type cross product above, but are currently broken + // see https://github.com/elastic/elasticsearch/issues/102935 + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "NotEqualsLongsEvaluator", + "lhs", + "rhs", + (l, r) -> false == l.equals(r), DataTypes.BOOLEAN, - equalTo(lhs != rhs) - ); - }))); - } - - @Override - protected > Matcher resultMatcher(T lhs, T rhs) { - return equalTo(false == lhs.equals(rhs)); - } - - @Override - protected BinaryComparison build(Source source, Expression lhs, Expression rhs) { - return new NotEquals(source, lhs, rhs, ZoneOffset.UTC); + TestCaseSupplier.ulongCases(BigInteger.ZERO, BigInteger.valueOf(Long.MAX_VALUE)), + TestCaseSupplier.ulongCases(BigInteger.ZERO, BigInteger.valueOf(Long.MAX_VALUE)), + List.of() + ) + ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "NotEqualsBoolsEvaluator", + "lhs", + "rhs", + (l, r) -> false == l.equals(r), + DataTypes.BOOLEAN, + TestCaseSupplier.booleanCases(), + TestCaseSupplier.booleanCases(), + List.of() + ) + ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "NotEqualsKeywordsEvaluator", + "lhs", + "rhs", + (l, r) -> false == l.equals(r), + DataTypes.BOOLEAN, + TestCaseSupplier.ipCases(), + TestCaseSupplier.ipCases(), + List.of() + ) + ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "NotEqualsKeywordsEvaluator", + "lhs", + "rhs", + (l, r) -> false == l.equals(r), + DataTypes.BOOLEAN, + TestCaseSupplier.versionCases(""), + TestCaseSupplier.versionCases(""), + List.of() + ) + ); + // Datetime + // TODO: I'm surprised this passes. Shouldn't there be a cast from DateTime to Long? + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "NotEqualsLongsEvaluator", + "lhs", + "rhs", + (l, r) -> false == l.equals(r), + DataTypes.BOOLEAN, + TestCaseSupplier.dateCases(), + TestCaseSupplier.dateCases(), + List.of() + ) + ); + suppliers.addAll( + TestCaseSupplier.stringCases( + (l, r) -> false == l.equals(r), + (lhsType, rhsType) -> "NotEqualsKeywordsEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]", + List.of(), + DataTypes.BOOLEAN + ) + ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "NotEqualsGeometriesEvaluator", + "lhs", + "rhs", + (l, r) -> false == l.equals(r), + DataTypes.BOOLEAN, + TestCaseSupplier.geoPointCases(), + TestCaseSupplier.geoPointCases(), + List.of() + ) + ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "NotEqualsGeometriesEvaluator", + "lhs", + "rhs", + (l, r) -> false == l.equals(r), + DataTypes.BOOLEAN, + TestCaseSupplier.geoShapeCases(), + TestCaseSupplier.geoShapeCases(), + List.of() + ) + ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "NotEqualsGeometriesEvaluator", + "lhs", + "rhs", + (l, r) -> false == l.equals(r), + DataTypes.BOOLEAN, + TestCaseSupplier.cartesianPointCases(), + TestCaseSupplier.cartesianPointCases(), + List.of() + ) + ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "NotEqualsGeometriesEvaluator", + "lhs", + "rhs", + (l, r) -> false == l.equals(r), + DataTypes.BOOLEAN, + TestCaseSupplier.cartesianShapeCases(), + TestCaseSupplier.cartesianShapeCases(), + List.of() + ) + ); + return parameterSuppliersFromTypedData( + errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), AbstractFunctionTestCase::errorMessageStringForBinaryOperators) + ); } @Override - protected boolean isEquality() { - return true; + protected Expression build(Source source, List args) { + return new NotEquals(source, args.get(0), args.get(1)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java index 01fcd222a514..28944252191b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java @@ -495,7 +495,7 @@ public void testPropagateEquals_VarEq2OrVarNeq5() { // a = 2 OR 3 < a < 4 OR a > 2 OR a!= 2 -> TRUE public void testPropagateEquals_VarEq2OrVarRangeGt3Lt4OrVarGt2OrVarNe2() { FieldAttribute fa = getFieldAttribute(); - org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals eq = equalsOf(fa, TWO); + Equals eq = equalsOf(fa, TWO); Range range = rangeOf(fa, THREE, false, FOUR, false); GreaterThan gt = greaterThanOf(fa, TWO); NotEquals neq = notEqualsOf(fa, TWO); From 69e3c22d0039b5380701e2e5e30298859a0b1cf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Tue, 16 Apr 2024 15:57:30 +0200 Subject: [PATCH 02/43] [Transform] Prevent the ResourceNotFoundException from being thrown on force-stop (#107520) --- .../integration/TransformRobustnessIT.java | 45 ++++++++ .../action/TransportStopTransformAction.java | 24 ++-- .../TransportStopTransformActionTests.java | 107 ++++++++++++++++++ 3 files changed, 169 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRobustnessIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRobustnessIT.java index a311237b826f..9c993b9dcb7d 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRobustnessIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRobustnessIT.java @@ -113,6 +113,51 @@ public void testBatchTransformLifecycltInALoop() throws IOException { } } + public void testInterruptedBatchTransformLifecycltInALoop() throws IOException { + createReviewsIndex(); + + String transformId = "test_interrupted_batch_lifecycle_in_a_loop"; + String destIndex = transformId + "-dest"; + for (int i = 0; i < 100; ++i) { + long sleepAfterStartMillis = randomLongBetween(0, 1_000); + boolean force = randomBoolean(); + try { + // Create the batch transform. + createPivotReviewsTransform(transformId, destIndex, null); + assertThat(getTransformTasks(), is(empty())); + assertThat(getTransformTasksFromClusterState(transformId), is(empty())); + + startTransform(transformId); + // There is 1 transform task after start. + assertThat(getTransformTasks(), hasSize(1)); + assertThat(getTransformTasksFromClusterState(transformId), hasSize(1)); + + Thread.sleep(sleepAfterStartMillis); + + // Stop the transform with force set randomly. + stopTransform(transformId, force); + // After the transform is stopped, there should be no transform task left. + if (force) { + // If the "force" has been used, then the persistent task is removed from the cluster state but the local task can still + // be seen by the PersistentTasksNodeService. We need to wait until PersistentTasksNodeService reconciles the state. + assertBusy(() -> assertThat(getTransformTasks(), is(empty()))); + } else { + // If the "force" hasn't been used then we can expect the local task to be already gone. + assertThat(getTransformTasks(), is(empty())); + } + assertThat(getTransformTasksFromClusterState(transformId), is(empty())); + + // Delete the transform. + deleteTransform(transformId); + } catch (AssertionError | Exception e) { + throw new AssertionError( + format("Failure at iteration %d (sleepAfterStart=%sms,force=%s): %s", i, sleepAfterStartMillis, force, e.getMessage()), + e + ); + } + } + } + public void testContinuousTransformLifecycleInALoop() throws Exception { createReviewsIndex(); diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStopTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStopTransformAction.java index 1996012ccdf5..6868948bb6f0 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStopTransformAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportStopTransformAction.java @@ -165,6 +165,7 @@ protected void doExecute(Task task, Request request, ActionListener li ); final ActionListener doExecuteListener = cancelTransformTasksListener( + persistentTasksService, transformNodeAssignments.getWaitingForAssignment(), finalListener ); @@ -173,9 +174,8 @@ protected void doExecute(Task task, Request request, ActionListener li // When force==true, we **do not** fan out to individual tasks (i.e. taskOperation method will not be called) as we // want to make sure that the persistent tasks will be removed from cluster state even if these tasks are no longer // visible by the PersistentTasksService. - cancelTransformTasksListener(transformNodeAssignments.getAssigned(), doExecuteListener).onResponse( - new Response(true) - ); + cancelTransformTasksListener(persistentTasksService, transformNodeAssignments.getAssigned(), doExecuteListener) + .onResponse(new Response(true)); } else if (transformNodeAssignments.getExecutorNodes().isEmpty()) { doExecuteListener.onResponse(new Response(true)); } else { @@ -195,6 +195,7 @@ protected void doExecute(Task task, Request request, ActionListener li // found transforms without a config } else if (request.isForce()) { final ActionListener doExecuteListener = cancelTransformTasksListener( + persistentTasksService, transformNodeAssignments.getWaitingForAssignment(), finalListener ); @@ -488,6 +489,7 @@ private void waitForTransformStopped( })); } + // Visible for testing /** * Creates and returns the listener that sends remove request for every task in the given set. * @@ -495,7 +497,8 @@ private void waitForTransformStopped( * @param finalListener listener that should be called once all the given tasks are removed * @return listener that removes given tasks in parallel */ - private ActionListener cancelTransformTasksListener( + static ActionListener cancelTransformTasksListener( + final PersistentTasksService persistentTasksService, final Set transformTasks, final ActionListener finalListener ) { @@ -505,16 +508,23 @@ private ActionListener cancelTransformTasksListener( return ActionListener.wrap(response -> { GroupedActionListener> groupedListener = new GroupedActionListener<>( transformTasks.size(), - ActionListener.wrap(r -> finalListener.onResponse(response), finalListener::onFailure) + ActionListener.wrap(unused -> finalListener.onResponse(response), finalListener::onFailure) ); for (String taskId : transformTasks) { - persistentTasksService.sendRemoveRequest(taskId, null, groupedListener); + persistentTasksService.sendRemoveRequest(taskId, null, ActionListener.wrap(groupedListener::onResponse, e -> { + // If we are about to remove a persistent task which does not exist, treat it as success. + if (e instanceof ResourceNotFoundException) { + groupedListener.onResponse(null); + return; + } + groupedListener.onFailure(e); + })); } }, e -> { GroupedActionListener> groupedListener = new GroupedActionListener<>( transformTasks.size(), - ActionListener.wrap(r -> finalListener.onFailure(e), finalListener::onFailure) + ActionListener.wrap(unused -> finalListener.onFailure(e), finalListener::onFailure) ); for (String taskId : transformTasks) { diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportStopTransformActionTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportStopTransformActionTests.java index 59959edc7232..08e0982b2ab8 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportStopTransformActionTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/action/TransportStopTransformActionTests.java @@ -8,24 +8,50 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.TaskOperationFailure; +import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.persistent.PersistentTaskResponse; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; +import org.elasticsearch.persistent.PersistentTasksService; +import org.elasticsearch.persistent.RemovePersistentTaskAction; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.indexing.IndexerState; import org.elasticsearch.xpack.core.transform.TransformConfigVersion; +import org.elasticsearch.xpack.core.transform.action.StopTransformAction; import org.elasticsearch.xpack.core.transform.transforms.TransformState; import org.elasticsearch.xpack.core.transform.transforms.TransformTaskParams; import org.elasticsearch.xpack.core.transform.transforms.TransformTaskState; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; +import org.mockito.stubbing.Answer; import java.util.ArrayList; import java.util.List; +import java.util.Set; import static org.elasticsearch.rest.RestStatus.CONFLICT; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.emptyArray; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class TransportStopTransformActionTests extends ESTestCase { @@ -198,4 +224,85 @@ public void testBuildException() { assertThat(statusException.getSuppressed().length, equalTo(0)); } + public void testCancelTransformTasksListener_NoTasks() { + StopTransformAction.Response responseTrue = new StopTransformAction.Response(true); + + PersistentTasksService persistentTasksService = mock(PersistentTasksService.class); + Set transformTasks = Set.of(); + ActionListener listener = Mockito.>mock(); + + ActionListener cancelTransformTasksListener = TransportStopTransformAction + .cancelTransformTasksListener(persistentTasksService, transformTasks, listener); + cancelTransformTasksListener.onResponse(responseTrue); + verify(listener, times(1)).onResponse(responseTrue); + } + + public void testCancelTransformTasksListener_ThreeTasksRemovedSuccessfully() { + ThreadPool threadPool = mock(ThreadPool.class); + when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY)); + Client client = mock(Client.class); + when(client.threadPool()).thenReturn(threadPool); + + // We treat NotFound as a successful removal of the task + doAnswer(randomBoolean() ? withResponse() : withException(new ResourceNotFoundException("task not found"))).when(client) + .execute(same(RemovePersistentTaskAction.INSTANCE), any(), any()); + + PersistentTasksService persistentTasksService = new PersistentTasksService(mock(ClusterService.class), threadPool, client); + Set transformTasks = Set.of("task-A", "task-B", "task-C"); + ActionListener listener = Mockito.>mock(); + + StopTransformAction.Response responseTrue = new StopTransformAction.Response(true); + ActionListener cancelTransformTasksListener = TransportStopTransformAction + .cancelTransformTasksListener(persistentTasksService, transformTasks, listener); + cancelTransformTasksListener.onResponse(responseTrue); + + verify(listener).onResponse(responseTrue); + } + + public void testCancelTransformTasksListener_OneTaskCouldNotBeRemoved() { + ThreadPool threadPool = mock(ThreadPool.class); + when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY)); + Client client = mock(Client.class); + when(client.threadPool()).thenReturn(threadPool); + + doAnswer(randomBoolean() ? withResponse() : withException(new ResourceNotFoundException("task not found"))).when(client) + .execute(same(RemovePersistentTaskAction.INSTANCE), eq(new RemovePersistentTaskAction.Request("task-A")), any()); + doAnswer(randomBoolean() ? withResponse() : withException(new ResourceNotFoundException("task not found"))).when(client) + .execute(same(RemovePersistentTaskAction.INSTANCE), eq(new RemovePersistentTaskAction.Request("task-B")), any()); + doAnswer(withException(new IllegalStateException("real issue while removing task"))).when(client) + .execute(same(RemovePersistentTaskAction.INSTANCE), eq(new RemovePersistentTaskAction.Request("task-C")), any()); + + PersistentTasksService persistentTasksService = new PersistentTasksService(mock(ClusterService.class), threadPool, client); + Set transformTasks = Set.of("task-A", "task-B", "task-C"); + ActionListener listener = Mockito.>mock(); + + StopTransformAction.Response responseTrue = new StopTransformAction.Response(true); + ActionListener cancelTransformTasksListener = TransportStopTransformAction + .cancelTransformTasksListener(persistentTasksService, transformTasks, listener); + cancelTransformTasksListener.onResponse(responseTrue); + + ArgumentCaptor exceptionArgumentCaptor = ArgumentCaptor.forClass(Exception.class); + verify(listener, times(1)).onFailure(exceptionArgumentCaptor.capture()); + Exception actualException = exceptionArgumentCaptor.getValue(); + assertThat(actualException.getMessage(), containsString("real issue while removing task")); + assertThat(actualException.getSuppressed(), is(emptyArray())); + } + + private static Answer withResponse() { + return invocationOnMock -> { + @SuppressWarnings("unchecked") + var l = (ActionListener) invocationOnMock.getArguments()[2]; + l.onResponse(new PersistentTaskResponse((PersistentTasksCustomMetadata.PersistentTask) null)); + return null; + }; + } + + private static Answer withException(Exception e) { + return invocationOnMock -> { + @SuppressWarnings("unchecked") + var l = (ActionListener) invocationOnMock.getArguments()[2]; + l.onFailure(e); + return null; + }; + } } From 7e18768bbf8dd5f54a2efeb8a0119c471a777158 Mon Sep 17 00:00:00 2001 From: Moritz Mack Date: Tue, 16 Apr 2024 16:22:28 +0200 Subject: [PATCH 03/43] Adjust log level for publish failure in TransportPutShutdownNodeAction (#107489) Publish failures are - to a certain degree - expected as the master might change before the corresponding task is executed. This reduces logging to INFO level for publish failures that will be retried by TransportMasterNodeAction. --- .../xpack/shutdown/TransportPutShutdownNodeAction.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeAction.java b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeAction.java index 750bb9227cff..b68a29604be2 100644 --- a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeAction.java +++ b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportPutShutdownNodeAction.java @@ -23,6 +23,7 @@ import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.cluster.service.MasterService; import org.elasticsearch.cluster.service.MasterServiceTaskQueue; import org.elasticsearch.common.Priority; import org.elasticsearch.common.inject.Inject; @@ -82,7 +83,11 @@ private static boolean putShutdownNodeState( record PutShutdownNodeTask(Request request, ActionListener listener) implements ClusterStateTaskListener { @Override public void onFailure(Exception e) { - logger.error(() -> "failed to put shutdown for node [" + request.getNodeId() + "]", e); + if (MasterService.isPublishFailureException(e)) { + logger.info(() -> "failed to put shutdown for node [" + request.getNodeId() + "], attempting retry", e); + } else { + logger.error(() -> "failed to put shutdown for node [" + request.getNodeId() + "]", e); + } listener.onFailure(e); } } From 225edaf6076770385b4d091af89a546020ec5c79 Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Tue, 16 Apr 2024 10:51:28 -0400 Subject: [PATCH 04/43] Revert "[ES|QL] Moving argument compatibility checking for Equals (#105217)" (#107537) * Revert "[ES|QL] Moving argument compatibility checking for Equals (#105217)" This reverts commit af8e4bf26d15af49824cdb94fcd149f8a984b48e. * Update docs/changelog/107537.yaml --- docs/changelog/107537.yaml | 5 + .../src/main/resources/conditional.csv-spec | 7 - .../predicate/operator/comparison/Equals.java | 47 ++--- .../comparison/EsqlBinaryComparison.java | 164 --------------- .../operator/comparison/GreaterThan.java | 33 +-- .../comparison/GreaterThanOrEqual.java | 34 +--- .../operator/comparison/LessThan.java | 36 ++-- .../operator/comparison/LessThanOrEqual.java | 31 +-- .../operator/comparison/NotEquals.java | 82 +++----- .../DateTimeArithmeticOperation.java | 8 +- .../arithmetic/EsqlArithmeticOperation.java | 23 +-- .../function/AbstractFunctionTestCase.java | 49 ----- .../expression/function/TestCaseSupplier.java | 84 ++------ .../operator/arithmetic/AddTests.java | 8 +- .../operator/comparison/EqualsTests.java | 188 +++--------------- .../comparison/GreaterThanOrEqualTests.java | 21 +- .../operator/comparison/GreaterThanTests.java | 21 +- .../comparison/LessThanOrEqualTests.java | 20 +- .../operator/comparison/LessThanTests.java | 20 +- .../operator/comparison/NotEqualsTests.java | 187 +++-------------- .../esql/optimizer/OptimizerRulesTests.java | 2 +- 21 files changed, 245 insertions(+), 825 deletions(-) create mode 100644 docs/changelog/107537.yaml delete mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EsqlBinaryComparison.java diff --git a/docs/changelog/107537.yaml b/docs/changelog/107537.yaml new file mode 100644 index 000000000000..d6d502b394c3 --- /dev/null +++ b/docs/changelog/107537.yaml @@ -0,0 +1,5 @@ +pr: 107537 +summary: "Revert \"[ES|QL] Moving argument compatibility checking for Equals\"" +area: ES|QL +type: bug +issues: [] diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/conditional.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/conditional.csv-spec index 64a8c1d9da31..f574722f691e 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/conditional.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/conditional.csv-spec @@ -156,9 +156,6 @@ nullOnMultivaluesComparisonOperation required_feature: esql.disable_nullable_opts ROW a = 5, b = [ 1, 2 ]| EVAL same = a == b| LIMIT 1 | WHERE same IS NULL; -warning:Line 1:38: evaluation of [a == b] failed, treating result as null. Only first 20 failures recorded. -warning:Line 1:38: java.lang.IllegalArgumentException: single-value function encountered multi-value - a:integer | b:integer | same:boolean 5 | [1, 2] | null @@ -169,8 +166,6 @@ notNullOnMultivaluesComparisonOperation required_feature: esql.disable_nullable_opts ROW a = 5, b = [ 1, 2 ]| EVAL same = a == b| LIMIT 1 | WHERE same IS NOT NULL; -warning:Line 1:38: evaluation of [a == b] failed, treating result as null. Only first 20 failures recorded. -warning:Line 1:38: java.lang.IllegalArgumentException: single-value function encountered multi-value a:integer | b:integer | same:boolean ; @@ -180,8 +175,6 @@ notNullOnMultivaluesComparisonOperationWithPartialMatch required_feature: esql.disable_nullable_opts ROW a = 5, b = [ 5, 2 ]| EVAL same = a == b| LIMIT 1 | WHERE same IS NOT NULL; -warning:Line 1:38: evaluation of [a == b] failed, treating result as null. Only first 20 failures recorded. -warning:Line 1:38: java.lang.IllegalArgumentException: single-value function encountered multi-value a:integer | b:integer | same:boolean ; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/Equals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/Equals.java index 62eec13af008..9fb899b8e36d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/Equals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/Equals.java @@ -8,48 +8,33 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Evaluator; -import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; +import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.predicate.Negatable; +import org.elasticsearch.xpack.ql.expression.TypeResolutions; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; -import org.elasticsearch.xpack.ql.type.DataType; -import org.elasticsearch.xpack.ql.type.DataTypes; import java.time.ZoneId; -import java.util.Map; - -public class Equals extends EsqlBinaryComparison implements Negatable { - private static final Map evaluatorMap = Map.ofEntries( - Map.entry(DataTypes.BOOLEAN, EqualsBoolsEvaluator.Factory::new), - Map.entry(DataTypes.INTEGER, EqualsIntsEvaluator.Factory::new), - Map.entry(DataTypes.DOUBLE, EqualsDoublesEvaluator.Factory::new), - Map.entry(DataTypes.LONG, EqualsLongsEvaluator.Factory::new), - Map.entry(DataTypes.UNSIGNED_LONG, EqualsLongsEvaluator.Factory::new), - Map.entry(DataTypes.DATETIME, EqualsLongsEvaluator.Factory::new), - Map.entry(EsqlDataTypes.GEO_POINT, EqualsGeometriesEvaluator.Factory::new), - Map.entry(EsqlDataTypes.CARTESIAN_POINT, EqualsGeometriesEvaluator.Factory::new), - Map.entry(EsqlDataTypes.GEO_SHAPE, EqualsGeometriesEvaluator.Factory::new), - Map.entry(EsqlDataTypes.CARTESIAN_SHAPE, EqualsGeometriesEvaluator.Factory::new), - Map.entry(DataTypes.KEYWORD, EqualsKeywordsEvaluator.Factory::new), - Map.entry(DataTypes.TEXT, EqualsKeywordsEvaluator.Factory::new), - Map.entry(DataTypes.VERSION, EqualsKeywordsEvaluator.Factory::new), - Map.entry(DataTypes.IP, EqualsKeywordsEvaluator.Factory::new) - ); +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; + +public class Equals extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals { public Equals(Source source, Expression left, Expression right) { - super(source, left, right, BinaryComparisonProcessor.BinaryComparisonOperation.EQ, evaluatorMap); + super(source, left, right); } public Equals(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, BinaryComparisonProcessor.BinaryComparisonOperation.EQ, zoneId, evaluatorMap); + super(source, left, right, zoneId); + } + + @Override + protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { + return EsqlTypeResolutions.isExact(e, sourceText(), DEFAULT); } @Override - protected NodeInfo info() { + protected NodeInfo info() { return NodeInfo.create(this, Equals::new, left(), right(), zoneId()); } @@ -63,11 +48,6 @@ public Equals swapLeftAndRight() { return new Equals(source(), right(), left(), zoneId()); } - @Override - public BinaryComparison reverse() { - return this; - } - @Override public BinaryComparison negate() { return new NotEquals(source(), left(), right(), zoneId()); @@ -102,5 +82,4 @@ static boolean processBools(boolean lhs, boolean rhs) { static boolean processGeometries(BytesRef lhs, BytesRef rhs) { return lhs.equals(rhs); } - } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EsqlBinaryComparison.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EsqlBinaryComparison.java deleted file mode 100644 index 58a808893c4c..000000000000 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EsqlBinaryComparison.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison; - -import org.elasticsearch.compute.operator.EvalOperator; -import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; -import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; -import org.elasticsearch.xpack.esql.expression.function.scalar.math.Cast; -import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; -import org.elasticsearch.xpack.esql.type.EsqlDataTypeRegistry; -import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.TypeResolutions; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor; -import org.elasticsearch.xpack.ql.tree.Source; -import org.elasticsearch.xpack.ql.type.DataType; -import org.elasticsearch.xpack.ql.type.DataTypes; - -import java.time.ZoneId; -import java.util.Map; -import java.util.function.Function; - -import static org.elasticsearch.common.logging.LoggerMessageFormat.format; -import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; - -public abstract class EsqlBinaryComparison extends BinaryComparison implements EvaluatorMapper { - - private final Map evaluatorMap; - - protected EsqlBinaryComparison( - Source source, - Expression left, - Expression right, - /* TODO: BinaryComparisonOperator is an enum with a bunch of functionality we don't really want. We should extract an interface and - create a symbol only version like we did for BinaryArithmeticOperation. Ideally, they could be the same class. - */ - BinaryComparisonProcessor.BinaryComparisonOperation operation, - Map evaluatorMap - ) { - this(source, left, right, operation, null, evaluatorMap); - } - - protected EsqlBinaryComparison( - Source source, - Expression left, - Expression right, - BinaryComparisonProcessor.BinaryComparisonOperation operation, - // TODO: We are definitely not doing the right thing with this zoneId - ZoneId zoneId, - Map evaluatorMap - ) { - super(source, left, right, operation, zoneId); - this.evaluatorMap = evaluatorMap; - } - - @Override - public EvalOperator.ExpressionEvaluator.Factory toEvaluator( - Function toEvaluator - ) { - // Our type is always boolean, so figure out the evaluator type from the inputs - DataType commonType = EsqlDataTypeRegistry.INSTANCE.commonType(left().dataType(), right().dataType()); - EvalOperator.ExpressionEvaluator.Factory lhs; - EvalOperator.ExpressionEvaluator.Factory rhs; - - if (commonType.isNumeric()) { - lhs = Cast.cast(source(), left().dataType(), commonType, toEvaluator.apply(left())); - rhs = Cast.cast(source(), right().dataType(), commonType, toEvaluator.apply(right())); - } else { - lhs = toEvaluator.apply(left()); - rhs = toEvaluator.apply(right()); - } - - if (evaluatorMap.containsKey(commonType) == false) { - throw new EsqlIllegalArgumentException("Unsupported type " + left().dataType()); - } - return evaluatorMap.get(commonType).apply(source(), lhs, rhs); - } - - @Override - public Boolean fold() { - return (Boolean) EvaluatorMapper.super.fold(); - } - - @Override - protected TypeResolution resolveType() { - TypeResolution typeResolution = super.resolveType(); - if (typeResolution.unresolved()) { - return typeResolution; - } - - return checkCompatibility(); - } - - @Override - protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { - return TypeResolutions.isType( - e, - evaluatorMap::containsKey, - sourceText(), - paramOrdinal, - evaluatorMap.keySet().stream().map(DataType::typeName).toArray(String[]::new) - ); - } - - /** - * Check if the two input types are compatible for this operation - * - * @return TypeResolution.TYPE_RESOLVED iff the types are compatible. Otherwise, an appropriate type resolution error. - */ - protected TypeResolution checkCompatibility() { - DataType leftType = left().dataType(); - DataType rightType = right().dataType(); - - // Unsigned long is only interoperable with other unsigned longs - if ((rightType == UNSIGNED_LONG && (false == (leftType == UNSIGNED_LONG || leftType == DataTypes.NULL))) - || (leftType == UNSIGNED_LONG && (false == (rightType == UNSIGNED_LONG || rightType == DataTypes.NULL)))) { - return new TypeResolution(formatIncompatibleTypesMessage()); - } - - if ((leftType.isNumeric() && rightType.isNumeric()) - || (DataTypes.isString(leftType) && DataTypes.isString(rightType)) - || leftType.equals(rightType) - || DataTypes.isNull(leftType) - || DataTypes.isNull(rightType)) { - return TypeResolution.TYPE_RESOLVED; - } - return new TypeResolution(formatIncompatibleTypesMessage()); - } - - public String formatIncompatibleTypesMessage() { - if (left().dataType().equals(UNSIGNED_LONG)) { - return format( - null, - "first argument of [{}] is [unsigned_long] and second is [{}]. " - + "[unsigned_long] can only be operated on together with another [unsigned_long]", - sourceText(), - right().dataType().typeName() - ); - } - if (right().dataType().equals(UNSIGNED_LONG)) { - return format( - null, - "first argument of [{}] is [{}] and second is [unsigned_long]. " - + "[unsigned_long] can only be operated on together with another [unsigned_long]", - sourceText(), - left().dataType().typeName() - ); - } - return format( - null, - "first argument of [{}] is [{}] so second argument must also be [{}] but was [{}]", - sourceText(), - left().dataType().isNumeric() ? "numeric" : left().dataType().typeName(), - left().dataType().isNumeric() ? "numeric" : left().dataType().typeName(), - right().dataType().typeName() - ); - } - -} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThan.java index 3eca0e858acb..5683a9d0d7e8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThan.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThan.java @@ -8,42 +8,29 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Evaluator; -import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; +import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.predicate.Negatable; +import org.elasticsearch.xpack.ql.expression.TypeResolutions; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; -import org.elasticsearch.xpack.ql.type.DataType; -import org.elasticsearch.xpack.ql.type.DataTypes; import java.time.ZoneId; -import java.util.Map; -public class GreaterThan extends EsqlBinaryComparison implements Negatable { - private static final Map evaluatorMap = Map.ofEntries( - Map.entry(DataTypes.INTEGER, GreaterThanIntsEvaluator.Factory::new), - Map.entry(DataTypes.DOUBLE, GreaterThanDoublesEvaluator.Factory::new), - Map.entry(DataTypes.LONG, GreaterThanLongsEvaluator.Factory::new), - Map.entry(DataTypes.UNSIGNED_LONG, GreaterThanLongsEvaluator.Factory::new), - Map.entry(DataTypes.DATETIME, GreaterThanLongsEvaluator.Factory::new), - Map.entry(DataTypes.KEYWORD, GreaterThanKeywordsEvaluator.Factory::new), - Map.entry(DataTypes.TEXT, GreaterThanKeywordsEvaluator.Factory::new), - Map.entry(DataTypes.VERSION, GreaterThanKeywordsEvaluator.Factory::new), - Map.entry(DataTypes.IP, GreaterThanKeywordsEvaluator.Factory::new) - ); +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; - public GreaterThan(Source source, Expression left, Expression right) { - super(source, left, right, BinaryComparisonProcessor.BinaryComparisonOperation.GT, evaluatorMap); +public class GreaterThan extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan { + public GreaterThan(Source source, Expression left, Expression right, ZoneId zoneId) { + super(source, left, right, zoneId); } - public GreaterThan(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, BinaryComparisonProcessor.BinaryComparisonOperation.GT, zoneId, evaluatorMap); + @Override + protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { + return EsqlTypeResolutions.isExact(e, sourceText(), DEFAULT); } @Override - protected NodeInfo info() { + protected NodeInfo info() { return NodeInfo.create(this, GreaterThan::new, left(), right(), zoneId()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqual.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqual.java index f99a85420870..ebb29998fb99 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqual.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqual.java @@ -8,42 +8,30 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Evaluator; -import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; +import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.predicate.Negatable; +import org.elasticsearch.xpack.ql.expression.TypeResolutions; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; -import org.elasticsearch.xpack.ql.type.DataType; -import org.elasticsearch.xpack.ql.type.DataTypes; import java.time.ZoneId; -import java.util.Map; -public class GreaterThanOrEqual extends EsqlBinaryComparison implements Negatable { - private static final Map evaluatorMap = Map.ofEntries( - Map.entry(DataTypes.INTEGER, GreaterThanOrEqualIntsEvaluator.Factory::new), - Map.entry(DataTypes.DOUBLE, GreaterThanOrEqualDoublesEvaluator.Factory::new), - Map.entry(DataTypes.LONG, GreaterThanOrEqualLongsEvaluator.Factory::new), - Map.entry(DataTypes.UNSIGNED_LONG, GreaterThanOrEqualLongsEvaluator.Factory::new), - Map.entry(DataTypes.DATETIME, GreaterThanOrEqualLongsEvaluator.Factory::new), - Map.entry(DataTypes.KEYWORD, GreaterThanOrEqualKeywordsEvaluator.Factory::new), - Map.entry(DataTypes.TEXT, GreaterThanOrEqualKeywordsEvaluator.Factory::new), - Map.entry(DataTypes.VERSION, GreaterThanOrEqualKeywordsEvaluator.Factory::new), - Map.entry(DataTypes.IP, GreaterThanOrEqualKeywordsEvaluator.Factory::new) - ); +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; - public GreaterThanOrEqual(Source source, Expression left, Expression right) { - super(source, left, right, BinaryComparisonProcessor.BinaryComparisonOperation.GTE, evaluatorMap); - } +public class GreaterThanOrEqual extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThanOrEqual { public GreaterThanOrEqual(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, BinaryComparisonProcessor.BinaryComparisonOperation.GTE, zoneId, evaluatorMap); + super(source, left, right, zoneId); + } + + @Override + protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { + return EsqlTypeResolutions.isExact(e, sourceText(), DEFAULT); } @Override - protected NodeInfo info() { + protected NodeInfo info() { return NodeInfo.create(this, GreaterThanOrEqual::new, left(), right(), zoneId()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThan.java index 6b82df1d67da..12f54270b65d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThan.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThan.java @@ -8,44 +8,38 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Evaluator; -import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; +import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.predicate.Negatable; +import org.elasticsearch.xpack.ql.expression.TypeResolutions; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; -import org.elasticsearch.xpack.ql.type.DataType; -import org.elasticsearch.xpack.ql.type.DataTypes; import java.time.ZoneId; -import java.util.Map; -public class LessThan extends EsqlBinaryComparison implements Negatable { +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; - private static final Map evaluatorMap = Map.ofEntries( - Map.entry(DataTypes.INTEGER, LessThanIntsEvaluator.Factory::new), - Map.entry(DataTypes.DOUBLE, LessThanDoublesEvaluator.Factory::new), - Map.entry(DataTypes.LONG, LessThanLongsEvaluator.Factory::new), - Map.entry(DataTypes.UNSIGNED_LONG, LessThanLongsEvaluator.Factory::new), - Map.entry(DataTypes.DATETIME, LessThanLongsEvaluator.Factory::new), - Map.entry(DataTypes.KEYWORD, LessThanKeywordsEvaluator.Factory::new), - Map.entry(DataTypes.TEXT, LessThanKeywordsEvaluator.Factory::new), - Map.entry(DataTypes.VERSION, LessThanKeywordsEvaluator.Factory::new), - Map.entry(DataTypes.IP, LessThanKeywordsEvaluator.Factory::new) - ); +public class LessThan extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan { public LessThan(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, BinaryComparisonProcessor.BinaryComparisonOperation.LT, zoneId, evaluatorMap); + super(source, left, right, zoneId); } @Override - protected NodeInfo info() { + protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { + return EsqlTypeResolutions.isExact(e, sourceText(), DEFAULT); + } + + @Override + protected NodeInfo info() { return NodeInfo.create(this, LessThan::new, left(), right(), zoneId()); } @Override - protected LessThan replaceChildren(Expression newLeft, Expression newRight) { + protected org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan replaceChildren( + Expression newLeft, + Expression newRight + ) { return new LessThan(source(), newLeft, newRight, zoneId()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqual.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqual.java index ac6a92aaf097..e75733a9e234 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqual.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqual.java @@ -8,38 +8,29 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Evaluator; -import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; +import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.predicate.Negatable; +import org.elasticsearch.xpack.ql.expression.TypeResolutions; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; -import org.elasticsearch.xpack.ql.type.DataType; -import org.elasticsearch.xpack.ql.type.DataTypes; import java.time.ZoneId; -import java.util.Map; -public class LessThanOrEqual extends EsqlBinaryComparison implements Negatable { - private static final Map evaluatorMap = Map.ofEntries( - Map.entry(DataTypes.INTEGER, LessThanOrEqualIntsEvaluator.Factory::new), - Map.entry(DataTypes.DOUBLE, LessThanOrEqualDoublesEvaluator.Factory::new), - Map.entry(DataTypes.LONG, LessThanOrEqualLongsEvaluator.Factory::new), - Map.entry(DataTypes.UNSIGNED_LONG, LessThanOrEqualLongsEvaluator.Factory::new), - Map.entry(DataTypes.DATETIME, LessThanOrEqualLongsEvaluator.Factory::new), - Map.entry(DataTypes.KEYWORD, LessThanOrEqualKeywordsEvaluator.Factory::new), - Map.entry(DataTypes.TEXT, LessThanOrEqualKeywordsEvaluator.Factory::new), - Map.entry(DataTypes.VERSION, LessThanOrEqualKeywordsEvaluator.Factory::new), - Map.entry(DataTypes.IP, LessThanOrEqualKeywordsEvaluator.Factory::new) - ); +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; +public class LessThanOrEqual extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThanOrEqual { public LessThanOrEqual(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, BinaryComparisonProcessor.BinaryComparisonOperation.LTE, zoneId, evaluatorMap); + super(source, left, right, zoneId); } @Override - protected NodeInfo info() { + protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { + return EsqlTypeResolutions.isExact(e, sourceText(), DEFAULT); + } + + @Override + protected NodeInfo info() { return NodeInfo.create(this, LessThanOrEqual::new, left(), right(), zoneId()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEquals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEquals.java index 9c931ec7433e..6fbed572cdc0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEquals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEquals.java @@ -8,44 +8,45 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Evaluator; -import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; +import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.predicate.Negatable; +import org.elasticsearch.xpack.ql.expression.TypeResolutions; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; -import org.elasticsearch.xpack.ql.type.DataType; -import org.elasticsearch.xpack.ql.type.DataTypes; import java.time.ZoneId; -import java.util.Map; -public class NotEquals extends EsqlBinaryComparison implements Negatable { - private static final Map evaluatorMap = Map.ofEntries( - Map.entry(DataTypes.BOOLEAN, NotEqualsBoolsEvaluator.Factory::new), - Map.entry(DataTypes.INTEGER, NotEqualsIntsEvaluator.Factory::new), - Map.entry(DataTypes.DOUBLE, NotEqualsDoublesEvaluator.Factory::new), - Map.entry(DataTypes.LONG, NotEqualsLongsEvaluator.Factory::new), - Map.entry(DataTypes.UNSIGNED_LONG, NotEqualsLongsEvaluator.Factory::new), - Map.entry(DataTypes.DATETIME, NotEqualsLongsEvaluator.Factory::new), - Map.entry(EsqlDataTypes.GEO_POINT, NotEqualsGeometriesEvaluator.Factory::new), - Map.entry(EsqlDataTypes.CARTESIAN_POINT, NotEqualsGeometriesEvaluator.Factory::new), - Map.entry(EsqlDataTypes.GEO_SHAPE, NotEqualsGeometriesEvaluator.Factory::new), - Map.entry(EsqlDataTypes.CARTESIAN_SHAPE, NotEqualsGeometriesEvaluator.Factory::new), - Map.entry(DataTypes.KEYWORD, NotEqualsKeywordsEvaluator.Factory::new), - Map.entry(DataTypes.TEXT, NotEqualsKeywordsEvaluator.Factory::new), - Map.entry(DataTypes.VERSION, NotEqualsKeywordsEvaluator.Factory::new), - Map.entry(DataTypes.IP, NotEqualsKeywordsEvaluator.Factory::new) - ); +import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; - public NotEquals(Source source, Expression left, Expression right) { - super(source, left, right, BinaryComparisonProcessor.BinaryComparisonOperation.NEQ, evaluatorMap); +public class NotEquals extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals { + public NotEquals(Source source, Expression left, Expression right, ZoneId zoneId) { + super(source, left, right, zoneId); } - public NotEquals(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, BinaryComparisonProcessor.BinaryComparisonOperation.NEQ, zoneId, evaluatorMap); + @Override + protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { + return EsqlTypeResolutions.isExact(e, sourceText(), DEFAULT); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, NotEquals::new, left(), right(), zoneId()); + } + + @Override + protected NotEquals replaceChildren(Expression newLeft, Expression newRight) { + return new NotEquals(source(), newLeft, newRight, zoneId()); + } + + @Override + public NotEquals swapLeftAndRight() { + return new NotEquals(source(), right(), left(), zoneId()); + } + + @Override + public BinaryComparison negate() { + return new Equals(source(), left(), right(), zoneId()); } @Evaluator(extraName = "Ints") @@ -77,29 +78,4 @@ static boolean processBools(boolean lhs, boolean rhs) { static boolean processGeometries(BytesRef lhs, BytesRef rhs) { return false == lhs.equals(rhs); } - - @Override - public BinaryComparison reverse() { - return this; - } - - @Override - protected NodeInfo info() { - return NodeInfo.create(this, NotEquals::new, left(), right(), zoneId()); - } - - @Override - protected NotEquals replaceChildren(Expression newLeft, Expression newRight) { - return new NotEquals(source(), newLeft, newRight, zoneId()); - } - - @Override - public NotEquals swapLeftAndRight() { - return new NotEquals(source(), right(), left(), zoneId()); - } - - @Override - public BinaryComparison negate() { - return new Equals(source(), left(), right(), zoneId()); - } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java index a45707a0197d..0f550862ed1f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java @@ -43,10 +43,10 @@ interface DatetimeArithmeticEvaluator { Expression left, Expression right, OperationSymbol op, - BinaryEvaluator ints, - BinaryEvaluator longs, - BinaryEvaluator ulongs, - BinaryEvaluator doubles, + ArithmeticEvaluator ints, + ArithmeticEvaluator longs, + ArithmeticEvaluator ulongs, + ArithmeticEvaluator doubles, DatetimeArithmeticEvaluator datetimes ) { super(source, left, right, op, ints, longs, ulongs, doubles); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java index ba283bc4d877..22f5798e5b1c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java @@ -71,15 +71,14 @@ public String symbol() { } /** Arithmetic (quad) function. */ - @FunctionalInterface - public interface BinaryEvaluator { + interface ArithmeticEvaluator { ExpressionEvaluator.Factory apply(Source source, ExpressionEvaluator.Factory lhs, ExpressionEvaluator.Factory rhs); } - private final BinaryEvaluator ints; - private final BinaryEvaluator longs; - private final BinaryEvaluator ulongs; - private final BinaryEvaluator doubles; + private final ArithmeticEvaluator ints; + private final ArithmeticEvaluator longs; + private final ArithmeticEvaluator ulongs; + private final ArithmeticEvaluator doubles; private DataType dataType; @@ -88,10 +87,10 @@ public interface BinaryEvaluator { Expression left, Expression right, OperationSymbol op, - BinaryEvaluator ints, - BinaryEvaluator longs, - BinaryEvaluator ulongs, - BinaryEvaluator doubles + ArithmeticEvaluator ints, + ArithmeticEvaluator longs, + ArithmeticEvaluator ulongs, + ArithmeticEvaluator doubles ) { super(source, left, right, op); this.ints = ints; @@ -140,7 +139,7 @@ protected TypeResolution checkCompatibility() { return TypeResolution.TYPE_RESOLVED; } - public static String formatIncompatibleTypesMessage(String symbol, DataType leftType, DataType rightType) { + static String formatIncompatibleTypesMessage(String symbol, DataType leftType, DataType rightType) { return format(null, "[{}] has arguments with incompatible types [{}] and [{}]", symbol, leftType.typeName(), rightType.typeName()); } @@ -153,7 +152,7 @@ public ExpressionEvaluator.Factory toEvaluator(Function errorsForCasesWithoutExamples( return suppliers; } - public static String errorMessageStringForBinaryOperators( - boolean includeOrdinal, - List> validPerPosition, - List types - ) { - try { - return typeErrorMessage(includeOrdinal, validPerPosition, types); - } catch (IllegalStateException e) { - // This means all the positional args were okay, so the expected error is from the combination - if (types.get(0).equals(DataTypes.UNSIGNED_LONG)) { - return "first argument of [] is [unsigned_long] and second is [" - + types.get(1).typeName() - + "]. [unsigned_long] can only be operated on together with another [unsigned_long]"; - - } - if (types.get(1).equals(DataTypes.UNSIGNED_LONG)) { - return "first argument of [] is [" - + types.get(0).typeName() - + "] and second is [unsigned_long]. [unsigned_long] can only be operated on together with another [unsigned_long]"; - } - return "first argument of [] is [" - + (types.get(0).isNumeric() ? "numeric" : types.get(0).typeName()) - + "] so second argument must also be [" - + (types.get(0).isNumeric() ? "numeric" : types.get(0).typeName()) - + "] but was [" - + types.get(1).typeName() - + "]"; - - } - } - /** * Adds test cases containing unsupported parameter types that immediately fail. */ @@ -962,24 +931,6 @@ protected static String typeErrorMessage(boolean includeOrdinal, List types) { return types.stream().map(t -> "<" + t.typeName() + ">").collect(Collectors.joining(", ")); } - public static List stringCases( - BinaryOperator expected, - BiFunction evaluatorToString, - List warnings, - DataType expectedType - ) { - List lhsSuppliers = new ArrayList<>(); - List rhsSuppliers = new ArrayList<>(); - List suppliers = new ArrayList<>(); - for (DataType type : AbstractConvertFunction.STRING_TYPES) { - lhsSuppliers.addAll(stringCases(type)); - rhsSuppliers.addAll(stringCases(type)); - casesCrossProduct(expected, lhsSuppliers, rhsSuppliers, evaluatorToString, warnings, suppliers, expectedType, true); - } - return suppliers; - } - @Override public TestCase get() { TestCase supplied = supplier.get(); @@ -275,14 +258,14 @@ public static List castToDoubleSuppliersFromRange(Double Min, return suppliers; } - public record NumericTypeTestConfig(Number min, Number max, BiFunction expected, String evaluatorName) {} + public record NumericTypeTestConfig(Number min, Number max, BinaryOperator expected, String evaluatorName) {} - public record NumericTypeTestConfigs( - NumericTypeTestConfig intStuff, - NumericTypeTestConfig longStuff, - NumericTypeTestConfig doubleStuff + public record NumericTypeTestConfigs( + NumericTypeTestConfig intStuff, + NumericTypeTestConfig longStuff, + NumericTypeTestConfig doubleStuff ) { - public NumericTypeTestConfig get(DataType type) { + public NumericTypeTestConfig get(DataType type) { if (type == DataTypes.INTEGER) { return intStuff; } @@ -329,8 +312,8 @@ public static List getSuppliersForNumericType(DataType type, throw new IllegalArgumentException("bogus numeric type [" + type + "]"); } - public static List forBinaryComparisonWithWidening( - NumericTypeTestConfigs typeStuff, + public static List forBinaryWithWidening( + NumericTypeTestConfigs typeStuff, String lhsName, String rhsName, BiFunction> warnings, @@ -342,45 +325,7 @@ public static List forBinaryComparisonWithWidening( for (DataType lhsType : numericTypes) { for (DataType rhsType : numericTypes) { DataType expected = widen(lhsType, rhsType); - NumericTypeTestConfig expectedTypeStuff = typeStuff.get(expected); - BiFunction evaluatorToString = (lhs, rhs) -> expectedTypeStuff.evaluatorName() - + "[" - + lhsName - + "=" - + getCastEvaluator("Attribute[channel=0]", lhs, expected) - + ", " - + rhsName - + "=" - + getCastEvaluator("Attribute[channel=1]", rhs, expected) - + "]"; - casesCrossProduct( - (l, r) -> expectedTypeStuff.expected().apply((Number) l, (Number) r), - getSuppliersForNumericType(lhsType, expectedTypeStuff.min(), expectedTypeStuff.max()), - getSuppliersForNumericType(rhsType, expectedTypeStuff.min(), expectedTypeStuff.max()), - evaluatorToString, - warnings, - suppliers, - DataTypes.BOOLEAN, - true - ); - } - } - return suppliers; - } - - public static List forBinaryWithWidening( - NumericTypeTestConfigs typeStuff, - String lhsName, - String rhsName, - List warnings - ) { - List suppliers = new ArrayList<>(); - List numericTypes = List.of(DataTypes.INTEGER, DataTypes.LONG, DataTypes.DOUBLE); - - for (DataType lhsType : numericTypes) { - for (DataType rhsType : numericTypes) { - DataType expected = widen(lhsType, rhsType); - NumericTypeTestConfig expectedTypeStuff = typeStuff.get(expected); + NumericTypeTestConfig expectedTypeStuff = typeStuff.get(expected); BiFunction evaluatorToString = (lhs, rhs) -> expectedTypeStuff.evaluatorName() + "[" + lhsName @@ -940,7 +885,7 @@ public static List doubleCases(double min, double max, boolea return cases; } - public static List booleanCases() { + private static List booleanCases() { return List.of( new TypedDataSupplier("", () -> true, DataTypes.BOOLEAN), new TypedDataSupplier("", () -> false, DataTypes.BOOLEAN) @@ -1322,14 +1267,9 @@ public Matcher evaluatorToString() { * exists because we can't generate random values from the test parameter generation functions, and instead need to return * suppliers which generate the random values at test execution time. */ - public record TypedDataSupplier(String name, Supplier supplier, DataType type, boolean forceLiteral) { - - public TypedDataSupplier(String name, Supplier supplier, DataType type) { - this(name, supplier, type, false); - } - + public record TypedDataSupplier(String name, Supplier supplier, DataType type) { public TypedData get() { - return new TypedData(supplier.get(), type, name, forceLiteral); + return new TypedData(supplier.get(), type, name); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java index 2596959c449d..c40d037890d5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java @@ -43,20 +43,20 @@ public static Iterable parameters() { List suppliers = new ArrayList<>(); suppliers.addAll( TestCaseSupplier.forBinaryWithWidening( - new TestCaseSupplier.NumericTypeTestConfigs( - new TestCaseSupplier.NumericTypeTestConfig<>( + new TestCaseSupplier.NumericTypeTestConfigs( + new TestCaseSupplier.NumericTypeTestConfig( (Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1, (l, r) -> l.intValue() + r.intValue(), "AddIntsEvaluator" ), - new TestCaseSupplier.NumericTypeTestConfig<>( + new TestCaseSupplier.NumericTypeTestConfig( (Long.MIN_VALUE >> 1) - 1, (Long.MAX_VALUE >> 1) - 1, (l, r) -> l.longValue() + r.longValue(), "AddLongsEvaluator" ), - new TestCaseSupplier.NumericTypeTestConfig<>( + new TestCaseSupplier.NumericTypeTestConfig( Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, (l, r) -> l.doubleValue() + r.doubleValue(), diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java index 6fcc4235f5b7..0a1e9bdfaf34 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java @@ -11,188 +11,52 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.Equals; -import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; -import org.elasticsearch.xpack.ql.util.NumericUtils; +import org.hamcrest.Matcher; -import java.math.BigInteger; -import java.util.ArrayList; import java.util.List; import java.util.function.Supplier; -public class EqualsTests extends AbstractFunctionTestCase { +import static org.hamcrest.Matchers.equalTo; + +public class EqualsTests extends AbstractBinaryComparisonTestCase { public EqualsTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); } @ParametersFactory public static Iterable parameters() { - List suppliers = new ArrayList<>(); - suppliers.addAll( - TestCaseSupplier.forBinaryComparisonWithWidening( - new TestCaseSupplier.NumericTypeTestConfigs<>( - new TestCaseSupplier.NumericTypeTestConfig<>( - (Integer.MIN_VALUE >> 1) - 1, - (Integer.MAX_VALUE >> 1) - 1, - (l, r) -> l.intValue() == r.intValue(), - "EqualsIntsEvaluator" - ), - new TestCaseSupplier.NumericTypeTestConfig<>( - (Long.MIN_VALUE >> 1) - 1, - (Long.MAX_VALUE >> 1) - 1, - (l, r) -> l.longValue() == r.longValue(), - "EqualsLongsEvaluator" - ), - new TestCaseSupplier.NumericTypeTestConfig<>( - Double.NEGATIVE_INFINITY, - Double.POSITIVE_INFINITY, - // NB: this has different behavior than Double::equals - (l, r) -> l.doubleValue() == r.doubleValue(), - "EqualsDoublesEvaluator" - ) + return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("Int == Int", () -> { + int rhs = randomInt(); + int lhs = randomInt(); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(lhs, DataTypes.INTEGER, "lhs"), + new TestCaseSupplier.TypedData(rhs, DataTypes.INTEGER, "rhs") ), - "lhs", - "rhs", - List.of() - ) - ); - - // Unsigned Long cases - // TODO: These should be integrated into the type cross product above, but are currently broken - // see https://github.com/elastic/elasticsearch/issues/102935 - suppliers.addAll( - TestCaseSupplier.forBinaryNotCasting( - "EqualsLongsEvaluator", - "lhs", - "rhs", - Object::equals, - DataTypes.BOOLEAN, - TestCaseSupplier.ulongCases(BigInteger.ZERO, NumericUtils.UNSIGNED_LONG_MAX), - TestCaseSupplier.ulongCases(BigInteger.ZERO, NumericUtils.UNSIGNED_LONG_MAX), - List.of() - ) - ); - suppliers.addAll( - TestCaseSupplier.forBinaryNotCasting( - "EqualsBoolsEvaluator", - "lhs", - "rhs", - Object::equals, - DataTypes.BOOLEAN, - TestCaseSupplier.booleanCases(), - TestCaseSupplier.booleanCases(), - List.of() - ) - ); - suppliers.addAll( - TestCaseSupplier.forBinaryNotCasting( - "EqualsKeywordsEvaluator", - "lhs", - "rhs", - Object::equals, - DataTypes.BOOLEAN, - TestCaseSupplier.ipCases(), - TestCaseSupplier.ipCases(), - List.of() - ) - ); - suppliers.addAll( - TestCaseSupplier.forBinaryNotCasting( - "EqualsKeywordsEvaluator", - "lhs", - "rhs", - Object::equals, - DataTypes.BOOLEAN, - TestCaseSupplier.versionCases(""), - TestCaseSupplier.versionCases(""), - List.of() - ) - ); - // Datetime - // TODO: I'm surprised this passes. Shouldn't there be a cast from DateTime to Long? - suppliers.addAll( - TestCaseSupplier.forBinaryNotCasting( - "EqualsLongsEvaluator", - "lhs", - "rhs", - Object::equals, - DataTypes.BOOLEAN, - TestCaseSupplier.dateCases(), - TestCaseSupplier.dateCases(), - List.of() - ) - ); - - suppliers.addAll( - TestCaseSupplier.stringCases( - Object::equals, - (lhsType, rhsType) -> "EqualsKeywordsEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]", - List.of(), - DataTypes.BOOLEAN - ) - ); - - suppliers.addAll( - TestCaseSupplier.forBinaryNotCasting( - "EqualsGeometriesEvaluator", - "lhs", - "rhs", - Object::equals, + "EqualsIntsEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]", DataTypes.BOOLEAN, - TestCaseSupplier.geoPointCases(), - TestCaseSupplier.geoPointCases(), - List.of() - ) - ); - - suppliers.addAll( - TestCaseSupplier.forBinaryNotCasting( - "EqualsGeometriesEvaluator", - "lhs", - "rhs", - Object::equals, - DataTypes.BOOLEAN, - TestCaseSupplier.geoShapeCases(), - TestCaseSupplier.geoShapeCases(), - List.of() - ) - ); - suppliers.addAll( - TestCaseSupplier.forBinaryNotCasting( - "EqualsGeometriesEvaluator", - "lhs", - "rhs", - Object::equals, - DataTypes.BOOLEAN, - TestCaseSupplier.cartesianPointCases(), - TestCaseSupplier.cartesianPointCases(), - List.of() - ) - ); + equalTo(lhs == rhs) + ); + }))); + } - suppliers.addAll( - TestCaseSupplier.forBinaryNotCasting( - "EqualsGeometriesEvaluator", - "lhs", - "rhs", - Object::equals, - DataTypes.BOOLEAN, - TestCaseSupplier.cartesianShapeCases(), - TestCaseSupplier.cartesianShapeCases(), - List.of() - ) - ); + @Override + protected > Matcher resultMatcher(T lhs, T rhs) { + return equalTo(lhs.equals(rhs)); + } - return parameterSuppliersFromTypedData( - errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), AbstractFunctionTestCase::errorMessageStringForBinaryOperators) - ); + @Override + protected BinaryComparison build(Source source, Expression lhs, Expression rhs) { + return new Equals(source, lhs, rhs); } @Override - protected Expression build(Source source, List args) { - return new Equals(source, args.get(0), args.get(1)); + protected boolean isEquality() { + return true; } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java index f45dedff837c..ad8dba7d6306 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java @@ -11,25 +11,26 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.GreaterThanOrEqual; -import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; +import org.hamcrest.Matcher; +import java.time.ZoneOffset; import java.util.List; import java.util.function.Supplier; import static org.hamcrest.Matchers.equalTo; -public class GreaterThanOrEqualTests extends AbstractFunctionTestCase { +public class GreaterThanOrEqualTests extends AbstractBinaryComparisonTestCase { public GreaterThanOrEqualTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); } @ParametersFactory public static Iterable parameters() { - // ToDo: Add the full set of typed test cases here return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("Int >= Int", () -> { int rhs = randomInt(); int lhs = randomInt(); @@ -46,7 +47,17 @@ public static Iterable parameters() { } @Override - protected Expression build(Source source, List args) { - return new GreaterThanOrEqual(source, args.get(0), args.get(1)); + protected > Matcher resultMatcher(T lhs, T rhs) { + return equalTo(lhs.compareTo(rhs) >= 0); + } + + @Override + protected BinaryComparison build(Source source, Expression lhs, Expression rhs) { + return new GreaterThanOrEqual(source, lhs, rhs, ZoneOffset.UTC); + } + + @Override + protected boolean isEquality() { + return false; } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java index e872af5b7c77..b631a742f788 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java @@ -11,25 +11,26 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.GreaterThan; -import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; +import org.hamcrest.Matcher; +import java.time.ZoneOffset; import java.util.List; import java.util.function.Supplier; import static org.hamcrest.Matchers.equalTo; -public class GreaterThanTests extends AbstractFunctionTestCase { +public class GreaterThanTests extends AbstractBinaryComparisonTestCase { public GreaterThanTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); } @ParametersFactory public static Iterable parameters() { - // ToDo: Add the full set of typed test cases here return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("Int > Int", () -> { int rhs = randomInt(); int lhs = randomInt(); @@ -46,7 +47,17 @@ public static Iterable parameters() { } @Override - protected Expression build(Source source, List args) { - return new GreaterThan(source, args.get(0), args.get(1)); + protected > Matcher resultMatcher(T lhs, T rhs) { + return equalTo(lhs.compareTo(rhs) > 0); + } + + @Override + protected BinaryComparison build(Source source, Expression lhs, Expression rhs) { + return new GreaterThan(source, lhs, rhs, ZoneOffset.UTC); + } + + @Override + protected boolean isEquality() { + return false; } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java index 8bba0c4a5afb..7864a0dda9fe 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java @@ -11,18 +11,20 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.LessThanOrEqual; -import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; +import org.hamcrest.Matcher; +import java.time.ZoneOffset; import java.util.List; import java.util.function.Supplier; import static org.hamcrest.Matchers.equalTo; -public class LessThanOrEqualTests extends AbstractFunctionTestCase { +public class LessThanOrEqualTests extends AbstractBinaryComparisonTestCase { public LessThanOrEqualTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); } @@ -45,7 +47,17 @@ public static Iterable parameters() { } @Override - protected Expression build(Source source, List args) { - return new LessThanOrEqual(source, args.get(0), args.get(1), null); + protected > Matcher resultMatcher(T lhs, T rhs) { + return equalTo(lhs.compareTo(rhs) <= 0); + } + + @Override + protected BinaryComparison build(Source source, Expression lhs, Expression rhs) { + return new LessThanOrEqual(source, lhs, rhs, ZoneOffset.UTC); + } + + @Override + protected boolean isEquality() { + return false; } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java index ab726dc51fbe..826e88551077 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java @@ -11,18 +11,20 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.LessThan; -import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; +import org.hamcrest.Matcher; +import java.time.ZoneOffset; import java.util.List; import java.util.function.Supplier; import static org.hamcrest.Matchers.equalTo; -public class LessThanTests extends AbstractFunctionTestCase { +public class LessThanTests extends AbstractBinaryComparisonTestCase { public LessThanTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); } @@ -45,7 +47,17 @@ public static Iterable parameters() { } @Override - protected Expression build(Source source, List args) { - return new LessThan(source, args.get(0), args.get(1), null); + protected > Matcher resultMatcher(T lhs, T rhs) { + return equalTo(lhs.compareTo(rhs) < 0); + } + + @Override + protected BinaryComparison build(Source source, Expression lhs, Expression rhs) { + return new LessThan(source, lhs, rhs, ZoneOffset.UTC); + } + + @Override + protected boolean isEquality() { + return false; } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java index d6ee5806e0c9..0d6bb32fe248 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java @@ -11,182 +11,53 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.NotEquals; -import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; +import org.hamcrest.Matcher; -import java.math.BigInteger; -import java.util.ArrayList; +import java.time.ZoneOffset; import java.util.List; import java.util.function.Supplier; -public class NotEqualsTests extends AbstractFunctionTestCase { +import static org.hamcrest.Matchers.equalTo; + +public class NotEqualsTests extends AbstractBinaryComparisonTestCase { public NotEqualsTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); } @ParametersFactory public static Iterable parameters() { - List suppliers = new ArrayList<>(); - suppliers.addAll( - - TestCaseSupplier.forBinaryComparisonWithWidening( - new TestCaseSupplier.NumericTypeTestConfigs<>( - new TestCaseSupplier.NumericTypeTestConfig<>( - (Integer.MIN_VALUE >> 1) - 1, - (Integer.MAX_VALUE >> 1) - 1, - (l, r) -> l.intValue() != r.intValue(), - "NotEqualsIntsEvaluator" - ), - new TestCaseSupplier.NumericTypeTestConfig<>( - (Long.MIN_VALUE >> 1) - 1, - (Long.MAX_VALUE >> 1) - 1, - (l, r) -> l.longValue() != r.longValue(), - "NotEqualsLongsEvaluator" - ), - new TestCaseSupplier.NumericTypeTestConfig<>( - Double.NEGATIVE_INFINITY, - Double.POSITIVE_INFINITY, - // NB: this has different behavior than Double::equals - (l, r) -> l.doubleValue() != r.doubleValue(), - "NotEqualsDoublesEvaluator" - ) + return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("Int != Int", () -> { + int rhs = randomInt(); + int lhs = randomInt(); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(lhs, DataTypes.INTEGER, "lhs"), + new TestCaseSupplier.TypedData(rhs, DataTypes.INTEGER, "rhs") ), - "lhs", - "rhs", - List.of() - ) - ); - // Unsigned Long cases - // TODO: These should be integrated into the type cross product above, but are currently broken - // see https://github.com/elastic/elasticsearch/issues/102935 - suppliers.addAll( - TestCaseSupplier.forBinaryNotCasting( - "NotEqualsLongsEvaluator", - "lhs", - "rhs", - (l, r) -> false == l.equals(r), - DataTypes.BOOLEAN, - TestCaseSupplier.ulongCases(BigInteger.ZERO, BigInteger.valueOf(Long.MAX_VALUE)), - TestCaseSupplier.ulongCases(BigInteger.ZERO, BigInteger.valueOf(Long.MAX_VALUE)), - List.of() - ) - ); - suppliers.addAll( - TestCaseSupplier.forBinaryNotCasting( - "NotEqualsBoolsEvaluator", - "lhs", - "rhs", - (l, r) -> false == l.equals(r), - DataTypes.BOOLEAN, - TestCaseSupplier.booleanCases(), - TestCaseSupplier.booleanCases(), - List.of() - ) - ); - suppliers.addAll( - TestCaseSupplier.forBinaryNotCasting( - "NotEqualsKeywordsEvaluator", - "lhs", - "rhs", - (l, r) -> false == l.equals(r), - DataTypes.BOOLEAN, - TestCaseSupplier.ipCases(), - TestCaseSupplier.ipCases(), - List.of() - ) - ); - suppliers.addAll( - TestCaseSupplier.forBinaryNotCasting( - "NotEqualsKeywordsEvaluator", - "lhs", - "rhs", - (l, r) -> false == l.equals(r), - DataTypes.BOOLEAN, - TestCaseSupplier.versionCases(""), - TestCaseSupplier.versionCases(""), - List.of() - ) - ); - // Datetime - // TODO: I'm surprised this passes. Shouldn't there be a cast from DateTime to Long? - suppliers.addAll( - TestCaseSupplier.forBinaryNotCasting( - "NotEqualsLongsEvaluator", - "lhs", - "rhs", - (l, r) -> false == l.equals(r), - DataTypes.BOOLEAN, - TestCaseSupplier.dateCases(), - TestCaseSupplier.dateCases(), - List.of() - ) - ); - suppliers.addAll( - TestCaseSupplier.stringCases( - (l, r) -> false == l.equals(r), - (lhsType, rhsType) -> "NotEqualsKeywordsEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]", - List.of(), - DataTypes.BOOLEAN - ) - ); - suppliers.addAll( - TestCaseSupplier.forBinaryNotCasting( - "NotEqualsGeometriesEvaluator", - "lhs", - "rhs", - (l, r) -> false == l.equals(r), - DataTypes.BOOLEAN, - TestCaseSupplier.geoPointCases(), - TestCaseSupplier.geoPointCases(), - List.of() - ) - ); - suppliers.addAll( - TestCaseSupplier.forBinaryNotCasting( - "NotEqualsGeometriesEvaluator", - "lhs", - "rhs", - (l, r) -> false == l.equals(r), + "NotEqualsIntsEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]", DataTypes.BOOLEAN, - TestCaseSupplier.geoShapeCases(), - TestCaseSupplier.geoShapeCases(), - List.of() - ) - ); - suppliers.addAll( - TestCaseSupplier.forBinaryNotCasting( - "NotEqualsGeometriesEvaluator", - "lhs", - "rhs", - (l, r) -> false == l.equals(r), - DataTypes.BOOLEAN, - TestCaseSupplier.cartesianPointCases(), - TestCaseSupplier.cartesianPointCases(), - List.of() - ) - ); - suppliers.addAll( - TestCaseSupplier.forBinaryNotCasting( - "NotEqualsGeometriesEvaluator", - "lhs", - "rhs", - (l, r) -> false == l.equals(r), - DataTypes.BOOLEAN, - TestCaseSupplier.cartesianShapeCases(), - TestCaseSupplier.cartesianShapeCases(), - List.of() - ) - ); - return parameterSuppliersFromTypedData( - errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), AbstractFunctionTestCase::errorMessageStringForBinaryOperators) - ); + equalTo(lhs != rhs) + ); + }))); + } + + @Override + protected > Matcher resultMatcher(T lhs, T rhs) { + return equalTo(false == lhs.equals(rhs)); + } + + @Override + protected BinaryComparison build(Source source, Expression lhs, Expression rhs) { + return new NotEquals(source, lhs, rhs, ZoneOffset.UTC); } @Override - protected Expression build(Source source, List args) { - return new NotEquals(source, args.get(0), args.get(1)); + protected boolean isEquality() { + return true; } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java index 28944252191b..01fcd222a514 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java @@ -495,7 +495,7 @@ public void testPropagateEquals_VarEq2OrVarNeq5() { // a = 2 OR 3 < a < 4 OR a > 2 OR a!= 2 -> TRUE public void testPropagateEquals_VarEq2OrVarRangeGt3Lt4OrVarGt2OrVarNe2() { FieldAttribute fa = getFieldAttribute(); - Equals eq = equalsOf(fa, TWO); + org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals eq = equalsOf(fa, TWO); Range range = rangeOf(fa, THREE, false, FOUR, false); GreaterThan gt = greaterThanOf(fa, TWO); NotEquals neq = notEqualsOf(fa, TWO); From 624a5b1fe52c7160f9a2cb4f892a947047ee6bdf Mon Sep 17 00:00:00 2001 From: "Mark J. Hoy" Date: Tue, 16 Apr 2024 10:52:27 -0400 Subject: [PATCH 05/43] Add Docs for Azure OpenAI Embeddings Inference (#107498) * Update docs for Azure OpenAI Embeddings inference * cleanups * update link for dot_product similarity * final cleanups --- docs/changelog/107178.yaml | 5 ++ .../inference/put-inference.asciidoc | 79 ++++++++++++++++++- .../infer-api-ingest-pipeline-widget.asciidoc | 19 ++++- .../infer-api-ingest-pipeline.asciidoc | 28 ++++++- .../infer-api-mapping-widget.asciidoc | 19 ++++- .../inference-api/infer-api-mapping.asciidoc | 43 +++++++++- .../infer-api-reindex-widget.asciidoc | 20 ++++- .../inference-api/infer-api-reindex.asciidoc | 30 ++++++- .../infer-api-requirements-widget.asciidoc | 19 ++++- .../infer-api-requirements.asciidoc | 10 ++- .../infer-api-search-widget.asciidoc | 19 ++++- .../inference-api/infer-api-search.asciidoc | 67 +++++++++++++++- .../infer-api-task-widget.asciidoc | 19 ++++- .../inference-api/infer-api-task.asciidoc | 37 ++++++++- 14 files changed, 394 insertions(+), 20 deletions(-) create mode 100644 docs/changelog/107178.yaml diff --git a/docs/changelog/107178.yaml b/docs/changelog/107178.yaml new file mode 100644 index 000000000000..94a91357d38e --- /dev/null +++ b/docs/changelog/107178.yaml @@ -0,0 +1,5 @@ +pr: 107178 +summary: "Add support for Azure OpenAI embeddings to inference service" +area: Machine Learning +type: feature +issues: [ ] diff --git a/docs/reference/inference/put-inference.asciidoc b/docs/reference/inference/put-inference.asciidoc index 332752e52f06..1f73cd08401e 100644 --- a/docs/reference/inference/put-inference.asciidoc +++ b/docs/reference/inference/put-inference.asciidoc @@ -7,8 +7,8 @@ experimental[] Creates an {infer} endpoint to perform an {infer} task. IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in -{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, or -Hugging Face. For built-in models and models uploaded though +{ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure +OpenAI or Hugging Face. For built-in models and models uploaded though Eland, the {infer} APIs offer an alternative way to use and manage trained models. However, if you do not plan to use the {infer} APIs to use these models or if you want to use non-NLP models, use the <>. @@ -42,6 +42,7 @@ The following services are available through the {infer} API: * ELSER * Hugging Face * OpenAI +* Azure OpenAI * Elasticsearch (for built-in models and models uploaded through Eland) @@ -78,6 +79,7 @@ Cohere service. service. * `openai`: specify the `completion` or `text_embedding` task type to use the OpenAI service. +* `azureopenai`: specify the `text_embedding` task type to use the Azure OpenAI service. * `elasticsearch`: specify the `text_embedding` task type to use the E5 built-in model or text embedding models uploaded by Eland. @@ -187,6 +189,41 @@ https://platform.openai.com/account/organization[**Settings** > **Organizations* (Optional, string) The URL endpoint to use for the requests. Can be changed for testing purposes. Defaults to `https://api.openai.com/v1/embeddings`. + +===== ++ +.`service_settings` for the `azureopenai` service +[%collapsible%closed] +===== + +`api_key` or `entra_id`::: +(Required, string) +You must provide _either_ an API key or an Entra ID. +If you do not provide either, or provide both, you will receive an error when trying to create your model. +See the https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#authentication[Azure OpenAI Authentication documentation] for more details on these authentication types. + +IMPORTANT: You need to provide the API key or Entra ID only once, during the {infer} model creation. +The <> does not retrieve your authentication credentials. +After creating the {infer} model, you cannot change the associated API key or Entra ID. +If you want to use a different API key or Entra ID, delete the {infer} model and recreate it with the same name and the updated API key. +You _must_ have either an `api_key` or an `entra_id` defined. +If neither are present, an error will occur. + +`resource_name`::: +(Required, string) +The name of your Azure OpenAI resource. +You can find this from the https://portal.azure.com/#view/HubsExtension/BrowseAll[list of resources] in the Azure Portal for your subscription. + +`deployment_id`::: +(Required, string) +The deployment name of your deployed models. +Your Azure OpenAI deployments can be found though the https://oai.azure.com/[Azure OpenAI Studio] portal that is linked to your subscription. + +`api_version`::: +(Required, string) +The Azure API version ID to use. +We recommend using the https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#embeddings[latest supported non-preview version]. + ===== + .`service_settings` for the `elasticsearch` service @@ -266,8 +303,17 @@ maximum token length. Defaults to `END`. Valid values are: `user`::: (optional, string) -For `openai` service only. Specifies the user issuing the request, which can be -used for abuse detection. +For `openai` and `azureopenai` service only. Specifies the user issuing the +request, which can be used for abuse detection. + +===== ++ +.`task_settings` for the `completion` task type +[%collapsible%closed] +===== +`user`::: +(optional, string) +For `openai` service only. Specifies the user issuing the request, which can be used for abuse detection. ===== @@ -491,3 +537,28 @@ PUT _inference/completion/openai-completion } ------------------------------------------------------------ // TEST[skip:TBD] + +[discrete] +[[inference-example-azureopenai]] +===== Azure OpenAI service + +The following example shows how to create an {infer} endpoint called +`azure_openai_embeddings` to perform a `text_embedding` task type. +Note that we do not specify a model here, as it is defined already via our Azure OpenAI deployment. + +The list of embeddings models that you can choose from in your deployment can be found in the https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models#embeddings[Azure models documentation]. + +[source,console] +------------------------------------------------------------ +PUT _inference/text_embedding/azure_openai_embeddings +{ + "service": "azureopenai", + "service_settings": { + "api_key": "", + "resource_name": "", + "deployment_id": "", + "api_version": "2024-02-01" + } +} +------------------------------------------------------------ +// TEST[skip:TBD] diff --git a/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc index 069dcb61f81b..4baada19998e 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc @@ -19,6 +19,12 @@ id="infer-api-ingest-openai"> OpenAI +
+
-++++ \ No newline at end of file +++++ diff --git a/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline.asciidoc index 869e41a4ca7d..f50b866e8a5b 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-ingest-pipeline.asciidoc @@ -85,4 +85,30 @@ PUT _ingest/pipeline/openai_embeddings <2> Configuration object that defines the `input_field` for the {infer} process and the `output_field` that will contain the {infer} results. -// end::openai[] \ No newline at end of file +// end::openai[] + +// tag::azure-openai[] + +[source,console] +-------------------------------------------------- +PUT _ingest/pipeline/azure_openai_embeddings +{ + "processors": [ + { + "inference": { + "model_id": "azure_openai_embeddings", <1> + "input_output": { <2> + "input_field": "content", + "output_field": "content_embedding" + } + } + } + ] +} +-------------------------------------------------- +<1> The name of the inference endpoint you created by using the +<>, it's referred to as `inference_id` in that step. +<2> Configuration object that defines the `input_field` for the {infer} process +and the `output_field` that will contain the {infer} results. + +// end::azure-openai[] diff --git a/docs/reference/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc index 9d94ce880988..e35ee712b8f5 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc @@ -19,6 +19,12 @@ id="infer-api-mapping-openai"> OpenAI +
+
-++++ \ No newline at end of file +++++ diff --git a/docs/reference/tab-widgets/inference-api/infer-api-mapping.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-mapping.asciidoc index 6803b73c0687..037c5957b01f 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-mapping.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-mapping.asciidoc @@ -84,7 +84,7 @@ PUT openai-embeddings } } -------------------------------------------------- -<1> The name of the field to contain the generated tokens. It must be refrenced +<1> The name of the field to contain the generated tokens. It must be referenced in the {infer} pipeline configuration in the next step. <2> The field to contain the tokens is a `dense_vector` field. <3> The output dimensions of the model. Find this value in the @@ -99,4 +99,43 @@ In this example, the name of the field is `content`. It must be referenced in the {infer} pipeline configuration in the next step. <6> The field type which is text in this example. -// end::openai[] \ No newline at end of file +// end::openai[] + +// tag::azure-openai[] + +[source,console] +-------------------------------------------------- +PUT azure-openai-embeddings +{ + "mappings": { + "properties": { + "content_embedding": { <1> + "type": "dense_vector", <2> + "dims": 1536, <3> + "element_type": "float", + "similarity": "dot_product" <4> + }, + "content": { <5> + "type": "text" <6> + } + } + } +} +-------------------------------------------------- +<1> The name of the field to contain the generated tokens. It must be referenced +in the {infer} pipeline configuration in the next step. +<2> The field to contain the tokens is a `dense_vector` field. +<3> The output dimensions of the model. Find this value in the +https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models#embeddings-models[Azure OpenAI documentation] +of the model you use. +<4> For Azure OpenAI embeddings, the `dot_product` function should be used to +calculate similarity as Azure OpenAI embeddings are normalised to unit length. +See the +https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/understand-embeddings[Azure OpenAI embeddings] +documentation for more information on the model specifications. +<5> The name of the field from which to create the dense vector representation. +In this example, the name of the field is `content`. It must be referenced in +the {infer} pipeline configuration in the next step. +<6> The field type which is text in this example. + +// end::azure-openai[] diff --git a/docs/reference/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc index 9a78868e44da..58dac586ba23 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc @@ -19,6 +19,12 @@ id="infer-api-reindex-openai"> OpenAI +
+
+ -++++ \ No newline at end of file +++++ diff --git a/docs/reference/tab-widgets/inference-api/infer-api-reindex.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-reindex.asciidoc index 118f7f046092..e97a7187415f 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-reindex.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-reindex.asciidoc @@ -75,4 +75,32 @@ https://platform.openai.com/account/limits[rate limit of your OpenAI account] may affect the throughput of the reindexing process. If this happens, change `size` to `3` or a similar value in magnitude. -// end::openai[] \ No newline at end of file +// end::openai[] + +// tag::azure-openai[] + +[source,console] +---- +POST _reindex?wait_for_completion=false +{ + "source": { + "index": "test-data", + "size": 50 <1> + }, + "dest": { + "index": "azure-openai-embeddings", + "pipeline": "azure_openai_embeddings" + } +} +---- +// TEST[skip:TBD] +<1> The default batch size for reindexing is 1000. Reducing `size` to a smaller +number makes the update of the reindexing process quicker which enables you to +follow the progress closely and detect errors early. + +NOTE: The +https://learn.microsoft.com/en-us/azure/ai-services/openai/quotas-limits#quotas-and-limits-reference[rate limit of your Azure OpenAI account] +may affect the throughput of the reindexing process. If this happens, change +`size` to `3` or a similar value in magnitude. + +// end::azure-openai[] diff --git a/docs/reference/tab-widgets/inference-api/infer-api-requirements-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-requirements-widget.asciidoc index c55056cd1a3d..781ddb43cb35 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-requirements-widget.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-requirements-widget.asciidoc @@ -19,6 +19,12 @@ id="infer-api-requirements-openai"> OpenAI +
+
-++++ \ No newline at end of file +++++ diff --git a/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc index 21a9d2111ef7..e67a905e1e97 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc @@ -17,4 +17,12 @@ API with the HuggingFace service. An https://openai.com/[OpenAI account] is required to use the {infer} API with the OpenAI service. -// end::openai[] \ No newline at end of file +// end::openai[] + +// tag::azure-openai[] +* An https://azure.microsoft.com/free/cognitive-services?azure-portal=true[Azure subscription] +* Access granted to Azure OpenAI in the desired Azure subscription. +You can apply for access to Azure OpenAI by completing the form at https://aka.ms/oai/access. +* An embedding model deployed in https://oai.azure.com/[Azure OpenAI Studio]. + +// end::azure-openai[] diff --git a/docs/reference/tab-widgets/inference-api/infer-api-search-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-search-widget.asciidoc index e945146e22ca..d3b7ba96bb19 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-search-widget.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-search-widget.asciidoc @@ -19,6 +19,12 @@ id="infer-api-search-openai"> OpenAI +
+
-++++ \ No newline at end of file +++++ diff --git a/docs/reference/tab-widgets/inference-api/infer-api-search.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-search.asciidoc index 1aa3b6f2f2ae..04515d0040ea 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-search.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-search.asciidoc @@ -209,4 +209,69 @@ query from the `openai-embeddings` index sorted by their proximity to the query: -------------------------------------------------- // NOTCONSOLE -// end::openai[] \ No newline at end of file +// end::openai[] + +// tag::azure-openai[] + +[source,console] +-------------------------------------------------- +GET azure-openai-embeddings/_search +{ + "knn": { + "field": "content_embedding", + "query_vector_builder": { + "text_embedding": { + "model_id": "azure_openai_embeddings", + "model_text": "Calculate fuel cost" + } + }, + "k": 10, + "num_candidates": 100 + }, + "_source": [ + "id", + "content" + ] +} +-------------------------------------------------- +// TEST[skip:TBD] + +As a result, you receive the top 10 documents that are closest in meaning to the +query from the `openai-embeddings` index sorted by their proximity to the query: + +[source,consol-result] +-------------------------------------------------- +"hits": [ + { + "_index": "azure-openai-embeddings", + "_id": "DDd5OowBHxQKHyc3TDSC", + "_score": 0.83704096, + "_source": { + "id": 862114, + "body": "How to calculate fuel cost for a road trip. By Tara Baukus Mello • Bankrate.com. Dear Driving for Dollars, My family is considering taking a long road trip to finish off the end of the summer, but I'm a little worried about gas prices and our overall fuel cost.It doesn't seem easy to calculate since we'll be traveling through many states and we are considering several routes.y family is considering taking a long road trip to finish off the end of the summer, but I'm a little worried about gas prices and our overall fuel cost. It doesn't seem easy to calculate since we'll be traveling through many states and we are considering several routes." + } + }, + { + "_index": "azure-openai-embeddings", + "_id": "ajd5OowBHxQKHyc3TDSC", + "_score": 0.8345704, + "_source": { + "id": 820622, + "body": "Home Heating Calculator. Typically, approximately 50% of the energy consumed in a home annually is for space heating. When deciding on a heating system, many factors will come into play: cost of fuel, installation cost, convenience and life style are all important.This calculator can help you estimate the cost of fuel for different heating appliances.hen deciding on a heating system, many factors will come into play: cost of fuel, installation cost, convenience and life style are all important. This calculator can help you estimate the cost of fuel for different heating appliances." + } + }, + { + "_index": "azure-openai-embeddings", + "_id": "Djd5OowBHxQKHyc3TDSC", + "_score": 0.8327426, + "_source": { + "id": 8202683, + "body": "Fuel is another important cost. This cost will depend on your boat, how far you travel, and how fast you travel. A 33-foot sailboat traveling at 7 knots should be able to travel 300 miles on 50 gallons of diesel fuel.If you are paying $4 per gallon, the trip would cost you $200.Most boats have much larger gas tanks than cars.uel is another important cost. This cost will depend on your boat, how far you travel, and how fast you travel. A 33-foot sailboat traveling at 7 knots should be able to travel 300 miles on 50 gallons of diesel fuel." + } + }, + (...) + ] +-------------------------------------------------- +// NOTCONSOLE + +// end::azure-openai[] diff --git a/docs/reference/tab-widgets/inference-api/infer-api-task-widget.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-task-widget.asciidoc index ebc8d093d01a..aac26913f955 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-task-widget.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-task-widget.asciidoc @@ -19,6 +19,12 @@ id="infer-api-task-openai"> OpenAI +
+
-++++ \ No newline at end of file +++++ diff --git a/docs/reference/tab-widgets/inference-api/infer-api-task.asciidoc b/docs/reference/tab-widgets/inference-api/infer-api-task.asciidoc index efbf1f8f25f5..07d5177b6034 100644 --- a/docs/reference/tab-widgets/inference-api/infer-api-task.asciidoc +++ b/docs/reference/tab-widgets/inference-api/infer-api-task.asciidoc @@ -13,7 +13,7 @@ PUT _inference/text_embedding/cohere_embeddings <1> } ------------------------------------------------------------ // TEST[skip:TBD] -<1> The task type is `text_embedding` in the path and the `inference_id` which +<1> The task type is `text_embedding` in the path and the `inference_id` which is the unique identifier of the {infer} endpoint is `cohere_embeddings`. <2> The API key of your Cohere account. You can find your API keys in your Cohere dashboard under the @@ -54,7 +54,7 @@ PUT _inference/text_embedding/hugging_face_embeddings <1> } ------------------------------------------------------------ // TEST[skip:TBD] -<1> The task type is `text_embedding` in the path and the `inference_id` which +<1> The task type is `text_embedding` in the path and the `inference_id` which is the unique identifier of the {infer} endpoint is `hugging_face_embeddings`. <2> A valid HuggingFace access token. You can find on the https://huggingface.co/settings/tokens[settings page of your account]. @@ -77,7 +77,7 @@ PUT _inference/text_embedding/openai_embeddings <1> } ------------------------------------------------------------ // TEST[skip:TBD] -<1> The task type is `text_embedding` in the path and the `inference_id` which +<1> The task type is `text_embedding` in the path and the `inference_id` which is the unique identifier of the {infer} endpoint is `openai_embeddings`. <2> The API key of your OpenAI account. You can find your OpenAI API keys in your OpenAI account under the @@ -93,4 +93,33 @@ NOTE: When using this model the recommended similarity measure to use in the embeddings are normalized to unit length in which case the `dot_product` and the `cosine` measures are equivalent. -// end::openai[] \ No newline at end of file +// end::openai[] + +// tag::azure-openai[] + +[source,console] +------------------------------------------------------------ +PUT _inference/text_embedding/azure_openai_embeddings <1> +{ + "service": "azureopenai", + "service_settings": { + "api_key": "", <2> + "resource_name": "", <3> + "deployment_id": "", <4> + "api_version": "2024-02-01" + } +} +------------------------------------------------------------ +// TEST[skip:TBD] +<1> The task type is `text_embedding` in the path and the `inference_id` which is the unique identifier of the {infer} endpoint is `azure_openai_embeddings`. +<2> The API key for accessing your Azure OpenAI services. +Alternately, you can provide an `entra_id` instead of an `api_key` here. +The <> does not return this information. +<3> The name our your Azure resource. +<4> The id of your deployed model. + +NOTE: When using this model the recommended similarity measure to use in the +`dense_vector` field mapping is `dot_product`. +In the case of Azure OpenAI models, the embeddings are normalized to unit length in which case the `dot_product` and the `cosine` measures are equivalent. + +// end::azure-openai[] From 1e4d4da483c441455a011f0d1b2ccfe398ceef41 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Tue, 16 Apr 2024 17:06:09 +0200 Subject: [PATCH 06/43] ESQL: Make esql version required in REST requests (#107433) * Enable corresponding validation in EsqlQueryRequest. * Add the ESQL version to requests to /_query in integration tests. * In mixed cluster tests for versions prior to 8.13.3, impersonate an 8.13 client and do not send any version. --------- Co-authored-by: Nik Everett --- .../xpack/esql/heap_attack/HeapAttackIT.java | 42 ++++++---- .../ImpersonateOfficialClientTestClient.java | 54 +++++++++++++ .../xpack/esql/EsqlAsyncSecurityIT.java | 1 + .../xpack/esql/EsqlSecurityIT.java | 2 + .../esql/qa/server/mixed-cluster/build.gradle | 9 ++- .../xpack/esql/qa/mixed/EsqlClientYamlIT.java | 76 +++++++++++++++++++ .../xpack/esql/ccq/MultiClustersIT.java | 5 +- .../xpack/esql/qa/single_node/RestEsqlIT.java | 8 +- .../esql/qa/single_node/TSDBRestEsqlIT.java | 5 +- .../single_node/AbstractEsqlClientYamlIT.java | 44 ++++++++++- .../qa/single_node/EsqlClientYamlAsyncIT.java | 44 +---------- .../EsqlClientYamlAsyncSubmitAndFetchIT.java | 2 +- .../xpack/esql/qa/rest/EsqlSpecTestCase.java | 20 +++-- .../esql/qa/rest/FieldExtractorTestCase.java | 68 ++++++----------- .../esql/qa/rest/RestEnrichTestCase.java | 29 +++---- .../xpack/esql/qa/rest/RestEsqlTestCase.java | 70 ++++++++++------- .../rest/generative/GenerativeRestTest.java | 5 +- .../xpack/esql/EsqlTestUtils.java | 7 +- .../action/AbstractEsqlIntegTestCase.java | 29 ++++++- .../action/CrossClustersCancellationIT.java | 2 +- .../esql/action/CrossClustersEnrichIT.java | 2 +- .../esql/action/CrossClustersQueryIT.java | 10 +-- .../xpack/esql/action/EnrichIT.java | 2 +- .../esql/action/EsqlActionBreakerIT.java | 2 +- .../xpack/esql/action/EsqlActionTaskIT.java | 2 +- .../xpack/esql/action/EsqlAsyncActionIT.java | 7 +- .../xpack/esql/action/TimeBasedIndicesIT.java | 14 ++-- .../xpack/esql/action/WarningsIT.java | 2 +- .../xpack/esql/action/EsqlQueryRequest.java | 4 +- .../esql/action/EsqlQueryRequestBuilder.java | 4 + .../esql/action/EsqlQueryRequestTests.java | 1 - .../RemoteClusterSecurityEsqlIT.java | 3 + .../rest-api-spec/test/esql/100_bug_fix.yml | 8 ++ .../rest-api-spec/test/esql/10_basic.yml | 13 ++++ .../rest-api-spec/test/esql/110_all_null.yml | 4 + .../test/esql/110_insensitive_equals.yml | 16 ++++ .../rest-api-spec/test/esql/120_profile.yml | 1 + .../rest-api-spec/test/esql/130_spatial.yml | 17 +++++ .../rest-api-spec/test/esql/20_aggs.yml | 26 +++++++ .../test/esql/25_aggs_on_null.yml | 7 ++ .../rest-api-spec/test/esql/30_types.yml | 34 ++++++++- .../rest-api-spec/test/esql/40_tsdb.yml | 8 ++ .../test/esql/40_unsupported_types.yml | 4 + .../test/esql/45_non_tsdb_counter.yml | 3 + .../test/esql/50_index_patterns.yml | 16 ++++ .../rest-api-spec/test/esql/60_enrich.yml | 4 + .../rest-api-spec/test/esql/60_usage.yml | 1 + .../rest-api-spec/test/esql/61_enrich_ip.yml | 4 + .../test/esql/62_extra_enrich.yml | 3 + .../rest-api-spec/test/esql/70_locale.yml | 2 + .../rest-api-spec/test/esql/80_text.yml | 24 ++++++ .../test/esql/81_text_exact_subfields.yml | 23 ++++-- .../test/esql/90_non_indexed.yml | 1 + .../xpack/restart/FullClusterRestartIT.java | 1 + .../test/querying_cluster/80_esql.yml | 3 + 55 files changed, 600 insertions(+), 198 deletions(-) create mode 100644 test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ImpersonateOfficialClientTestClient.java diff --git a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java index 4f43817b7b92..8c8eb942f891 100644 --- a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java +++ b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java @@ -56,12 +56,13 @@ * crash Elasticsearch. */ public class HeapAttackIT extends ESRestTestCase { - @ClassRule public static ElasticsearchCluster cluster = Clusters.buildCluster(); static volatile boolean SUITE_ABORTED = false; + private static String ESQL_VERSION = "2024.04.01"; + @Override protected String getTestRestCluster() { return cluster.getHttpAddresses(); @@ -155,8 +156,8 @@ private Response groupOnManyLongs(int count) throws IOException { } private StringBuilder makeManyLongs(int count) { - StringBuilder query = new StringBuilder(); - query.append("{\"query\":\"FROM manylongs\\n| EVAL i0 = a + b, i1 = b + i0"); + StringBuilder query = startQueryWithVersion(ESQL_VERSION); + query.append("FROM manylongs\\n| EVAL i0 = a + b, i1 = b + i0"); for (int i = 2; i < count; i++) { query.append(", i").append(i).append(" = i").append(i - 2).append(" + ").append(i - 1); } @@ -186,8 +187,8 @@ public void testHugeConcat() throws IOException { } private Response concat(int evals) throws IOException { - StringBuilder query = new StringBuilder(); - query.append("{\"query\":\"FROM single | EVAL str = TO_STRING(a)"); + StringBuilder query = startQueryWithVersion(ESQL_VERSION); + query.append("FROM single | EVAL str = TO_STRING(a)"); for (int e = 0; e < evals; e++) { query.append("\n| EVAL str=CONCAT(") .append(IntStream.range(0, 10).mapToObj(i -> "str").collect(Collectors.joining(", "))) @@ -223,8 +224,8 @@ public void testHugeManyConcat() throws IOException { * Tests that generate many moderately long strings. */ private Response manyConcat(int strings) throws IOException { - StringBuilder query = new StringBuilder(); - query.append("{\"query\":\"FROM manylongs | EVAL str = CONCAT("); + StringBuilder query = startQueryWithVersion(ESQL_VERSION); + query.append("FROM manylongs | EVAL str = CONCAT("); query.append( Arrays.stream(new String[] { "a", "b", "c", "d", "e" }) .map(f -> "TO_STRING(" + f + ")") @@ -274,8 +275,8 @@ public void testTooManyEval() throws IOException { } private Response manyEval(int evalLines) throws IOException { - StringBuilder query = new StringBuilder(); - query.append("{\"query\":\"FROM manylongs"); + StringBuilder query = startQueryWithVersion(ESQL_VERSION); + query.append("FROM manylongs"); for (int e = 0; e < evalLines; e++) { query.append("\n| EVAL "); for (int i = 0; i < 10; i++) { @@ -356,7 +357,9 @@ public void testFetchTooManyBigFields() throws IOException { * Fetches documents containing 1000 fields which are {@code 1kb} each. */ private void fetchManyBigFields(int docs) throws IOException { - Response response = query("{\"query\": \"FROM manybigfields | SORT f000 | LIMIT " + docs + "\"}", "columns"); + StringBuilder query = startQueryWithVersion(ESQL_VERSION); + query.append("FROM manybigfields | SORT f000 | LIMIT " + docs + "\"}"); + Response response = query(query.toString(), "columns"); Map map = responseAsMap(response); ListMatcher columns = matchesList(); for (int f = 0; f < 1000; f++) { @@ -383,11 +386,12 @@ public void testAggTooManyMvLongs() throws IOException { } private Response aggMvLongs(int fields) throws IOException { - StringBuilder builder = new StringBuilder("{\"query\": \"FROM mv_longs | STATS MAX(f00) BY f00"); + StringBuilder query = startQueryWithVersion(ESQL_VERSION); + query.append("FROM mv_longs | STATS MAX(f00) BY f00"); for (int f = 1; f < fields; f++) { - builder.append(", f").append(String.format(Locale.ROOT, "%02d", f)); + query.append(", f").append(String.format(Locale.ROOT, "%02d", f)); } - return query(builder.append("\"}").toString(), "columns"); + return query(query.append("\"}").toString(), "columns"); } public void testFetchMvLongs() throws IOException { @@ -408,7 +412,9 @@ public void testFetchTooManyMvLongs() throws IOException { } private Response fetchMvLongs() throws IOException { - return query("{\"query\": \"FROM mv_longs\"}", "columns"); + StringBuilder query = startQueryWithVersion(ESQL_VERSION); + query.append("FROM mv_longs\"}"); + return query(query.toString(), "columns"); } private void initManyLongs() throws IOException { @@ -576,4 +582,12 @@ public void assertRequestBreakerEmpty() throws Exception { } }); } + + private static StringBuilder startQueryWithVersion(String version) { + StringBuilder query = new StringBuilder(); + query.append("{\"version\":\"" + version + "\","); + query.append("\"query\":\""); + + return query; + } } diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ImpersonateOfficialClientTestClient.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ImpersonateOfficialClientTestClient.java new file mode 100644 index 000000000000..34856c8ca93c --- /dev/null +++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ImpersonateOfficialClientTestClient.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.test.rest.yaml; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpHost; +import org.elasticsearch.client.NodeSelector; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.common.CheckedSupplier; +import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestApi; +import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.function.BiPredicate; + +/** + * Impersonates an official test client by setting the @{code x-elastic-client-meta} header. + */ +public class ImpersonateOfficialClientTestClient extends ClientYamlTestClient { + private final String meta; + + public ImpersonateOfficialClientTestClient( + ClientYamlSuiteRestSpec restSpec, + RestClient restClient, + List hosts, + CheckedSupplier clientBuilderWithSniffedNodes, + String meta + ) { + super(restSpec, restClient, hosts, clientBuilderWithSniffedNodes); + this.meta = meta; + } + + @Override + public ClientYamlTestResponse callApi( + String apiName, + Map params, + HttpEntity entity, + Map headers, + NodeSelector nodeSelector, + BiPredicate pathPredicate + ) throws IOException { + headers.put("x-elastic-client-meta", meta); + return super.callApi(apiName, params, entity, headers, nodeSelector, pathPredicate); + } +} diff --git a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlAsyncSecurityIT.java b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlAsyncSecurityIT.java index 544eb82fb5ac..c7e9c3994ee4 100644 --- a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlAsyncSecurityIT.java +++ b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlAsyncSecurityIT.java @@ -90,6 +90,7 @@ private Response runAsync(String user, String command) throws IOException { } XContentBuilder json = JsonXContent.contentBuilder(); json.startObject(); + json.field("version", ESQL_VERSION); json.field("query", command); addRandomPragmas(json); json.field("wait_for_completion_timeout", timeValueNanos(randomIntBetween(1, 1000))); diff --git a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java index 7a9b90baa0d3..41df233af645 100644 --- a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java +++ b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java @@ -38,6 +38,7 @@ import static org.hamcrest.Matchers.equalTo; public class EsqlSecurityIT extends ESRestTestCase { + static String ESQL_VERSION = "2024.04.01.🚀"; @ClassRule public static ElasticsearchCluster cluster = ElasticsearchCluster.local() @@ -354,6 +355,7 @@ protected Response runESQLCommand(String user, String command) throws IOExceptio } XContentBuilder json = JsonXContent.contentBuilder(); json.startObject(); + json.field("version", ESQL_VERSION); json.field("query", command); addRandomPragmas(json); json.endObject(); diff --git a/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle b/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle index 09397710bb85..c25ef858534e 100644 --- a/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle +++ b/x-pack/plugin/esql/qa/server/mixed-cluster/build.gradle @@ -25,20 +25,27 @@ dependencies { GradleUtils.extendSourceSet(project, "javaRestTest", "yamlRestTest") +// ESQL is available in 8.11 or later def supportedVersion = bwcVersion -> { - // ESQL is available in 8.11 or later return bwcVersion.onOrAfter(Version.fromString("8.11.0")); } +// Versions on and after 8.13.3 will get a `version` parameter +def versionUnsupported = bwcVersion -> { + return bwcVersion.before(Version.fromString("8.13.3")); +} + BuildParams.bwcVersions.withWireCompatible(supportedVersion) { bwcVersion, baseName -> def javaRestTest = tasks.register("v${bwcVersion}#javaRestTest", StandaloneRestIntegTestTask) { usesBwcDistribution(bwcVersion) systemProperty("tests.old_cluster_version", bwcVersion) + systemProperty("tests.version_parameter_unsupported", versionUnsupported(bwcVersion)) } def yamlRestTest = tasks.register("v${bwcVersion}#yamlRestTest", StandaloneRestIntegTestTask) { usesBwcDistribution(bwcVersion) systemProperty("tests.old_cluster_version", bwcVersion) + systemProperty("tests.version_parameter_unsupported", versionUnsupported(bwcVersion)) testClassesDirs = sourceSets.yamlRestTest.output.classesDirs classpath = sourceSets.yamlRestTest.runtimeClasspath } diff --git a/x-pack/plugin/esql/qa/server/mixed-cluster/src/yamlRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/EsqlClientYamlIT.java b/x-pack/plugin/esql/qa/server/mixed-cluster/src/yamlRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/EsqlClientYamlIT.java index 2c9833ba0793..9bb114aaa6f6 100644 --- a/x-pack/plugin/esql/qa/server/mixed-cluster/src/yamlRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/EsqlClientYamlIT.java +++ b/x-pack/plugin/esql/qa/server/mixed-cluster/src/yamlRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/EsqlClientYamlIT.java @@ -9,14 +9,28 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.apache.http.HttpHost; +import org.elasticsearch.client.RestClient; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ClientYamlTestClient; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.elasticsearch.test.rest.yaml.ImpersonateOfficialClientTestClient; +import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec; +import org.elasticsearch.test.rest.yaml.section.ApiCallSection; +import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection; +import org.elasticsearch.test.rest.yaml.section.DoSection; +import org.elasticsearch.test.rest.yaml.section.ExecutableSection; import org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase; import org.junit.After; import org.junit.Before; import org.junit.ClassRule; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + public class EsqlClientYamlIT extends ESClientYamlSuiteTestCase { @ClassRule public static ElasticsearchCluster cluster = Clusters.mixedVersionCluster(); @@ -32,6 +46,9 @@ public EsqlClientYamlIT(final ClientYamlTestCandidate testCandidate) { @ParametersFactory public static Iterable parameters() throws Exception { + if (EsqlSpecTestCase.availableVersions().isEmpty()) { + return updateEsqlQueryDoSections(createParameters(), EsqlClientYamlIT::stripVersion); + } return createParameters(); } @@ -40,4 +57,63 @@ public static Iterable parameters() throws Exception { public void assertRequestBreakerEmpty() throws Exception { EsqlSpecTestCase.assertRequestBreakerEmpty(); } + + @Override + protected ClientYamlTestClient initClientYamlTestClient( + final ClientYamlSuiteRestSpec restSpec, + final RestClient restClient, + final List hosts + ) { + if (EsqlSpecTestCase.availableVersions().isEmpty()) { + return new ImpersonateOfficialClientTestClient(restSpec, restClient, hosts, this::getClientBuilderWithSniffedHosts, "es=8.13"); + } + return super.initClientYamlTestClient(restSpec, restClient, hosts); + } + + static DoSection stripVersion(DoSection doSection) { + ApiCallSection copy = doSection.getApiCallSection().copyWithNewApi(doSection.getApiCallSection().getApi()); + for (Map body : copy.getBodies()) { + body.remove("version"); + } + doSection.setApiCallSection(copy); + return doSection; + } + + // TODO: refactor, copied from single-node's AbstractEsqlClientYamlIt + public static Iterable updateEsqlQueryDoSections(Iterable parameters, Function modify) + throws Exception { + List result = new ArrayList<>(); + for (Object[] orig : parameters) { + assert orig.length == 1; + ClientYamlTestCandidate candidate = (ClientYamlTestCandidate) orig[0]; + try { + ClientYamlTestSection modified = new ClientYamlTestSection( + candidate.getTestSection().getLocation(), + candidate.getTestSection().getName(), + candidate.getTestSection().getPrerequisiteSection(), + candidate.getTestSection().getExecutableSections().stream().map(e -> modifyExecutableSection(e, modify)).toList() + ); + result.add(new Object[] { new ClientYamlTestCandidate(candidate.getRestTestSuite(), modified) }); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("error modifying " + candidate + ": " + e.getMessage(), e); + } + } + return result; + } + + // TODO: refactor, copied from single-node's AbstractEsqlClientYamlIt + private static ExecutableSection modifyExecutableSection(ExecutableSection e, Function modify) { + if (false == (e instanceof DoSection)) { + return e; + } + DoSection doSection = (DoSection) e; + String api = doSection.getApiCallSection().getApi(); + return switch (api) { + case "esql.query" -> modify.apply(doSection); + // case "esql.async_query", "esql.async_query_get" -> throw new IllegalArgumentException( + // "The esql yaml tests can't contain async_query or async_query_get because we modify them on the fly and *add* those." + // ); + default -> e; + }; + } } diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java index 2f681fc23bf3..3a3fbdba74ae 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClustersIT.java @@ -18,6 +18,7 @@ import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.rest.TestFeatureService; +import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase; import org.junit.After; import org.junit.Before; @@ -122,7 +123,9 @@ void indexDocs(RestClient client, String index, List docs) throws IOExcepti } private Map run(String query) throws IOException { - Map resp = runEsql(new RestEsqlTestCase.RequestObjectBuilder().query(query).build()); + Map resp = runEsql( + new RestEsqlTestCase.RequestObjectBuilder().query(query).version(EsqlTestUtils.latestEsqlVersionOrSnapshot()).build() + ); logger.info("--> query {} response {}", query, resp); return resp; } diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java index 6743657e8687..4de2a0f565c7 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java @@ -69,7 +69,7 @@ public void testBasicEsql() throws IOException { Response response = client().performRequest(bulk); Assert.assertEquals("{\"errors\":false}", EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8)); - RequestObjectBuilder builder = new RequestObjectBuilder().query(fromIndex() + " | stats avg(value)"); + RequestObjectBuilder builder = requestObjectBuilder().query(fromIndex() + " | stats avg(value)"); if (Build.current().isSnapshot()) { builder.pragmas(Settings.builder().put("data_partitioning", "shard").build()); } @@ -89,7 +89,7 @@ public void testInvalidPragma() throws IOException { request.setJsonEntity("{\"f\":" + i + "}"); assertOK(client().performRequest(request)); } - RequestObjectBuilder builder = new RequestObjectBuilder().query("from test-index | limit 1 | keep f"); + RequestObjectBuilder builder = requestObjectBuilder().query("from test-index | limit 1 | keep f"); builder.pragmas(Settings.builder().put("data_partitioning", "invalid-option").build()); ResponseException re = expectThrows(ResponseException.class, () -> runEsqlSync(builder)); assertThat(EntityUtils.toString(re.getResponse().getEntity()), containsString("No enum constant")); @@ -99,7 +99,7 @@ public void testInvalidPragma() throws IOException { public void testPragmaNotAllowed() throws IOException { assumeFalse("pragma only disabled on release builds", Build.current().isSnapshot()); - RequestObjectBuilder builder = new RequestObjectBuilder().query("row a = 1, b = 2"); + RequestObjectBuilder builder = requestObjectBuilder().query("row a = 1, b = 2"); builder.pragmas(Settings.builder().put("data_partitioning", "shard").build()); ResponseException re = expectThrows(ResponseException.class, () -> runEsqlSync(builder)); assertThat(EntityUtils.toString(re.getResponse().getEntity()), containsString("[pragma] only allowed in snapshot builds")); @@ -197,7 +197,7 @@ public void testIncompatibleMappingsErrors() throws IOException { } private void assertException(String query, String... errorMessages) throws IOException { - ResponseException re = expectThrows(ResponseException.class, () -> runEsqlSync(new RequestObjectBuilder().query(query))); + ResponseException re = expectThrows(ResponseException.class, () -> runEsqlSync(requestObjectBuilder().query(query))); assertThat(re.getResponse().getStatusLine().getStatusCode(), equalTo(400)); for (var error : errorMessages) { assertThat(re.getMessage(), containsString(error)); diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/TSDBRestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/TSDBRestEsqlIT.java index b7ab7b623d46..057119103f0e 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/TSDBRestEsqlIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/TSDBRestEsqlIT.java @@ -61,9 +61,8 @@ public void testTimeSeriesQuerying() throws IOException { Response response = client().performRequest(bulk); assertEquals("{\"errors\":false}", EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8)); - RestEsqlTestCase.RequestObjectBuilder builder = new RestEsqlTestCase.RequestObjectBuilder().query( - "FROM k8s | KEEP k8s.pod.name, @timestamp" - ); + RestEsqlTestCase.RequestObjectBuilder builder = RestEsqlTestCase.requestObjectBuilder() + .query("FROM k8s | KEEP k8s.pod.name, @timestamp"); builder.pragmas(Settings.builder().put("time_series", true).build()); Map result = runEsqlSync(builder); @SuppressWarnings("unchecked") diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/AbstractEsqlClientYamlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/AbstractEsqlClientYamlIT.java index 70afdf32d380..b2a3b12c2a02 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/AbstractEsqlClientYamlIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/AbstractEsqlClientYamlIT.java @@ -11,13 +11,19 @@ import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection; +import org.elasticsearch.test.rest.yaml.section.DoSection; +import org.elasticsearch.test.rest.yaml.section.ExecutableSection; import org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase; import org.junit.After; import org.junit.Before; import org.junit.ClassRule; -abstract class AbstractEsqlClientYamlIT extends ESClientYamlSuiteTestCase { +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +abstract class AbstractEsqlClientYamlIT extends ESClientYamlSuiteTestCase { @ClassRule public static ElasticsearchCluster cluster = ElasticsearchCluster.local() .distribution(DistributionType.DEFAULT) @@ -44,4 +50,40 @@ private void assertRequestBreakerEmpty() throws Exception { */ EsqlSpecTestCase.assertRequestBreakerEmpty(); } + + public static Iterable updateEsqlQueryDoSections(Iterable parameters, Function modify) + throws Exception { + List result = new ArrayList<>(); + for (Object[] orig : parameters) { + assert orig.length == 1; + ClientYamlTestCandidate candidate = (ClientYamlTestCandidate) orig[0]; + try { + ClientYamlTestSection modified = new ClientYamlTestSection( + candidate.getTestSection().getLocation(), + candidate.getTestSection().getName(), + candidate.getTestSection().getPrerequisiteSection(), + candidate.getTestSection().getExecutableSections().stream().map(e -> modifyExecutableSection(e, modify)).toList() + ); + result.add(new Object[] { new ClientYamlTestCandidate(candidate.getRestTestSuite(), modified) }); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("error modifying " + candidate + ": " + e.getMessage(), e); + } + } + return result; + } + + private static ExecutableSection modifyExecutableSection(ExecutableSection e, Function modify) { + if (false == (e instanceof DoSection)) { + return e; + } + DoSection doSection = (DoSection) e; + String api = doSection.getApiCallSection().getApi(); + return switch (api) { + case "esql.query" -> modify.apply(doSection); + case "esql.async_query", "esql.async_query_get" -> throw new IllegalArgumentException( + "The esql yaml tests can't contain async_query or async_query_get because we modify them on the fly and *add* those." + ); + default -> e; + }; + } } diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlClientYamlAsyncIT.java b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlClientYamlAsyncIT.java index 657f396b2857..f5bd1efb106a 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlClientYamlAsyncIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlClientYamlAsyncIT.java @@ -10,16 +10,9 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; -import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; import org.elasticsearch.test.rest.yaml.section.ApiCallSection; -import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection; -import org.elasticsearch.test.rest.yaml.section.DoSection; -import org.elasticsearch.test.rest.yaml.section.ExecutableSection; -import java.util.ArrayList; -import java.util.List; import java.util.Map; -import java.util.function.Function; /** * Run the ESQL yaml tests against the async esql endpoint with a 30 minute {@code wait_until_completion_timeout}. @@ -33,7 +26,7 @@ public EsqlClientYamlAsyncIT(final ClientYamlTestCandidate testCandidate) { @ParametersFactory public static Iterable parameters() throws Exception { - return parameters(doSection -> { + return updateEsqlQueryDoSections(createParameters(), doSection -> { ApiCallSection copy = doSection.getApiCallSection().copyWithNewApi("esql.async_query"); for (Map body : copy.getBodies()) { body.put("wait_for_completion_timeout", "30m"); @@ -42,39 +35,4 @@ public static Iterable parameters() throws Exception { return doSection; }); } - - public static Iterable parameters(Function modify) throws Exception { - List result = new ArrayList<>(); - for (Object[] orig : ESClientYamlSuiteTestCase.createParameters()) { - assert orig.length == 1; - ClientYamlTestCandidate candidate = (ClientYamlTestCandidate) orig[0]; - try { - ClientYamlTestSection modified = new ClientYamlTestSection( - candidate.getTestSection().getLocation(), - candidate.getTestSection().getName(), - candidate.getTestSection().getPrerequisiteSection(), - candidate.getTestSection().getExecutableSections().stream().map(e -> modifyExecutableSection(e, modify)).toList() - ); - result.add(new Object[] { new ClientYamlTestCandidate(candidate.getRestTestSuite(), modified) }); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("error modifying " + candidate + ": " + e.getMessage(), e); - } - } - return result; - } - - private static ExecutableSection modifyExecutableSection(ExecutableSection e, Function modify) { - if (false == (e instanceof DoSection)) { - return e; - } - DoSection doSection = (DoSection) e; - String api = doSection.getApiCallSection().getApi(); - return switch (api) { - case "esql.query" -> modify.apply(doSection); - case "esql.async_query", "esql.async_query_get" -> throw new IllegalArgumentException( - "The esql yaml tests can't contain async_query or async_query_get because we modify them on the fly and *add* those." - ); - default -> e; - }; - } } diff --git a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlClientYamlAsyncSubmitAndFetchIT.java b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlClientYamlAsyncSubmitAndFetchIT.java index b32a7385d12c..38051007568e 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlClientYamlAsyncSubmitAndFetchIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/yamlRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlClientYamlAsyncSubmitAndFetchIT.java @@ -32,7 +32,7 @@ public EsqlClientYamlAsyncSubmitAndFetchIT(final ClientYamlTestCandidate testCan @ParametersFactory public static Iterable parameters() throws Exception { - return EsqlClientYamlAsyncIT.parameters(DoEsqlAsync::new); + return updateEsqlQueryDoSections(createParameters(), DoEsqlAsync::new); } private static class DoEsqlAsync implements ExecutableSection { diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java index 75723ed1d531..14579dfb537d 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java @@ -68,7 +68,14 @@ public abstract class EsqlSpecTestCase extends ESRestTestCase { private final Integer lineNumber; protected final CsvTestCase testCase; protected final Mode mode; - protected final Set versions; + + public static Set availableVersions() { + if ("true".equals(System.getProperty("tests.version_parameter_unsupported"))) { + // TODO: skip tests with explicitly set version and/or strip the version if it's 2024.04.01. + return Set.of(); + } + return Build.current().isSnapshot() ? Set.of(EsqlVersion.values()) : Set.of(EsqlVersion.releasedAscending()); + } public enum Mode { SYNC, @@ -101,8 +108,6 @@ protected EsqlSpecTestCase(String fileName, String groupName, String testName, I this.lineNumber = lineNumber; this.testCase = testCase; this.mode = mode; - // TODO: Read applicable versions from csv-spec files/make it part of testCase. - this.versions = Build.current().isSnapshot() ? Set.of(EsqlVersion.values()) : Set.of(EsqlVersion.releasedAscending()); } @Before @@ -150,8 +155,13 @@ protected void shouldSkipTest(String testName) throws IOException { protected final void doTest() throws Throwable { RequestObjectBuilder builder = new RequestObjectBuilder(randomFrom(XContentType.values())); - EsqlVersion version = randomFrom(versions); - String versionString = randomBoolean() ? version.toString() : version.versionStringWithoutEmoji(); + + String versionString = null; + if (availableVersions().isEmpty() == false) { + EsqlVersion version = randomFrom(availableVersions()); + versionString = randomBoolean() ? version.toString() : version.versionStringWithoutEmoji(); + } + Map answer = runEsql( builder.query(testCase.query).version(versionString), testCase.expectedWarnings(false), diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java index d107f8a147fd..d5daab2d46a8 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java @@ -26,6 +26,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.hamcrest.Matcher; import org.junit.Before; @@ -296,7 +297,7 @@ public void testFlattenedUnsupported() throws IOException { new Test("flattened").createIndex("test", "flattened"); index("test", """ {"flattened": {"a": "foo"}}"""); - Map result = runEsqlSync(new RestEsqlTestCase.RequestObjectBuilder().query("FROM test* | LIMIT 2")); + Map result = runEsql("FROM test* | LIMIT 2"); assertMap( result, @@ -310,10 +311,7 @@ public void testEmptyMapping() throws IOException { index("test", """ {}"""); - ResponseException e = expectThrows( - ResponseException.class, - () -> runEsqlSync(new RestEsqlTestCase.RequestObjectBuilder().query("FROM test* | SORT missing | LIMIT 3")) - ); + ResponseException e = expectThrows(ResponseException.class, () -> runEsql("FROM test* | SORT missing | LIMIT 3")); String err = EntityUtils.toString(e.getResponse().getEntity()); assertThat(err, containsString("Unknown column [missing]")); @@ -674,16 +672,13 @@ public void testIncompatibleTypes() throws IOException { index("test2", """ {"f": 1}"""); - Map result = runEsqlSync(new RestEsqlTestCase.RequestObjectBuilder().query("FROM test*")); + Map result = runEsql("FROM test*"); assertMap( result, matchesMap().entry("columns", List.of(columnInfo("f", "unsupported"))) .entry("values", List.of(matchesList().item(null), matchesList().item(null))) ); - ResponseException e = expectThrows( - ResponseException.class, - () -> runEsqlSync(new RestEsqlTestCase.RequestObjectBuilder().query("FROM test* | SORT f | LIMIT 3")) - ); + ResponseException e = expectThrows(ResponseException.class, () -> runEsql("FROM test* | SORT f | LIMIT 3")); String err = EntityUtils.toString(e.getResponse().getEntity()); assertThat( deyaml(err), @@ -715,7 +710,7 @@ public void testDistinctInEachIndex() throws IOException { index("test2", """ {"other": "o2"}"""); - Map result = runEsqlSync(new RestEsqlTestCase.RequestObjectBuilder().query("FROM test* | SORT file, other")); + Map result = runEsql("FROM test* | SORT file, other"); assertMap( result, matchesMap().entry("columns", List.of(columnInfo("file", "keyword"), columnInfo("other", "keyword"))) @@ -769,10 +764,7 @@ public void testMergeKeywordAndObject() throws IOException { index("test2", """ {"file": {"raw": "o2"}}"""); - ResponseException e = expectThrows( - ResponseException.class, - () -> runEsqlSync(new RestEsqlTestCase.RequestObjectBuilder().query("FROM test* | SORT file, file.raw | LIMIT 3")) - ); + ResponseException e = expectThrows(ResponseException.class, () -> runEsql("FROM test* | SORT file, file.raw | LIMIT 3")); String err = EntityUtils.toString(e.getResponse().getEntity()); assertThat( deyaml(err), @@ -782,7 +774,7 @@ public void testMergeKeywordAndObject() throws IOException { ) ); - Map result = runEsqlSync(new RestEsqlTestCase.RequestObjectBuilder().query("FROM test* | SORT file.raw | LIMIT 2")); + Map result = runEsql("FROM test* | SORT file.raw | LIMIT 2"); assertMap( result, matchesMap().entry("columns", List.of(columnInfo("file", "unsupported"), columnInfo("file.raw", "keyword"))) @@ -822,15 +814,12 @@ public void testPropagateUnsupportedToSubFields() throws IOException { index("test", """ {"f": "192.168.0.1/24"}"""); - ResponseException e = expectThrows( - ResponseException.class, - () -> runEsqlSync(new RestEsqlTestCase.RequestObjectBuilder().query("FROM test* | SORT f, f.raw | LIMIT 3")) - ); + ResponseException e = expectThrows(ResponseException.class, () -> runEsql("FROM test* | SORT f, f.raw | LIMIT 3")); String err = EntityUtils.toString(e.getResponse().getEntity()); assertThat(err, containsString("Cannot use field [f] with unsupported type [ip_range]")); assertThat(err, containsString("Cannot use field [f.raw] with unsupported type [ip_range]")); - Map result = runEsqlSync(new RestEsqlTestCase.RequestObjectBuilder().query("FROM test* | LIMIT 2")); + Map result = runEsql("FROM test* | LIMIT 2"); assertMap( result, matchesMap().entry("columns", List.of(columnInfo("f", "unsupported"), columnInfo("f.raw", "unsupported"))) @@ -888,15 +877,12 @@ public void testMergeUnsupportedAndObject() throws IOException { index("test2", """ {"f": {"raw": "o2"}}"""); - ResponseException e = expectThrows( - ResponseException.class, - () -> runEsqlSync(new RestEsqlTestCase.RequestObjectBuilder().query("FROM test* | SORT f, f.raw | LIMIT 3")) - ); + ResponseException e = expectThrows(ResponseException.class, () -> runEsql("FROM test* | SORT f, f.raw | LIMIT 3")); String err = EntityUtils.toString(e.getResponse().getEntity()); assertThat(err, containsString("Cannot use field [f] with unsupported type [ip_range]")); assertThat(err, containsString("Cannot use field [f.raw] with unsupported type [ip_range]")); - Map result = runEsqlSync(new RestEsqlTestCase.RequestObjectBuilder().query("FROM test* | LIMIT 2")); + Map result = runEsql("FROM test* | LIMIT 2"); assertMap( result, matchesMap().entry("columns", List.of(columnInfo("f", "unsupported"), columnInfo("f.raw", "unsupported"))) @@ -931,7 +917,7 @@ public void testIntegerDocValuesConflict() throws IOException { index("test2", """ {"emp_no": 2}"""); - Map result = runEsqlSync(new RestEsqlTestCase.RequestObjectBuilder().query("FROM test* | SORT emp_no | LIMIT 2")); + Map result = runEsql("FROM test* | SORT emp_no | LIMIT 2"); assertMap( result, matchesMap().entry("columns", List.of(columnInfo("emp_no", "integer"))) @@ -967,10 +953,7 @@ public void testLongIntegerConflict() throws IOException { index("test2", """ {"emp_no": 2}"""); - ResponseException e = expectThrows( - ResponseException.class, - () -> runEsqlSync(new RestEsqlTestCase.RequestObjectBuilder().query("FROM test* | SORT emp_no | LIMIT 3")) - ); + ResponseException e = expectThrows(ResponseException.class, () -> runEsql("FROM test* | SORT emp_no | LIMIT 3")); String err = EntityUtils.toString(e.getResponse().getEntity()); assertThat( deyaml(err), @@ -980,7 +963,7 @@ public void testLongIntegerConflict() throws IOException { ) ); - Map result = runEsqlSync(new RestEsqlTestCase.RequestObjectBuilder().query("FROM test* | LIMIT 2")); + Map result = runEsql("FROM test* | LIMIT 2"); assertMap( result, matchesMap().entry("columns", List.of(columnInfo("emp_no", "unsupported"))) @@ -1016,10 +999,7 @@ public void testIntegerShortConflict() throws IOException { index("test2", """ {"emp_no": 2}"""); - ResponseException e = expectThrows( - ResponseException.class, - () -> runEsqlSync(new RestEsqlTestCase.RequestObjectBuilder().query("FROM test* | SORT emp_no | LIMIT 3")) - ); + ResponseException e = expectThrows(ResponseException.class, () -> runEsql("FROM test* | SORT emp_no | LIMIT 3")); String err = EntityUtils.toString(e.getResponse().getEntity()); assertThat( deyaml(err), @@ -1029,7 +1009,7 @@ public void testIntegerShortConflict() throws IOException { ) ); - Map result = runEsqlSync(new RestEsqlTestCase.RequestObjectBuilder().query("FROM test* | LIMIT 2")); + Map result = runEsql("FROM test* | LIMIT 2"); assertMap( result, matchesMap().entry("columns", List.of(columnInfo("emp_no", "unsupported"))) @@ -1071,13 +1051,10 @@ public void testTypeConflictInObject() throws IOException { index("test2", """ {"foo": {"emp_no": "cat"}}"""); - Map result = runEsqlSync(new RestEsqlTestCase.RequestObjectBuilder().query("FROM test* | LIMIT 3")); + Map result = runEsql("FROM test* | LIMIT 3"); assertMap(result, matchesMap().entry("columns", List.of(columnInfo("foo.emp_no", "unsupported"))).extraOk()); - ResponseException e = expectThrows( - ResponseException.class, - () -> runEsqlSync(new RestEsqlTestCase.RequestObjectBuilder().query("FROM test* | SORT foo.emp_no | LIMIT 3")) - ); + ResponseException e = expectThrows(ResponseException.class, () -> runEsql("FROM test* | SORT foo.emp_no | LIMIT 3")); String err = EntityUtils.toString(e.getResponse().getEntity()); assertThat( deyaml(err), @@ -1413,7 +1390,7 @@ private void fieldMapping(XContentBuilder builder) throws IOException { } private Map fetchAll() throws IOException { - return runEsqlSync(new RestEsqlTestCase.RequestObjectBuilder().query("FROM test* | LIMIT 10")); + return runEsql("FROM test* | LIMIT 10"); } } @@ -1456,4 +1433,9 @@ private static void createIndex(String name, CheckedConsumer runEsql(String query) throws IOException { + return runEsqlSync(new RestEsqlTestCase.RequestObjectBuilder().query(query).version(EsqlTestUtils.latestEsqlVersionOrSnapshot())); + } + } diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEnrichTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEnrichTestCase.java index a670b11c6178..07abc26e8c78 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEnrichTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEnrichTestCase.java @@ -13,6 +13,7 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.ResponseException; import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.junit.After; import org.junit.Before; @@ -142,10 +143,7 @@ public void wipeTestData() throws IOException { } public void testNonExistentEnrichPolicy() throws IOException { - ResponseException re = expectThrows( - ResponseException.class, - () -> RestEsqlTestCase.runEsqlSync(new RestEsqlTestCase.RequestObjectBuilder().query("from test | enrich countris")) - ); + ResponseException re = expectThrows(ResponseException.class, () -> runEsql("from test | enrich countris", Mode.SYNC)); assertThat( EntityUtils.toString(re.getResponse().getEntity()), containsString("cannot find enrich policy [countris], did you mean [countries]?") @@ -153,12 +151,7 @@ public void testNonExistentEnrichPolicy() throws IOException { } public void testNonExistentEnrichPolicy_KeepField() throws IOException { - ResponseException re = expectThrows( - ResponseException.class, - () -> RestEsqlTestCase.runEsqlSync( - new RestEsqlTestCase.RequestObjectBuilder().query("from test | enrich countris | keep number") - ) - ); + ResponseException re = expectThrows(ResponseException.class, () -> runEsql("from test | enrich countris | keep number", Mode.SYNC)); assertThat( EntityUtils.toString(re.getResponse().getEntity()), containsString("cannot find enrich policy [countris], did you mean [countries]?") @@ -166,9 +159,7 @@ public void testNonExistentEnrichPolicy_KeepField() throws IOException { } public void testMatchField_ImplicitFieldsList() throws IOException { - Map result = runEsql( - new RestEsqlTestCase.RequestObjectBuilder().query("from test | enrich countries | keep number | sort number") - ); + Map result = runEsql("from test | enrich countries | keep number | sort number"); var columns = List.of(Map.of("name", "number", "type", "long")); var values = List.of(List.of(1000), List.of(1000), List.of(5000)); @@ -176,17 +167,19 @@ public void testMatchField_ImplicitFieldsList() throws IOException { } public void testMatchField_ImplicitFieldsList_WithStats() throws IOException { - Map result = runEsql( - new RestEsqlTestCase.RequestObjectBuilder().query("from test | enrich countries | stats s = sum(number) by country_name") - - ); + Map result = runEsql("from test | enrich countries | stats s = sum(number) by country_name"); var columns = List.of(Map.of("name", "s", "type", "long"), Map.of("name", "country_name", "type", "keyword")); var values = List.of(List.of(2000, "United States of America"), List.of(5000, "China")); assertMap(result, matchesMap().entry("columns", columns).entry("values", values)); } - private Map runEsql(RestEsqlTestCase.RequestObjectBuilder requestObject) throws IOException { + private Map runEsql(String query) throws IOException { + return runEsql(query, mode); + } + + private Map runEsql(String query, Mode mode) throws IOException { + var requestObject = new RestEsqlTestCase.RequestObjectBuilder().query(query).version(EsqlTestUtils.latestEsqlVersionOrSnapshot()); if (mode == Mode.ASYNC) { return RestEsqlTestCase.runEsqlAsync(requestObject); } else { diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java index 86d48aca3bae..76fbbcfb71d7 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java @@ -113,6 +113,7 @@ protected RestEsqlTestCase(Mode mode) { public static class RequestObjectBuilder { private final XContentBuilder builder; private boolean isBuilt = false; + private String version; private Boolean keepOnCompletion = null; @@ -131,7 +132,7 @@ public RequestObjectBuilder query(String query) throws IOException { } public RequestObjectBuilder version(String version) throws IOException { - builder.field("version", version); + this.version = version; return this; } @@ -179,6 +180,9 @@ public RequestObjectBuilder pragmas(Settings pragmas) throws IOException { public RequestObjectBuilder build() throws IOException { if (isBuilt == false) { + if (version != null) { + builder.field("version", version); + } builder.endObject(); isBuilt = true; } @@ -203,7 +207,7 @@ public static RequestObjectBuilder jsonBuilder() throws IOException { } public void testGetAnswer() throws IOException { - Map answer = runEsql(builder().query("row a = 1, b = 2")); + Map answer = runEsql(requestObjectBuilder().query("row a = 1, b = 2")); assertEquals(2, answer.size()); Map colA = Map.of("name", "a", "type", "integer"); Map colB = Map.of("name", "b", "type", "integer"); @@ -212,7 +216,7 @@ public void testGetAnswer() throws IOException { } public void testUseUnknownIndex() throws IOException { - ResponseException e = expectThrows(ResponseException.class, () -> runEsql(builder().query("from doesNotExist"))); + ResponseException e = expectThrows(ResponseException.class, () -> runEsql(requestObjectBuilder().query("from doesNotExist"))); assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); assertThat(e.getMessage(), containsString("verification_exception")); assertThat(e.getMessage(), containsString("Unknown index [doesNotExist]")); @@ -236,7 +240,7 @@ private void useKnownIndexWithOther(String other, String option) throws IOExcept String q = fromIndex() + ',' + other; q += " OPTIONS \"" + option + "\"=\"" + o + "\""; q += " | KEEP keyword, integer | SORT integer asc | LIMIT 10"; - return builder().query(q); + return requestObjectBuilder().query(q); }; // test failure @@ -258,7 +262,7 @@ private void useUnknownIndex(String option) { CheckedFunction builder = o -> { String q = "FROM doesnotexist OPTIONS \"" + option + "\"=\"" + o + "\""; q += " | KEEP keyword, integer | SORT integer asc | LIMIT 10"; - return builder().query(q); + return requestObjectBuilder().query(q); }; // test failure 404 from resolver @@ -285,7 +289,7 @@ public void testSearchPreference() throws IOException { q += " OPTIONS " + o; } q += " | KEEP keyword, integer | SORT integer asc | LIMIT 10"; - return builder().query(q); + return requestObjectBuilder().query(q); }; // verify that it returns as expected @@ -319,14 +323,14 @@ public void testNullInAggs() throws IOException { Response response = client().performRequest(bulk); assertThat(EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8), equalTo("{\"errors\":false}")); - RequestObjectBuilder builder = new RequestObjectBuilder().query(fromIndex() + " | stats min(value)"); + RequestObjectBuilder builder = requestObjectBuilder().query(fromIndex() + " | stats min(value)"); Map result = runEsql(builder); assertMap( result, matchesMap().entry("values", List.of(List.of(1))).entry("columns", List.of(Map.of("name", "min(value)", "type", "long"))) ); - builder = new RequestObjectBuilder().query(fromIndex() + " | stats min(value) by group | sort group, `min(value)`"); + builder = requestObjectBuilder().query(fromIndex() + " | stats min(value) by group | sort group, `min(value)`"); result = runEsql(builder); assertMap( result, @@ -340,7 +344,7 @@ public void testColumnarMode() throws IOException { bulkLoadTestData(docCount); boolean columnar = randomBoolean(); - var query = builder().query(fromIndex() + " | keep keyword, integer | sort integer asc"); + var query = requestObjectBuilder().query(fromIndex() + " | keep keyword, integer | sort integer asc"); if (columnar || randomBoolean()) { query.columnar(columnar); } @@ -370,27 +374,27 @@ public void testColumnarMode() throws IOException { public void testTextMode() throws IOException { int count = randomIntBetween(0, 100); bulkLoadTestData(count); - var builder = builder().query(fromIndex() + " | keep keyword, integer | sort integer asc | limit 100"); + var builder = requestObjectBuilder().query(fromIndex() + " | keep keyword, integer | sort integer asc | limit 100"); assertEquals(expectedTextBody("txt", count, null), runEsqlAsTextWithFormat(builder, "txt", null)); } public void testCSVMode() throws IOException { int count = randomIntBetween(0, 100); bulkLoadTestData(count); - var builder = builder().query(fromIndex() + " | keep keyword, integer | sort integer asc | limit 100"); + var builder = requestObjectBuilder().query(fromIndex() + " | keep keyword, integer | sort integer asc | limit 100"); assertEquals(expectedTextBody("csv", count, '|'), runEsqlAsTextWithFormat(builder, "csv", '|')); } public void testTSVMode() throws IOException { int count = randomIntBetween(0, 100); bulkLoadTestData(count); - var builder = builder().query(fromIndex() + " | keep keyword, integer | sort integer asc | limit 100"); + var builder = requestObjectBuilder().query(fromIndex() + " | keep keyword, integer | sort integer asc | limit 100"); assertEquals(expectedTextBody("tsv", count, null), runEsqlAsTextWithFormat(builder, "tsv", null)); } public void testCSVNoHeaderMode() throws IOException { bulkLoadTestData(1); - var builder = builder().query(fromIndex() + " | keep keyword, integer | sort integer asc | limit 100"); + var builder = requestObjectBuilder().query(fromIndex() + " | keep keyword, integer | sort integer asc | limit 100"); Request request = prepareRequest(SYNC); String mediaType = attachBody(builder.build(), request); RequestOptions.Builder options = request.getOptions().toBuilder(); @@ -448,7 +452,7 @@ public void testOutOfRangeComparisons() throws IOException { for (String fieldWithType : dataTypes) { for (String truePredicate : trueForSingleValuesPredicates) { String comparison = fieldWithType + truePredicate; - var query = builder().query(format(null, "from {} | where {}", testIndexName(), comparison)); + var query = requestObjectBuilder().query(format(null, "from {} | where {}", testIndexName(), comparison)); List expectedWarnings = List.of( "Line 1:29: evaluation of [" + comparison + "] failed, treating result as null. Only first 20 failures recorded.", "Line 1:29: java.lang.IllegalArgumentException: single-value function encountered multi-value" @@ -465,7 +469,7 @@ public void testOutOfRangeComparisons() throws IOException { for (String falsePredicate : alwaysFalsePredicates) { String comparison = fieldWithType + falsePredicate; - var query = builder().query(format(null, "from {} | where {}", testIndexName(), comparison)); + var query = requestObjectBuilder().query(format(null, "from {} | where {}", testIndexName(), comparison)); var result = runEsql(query); var values = as(result.get("values"), ArrayList.class); @@ -481,7 +485,7 @@ public void testWarningHeadersOnFailedConversions() throws IOException { Request request = prepareRequest(SYNC); var query = fromIndex() + " | sort integer asc | eval asInt = to_int(case(integer % 2 == 0, to_str(integer), keyword)) | limit 1000"; - var mediaType = attachBody(new RequestObjectBuilder().query(query).build(), request); + var mediaType = attachBody(requestObjectBuilder().query(query).build(), request); RequestOptions.Builder options = request.getOptions().toBuilder(); options.setWarningsHandler(WarningsHandler.PERMISSIVE); @@ -521,7 +525,7 @@ public void testMetadataFieldsOnMultipleIndices() throws IOException { assertEquals(201, client().performRequest(request).getStatusLine().getStatusCode()); var query = fromIndex() + "* metadata _index, _version, _id | sort _version"; - Map result = runEsql(new RequestObjectBuilder().query(query)); + Map result = runEsql(requestObjectBuilder().query(query)); var columns = List.of( Map.of("name", "a", "type", "long"), Map.of("name", "_index", "type", "keyword"), @@ -539,7 +543,7 @@ public void testMetadataFieldsOnMultipleIndices() throws IOException { public void testErrorMessageForEmptyParams() throws IOException { ResponseException re = expectThrows( ResponseException.class, - () -> runEsql(new RequestObjectBuilder().query("row a = 1 | eval x = ?").params("[]")) + () -> runEsql(requestObjectBuilder().query("row a = 1 | eval x = ?").params("[]")) ); assertThat(EntityUtils.toString(re.getResponse().getEntity()), containsString("Not enough actual parameters 0")); } @@ -547,7 +551,7 @@ public void testErrorMessageForEmptyParams() throws IOException { public void testErrorMessageForInvalidParams() throws IOException { ResponseException re = expectThrows( ResponseException.class, - () -> runEsql(new RequestObjectBuilder().query("row a = 1").params("[{\"x\":\"y\"}]")) + () -> runEsql(requestObjectBuilder().query("row a = 1").params("[{\"x\":\"y\"}]")) ); assertThat(EntityUtils.toString(re.getResponse().getEntity()), containsString("Required [value, type]")); } @@ -555,7 +559,7 @@ public void testErrorMessageForInvalidParams() throws IOException { public void testErrorMessageForMissingTypeInParams() throws IOException { ResponseException re = expectThrows( ResponseException.class, - () -> runEsql(new RequestObjectBuilder().query("row a = 1").params("[\"x\", 123, true, {\"value\": \"y\"}]")) + () -> runEsql(requestObjectBuilder().query("row a = 1").params("[\"x\", 123, true, {\"value\": \"y\"}]")) ); assertThat(EntityUtils.toString(re.getResponse().getEntity()), containsString("Required [type]")); } @@ -563,7 +567,7 @@ public void testErrorMessageForMissingTypeInParams() throws IOException { public void testErrorMessageForMissingValueInParams() throws IOException { ResponseException re = expectThrows( ResponseException.class, - () -> runEsql(new RequestObjectBuilder().query("row a = 1").params("[\"x\", 123, true, {\"type\": \"y\"}]")) + () -> runEsql(requestObjectBuilder().query("row a = 1").params("[\"x\", 123, true, {\"type\": \"y\"}]")) ); assertThat(EntityUtils.toString(re.getResponse().getEntity()), containsString("Required [value]")); } @@ -571,7 +575,7 @@ public void testErrorMessageForMissingValueInParams() throws IOException { public void testErrorMessageForInvalidTypeInParams() throws IOException { ResponseException re = expectThrows( ResponseException.class, - () -> runEsqlSync(new RequestObjectBuilder().query("row a = 1 | eval x = ?").params("[{\"type\": \"byte\", \"value\": 5}]")) + () -> runEsqlSync(requestObjectBuilder().query("row a = 1 | eval x = ?").params("[{\"type\": \"byte\", \"value\": 5}]")) ); assertThat( EntityUtils.toString(re.getResponse().getEntity()), @@ -608,7 +612,7 @@ public void testErrorMessageForLiteralDateMathOverflowOnNegation() throws IOExce private void assertExceptionForDateMath(String dateMathString, String errorSubstring) throws IOException { ResponseException re = expectThrows( ResponseException.class, - () -> runEsql(new RequestObjectBuilder().query("row a = 1 | eval x = now() + (" + dateMathString + ")")) + () -> runEsql(requestObjectBuilder().query("row a = 1 | eval x = now() + (" + dateMathString + ")")) ); String responseMessage = EntityUtils.toString(re.getResponse().getEntity()); @@ -621,9 +625,7 @@ private void assertExceptionForDateMath(String dateMathString, String errorSubst public void testErrorMessageForArrayValuesInParams() throws IOException { ResponseException re = expectThrows( ResponseException.class, - () -> runEsql( - new RequestObjectBuilder().query("row a = 1 | eval x = ?").params("[{\"type\": \"integer\", \"value\": [5, 6, 7]}]") - ) + () -> runEsql(requestObjectBuilder().query("row a = 1 | eval x = ?").params("[{\"type\": \"integer\", \"value\": [5, 6, 7]}]")) ); assertThat( EntityUtils.toString(re.getResponse().getEntity()), @@ -698,6 +700,10 @@ public static Map runEsqlSync( RequestOptions.Builder options = request.getOptions().toBuilder(); options.setWarningsHandler(WarningsHandler.PERMISSIVE); // We assert the warnings ourselves options.addHeader("Content-Type", mediaType); + if ("true".equals(System.getProperty("tests.version_parameter_unsupported"))) { + // Masquerade as an old version of the official client, so we get the oldest version by default + options.addHeader("x-elastic-client-meta", "es=8.13"); + } if (randomBoolean()) { options.addHeader("Accept", mediaType); @@ -723,6 +729,10 @@ public static Map runEsqlAsync( RequestOptions.Builder options = request.getOptions().toBuilder(); options.setWarningsHandler(WarningsHandler.PERMISSIVE); // We assert the warnings ourselves options.addHeader("Content-Type", mediaType); + if ("true".equals(System.getProperty("tests.version_parameter_unsupported"))) { + // Masquerade as an old version of the official client, so we get the oldest version by default + options.addHeader("x-elastic-client-meta", "es=8.13"); + } if (randomBoolean()) { options.addHeader("Accept", mediaType); @@ -1005,8 +1015,12 @@ private static String repeatValueAsMV(Object value) { return "[" + value + ", " + value + "]"; } - private static RequestObjectBuilder builder() throws IOException { - return new RequestObjectBuilder(); + public static RequestObjectBuilder requestObjectBuilder(String version) throws IOException { + return new RequestObjectBuilder().version(version); + } + + public static RequestObjectBuilder requestObjectBuilder() throws IOException { + return requestObjectBuilder(EsqlTestUtils.latestEsqlVersionOrSnapshot()); } @After diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java index cdec7752aef5..e562c1cfa72e 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java @@ -12,6 +12,7 @@ import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xpack.esql.CsvTestsDataLoader; import org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase; +import org.elasticsearch.xpack.esql.version.EsqlVersion; import org.junit.AfterClass; import org.junit.Before; @@ -96,7 +97,9 @@ private void checkException(EsqlQueryGenerator.QueryExecuted query) { private EsqlQueryGenerator.QueryExecuted execute(String command, int depth) { try { - Map a = RestEsqlTestCase.runEsqlSync(new RestEsqlTestCase.RequestObjectBuilder().query(command).build()); + Map a = RestEsqlTestCase.runEsqlSync( + new RestEsqlTestCase.RequestObjectBuilder().query(command).version(EsqlVersion.ROCKET.toString()).build() + ); List outputSchema = outputSchema(a); return new EsqlQueryGenerator.QueryExecuted(command, depth, outputSchema, null); } catch (Exception e) { diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java index e6470e0eb2d0..5113346baf0a 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Build; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.BlockUtils; @@ -23,6 +24,7 @@ import org.elasticsearch.xpack.esql.stats.Metrics; import org.elasticsearch.xpack.esql.stats.SearchStats; import org.elasticsearch.xpack.esql.type.EsqlDataTypeRegistry; +import org.elasticsearch.xpack.esql.version.EsqlVersion; import org.elasticsearch.xpack.ql.expression.Attribute; import org.elasticsearch.xpack.ql.expression.Literal; import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; @@ -54,9 +56,12 @@ import static org.junit.Assert.assertTrue; public final class EsqlTestUtils { + public static String latestEsqlVersionOrSnapshot() { + EsqlVersion version = Build.current().isSnapshot() ? EsqlVersion.SNAPSHOT : EsqlVersion.latestReleased(); + return version.toString(); + } public static class TestSearchStats extends SearchStats { - public TestSearchStats() { super(emptyList()); } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractEsqlIntegTestCase.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractEsqlIntegTestCase.java index a9238d202e5b..04a752e79b2f 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractEsqlIntegTestCase.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/AbstractEsqlIntegTestCase.java @@ -25,6 +25,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.junit.annotations.TestLogging; +import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; import org.elasticsearch.xpack.esql.plugin.QueryPragmas; import org.elasticsearch.xpack.esql.plugin.TransportEsqlQueryAction; @@ -39,6 +40,21 @@ @TestLogging(value = "org.elasticsearch.xpack.esql.session:DEBUG", reason = "to better understand planning") public abstract class AbstractEsqlIntegTestCase extends ESIntegTestCase { + public static EsqlQueryRequest asyncSyncRequestOnLatestVersion() { + EsqlQueryRequest request = EsqlQueryRequest.asyncEsqlQueryRequest(); + applyLatestVersion(request); + return request; + } + + public static EsqlQueryRequest syncRequestOnLatestVersion() { + EsqlQueryRequest request = EsqlQueryRequest.syncEsqlQueryRequest(); + applyLatestVersion(request); + return request; + } + + private static void applyLatestVersion(EsqlQueryRequest request) { + request.esqlVersion(EsqlTestUtils.latestEsqlVersionOrSnapshot()); + } @After public void ensureExchangesAreReleased() throws Exception { @@ -138,9 +154,18 @@ protected EsqlQueryResponse run(String esqlCommands, QueryPragmas pragmas) { } protected EsqlQueryResponse run(String esqlCommands, QueryPragmas pragmas, QueryBuilder filter) { - EsqlQueryRequest request = new EsqlQueryRequest(); + return run(esqlCommands, pragmas, filter, null); + } + + protected EsqlQueryResponse run(String esqlCommands, QueryPragmas pragmas, QueryBuilder filter, String version) { + EsqlQueryRequest request = syncRequestOnLatestVersion(); + if (version != null) { + request.esqlVersion(version); + } request.query(esqlCommands); - request.pragmas(pragmas); + if (pragmas != null) { + request.pragmas(pragmas); + } if (filter != null) { request.filter(filter); } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersCancellationIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersCancellationIT.java index bc4708cc19c1..736a20b367b7 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersCancellationIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersCancellationIT.java @@ -159,7 +159,7 @@ private void createRemoteIndex(int numDocs) throws Exception { public void testCancel() throws Exception { createRemoteIndex(between(10, 100)); - EsqlQueryRequest request = new EsqlQueryRequest(); + EsqlQueryRequest request = AbstractEsqlIntegTestCase.syncRequestOnLatestVersion(); request.query("FROM *:test | STATS total=sum(const) | LIMIT 1"); request.pragmas(randomPragmas()); PlainActionFuture requestFuture = new PlainActionFuture<>(); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java index 2b59e6dd1957..77fc6987e07c 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersEnrichIT.java @@ -457,7 +457,7 @@ public void testEnrichCoordinatorThenEnrichRemote() { } protected EsqlQueryResponse runQuery(String query) { - EsqlQueryRequest request = new EsqlQueryRequest(); + EsqlQueryRequest request = AbstractEsqlIntegTestCase.syncRequestOnLatestVersion(); request.query(query); request.pragmas(AbstractEsqlIntegTestCase.randomPragmas()); if (randomBoolean()) { diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java index ac2abf21a8f8..9021a1056212 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java @@ -156,7 +156,7 @@ public void testProfile() { waitForNoInitializingShards(client(REMOTE_CLUSTER), TimeValue.timeValueSeconds(30), "logs-2"); final int localOnlyProfiles; { - EsqlQueryRequest request = new EsqlQueryRequest(); + EsqlQueryRequest request = AbstractEsqlIntegTestCase.syncRequestOnLatestVersion(); request.query("FROM logs* | stats sum(v)"); request.pragmas(pragmas); request.profile(true); @@ -171,7 +171,7 @@ public void testProfile() { } final int remoteOnlyProfiles; { - EsqlQueryRequest request = new EsqlQueryRequest(); + EsqlQueryRequest request = AbstractEsqlIntegTestCase.syncRequestOnLatestVersion(); request.query("FROM *:logs* | stats sum(v)"); request.pragmas(pragmas); request.profile(true); @@ -186,7 +186,7 @@ public void testProfile() { } final int allProfiles; { - EsqlQueryRequest request = new EsqlQueryRequest(); + EsqlQueryRequest request = AbstractEsqlIntegTestCase.syncRequestOnLatestVersion(); request.query("FROM logs*,*:logs* | stats total = sum(v)"); request.pragmas(pragmas); request.profile(true); @@ -203,7 +203,7 @@ public void testProfile() { } public void testWarnings() throws Exception { - EsqlQueryRequest request = new EsqlQueryRequest(); + EsqlQueryRequest request = AbstractEsqlIntegTestCase.syncRequestOnLatestVersion(); request.query("FROM logs*,*:logs* | EVAL ip = to_ip(id) | STATS total = sum(v) by ip | LIMIT 10"); PlainActionFuture future = new PlainActionFuture<>(); InternalTestCluster cluster = cluster(LOCAL_CLUSTER); @@ -229,7 +229,7 @@ public void testWarnings() throws Exception { } protected EsqlQueryResponse runQuery(String query) { - EsqlQueryRequest request = new EsqlQueryRequest(); + EsqlQueryRequest request = AbstractEsqlIntegTestCase.syncRequestOnLatestVersion(); request.query(query); request.pragmas(AbstractEsqlIntegTestCase.randomPragmas()); return runQuery(request); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java index 9b3f61175c3f..43c282e9361f 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java @@ -328,7 +328,7 @@ public void testTopN() { } public void testProfile() { - EsqlQueryRequest request = new EsqlQueryRequest(); + EsqlQueryRequest request = AbstractEsqlIntegTestCase.syncRequestOnLatestVersion(); request.pragmas(randomPragmas()); request.query("from listens* | sort timestamp DESC | limit 1 | " + enrichSongCommand() + " | KEEP timestamp, artist"); request.profile(true); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionBreakerIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionBreakerIT.java index 85eb0c02625a..f16f5808da89 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionBreakerIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionBreakerIT.java @@ -130,7 +130,7 @@ public void testBreaker() { setRequestCircuitBreakerLimit(ByteSizeValue.ofBytes(between(256, 512))); try { final ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> { - var request = new EsqlQueryRequest(); + var request = AbstractEsqlIntegTestCase.syncRequestOnLatestVersion(); request.query("from test_breaker | stats count_distinct(foo) by bar"); request.pragmas(randomPragmas()); try (var ignored = client().execute(EsqlQueryAction.INSTANCE, request).actionGet(2, TimeUnit.MINUTES)) { diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java index 4524c4c889fa..82ab52ca5a1b 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionTaskIT.java @@ -367,7 +367,7 @@ protected void doRun() throws Exception { try { scriptPermits.release(numberOfDocs()); // do not block Lucene operators Client client = client(coordinator); - EsqlQueryRequest request = new EsqlQueryRequest(); + EsqlQueryRequest request = AbstractEsqlIntegTestCase.syncRequestOnLatestVersion(); client().admin() .indices() .prepareUpdateSettings("test") diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlAsyncActionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlAsyncActionIT.java index e2e635917ed1..27edadb25ab2 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlAsyncActionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlAsyncActionIT.java @@ -52,8 +52,11 @@ protected Collection> nodePlugins() { } @Override - protected EsqlQueryResponse run(String esqlCommands, QueryPragmas pragmas, QueryBuilder filter) { - EsqlQueryRequest request = EsqlQueryRequest.asyncEsqlQueryRequest(); + protected EsqlQueryResponse run(String esqlCommands, QueryPragmas pragmas, QueryBuilder filter, String version) { + EsqlQueryRequest request = AbstractEsqlIntegTestCase.asyncSyncRequestOnLatestVersion(); + if (version != null) { + request.esqlVersion(version); + } request.query(esqlCommands); request.pragmas(pragmas); // deliberately small timeout, to frequently trigger incomplete response diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeBasedIndicesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeBasedIndicesIT.java index a1fbee17ef8e..150d617bb4e2 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeBasedIndicesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeBasedIndicesIT.java @@ -37,19 +37,17 @@ public void testFilter() { } bulk.get(); { - EsqlQueryRequest request = new EsqlQueryRequest(); - request.query("FROM test | limit 1000"); - request.filter(new RangeQueryBuilder("@timestamp").from(epoch - TimeValue.timeValueHours(3).millis()).to("now")); - try (var resp = run(request)) { + String query = "FROM test | limit 1000"; + var filter = new RangeQueryBuilder("@timestamp").from(epoch - TimeValue.timeValueHours(3).millis()).to("now"); + try (var resp = run(query, null, filter)) { List> values = getValuesList(resp); assertThat(values, hasSize(oldDocs)); } } { - EsqlQueryRequest request = new EsqlQueryRequest(); - request.query("FROM test | limit 1000"); - request.filter(new RangeQueryBuilder("@timestamp").from("now").to(epoch + TimeValue.timeValueHours(3).millis())); - try (var resp = run(request)) { + String query = "FROM test | limit 1000"; + var filter = new RangeQueryBuilder("@timestamp").from("now").to(epoch + TimeValue.timeValueHours(3).millis()); + try (var resp = run(query, null, filter)) { List> values = getValuesList(resp); assertThat(values, hasSize(newDocs)); } diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/WarningsIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/WarningsIT.java index 0f05add15da5..445ca0414ed8 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/WarningsIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/WarningsIT.java @@ -68,7 +68,7 @@ public void testCollectWarnings() throws Exception { DiscoveryNode coordinator = randomFrom(clusterService().state().nodes().stream().toList()); client().admin().indices().prepareRefresh("index-1", "index-2").get(); - EsqlQueryRequest request = new EsqlQueryRequest(); + EsqlQueryRequest request = AbstractEsqlIntegTestCase.syncRequestOnLatestVersion(); request.query("FROM index-* | EVAL ip = to_ip(host) | STATS s = COUNT(*) by ip | KEEP ip | LIMIT 100"); request.pragmas(randomPragmas()); CountDownLatch latch = new CountDownLatch(1); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java index 54ae2f4c90fc..32ff0cf7bc6a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequest.java @@ -69,9 +69,7 @@ public EsqlQueryRequest(StreamInput in) throws IOException { public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; if (Strings.hasText(esqlVersion) == false) { - // TODO: make this required - // "https://github.com/elastic/elasticsearch/issues/104890" - // validationException = addValidationError(invalidVersion("is required"), validationException); + validationException = addValidationError(invalidVersion("is required"), validationException); } else { EsqlVersion version = EsqlVersion.parse(esqlVersion); if (version == null) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestBuilder.java index 511fbd9f1c27..9eeffbb35c10 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestBuilder.java @@ -7,11 +7,13 @@ package org.elasticsearch.xpack.esql.action; +import org.elasticsearch.Build; import org.elasticsearch.client.internal.ElasticsearchClient; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.xpack.core.esql.action.internal.SharedSecrets; import org.elasticsearch.xpack.esql.plugin.QueryPragmas; +import org.elasticsearch.xpack.esql.version.EsqlVersion; public class EsqlQueryRequestBuilder extends org.elasticsearch.xpack.core.esql.action.EsqlQueryRequestBuilder< EsqlQueryRequest, @@ -27,6 +29,8 @@ public static EsqlQueryRequestBuilder newSyncEsqlQueryRequestBuilder(Elasticsear private EsqlQueryRequestBuilder(ElasticsearchClient client, EsqlQueryRequest request) { super(client, EsqlQueryAction.INSTANCE, request); + EsqlVersion version = Build.current().isSnapshot() ? EsqlVersion.SNAPSHOT : EsqlVersion.latestReleased(); + esqlVersion(version.versionStringWithoutEmoji()); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java index 6ec1af033f86..0c9bfa2054b1 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryRequestTests.java @@ -238,7 +238,6 @@ public void testSnapshotVersionIsOnlyValidOnSnapshot() throws IOException { assertThat(request.validate().getMessage(), containsString(errorOnNonSnapshotBuilds)); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/104890") public void testMissingVersionIsNotValid() throws IOException { String missingVersion = randomBoolean() ? "" : ", \"version\": \"\""; String json = String.format(Locale.ROOT, """ diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java index 2c393ea7ed1d..6522196eb76a 100644 --- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java @@ -690,6 +690,9 @@ protected Request esqlRequest(String command) throws IOException { body.endObject(); } } + // TODO: we should use the latest or a random version, even when new versions are released. + String version = Build.current().isSnapshot() ? "snapshot" : "2024.04.01"; + body.field("version", version); body.endObject(); Request request = new Request("POST", "_query"); request.setJsonEntity(org.elasticsearch.common.Strings.toString(body)); diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/100_bug_fix.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/100_bug_fix.yml index 412e23a76853..bc1bf5987a6d 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/100_bug_fix.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/100_bug_fix.yml @@ -22,6 +22,7 @@ esql.query: body: query: 'FROM test | sort emp_no | eval ip = to_ip(coalesce(ip1.keyword, "255.255.255.255")) | keep emp_no, ip' + version: 2024.04.01 - match: { columns.0.name: "emp_no" } - match: { columns.0.type: "long" } @@ -41,6 +42,7 @@ esql.query: body: query: 'FROM test | sort emp_no | eval x1 = concat(ip1, ip2), x2 = coalesce(x1, "255.255.255.255"), x3 = to_ip(x2) | keep emp_no, x*' + version: 2024.04.01 - match: { columns.0.name: "emp_no" } - match: { columns.0.type: "long" } @@ -109,6 +111,7 @@ esql.query: body: query: 'from index* metadata _index | limit 5 | sort _index desc' + version: 2024.04.01 - match: { columns.0.name: http.headers } - match: { columns.0.type: unsupported } - match: { columns.1.name: http.headers.location } @@ -171,6 +174,7 @@ esql.query: body: query: 'from npe_single_value* | stats x = avg(field1) | limit 10' + version: 2024.04.01 - match: { columns.0.name: x } - match: { columns.0.type: double } - length: { values: 1 } @@ -180,6 +184,7 @@ esql.query: body: query: 'from npe_single_value* | stats x = avg(field2) | limit 10' + version: 2024.04.01 - match: { columns.0.name: x } - match: { columns.0.type: double } - length: { values: 1 } @@ -189,6 +194,7 @@ esql.query: body: query: 'from npe_single_value* | stats x = avg(field3) | limit 10' + version: 2024.04.01 - match: { columns.0.name: x } - match: { columns.0.type: double } - length: { values: 1 } @@ -232,6 +238,7 @@ esql.query: body: query: 'from idx_with_date_ip_txt | where id == 1 | eval x = date_format(text, date), y = date_extract(text2, date), p = date_parse(text, "2024-03-14") | keep x, y, p | limit 1' + version: 2024.04.01 - match: { columns.0.name: x } - match: { columns.0.type: keyword } - match: { columns.1.name: y } @@ -245,6 +252,7 @@ esql.query: body: query: 'from idx_with_date_ip_txt | where id > 1 | eval x = cidr_match(ip, text) | sort id | keep id, x | limit 2' + version: 2024.04.01 - match: { columns.0.name: id } - match: { columns.0.type: long } - match: { columns.1.name: x } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/10_basic.yml index 52d390e7b288..da87251c3596 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/10_basic.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/10_basic.yml @@ -118,6 +118,7 @@ setup: esql.query: body: query: 'from test' + version: 2024.04.01 - match: {columns.0.name: "color"} - match: {columns.0.type: "keyword"} @@ -139,6 +140,7 @@ setup: esql.query: body: query: 'from test | sort count | limit 1' + version: 2024.04.01 - match: {columns.1.name: "count"} - match: {columns.1.type: "long"} @@ -151,6 +153,7 @@ setup: body: query: 'from test | keep data | sort data | limit 2' columnar: true + version: 2024.04.01 - match: {columns.0.name: "data"} - match: {columns.0.type: "long"} @@ -162,6 +165,7 @@ setup: esql.query: body: query: 'from test | eval x = count + 7 | sort x | limit 1' + version: 2024.04.01 - match: {columns.0.name: "color"} - match: {columns.1.name: "count"} @@ -179,6 +183,7 @@ setup: esql.query: body: query: 'from test | sort time | eval x = data + 1, y = data_d + count, z = x + y | keep data, x, y, z, time | limit 2' + version: 2024.04.01 - match: {columns.0.name: "data"} - match: {columns.0.type: "long"} @@ -209,6 +214,7 @@ setup: body: query: 'from test | sort time | limit 2 | keep count' columnar: true + version: 2024.04.01 - length: {columns: 1} - match: {columns.0.name: "count"} @@ -222,6 +228,7 @@ setup: body: query: 'from test | sort time desc | limit 2 | keep count' columnar: true + version: 2024.04.01 - length: {columns: 1} - match: {columns.0.name: "count"} @@ -235,6 +242,7 @@ setup: body: query: 'from test | sort time | limit 2 | keep count | eval x = count + 1' columnar: true + version: 2024.04.01 - length: {columns: 2} - match: {columns.0.name: "count"} @@ -252,6 +260,7 @@ setup: body: query: 'from test | sort time | limit 2 | keep count | eval x = count + 1 | keep x' columnar: true + version: 2024.04.01 - length: {columns: 1} - match: {columns.0.name: "x"} @@ -265,6 +274,7 @@ setup: esql.query: body: query: 'from test | limit 10 | sort time | limit 1' + version: 2024.04.01 - length: {columns: 6} - length: {values: 1} @@ -278,6 +288,7 @@ setup: body: query: 'row a = ? | eval b = ?, c = 1 + ?' params: ["foo", 15, 10] + version: 2024.04.01 - length: {columns: 3} - match: {columns.0.name: "a"} @@ -297,6 +308,7 @@ setup: body: query: 'from test | where color == ? and count == ? and time == ? | keep data, count, color' params: ["green", 44, 1674835275193] + version: 2024.04.01 - length: {columns: 3} - match: {columns.0.name: "data"} @@ -315,6 +327,7 @@ setup: body: query: 'from test | eval x = ?, y = ?, z = ?, t = ?, u = ?, v = ? | keep x, y, z, t, u, v | limit 3' params: [{"value": 1, "type": "keyword"}, {"value": 2, "type": "double"}, null, true, 123, {"value": 123, "type": "long"}] + version: 2024.04.01 - length: {columns: 6} - match: {columns.0.name: "x"} diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/110_all_null.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/110_all_null.yml index a18dbba1abfa..f6271ab02b81 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/110_all_null.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/110_all_null.yml @@ -122,6 +122,7 @@ row wise and keep null: body: query: 'FROM test | WHERE time <= 1674835275188 | SORT time ASC | LIMIT 2' columnar: false + version: 2024.04.01 - length: {columns: 8} - match: {columns.0.name: "always_null"} @@ -153,6 +154,7 @@ row wise and drop null: body: query: 'FROM test | WHERE time <= 1674835275188 | SORT time ASC | LIMIT 2' columnar: false + version: 2024.04.01 - length: {all_columns: 8} - match: {all_columns.0.name: "always_null"} @@ -196,6 +198,7 @@ columnar and keep null: body: query: 'FROM test | WHERE time <= 1674835275188 | SORT time ASC | LIMIT 2' columnar: true + version: 2024.04.01 - length: {columns: 8} - match: {columns.0.name: "always_null"} @@ -227,6 +230,7 @@ columnar and drop null: body: query: 'FROM test | WHERE time <= 1674835275188 | SORT time ASC | LIMIT 2' columnar: true + version: 2024.04.01 - length: {all_columns: 8} - match: {all_columns.0.name: "always_null"} diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/110_insensitive_equals.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/110_insensitive_equals.yml index b40564cdac1d..e505d11cbe13 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/110_insensitive_equals.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/110_insensitive_equals.yml @@ -47,6 +47,7 @@ setup: esql.query: body: query: 'FROM test | where keyword =~ keywordUpper | keep id, keyword, keywordUpper' + version: 2024.04.01 - match: { columns.0.name: "id" } - match: { columns.0.type: "long" } @@ -66,6 +67,7 @@ setup: esql.query: body: query: 'FROM test | where text =~ textCamel | keep id, text, textCamel' + version: 2024.04.01 - match: { columns.0.name: "id" } - match: { columns.0.type: "long" } @@ -86,6 +88,7 @@ setup: esql.query: body: query: 'FROM test | where keyword =~ text | keep id, keyword, text' + version: 2024.04.01 - match: { columns.0.name: "id" } - match: { columns.0.type: "long" } @@ -106,6 +109,7 @@ setup: esql.query: body: query: 'FROM test | where keywordUpper =~ textCamel | keep id, keywordUpper, textCamel | sort id' + version: 2024.04.01 - match: { columns.0.name: "id" } - match: { columns.0.type: "long" } @@ -127,6 +131,7 @@ setup: esql.query: body: query: 'FROM test | where keywordUpper =~ "fo*" | keep id, keywordUpper' + version: 2024.04.01 - match: { columns.0.name: "id" } - match: { columns.0.type: "long" } @@ -141,6 +146,7 @@ setup: esql.query: body: query: 'FROM test | where wildcard =~ "foo*" | keep id, wildcard' + version: 2024.04.01 - match: { columns.0.name: "id" } - match: { columns.0.type: "long" } @@ -156,6 +162,7 @@ setup: esql.query: body: query: 'FROM test | where wildcard =~ "fOo*" | keep id, wildcard' + version: 2024.04.01 - match: { columns.0.name: "id" } - match: { columns.0.type: "long" } @@ -172,6 +179,7 @@ setup: esql.query: body: query: 'FROM test | where keywordUpper =~ "fo?" | keep id, keywordUpper' + version: 2024.04.01 - match: { columns.0.name: "id" } - match: { columns.0.type: "long" } @@ -186,6 +194,7 @@ setup: esql.query: body: query: 'FROM test | where wildcard =~ "bar?" | keep id, wildcard' + version: 2024.04.01 - match: { columns.0.name: "id" } - match: { columns.0.type: "long" } @@ -201,6 +210,7 @@ setup: esql.query: body: query: 'FROM test | where wildcard =~ "bAr?" | keep id, wildcard' + version: 2024.04.01 - match: { columns.0.name: "id" } - match: { columns.0.type: "long" } @@ -219,6 +229,7 @@ setup: esql.query: body: query: 'FROM test | where text =~ "Fo*" | keep id, text | sort id' + version: 2024.04.01 - match: { columns.0.name: "id" } - match: { columns.0.type: "long" } @@ -233,6 +244,7 @@ setup: esql.query: body: query: 'FROM test | where wildcardText =~ "fOo*" | keep id, wildcardText' + version: 2024.04.01 - match: { columns.0.name: "id" } - match: { columns.0.type: "long" } @@ -248,6 +260,7 @@ setup: esql.query: body: query: 'FROM test | where wildcardText =~ "bAr?" | keep id, wildcardText' + version: 2024.04.01 - match: { columns.0.name: "id" } - match: { columns.0.type: "long" } @@ -266,6 +279,7 @@ setup: esql.query: body: query: 'FROM test | where text =~ "fo\\*" | keep id, text' + version: 2024.04.01 - match: { columns.0.name: "id" } - match: { columns.0.type: "long" } @@ -283,6 +297,7 @@ setup: esql.query: body: query: 'FROM test | where wildcard =~ wildcardText | keep id, wildcard, wildcardText | sort id' + version: 2024.04.01 - match: { columns.0.name: "id" } - match: { columns.0.type: "long" } @@ -302,6 +317,7 @@ setup: esql.query: body: query: 'FROM test | where NOT wildcard =~ wildcardText | keep id, wildcard, wildcardText | sort id' + version: 2024.04.01 - match: { columns.0.name: "id" } - match: { columns.0.type: "long" } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/120_profile.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/120_profile.yml index 17034de677b8..ec415cbfa12d 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/120_profile.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/120_profile.yml @@ -130,6 +130,7 @@ avg 8.14 or after: query: 'FROM test | STATS AVG(data) | LIMIT 1' columnar: true profile: true + version: 2024.04.01 - match: {columns.0.name: "AVG(data)"} - match: {columns.0.type: "double"} diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/130_spatial.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/130_spatial.yml index 053d33ee9bf4..2274d5973087 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/130_spatial.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/130_spatial.yml @@ -97,6 +97,7 @@ geo_point: esql.query: body: query: 'from geo_points | sort id' + version: 2024.04.01 - match: { columns.0.name: id } - match: { columns.0.type: integer } - match: { columns.1.name: location } @@ -114,6 +115,7 @@ geo_point unsortable: esql.query: body: query: 'from geo_points | sort location' + version: 2024.04.01 --- geo_point unsortable with limit: @@ -122,6 +124,7 @@ geo_point unsortable with limit: esql.query: body: query: 'from geo_points | LIMIT 10 | sort location' + version: 2024.04.01 --- geo_point unsortable with limit from row: @@ -130,6 +133,7 @@ geo_point unsortable with limit from row: esql.query: body: query: 'ROW wkt = ["POINT(42.9711 -14.7553)", "POINT(75.8093 22.7277)"] | MV_EXPAND wkt | EVAL pt = TO_GEOPOINT(wkt) | limit 5 | sort pt' + version: 2024.04.01 --- values unsupported for geo_point: @@ -138,6 +142,7 @@ values unsupported for geo_point: esql.query: body: query: 'FROM geo_points | STATS VALUES(location)' + version: 2024.04.01 --- cartesian_point: @@ -147,6 +152,7 @@ cartesian_point: esql.query: body: query: 'from cartesian_points | sort id' + version: 2024.04.01 - match: { columns.0.name: id } - match: { columns.0.type: integer } - match: { columns.1.name: location } @@ -164,6 +170,7 @@ cartesian_point unsortable: esql.query: body: query: 'from cartesian_points | sort location' + version: 2024.04.01 --- cartesian_point unsortable with limit: @@ -172,6 +179,7 @@ cartesian_point unsortable with limit: esql.query: body: query: 'from cartesian_points | LIMIT 10 | sort location' + version: 2024.04.01 --- cartesian_point unsortable with limit from row: @@ -180,6 +188,7 @@ cartesian_point unsortable with limit from row: esql.query: body: query: 'ROW wkt = ["POINT(4297.11 -1475.53)", "POINT(7580.93 2272.77)"] | MV_EXPAND wkt | EVAL pt = TO_CARTESIANPOINT(wkt) | limit 5 | sort pt' + version: 2024.04.01 --- geo_shape: @@ -189,6 +198,7 @@ geo_shape: esql.query: body: query: 'from geo_shapes | sort id' + version: 2024.04.01 - match: { columns.0.name: id } - match: { columns.0.type: integer } - match: { columns.1.name: shape } @@ -206,6 +216,7 @@ geo_shape unsortable: esql.query: body: query: 'from geo_shapes | sort shape' + version: 2024.04.01 --- geo_shape unsortable with limit: @@ -214,6 +225,7 @@ geo_shape unsortable with limit: esql.query: body: query: 'from geo_shapes | LIMIT 10 | sort shape' + version: 2024.04.01 --- geo_shape unsortable with limit from row: @@ -222,6 +234,7 @@ geo_shape unsortable with limit from row: esql.query: body: query: 'ROW wkt = ["POINT(42.9711 -14.7553)", "POINT(75.8093 22.7277)"] | MV_EXPAND wkt | EVAL shape = TO_GEOSHAPE(wkt) | limit 5 | sort shape' + version: 2024.04.01 --- cartesian_shape: @@ -231,6 +244,7 @@ cartesian_shape: esql.query: body: query: 'from cartesian_shapes | sort id' + version: 2024.04.01 - match: { columns.0.name: id } - match: { columns.0.type: integer } - match: { columns.1.name: shape } @@ -248,6 +262,7 @@ cartesian_shape unsortable: esql.query: body: query: 'from cartesian_shapes | sort shape' + version: 2024.04.01 --- cartesian_shape unsortable with limit: @@ -256,6 +271,7 @@ cartesian_shape unsortable with limit: esql.query: body: query: 'from cartesian_shapes | LIMIT 10 | sort shape' + version: 2024.04.01 --- cartesian_shape unsortable with limit from row: @@ -264,3 +280,4 @@ cartesian_shape unsortable with limit from row: esql.query: body: query: 'ROW wkt = ["POINT(4297.11 -1475.53)", "POINT(7580.93 2272.77)"] | MV_EXPAND wkt | EVAL shape = TO_CARTESIANSHAPE(wkt) | limit 5 | sort shape' + version: 2024.04.01 diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/20_aggs.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/20_aggs.yml index 672dfa1503c4..69a9213980f9 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/20_aggs.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/20_aggs.yml @@ -120,6 +120,7 @@ setup: esql.query: body: query: 'from test' + version: 2024.04.01 - match: {columns.0.name: "color"} - match: {columns.0.type: "keyword"} @@ -146,6 +147,7 @@ setup: body: query: 'from test | where color == "red" | stats avg(data) by color' columnar: true + version: 2024.04.01 - match: {columns.0.name: "avg(data)"} - match: {columns.0.type: "double"} @@ -162,6 +164,7 @@ setup: body: query: 'from test | stats avg(count)' columnar: true + version: 2024.04.01 - match: {columns.0.name: "avg(count)"} - match: {columns.0.type: "double"} @@ -176,6 +179,7 @@ setup: body: query: 'from test | stats f1 = avg(count)' columnar: true + version: 2024.04.01 - match: {columns.0.name: "f1"} - match: {columns.0.type: "double"} @@ -190,6 +194,7 @@ setup: body: query: 'from test | stats count(data)' columnar: true + version: 2024.04.01 - match: {columns.0.name: "count(data)"} - match: {columns.0.type: "long"} @@ -204,6 +209,7 @@ setup: body: query: 'from test | stats dataCount = count(data)' columnar: true + version: 2024.04.01 - match: {columns.0.name: "dataCount"} - match: {columns.0.type: "long"} @@ -218,6 +224,7 @@ setup: body: query: 'from test | stats min(count)' columnar: true + version: 2024.04.01 - match: {columns.0.name: "min(count)"} - match: {columns.0.type: "long"} @@ -232,6 +239,7 @@ setup: body: query: 'from test | stats minCount=min(count)' columnar: true + version: 2024.04.01 - match: {columns.0.name: "minCount"} - match: {columns.0.type: "long"} @@ -246,6 +254,7 @@ setup: body: query: 'from test | stats max(count)' columnar: true + version: 2024.04.01 - match: {columns.0.name: "max(count)"} - match: {columns.0.type: "long"} @@ -260,6 +269,7 @@ setup: body: query: 'from test | stats maxCount=max(count)' columnar: true + version: 2024.04.01 - match: {columns.0.name: "maxCount"} - match: {columns.0.type: "long"} @@ -272,6 +282,7 @@ setup: body: query: 'from test | stats avg(count) by color | sort color | limit 2' columnar: true + version: 2024.04.01 - match: {columns.0.name: "avg(count)"} - match: {columns.0.type: "double"} @@ -289,6 +300,7 @@ setup: body: query: 'from test | stats med=median(count)' columnar: true + version: 2024.04.01 - match: {columns.0.name: "med"} - match: {columns.0.type: "double"} @@ -303,6 +315,7 @@ setup: body: query: 'from test | stats med=median(count_d)' columnar: true + version: 2024.04.01 - match: {columns.0.name: "med"} - match: {columns.0.type: "double"} @@ -317,6 +330,7 @@ setup: body: query: 'from test | stats med=median(count) by color | sort med' columnar: true + version: 2024.04.01 - match: {columns.0.name: "med"} - match: {columns.0.type: "double"} @@ -334,6 +348,7 @@ setup: body: query: 'from test | stats med=median(count_d) by color | sort med' columnar: true + version: 2024.04.01 - match: {columns.0.name: "med"} - match: {columns.0.type: "double"} @@ -351,6 +366,7 @@ setup: body: query: 'from test | stats med=median_absolute_deviation(count)' columnar: true + version: 2024.04.01 - match: {columns.0.name: "med"} - match: {columns.0.type: "double"} @@ -365,6 +381,7 @@ setup: body: query: 'from test | stats med=median_absolute_deviation(count_d)' columnar: true + version: 2024.04.01 - match: {columns.0.name: "med"} - match: {columns.0.type: "double"} @@ -379,6 +396,7 @@ setup: body: query: 'from test | stats med=median_absolute_deviation(count) by color | sort color' columnar: true + version: 2024.04.01 - match: {columns.0.name: "med"} - match: {columns.0.type: "double"} @@ -396,6 +414,7 @@ setup: body: query: 'from test | stats med=median_absolute_deviation(count_d) by color | sort color' columnar: true + version: 2024.04.01 - match: {columns.0.name: "med"} - match: {columns.0.type: "double"} @@ -412,6 +431,7 @@ setup: esql.query: body: query: 'from test | stats avg_count = avg(count) | eval x = avg_count + 7' + version: 2024.04.01 - length: {values: 1} - length: {values.0: 2} @@ -425,6 +445,7 @@ setup: esql.query: body: query: 'from test | stats x = avg(count) | where x > 100' + version: 2024.04.01 - length: {values: 0} @@ -434,6 +455,7 @@ setup: esql.query: body: query: 'from test | eval nullsum = count_d + null | sort nullsum | limit 1' + version: 2024.04.01 - length: {columns: 8} - length: {values: 1} @@ -449,6 +471,7 @@ setup: esql.query: body: query: 'row a = 1, b = 2, c = null | eval z = c + b + a' + version: 2024.04.01 - length: {columns: 4} - length: {values: 1} @@ -474,6 +497,7 @@ setup: esql.query: body: query: 'from test | eval nullsum = count_d + null | stats count(nullsum)' + version: 2024.04.01 - length: {columns: 1} - length: {values: 1} @@ -490,6 +514,7 @@ setup: esql.query: body: query: 'row l=1, d=1.0, ln=1 + null, dn=1.0 + null | stats sum(l), sum(d), sum(ln), sum(dn)' + version: 2024.04.01 - length: {columns: 4} - length: {values: 1} @@ -516,6 +541,7 @@ grouping on text: body: query: 'FROM test | STATS med=median(count) BY text | SORT med' columnar: true + version: 2024.04.01 - match: {columns.0.name: "med"} - match: {columns.0.type: "double"} diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/25_aggs_on_null.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/25_aggs_on_null.yml index 068493993277..1980ed8bb040 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/25_aggs_on_null.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/25_aggs_on_null.yml @@ -39,6 +39,7 @@ group on null: body: query: 'FROM test | STATS med=median(never_null) BY always_null | LIMIT 1' columnar: true + version: 2024.04.01 - match: {columns.0.name: "med"} - match: {columns.0.type: "double"} - match: {columns.1.name: "always_null"} @@ -54,6 +55,7 @@ group on null, long: body: query: 'FROM test | STATS med=median(sometimes_null) BY always_null, never_null | SORT always_null, never_null | LIMIT 10' columnar: true + version: 2024.04.01 - match: {columns.0.name: "med"} - match: {columns.0.type: "double"} - match: {columns.1.name: "always_null"} @@ -72,6 +74,7 @@ agg on null: body: query: 'FROM test | STATS med=median(always_null) | LIMIT 1' columnar: true + version: 2024.04.01 - match: {columns.0.name: "med"} - match: {columns.0.type: "double"} - length: {values: 1} @@ -85,6 +88,7 @@ agg on missing: body: query: 'FROM test | STATS med=median(missing) | LIMIT 1' columnar: true + version: 2024.04.01 --- group on missing: @@ -94,6 +98,7 @@ group on missing: body: query: 'FROM test | STATS med=median(never_null) BY missing | LIMIT 1' columnar: true + version: 2024.04.01 --- agg on half missing: @@ -119,6 +124,7 @@ agg on half missing: body: query: 'FROM test* | STATS med=median(missing) | LIMIT 1' columnar: true + version: 2024.04.01 - match: {columns.0.name: "med"} - match: {columns.0.type: "double"} - length: {values: 1} @@ -148,6 +154,7 @@ group on half missing: body: query: 'FROM test,test2 | STATS med=median(never_null) BY missing | LIMIT 1' columnar: true + version: 2024.04.01 - match: {columns.0.name: "med"} - match: {columns.0.type: "double"} - match: {columns.1.name: "missing"} diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/30_types.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/30_types.yml index 9dded5d0855c..bbf8b33445fa 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/30_types.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/30_types.yml @@ -35,6 +35,7 @@ constant_keyword: esql.query: body: query: 'from test' + version: 2024.04.01 - match: { columns.0.name: color } - match: { columns.0.type: keyword } - match: { columns.1.name: kind } @@ -49,7 +50,7 @@ constant_keyword: esql.query: body: query: 'from test | eval l=length(kind) | keep l' - + version: 2024.04.01 - match: {columns.0.name: l} - match: {columns.0.type: integer} - length: {values: 1} @@ -80,6 +81,7 @@ constant_keyword with null value: esql.query: body: query: 'from test | limit 1' + version: 2024.04.01 - match: { columns.0.name: color } - match: { columns.0.type: keyword } - match: { columns.1.name: kind } @@ -113,6 +115,7 @@ multivalued keyword: esql.query: body: query: 'from test' + version: 2024.04.01 - match: {columns.0.name: card} - match: {columns.0.type: keyword} - length: {values: 1} @@ -144,6 +147,7 @@ keyword no doc_values: esql.query: body: query: 'from test' + version: 2024.04.01 - match: {columns.0.name: card} - match: {columns.0.type: keyword} - length: {values: 1} @@ -174,6 +178,7 @@ wildcard: esql.query: body: query: 'from test' + version: 2024.04.01 - match: {columns.0.name: card} - match: {columns.0.type: keyword} - length: {values: 1} @@ -185,6 +190,7 @@ wildcard: esql.query: body: query: 'from test | eval l=length(card) | keep l' + version: 2024.04.01 - match: {columns.0.name: l} - match: {columns.0.type: integer} - length: {values: 1} @@ -225,6 +231,7 @@ numbers: esql.query: body: query: 'from test' + version: 2024.04.01 - match: {columns.0.name: d} - match: {columns.0.type: double} - match: {columns.1.name: i} @@ -276,6 +283,7 @@ small_numbers: esql.query: body: query: 'from test' + version: 2024.04.01 - match: {columns.0.name: b} - match: {columns.0.type: integer} - match: {columns.1.name: f} @@ -296,6 +304,7 @@ small_numbers: esql.query: body: query: 'from test | eval sum_d = b + f + hf + s, sum_i = b + s | keep sum_d, sum_i' + version: 2024.04.01 - match: {columns.0.name: sum_d} - match: {columns.0.type: double} - match: {columns.1.name: sum_i} @@ -310,6 +319,7 @@ small_numbers: esql.query: body: query: 'from test | eval r_f = round(f), r_hf = round(hf) | keep r_f, r_hf' + version: 2024.04.01 - match: {columns.0.name: r_f} - match: {columns.0.type: double} - match: {columns.1.name: r_hf} @@ -346,6 +356,7 @@ scaled_float: esql.query: body: query: 'from test' + version: 2024.04.01 - match: {columns.0.name: d} - match: {columns.0.type: double} - match: {columns.1.name: f} @@ -360,6 +371,7 @@ scaled_float: esql.query: body: query: 'from test | eval sum = d + f | keep sum' + version: 2024.04.01 - match: {columns.0.name: sum} - match: {columns.0.type: double} - length: {values: 1} @@ -390,6 +402,7 @@ multivalued boolean: esql.query: body: query: 'from test' + version: 2024.04.01 - match: { columns.0.name: booleans } - match: { columns.0.type: boolean } - length: { values: 1 } @@ -422,6 +435,7 @@ ip: esql.query: body: query: 'from test' + version: 2024.04.01 - match: { columns.0.name: ip } - match: { columns.0.type: ip } - match: { columns.1.name: keyword } @@ -436,7 +450,7 @@ ip: esql.query: body: query: 'from test | where keyword == "127.0.0.2" | rename ip as IP | drop keyword' - + version: 2024.04.01 - match: {columns.0.name: IP } - match: {columns.0.type: ip } - length: {values: 1 } @@ -492,6 +506,7 @@ alias: esql.query: body: query: 'from test | keep foo, bar, level1.level2, level2_alias, some_long, some_long_alias, some_long_alias2, some_date, some_date_alias | sort level2_alias' + version: 2024.04.01 - match: { columns.0.name: foo } - match: { columns.0.type: keyword } - match: { columns.1.name: bar } @@ -536,6 +551,7 @@ alias: esql.query: body: query: 'from test | where bar == "abc" | keep foo, bar, level1.level2, level2_alias' + version: 2024.04.01 - match: { columns.0.name: foo } - match: { columns.0.type: keyword } - match: { columns.1.name: bar } @@ -556,6 +572,7 @@ alias: esql.query: body: query: 'from test | where level2_alias == 10 | keep foo, bar, level1.level2, level2_alias' + version: 2024.04.01 - match: { columns.0.name: foo } - match: { columns.0.type: keyword } - match: { columns.1.name: bar } @@ -576,6 +593,7 @@ alias: esql.query: body: query: 'from test | where level2_alias == 20' + version: 2024.04.01 - length: { values: 0 } - do: @@ -584,6 +602,7 @@ alias: esql.query: body: query: 'from test | stats x = max(level2_alias)' + version: 2024.04.01 - match: { columns.0.name: x } - match: { columns.0.type: long } - length: { values: 1 } @@ -614,6 +633,7 @@ version: esql.query: body: query: 'from test' + version: 2024.04.01 - match: { columns.0.name: version } - match: { columns.0.type: version } - length: { values: 1 } @@ -647,6 +667,7 @@ id: esql.query: body: query: 'from test metadata _id | keep _id, kw' + version: 2024.04.01 - match: { columns.0.name: _id } - match: { columns.0.type: keyword } - length: { values: 1 } @@ -678,6 +699,7 @@ unsigned_long: esql.query: body: query: 'from test' + version: 2024.04.01 - match: { columns.0.name: number } - match: { columns.0.type: unsigned_long } - length: { values: 1 } @@ -701,6 +723,7 @@ _source: esql.query: body: query: 'FROM test METADATA _source | KEEP _source | LIMIT 1' + version: 2024.04.01 - match: { columns.0.name: _source } - match: { columns.0.type: _source } - length: { values: 1 } @@ -736,6 +759,7 @@ _source keep all: esql.query: body: query: 'FROM test METADATA _source | LIMIT 1' + version: 2024.04.01 - match: { columns.0.name: _source } - match: { columns.0.type: _source } - length: { values: 1 } @@ -772,6 +796,7 @@ _source disabled: esql.query: body: query: 'FROM test METADATA _source | KEEP _source | LIMIT 1' + version: 2024.04.01 - match: { columns.0.name: _source } - match: { columns.0.type: _source } - length: { values: 1 } @@ -800,6 +825,7 @@ text: esql.query: body: query: 'FROM test | LIMIT 1' + version: 2024.04.01 - match: {columns.0.name: card} - match: {columns.0.type: text} - length: {values: 1} @@ -831,6 +857,7 @@ synthetic _source text stored: esql.query: body: query: 'FROM test | LIMIT 1' + version: 2024.04.01 - match: {columns.0.name: card} - match: {columns.0.type: text} - length: {values: 1} @@ -864,6 +891,7 @@ synthetic _source text with parent keyword: esql.query: body: query: 'FROM test | KEEP card.text | LIMIT 1' + version: 2024.04.01 - match: {columns.0.name: card.text} - match: {columns.0.type: text} - length: {values: 1} @@ -897,6 +925,7 @@ geo_point: esql.query: body: query: 'from test' + version: 2024.04.01 - match: { columns.0.name: location } - match: { columns.0.type: geo_point } - length: { values: 1 } @@ -930,6 +959,7 @@ cartesian_point: esql.query: body: query: 'from test' + version: 2024.04.01 - match: { columns.0.name: location } - match: { columns.0.type: cartesian_point } - length: { values: 1 } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml index 7549d33b4de1..30b81860f014 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml @@ -117,6 +117,7 @@ load everything: esql.query: body: query: 'from test metadata _id' + version: 2024.04.01 - match: {columns.0.name: "@timestamp"} - match: {columns.0.type: "date"} @@ -143,6 +144,7 @@ load a document: esql.query: body: query: 'from test | where @timestamp == "2021-04-28T18:50:23.142Z"' + version: 2024.04.01 - length: {values: 1} - length: {values.0: 7} @@ -161,6 +163,7 @@ filter on counter: esql.query: body: query: 'from test | where k8s.pod.network.tx == 1434577921' + version: 2024.04.01 --- from doc with aggregate_metric_double: @@ -171,6 +174,7 @@ from doc with aggregate_metric_double: esql.query: body: query: 'from test2' + version: 2024.04.01 - match: {columns.0.name: "@timestamp"} - match: {columns.0.type: "date"} @@ -191,6 +195,7 @@ stats on aggregate_metric_double: esql.query: body: query: 'FROM test2 | STATS max(agg_metric) BY dim' + version: 2024.04.01 --- from index pattern unsupported counter: @@ -201,6 +206,7 @@ from index pattern unsupported counter: esql.query: body: query: 'FROM test*' + version: 2024.04.01 - match: {columns.0.name: "@timestamp"} - match: {columns.0.type: "date"} @@ -229,6 +235,7 @@ from index pattern explicit counter use: esql.query: body: query: 'FROM test* | keep *.tx' + version: 2024.04.01 --- @@ -249,6 +256,7 @@ _source: esql.query: body: query: 'FROM test METADATA _source | WHERE @timestamp == "2021-04-28T18:50:23.142Z" | KEEP _source | LIMIT 1' + version: 2024.04.01 - match: { columns.0.name: _source } - match: { columns.0.type: _source } - length: { values: 1 } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml index c34666bb12b0..1ff0b8763c2e 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml @@ -120,6 +120,7 @@ unsupported: esql.query: body: query: 'from test' + version: 2024.04.01 - match: { columns.0.name: aggregate_metric_double } - match: { columns.0.type: unsupported } @@ -217,6 +218,7 @@ unsupported: esql.query: body: query: 'from test | limit 0' + version: 2024.04.01 - match: { columns.0.name: aggregate_metric_double } - match: { columns.0.type: unsupported } - match: { columns.1.name: binary } @@ -283,6 +285,7 @@ unsupported: esql.query: body: query: 'from test | keep histogram | limit 0' + version: 2024.04.01 - match: { columns.0.name: histogram } - match: { columns.0.type: unsupported } - length: { values: 0 } @@ -300,6 +303,7 @@ unsupported with sort: esql.query: body: query: 'from test | sort some_doc.bar' + version: 2024.04.01 - match: { columns.0.name: aggregate_metric_double } - match: { columns.0.type: unsupported } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/45_non_tsdb_counter.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/45_non_tsdb_counter.yml index 05ba568838fe..7f78ee1c7b09 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/45_non_tsdb_counter.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/45_non_tsdb_counter.yml @@ -66,6 +66,7 @@ load everything: esql.query: body: query: 'from test' + version: 2024.04.01 - match: {columns.0.name: "@timestamp"} - match: {columns.0.type: "date"} @@ -91,6 +92,7 @@ load a document: esql.query: body: query: 'from test | where @timestamp == "2021-04-28T18:50:23.142Z"' + version: 2024.04.01 - length: {values: 1} - length: {values.0: 7} @@ -110,6 +112,7 @@ filter on counter: esql.query: body: query: 'from test | where k8s.pod.network.tx == 1434577921' + version: 2024.04.01 - length: {values: 1} - length: {values.0: 7} diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/50_index_patterns.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/50_index_patterns.yml index d8aad2753443..ff04eec1d173 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/50_index_patterns.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/50_index_patterns.yml @@ -51,6 +51,7 @@ disjoint_mappings: esql.query: body: query: 'from test1,test2 | keep message1, message2 | sort message1' + version: 2024.04.01 - match: { columns.0.name: message1 } - match: { columns.0.type: keyword } - match: { columns.1.name: message2 } @@ -67,6 +68,7 @@ disjoint_mappings: esql.query: body: query: 'from test1,test2 | keep message1, message2 | sort message1 | limit 2' + version: 2024.04.01 - match: { columns.0.name: message1 } - match: { columns.0.type: keyword } - match: { columns.1.name: message2 } @@ -81,6 +83,7 @@ disjoint_mappings: esql.query: body: query: 'from test1,test2 | keep message1, message2 | sort message1 desc nulls last | limit 1' + version: 2024.04.01 - match: { columns.0.name: message1 } - match: { columns.0.type: keyword } - match: { columns.1.name: message2 } @@ -95,6 +98,7 @@ disjoint_mappings: esql.query: body: query: 'from test1,test2 | keep message1, message2 | sort message1, message2' + version: 2024.04.01 - match: { columns.0.name: message1 } - match: { columns.0.type: keyword } - match: { columns.1.name: message2 } @@ -113,6 +117,7 @@ disjoint_mappings: esql.query: body: query: 'from test1,test2 | keep message1, message2 | sort message1, message2 | limit 3' + version: 2024.04.01 - match: { columns.0.name: message1 } - match: { columns.0.type: keyword } - match: { columns.1.name: message2 } @@ -130,6 +135,7 @@ disjoint_mappings: esql.query: body: query: 'from test1,test2 | keep message1, message2 | sort message1 desc nulls first, message2 | limit 3' + version: 2024.04.01 - match: { columns.0.name: message1 } - match: { columns.0.type: keyword } - match: { columns.1.name: message2 } @@ -146,6 +152,7 @@ disjoint_mappings: esql.query: body: query: 'from test1,test2 | keep message1, message2 | sort message1, message2 | limit 2' + version: 2024.04.01 - match: { columns.0.name: message1 } - match: { columns.0.type: keyword } - match: { columns.1.name: message2 } @@ -162,6 +169,7 @@ disjoint_mappings: esql.query: body: query: 'from test1,test2 | keep message1, message2 | sort message1 nulls first, message2' + version: 2024.04.01 - match: { columns.0.name: message1 } - match: { columns.0.type: keyword } - match: { columns.1.name: message2 } @@ -182,6 +190,7 @@ disjoint_mappings: esql.query: body: query: 'from test1,test2 | keep message1, message2 | sort message1 nulls first, message2 nulls first' + version: 2024.04.01 - match: { columns.0.name: message1 } - match: { columns.0.type: keyword } - match: { columns.1.name: message2 } @@ -202,6 +211,7 @@ disjoint_mappings: esql.query: body: query: 'from test1,test2 | keep message1, message2 | sort message1 desc nulls first, message2 desc nulls first' + version: 2024.04.01 - match: { columns.0.name: message1 } - match: { columns.0.type: keyword } - match: { columns.1.name: message2 } @@ -222,6 +232,7 @@ disjoint_mappings: esql.query: body: query: 'from test1,test2 | where message1 == "foo1" | keep message1, message2 | sort message1, message2' + version: 2024.04.01 - match: { columns.0.name: message1 } - match: { columns.0.type: keyword } - match: { columns.1.name: message2 } @@ -236,6 +247,7 @@ disjoint_mappings: esql.query: body: query: 'from test1,test2 | where message1 == "foo1" or message2 == 2 | keep message1, message2 | sort message1, message2' + version: 2024.04.01 - match: { columns.0.name: message1 } - match: { columns.0.type: keyword } - match: { columns.1.name: message2 } @@ -252,6 +264,7 @@ disjoint_mappings: esql.query: body: query: 'from test1,test2 | stats x = max(message2)' + version: 2024.04.01 - match: { columns.0.name: x } - match: { columns.0.type: long } - length: { values: 1 } @@ -263,6 +276,7 @@ disjoint_mappings: esql.query: body: query: 'from test1,test2 | sort message1, message2 | eval x = message1, y = message2 + 1 | keep message1, message2, x, y' + version: 2024.04.01 - match: { columns.0.name: message1 } - match: { columns.0.type: keyword } - match: { columns.1.name: message2 } @@ -338,6 +352,7 @@ same_name_different_type: esql.query: body: query: 'from test1,test2' + version: 2024.04.01 - match: { columns.0.name: message } - match: { columns.0.type: unsupported } - length: { values: 4 } @@ -389,6 +404,7 @@ same_name_different_type_same_family: esql.query: body: query: 'from test1,test2 | sort message | keep message' + version: 2024.04.01 - match: { columns.0.name: message } - match: { columns.0.type: keyword } - length: { values: 4 } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_enrich.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_enrich.yml index 8a5d3be6758e..8fbc4be3cfb3 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_enrich.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_enrich.yml @@ -103,6 +103,7 @@ teardown: esql.query: body: query: 'from test | enrich city_codes_policy on city_id | keep name, city, country | sort name' + version: 2024.04.01 - match: { columns.0.name: "name" } - match: { columns.0.type: "keyword" } @@ -126,6 +127,7 @@ teardown: esql.query: body: query: 'from test | keep name, city_id | enrich city_codes_policy on city_id with country | sort name' + version: 2024.04.01 - match: { columns.0.name: "name" } - match: { columns.0.type: "keyword" } @@ -149,6 +151,7 @@ teardown: esql.query: body: query: 'from test | keep name, city_id | enrich city_codes_policy on city_id with country_name = country | sort name' + version: 2024.04.01 - match: { columns.0.name: "name" } - match: { columns.0.type: "keyword" } @@ -176,6 +179,7 @@ teardown: esql.query: body: query: 'from test | keep name, city_name | enrich city_names_policy on city_name | sort name' + version: 2024.04.01 - match: { columns.0.name: "name" } - match: { columns.0.type: "keyword" } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml index 74c0e9ef1bb3..018106cf1aa1 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml @@ -52,6 +52,7 @@ setup: esql.query: body: query: 'from test | where data > 2 | sort count desc | limit 5 | stats m = max(data)' + version: 2024.04.01 - do: {xpack.usage: {}} - match: { esql.available: true } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/61_enrich_ip.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/61_enrich_ip.yml index 076bf116292d..a9ea9c704e6e 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/61_enrich_ip.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/61_enrich_ip.yml @@ -115,6 +115,7 @@ teardown: esql.query: body: query: 'FROM events | eval ip_str = to_string(ip) | ENRICH networks-policy ON ip_str | sort @timestamp | KEEP ip, name, department, message' + version: 2024.04.01 - match: { columns.0.name: "ip" } - match: { columns.0.type: "ip" } @@ -143,6 +144,7 @@ teardown: esql.query: body: query: 'FROM events_text | ENRICH networks-policy ON ip_text | sort @timestamp | KEEP ip_text, name, department, message' + version: 2024.04.01 - match: { columns.0.name: "ip_text" } - match: { columns.0.type: "text" } @@ -170,6 +172,7 @@ teardown: esql.query: body: query: 'FROM events | eval ip_str = concat("invalid_", to_string(ip)) | ENRICH networks-policy ON ip_str | sort @timestamp | KEEP ip, name, department, message' + version: 2024.04.01 --- "IP": @@ -183,6 +186,7 @@ teardown: esql.query: body: query: 'FROM events | ENRICH networks-policy ON ip | sort @timestamp | KEEP ip, name, department, message' + version: 2024.04.01 - match: { columns.0.name: "ip" } - match: { columns.0.type: "ip" } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/62_extra_enrich.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/62_extra_enrich.yml index 19b08007fe18..288c17bac1d1 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/62_extra_enrich.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/62_extra_enrich.yml @@ -44,6 +44,7 @@ esql.query: body: query: 'ROW name="engineering" | ENRICH departments-policy | LIMIT 10 | KEEP name, employees' + version: 2024.04.01 - match: { columns.0.name: "name" } - match: { columns.0.type: "keyword" } @@ -58,6 +59,7 @@ esql.query: body: query: 'ROW name="sales" | ENRICH departments-policy ON name WITH department=name | WHERE name==department | KEEP name, department | LIMIT 10' + version: 2024.04.01 - match: { columns.0.name: "name" } - match: { columns.0.type: "keyword" } @@ -257,6 +259,7 @@ movies: SORT total DESC, title ASC | KEEP total, title | LIMIT 10 + version: 2024.04.01 - match: { columns.0.name: "total" } - match: { columns.0.type: "long" } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/70_locale.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/70_locale.yml index e181f77f2bce..a0ec659b21d0 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/70_locale.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/70_locale.yml @@ -32,6 +32,7 @@ setup: esql.query: body: query: 'FROM events | eval fixed_format = date_format("MMMM", @timestamp), variable_format = date_format(format, @timestamp) | sort @timestamp | keep @timestamp, fixed_format, variable_format' + version: 2024.04.01 - match: { columns.0.name: "@timestamp" } - match: { columns.0.type: "date" } @@ -54,6 +55,7 @@ setup: body: query: 'FROM events | eval fixed_format = date_format("MMMM", @timestamp), variable_format = date_format(format, @timestamp) | sort @timestamp | keep @timestamp, fixed_format, variable_format' locale: "it-IT" + version: 2024.04.01 - match: { columns.0.name: "@timestamp" } - match: { columns.0.type: "date" } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml index 9607b6438572..c8867b2d1bf8 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/80_text.yml @@ -41,6 +41,7 @@ setup: esql.query: body: query: 'from test | sort emp_no' + version: 2024.04.01 - match: { columns.0.name: "emp_no" } - match: { columns.0.type: "long" } @@ -66,6 +67,7 @@ setup: esql.query: body: query: 'from test | where tag == "baz" | keep emp_no, name, job, tag' + version: 2024.04.01 - match: { columns.0.name: "emp_no" } - match: { columns.0.type: "long" } @@ -87,6 +89,7 @@ setup: esql.query: body: query: 'from test | where tag LIKE "*az" | keep emp_no, name, job, tag' + version: 2024.04.01 - match: { columns.0.name: "emp_no" } - match: { columns.0.type: "long" } @@ -108,6 +111,7 @@ setup: esql.query: body: query: 'from test | where tag RLIKE ".*az" | keep emp_no, name, job, tag' + version: 2024.04.01 - match: { columns.0.name: "emp_no" } - match: { columns.0.type: "long" } @@ -133,6 +137,7 @@ setup: esql.query: body: query: 'from test | where tag IN ("abc", "baz") | keep emp_no, name, job, tag' + version: 2024.04.01 - match: { columns.0.name: "emp_no" } - match: { columns.0.type: "long" } @@ -158,6 +163,7 @@ setup: esql.query: body: query: 'from test | where tag IN ("abc", tag) | keep emp_no, name, job, tag | sort emp_no' + version: 2024.04.01 - match: { columns.0.name: "emp_no" } - match: { columns.0.type: "long" } @@ -184,6 +190,7 @@ setup: esql.query: body: query: 'from test | where tag NOT IN ("abc", "baz") | keep emp_no, name, job, tag' + version: 2024.04.01 - match: { columns.0.name: "emp_no" } - match: { columns.0.type: "long" } @@ -205,6 +212,7 @@ setup: esql.query: body: query: 'from test | eval x = tag | where x == "baz" | keep emp_no, name, job, x' + version: 2024.04.01 - match: { columns.0.name: "emp_no" } - match: { columns.0.type: "long" } @@ -226,6 +234,7 @@ setup: esql.query: body: query: 'from test | where job == "IT Director" | keep emp_no, name, job, tag' + version: 2024.04.01 - match: { columns.0.name: "emp_no" } - match: { columns.0.type: "long" } @@ -247,6 +256,7 @@ setup: esql.query: body: query: 'from test | where job LIKE "*Specialist" | keep emp_no, name, job, tag' + version: 2024.04.01 - match: { columns.0.name: "emp_no" } - match: { columns.0.type: "long" } @@ -268,6 +278,7 @@ setup: esql.query: body: query: 'from test | where job RLIKE ".*Specialist" | keep emp_no, name, job, tag' + version: 2024.04.01 - match: { columns.0.name: "emp_no" } - match: { columns.0.type: "long" } @@ -290,6 +301,7 @@ setup: esql.query: body: query: 'from test | sort tag | keep emp_no, name, job, tag' + version: 2024.04.01 - match: { columns.0.name: "emp_no" } - match: { columns.0.type: "long" } @@ -313,6 +325,7 @@ setup: esql.query: body: query: 'from test | sort job | keep emp_no, name, job, tag' + version: 2024.04.01 - match: { columns.0.name: "emp_no" } - match: { columns.0.type: "long" } @@ -335,6 +348,7 @@ setup: esql.query: body: query: 'from test | sort job desc | keep emp_no, name, job, tag' + version: 2024.04.01 - match: { columns.0.name: "emp_no" } - match: { columns.0.type: "long" } @@ -358,6 +372,7 @@ setup: esql.query: body: query: 'from test | sort name | eval description = concat(name, " - ", job) | keep description' + version: 2024.04.01 - match: { columns.0.name: "description" } - match: { columns.0.type: "keyword" } @@ -378,6 +393,7 @@ setup: esql.query: body: query: 'from test | sort emp_no | eval split = split(tag, " ") | keep split' + version: 2024.04.01 - match: { columns.0.name: "split" } - match: { columns.0.type: "keyword" } @@ -395,6 +411,7 @@ setup: esql.query: body: query: 'from test | stats jobs = count(job) | keep jobs' + version: 2024.04.01 - match: { columns.0.name: "jobs" } - match: { columns.0.type: "long" } @@ -411,6 +428,7 @@ setup: esql.query: body: query: 'from test | stats tags = count(tag) | keep tags' + version: 2024.04.01 - match: { columns.0.name: "tags" } - match: { columns.0.type: "long" } @@ -427,6 +445,7 @@ setup: esql.query: body: query: 'from test | stats names = count(name) by job | keep names' + version: 2024.04.01 - match: { columns.0.name: "names" } - match: { columns.0.type: "long" } @@ -444,6 +463,7 @@ setup: esql.query: body: query: 'from test | stats names = count(name) by tag | keep names' + version: 2024.04.01 - match: { columns.0.name: "names" } - match: { columns.0.type: "long" } @@ -488,6 +508,7 @@ setup: esql.query: body: query: 'from test2 | sort emp_no | keep job' + version: 2024.04.01 - match: { columns.0.name: "job" } - match: { columns.0.type: "text" } @@ -531,6 +552,7 @@ setup: esql.query: body: query: 'from test2 | sort emp_no | keep job' + version: 2024.04.01 - match: { columns.0.name: "job" } - match: { columns.0.type: "text" } @@ -549,6 +571,7 @@ values: esql.query: body: query: 'FROM test | STATS job = VALUES(job) | EVAL job = MV_SORT(job) | LIMIT 1' + version: 2024.04.01 - match: { columns.0.name: "job" } - match: { columns.0.type: "text" } - length: { values: 1 } @@ -566,6 +589,7 @@ values: esql.query: body: query: 'FROM test | STATS job = VALUES(job) BY tag | EVAL job = MV_SORT(job) | SORT tag | LIMIT 10' + version: 2024.04.01 - match: { columns.0.name: "tag" } - match: { columns.0.type: "text" } - match: { columns.1.name: "job" } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/81_text_exact_subfields.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/81_text_exact_subfields.yml index a3be4221712f..20dd668e0f8c 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/81_text_exact_subfields.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/81_text_exact_subfields.yml @@ -57,6 +57,7 @@ setup: esql.query: body: query: 'from test | sort emp_no | keep text_ignore_above, text_ignore_above.raw, text_normalizer, text_normalizer.raw, non_indexed, non_indexed.raw' + version: 2024.04.01 - match: { columns.0.name: "text_ignore_above" } - match: { columns.0.type: "text" } @@ -83,6 +84,7 @@ setup: esql.query: body: query: 'from test | where text_ignore_above == "this" | keep text_ignore_above, text_ignore_above.raw, text_normalizer, text_normalizer.raw, non_indexed, non_indexed.raw' + version: 2024.04.01 - match: { columns.0.name: "text_ignore_above" } - match: { columns.0.type: "text" } @@ -106,6 +108,7 @@ setup: esql.query: body: query: 'from test | where text_ignore_above == "this is a long text" | keep text_ignore_above, text_ignore_above.raw, text_normalizer, text_normalizer.raw, non_indexed, non_indexed.raw' + version: 2024.04.01 - match: { columns.0.name: "text_ignore_above" } - match: { columns.0.type: "text" } @@ -130,6 +133,7 @@ setup: esql.query: body: query: 'from test | where text_ignore_above is null | keep text_ignore_above, text_ignore_above.raw, text_normalizer, text_normalizer.raw, non_indexed, non_indexed.raw' + version: 2024.04.01 - match: { columns.0.name: "text_ignore_above" } - match: { columns.0.type: "text" } @@ -153,6 +157,7 @@ setup: esql.query: body: query: 'from test | where text_ignore_above is not null | keep text_ignore_above, text_ignore_above.raw, text_normalizer, text_normalizer.raw, non_indexed, non_indexed.raw' + version: 2024.04.01 - match: { columns.0.name: "text_ignore_above" } - match: { columns.0.type: "text" } @@ -176,6 +181,7 @@ setup: esql.query: body: query: 'from test | where text_ignore_above LIKE "*long*" | keep text_ignore_above, text_ignore_above.raw, text_normalizer, text_normalizer.raw, non_indexed, non_indexed.raw' + version: 2024.04.01 - match: { columns.0.name: "text_ignore_above" } - match: { columns.0.type: "text" } @@ -201,6 +207,7 @@ setup: esql.query: body: query: 'from test | where text_normalizer == "CamelCase" | keep text_ignore_above, text_ignore_above.raw, text_normalizer, text_normalizer.raw, non_indexed, non_indexed.raw' + version: 2024.04.01 - match: { columns.0.name: "text_ignore_above" } - match: { columns.0.type: "text" } @@ -225,6 +232,7 @@ setup: esql.query: body: query: 'from test | where text_normalizer == text_normalizer.raw | keep text_ignore_above, text_ignore_above.raw, text_normalizer, text_normalizer.raw, non_indexed, non_indexed.raw' + version: 2024.04.01 - match: { columns.0.name: "text_ignore_above" } - match: { columns.0.type: "text" } @@ -242,7 +250,6 @@ setup: - length: { values: 1 } - match: { values.0: [ "this", "this", "abc", "abc", "bar", "bar" ]} - --- "sort ignore above": - do: @@ -251,6 +258,7 @@ setup: esql.query: body: query: 'from test | sort text_ignore_above asc | keep text_ignore_above, text_ignore_above.raw, text_normalizer, text_normalizer.raw, non_indexed, non_indexed.raw' + version: 2024.04.01 - match: { columns.0.name: "text_ignore_above" } - match: { columns.0.type: "text" } @@ -275,6 +283,7 @@ setup: esql.query: body: query: 'from test | sort text_ignore_above desc | keep text_ignore_above, text_ignore_above.raw, text_normalizer, text_normalizer.raw, non_indexed, non_indexed.raw' + version: 2024.04.01 - match: { columns.0.name: "text_ignore_above" } - match: { columns.0.type: "text" } @@ -299,6 +308,7 @@ setup: esql.query: body: query: 'from test | sort text_ignore_above asc nulls first | keep text_ignore_above, text_ignore_above.raw, text_normalizer, text_normalizer.raw, non_indexed, non_indexed.raw' + version: 2024.04.01 - match: { columns.0.name: "text_ignore_above" } - match: { columns.0.type: "text" } @@ -323,6 +333,7 @@ setup: esql.query: body: query: 'from test | sort text_ignore_above asc nulls last | keep text_ignore_above, text_ignore_above.raw, text_normalizer, text_normalizer.raw, non_indexed, non_indexed.raw' + version: 2024.04.01 - match: { columns.0.name: "text_ignore_above" } - match: { columns.0.type: "text" } @@ -341,8 +352,6 @@ setup: - match: { values.0: [ "this", "this", "abc", "abc", "bar", "bar" ]} - match: { values.1: [ "this is a long text", null, "CamelCase", "camelcase", "foo", "foo"] } - - --- "sort normalizer": - do: @@ -351,6 +360,7 @@ setup: esql.query: body: query: 'from test | sort text_normalizer asc | keep text_ignore_above, text_ignore_above.raw, text_normalizer, text_normalizer.raw, non_indexed, non_indexed.raw' + version: 2024.04.01 - match: { columns.0.name: "text_ignore_above" } - match: { columns.0.type: "text" } @@ -369,13 +379,13 @@ setup: - match: { values.0: [ "this is a long text", null, "CamelCase", "camelcase", "foo", "foo"] } - match: { values.1: [ "this", "this", "abc", "abc", "bar", "bar" ]} - - do: allowed_warnings_regex: - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test | sort text_normalizer desc | keep text_ignore_above, text_ignore_above.raw, text_normalizer, text_normalizer.raw, non_indexed, non_indexed.raw' + version: 2024.04.01 - match: { columns.0.name: "text_ignore_above" } - match: { columns.0.type: "text" } @@ -394,13 +404,13 @@ setup: - match: { values.0: [ "this", "this", "abc", "abc", "bar", "bar" ]} - match: { values.1: [ "this is a long text", null, "CamelCase", "camelcase", "foo", "foo"] } - - do: allowed_warnings_regex: - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: query: 'from test | sort text_normalizer.raw asc | keep text_ignore_above, text_ignore_above.raw, text_normalizer, text_normalizer.raw, non_indexed, non_indexed.raw' + version: 2024.04.01 - match: { columns.0.name: "text_ignore_above" } - match: { columns.0.type: "text" } @@ -428,6 +438,7 @@ setup: esql.query: body: query: 'from test | sort non_indexed asc | keep text_ignore_above, text_ignore_above.raw, text_normalizer, text_normalizer.raw, non_indexed, non_indexed.raw' + version: 2024.04.01 - match: { columns.0.name: "text_ignore_above" } - match: { columns.0.type: "text" } @@ -452,6 +463,7 @@ setup: esql.query: body: query: 'from test | sort non_indexed desc | keep text_ignore_above, text_ignore_above.raw, text_normalizer, text_normalizer.raw, non_indexed, non_indexed.raw' + version: 2024.04.01 - match: { columns.0.name: "text_ignore_above" } - match: { columns.0.type: "text" } @@ -476,6 +488,7 @@ setup: esql.query: body: query: 'from test | where non_indexed == "foo" | keep text_ignore_above, text_ignore_above.raw, text_normalizer, text_normalizer.raw, non_indexed, non_indexed.raw' + version: 2024.04.01 - match: { columns.0.name: "text_ignore_above" } - match: { columns.0.type: "text" } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/90_non_indexed.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/90_non_indexed.yml index f69854388baf..86ff9626e007 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/90_non_indexed.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/90_non_indexed.yml @@ -102,6 +102,7 @@ fetch: esql.query: body: query: 'from test' + version: 2024.04.01 - length: { columns: 18 } - match: { columns.0.name: boolean } diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java index d7760eb42a1d..9c9ebf24bda8 100644 --- a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java @@ -1041,6 +1041,7 @@ public void testDisableFieldNameField() throws IOException { Request esql = new Request("POST", "_query"); esql.setJsonEntity(""" { + "version": "2024.04.01", "query": "FROM nofnf | LIMIT 1" }"""); // {"columns":[{"name":"dv","type":"keyword"},{"name":"no_dv","type":"keyword"}],"values":[["test",null]]} diff --git a/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/querying_cluster/80_esql.yml b/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/querying_cluster/80_esql.yml index 4c0bbfd7ec13..e8cd1321db73 100644 --- a/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/querying_cluster/80_esql.yml +++ b/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/querying_cluster/80_esql.yml @@ -97,6 +97,7 @@ teardown: esql.query: body: query: 'FROM *:esql*,esql_* | STATS total = sum(cost) by tag | SORT tag | LIMIT 10' + version: '2024.04.01' - match: {columns.0.name: "total"} - match: {columns.0.type: "long"} @@ -127,6 +128,7 @@ teardown: gte: "2023-01-02" lte: "2023-01-03" format: "yyyy-MM-dd" + version: '2024.04.01' - match: {columns.0.name: "_index"} - match: {columns.0.type: "keyword"} @@ -198,6 +200,7 @@ teardown: esql.query: body: query: 'FROM *:esql*,esql_* | STATS total = sum(cost) by tag | SORT total DESC | LIMIT 3 | ENRICH suggestions | KEEP tag, total, phrase' + version: '2024.04.01' - match: {columns.0.name: "tag"} - match: {columns.0.type: "keyword"} From c2a3ec42632b0339387121efdef13f52c6c66848 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Tue, 16 Apr 2024 08:46:21 -0700 Subject: [PATCH 07/43] Leverage ordinals in enrich lookup (#107449) This change leverages ordinals in enrich lookup. Instead of looking up and extracting enrich fields for all input terms, this improvement only looks up and extracts the dictionary, then applies the ordinals to the enrich results. ``` | 50th percentile | esql_stats_enrich_rates_fares | 242.949 | 34.7007 | -208.248 | ms | -85.72% | | 90th percentile | esql_stats_enrich_rates_fares | 245.479 | 36.3419 | -209.137 | ms | -85.20% | |100th percentile | esql_stats_enrich_rates_fares | 252.877 | 49.0826 | -203.795 | ms | -80.59% | ``` --- docs/changelog/107449.yaml | 5 + .../compute/data/OrdinalBytesRefBlock.java | 8 ++ .../xpack/esql/action/EnrichIT.java | 33 ++++++ .../enrich/EnrichResultBuilderForBoolean.java | 96 ++++++++++++---- .../EnrichResultBuilderForBytesRef.java | 100 ++++++++++++---- .../enrich/EnrichResultBuilderForDouble.java | 96 ++++++++++++---- .../enrich/EnrichResultBuilderForInt.java | 95 +++++++++++---- .../enrich/EnrichResultBuilderForLong.java | 96 ++++++++++++---- .../esql/enrich/EnrichLookupService.java | 15 ++- .../esql/enrich/EnrichResultBuilder.java | 29 +++-- .../esql/enrich/MergePositionsOperator.java | 14 ++- .../esql/enrich/X-EnrichResultBuilder.java.st | 108 ++++++++++++++---- .../esql/enrich/EnrichResultBuilderTests.java | 100 +++++++++++----- .../enrich/MergePositionsOperatorTests.java | 8 +- 14 files changed, 619 insertions(+), 184 deletions(-) create mode 100644 docs/changelog/107449.yaml diff --git a/docs/changelog/107449.yaml b/docs/changelog/107449.yaml new file mode 100644 index 000000000000..7f0b1bb826e9 --- /dev/null +++ b/docs/changelog/107449.yaml @@ -0,0 +1,5 @@ +pr: 107449 +summary: Leverage ordinals in enrich lookup +area: ES|QL +type: enhancement +issues: [] diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/OrdinalBytesRefBlock.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/OrdinalBytesRefBlock.java index 4e409a7d214e..64e3faca1f51 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/OrdinalBytesRefBlock.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/OrdinalBytesRefBlock.java @@ -55,6 +55,14 @@ public boolean isDense() { return ordinals.getTotalValueCount() * 2 / 3 >= bytes.getPositionCount(); } + public IntBlock getOrdinalsBlock() { + return ordinals; + } + + public BytesRefVector getDictionaryVector() { + return bytes; + } + @Override public BytesRef getBytesRef(int valueIndex, BytesRef dest) { return bytes.getBytesRef(ordinals.getInt(valueIndex), dest); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java index 43c282e9361f..c4adfb688526 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EnrichIT.java @@ -385,6 +385,39 @@ public void testForPushDownEnrichRule() { } } + /** + * To enable enrich lookup using ordinals + */ + public void testManyDocuments() { + int numDocs = between(200, 2000); + var artists = Map.of("s1", "Eagles", "s2", "Linkin Park", "s3", "Linkin Park", "s4", "Disturbed"); + client().admin() + .indices() + .prepareCreate("many_docs") + .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)) + .setMapping("song_id", "type=keyword") + .get(); + Map songs = new HashMap<>(); + for (int i = 0; i < numDocs; i++) { + String song = randomFrom(artists.keySet()); + client().prepareIndex("many_docs").setSource("song_id", song).get(); + songs.merge(song, 1L, Long::sum); + } + client().admin().indices().prepareRefresh("many_docs").get(); + try (EsqlQueryResponse resp = run("FROM many_docs | ENRICH songs | STATS count(*) BY artist")) { + List> values = EsqlTestUtils.getValuesList(resp); + Map actual = new HashMap<>(); + for (List value : values) { + actual.merge((String) value.get(1), (Long) value.get(0), Long::sum); + } + Map expected = new HashMap<>(); + for (Map.Entry e : songs.entrySet()) { + expected.merge(artists.get(e.getKey()), e.getValue(), Long::sum); + } + assertThat(actual, equalTo(expected)); + } + } + public static class LocalStateEnrich extends LocalStateCompositeXPackPlugin { public LocalStateEnrich(final Settings settings, final Path configPath) throws Exception { diff --git a/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/enrich/EnrichResultBuilderForBoolean.java b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/enrich/EnrichResultBuilderForBoolean.java index 0427afb6d80c..7978baf0c5f2 100644 --- a/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/enrich/EnrichResultBuilderForBoolean.java +++ b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/enrich/EnrichResultBuilderForBoolean.java @@ -12,6 +12,7 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.core.Releasables; @@ -25,9 +26,9 @@ final class EnrichResultBuilderForBoolean extends EnrichResultBuilder { private ObjectArray cells; - EnrichResultBuilderForBoolean(BlockFactory blockFactory, int channel, int totalPositions) { - super(blockFactory, channel, totalPositions); - this.cells = blockFactory.bigArrays().newObjectArray(totalPositions); + EnrichResultBuilderForBoolean(BlockFactory blockFactory, int channel) { + super(blockFactory, channel); + this.cells = blockFactory.bigArrays().newObjectArray(1); } @Override @@ -39,6 +40,7 @@ void addInputPage(IntVector positions, Page page) { continue; } int cellPosition = positions.getInt(i); + cells = blockFactory.bigArrays().grow(cells, cellPosition + 1); final var oldCell = cells.get(cellPosition); final var newCell = extendCell(oldCell, valueCount); cells.set(cellPosition, newCell); @@ -59,30 +61,82 @@ private boolean[] extendCell(boolean[] oldCell, int newValueCount) { } } - @Override - Block build() { - try (BooleanBlock.Builder builder = blockFactory.newBooleanBlockBuilder(totalPositions)) { - for (int i = 0; i < totalPositions; i++) { - final var cell = cells.get(i); - if (cell == null) { - builder.appendNull(); - continue; - } - if (cell.length > 1) { - builder.beginPositionEntry(); - } - // TODO: sort and dedup - for (var v : cell) { - builder.appendBoolean(v); - } - if (cell.length > 1) { - builder.endPositionEntry(); + private boolean[] combineCell(boolean[] first, boolean[] second) { + if (first == null) { + return second; + } + if (second == null) { + return first; + } + var result = new boolean[first.length + second.length]; + System.arraycopy(first, 0, result, 0, first.length); + System.arraycopy(second, 0, result, first.length, second.length); + return result; + } + + private void appendGroupToBlockBuilder(BooleanBlock.Builder builder, boolean[] group) { + if (group == null) { + builder.appendNull(); + } else if (group.length == 1) { + builder.appendBoolean(group[0]); + } else { + builder.beginPositionEntry(); + // TODO: sort and dedup and set MvOrdering + for (var v : group) { + builder.appendBoolean(v); + } + builder.endPositionEntry(); + } + } + + private boolean[] getCellOrNull(int position) { + return position < cells.size() ? cells.get(position) : null; + } + + private Block buildWithSelected(IntBlock selected) { + try (BooleanBlock.Builder builder = blockFactory.newBooleanBlockBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int selectedCount = selected.getValueCount(i); + switch (selectedCount) { + case 0 -> builder.appendNull(); + case 1 -> { + int groupId = selected.getInt(selected.getFirstValueIndex(i)); + appendGroupToBlockBuilder(builder, getCellOrNull(groupId)); + } + default -> { + int firstValueIndex = selected.getFirstValueIndex(i); + var cell = getCellOrNull(selected.getInt(firstValueIndex)); + for (int p = 1; p < selectedCount; p++) { + int groupId = selected.getInt(firstValueIndex + p); + cell = combineCell(cell, getCellOrNull(groupId)); + } + appendGroupToBlockBuilder(builder, cell); + } } } return builder.build(); } } + private Block buildWithSelected(IntVector selected) { + try (BooleanBlock.Builder builder = blockFactory.newBooleanBlockBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + appendGroupToBlockBuilder(builder, getCellOrNull(selected.getInt(i))); + } + return builder.build(); + } + } + + @Override + Block build(IntBlock selected) { + var vector = selected.asVector(); + if (vector != null) { + return buildWithSelected(vector); + } else { + return buildWithSelected(selected); + } + } + @Override public void close() { Releasables.close(cells, super::close); diff --git a/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/enrich/EnrichResultBuilderForBytesRef.java b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/enrich/EnrichResultBuilderForBytesRef.java index ff881da5baf4..28326568af63 100644 --- a/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/enrich/EnrichResultBuilderForBytesRef.java +++ b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/enrich/EnrichResultBuilderForBytesRef.java @@ -14,6 +14,7 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.core.Releasables; @@ -26,14 +27,15 @@ */ final class EnrichResultBuilderForBytesRef extends EnrichResultBuilder { private final BytesRefArray bytes; // shared between all cells + private BytesRef scratch = new BytesRef(); private ObjectArray cells; - EnrichResultBuilderForBytesRef(BlockFactory blockFactory, int channel, int totalPositions) { - super(blockFactory, channel, totalPositions); - this.cells = blockFactory.bigArrays().newObjectArray(totalPositions); + EnrichResultBuilderForBytesRef(BlockFactory blockFactory, int channel) { + super(blockFactory, channel); + this.cells = blockFactory.bigArrays().newObjectArray(1); BytesRefArray bytes = null; try { - bytes = new BytesRefArray(totalPositions * 3L, blockFactory.bigArrays()); + bytes = new BytesRefArray(1L, blockFactory.bigArrays()); this.bytes = bytes; } finally { if (bytes == null) { @@ -52,6 +54,7 @@ void addInputPage(IntVector positions, Page page) { continue; } int cellPosition = positions.getInt(i); + cells = blockFactory.bigArrays().grow(cells, cellPosition + 1); final var oldCell = cells.get(cellPosition); final var newCell = extendCell(oldCell, valueCount); cells.set(cellPosition, newCell); @@ -75,31 +78,82 @@ private int[] extendCell(int[] oldCell, int newValueCount) { } } - @Override - Block build() { - try (BytesRefBlock.Builder builder = blockFactory.newBytesRefBlockBuilder(totalPositions)) { - BytesRef scratch = new BytesRef(); - for (int i = 0; i < totalPositions; i++) { - final var cell = cells.get(i); - if (cell == null) { - builder.appendNull(); - continue; - } - if (cell.length > 1) { - builder.beginPositionEntry(); - } - // TODO: sort and dedup - for (var v : cell) { - builder.appendBytesRef(bytes.get(v, scratch)); - } - if (cell.length > 1) { - builder.endPositionEntry(); + private int[] combineCell(int[] first, int[] second) { + if (first == null) { + return second; + } + if (second == null) { + return first; + } + var result = new int[first.length + second.length]; + System.arraycopy(first, 0, result, 0, first.length); + System.arraycopy(second, 0, result, first.length, second.length); + return result; + } + + private void appendGroupToBlockBuilder(BytesRefBlock.Builder builder, int[] group) { + if (group == null) { + builder.appendNull(); + } else if (group.length == 1) { + builder.appendBytesRef(bytes.get(group[0], scratch)); + } else { + builder.beginPositionEntry(); + // TODO: sort and dedup and set MvOrdering + for (var v : group) { + builder.appendBytesRef(bytes.get(v, scratch)); + } + builder.endPositionEntry(); + } + } + + private int[] getCellOrNull(int position) { + return position < cells.size() ? cells.get(position) : null; + } + + private Block buildWithSelected(IntBlock selected) { + try (BytesRefBlock.Builder builder = blockFactory.newBytesRefBlockBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int selectedCount = selected.getValueCount(i); + switch (selectedCount) { + case 0 -> builder.appendNull(); + case 1 -> { + int groupId = selected.getInt(selected.getFirstValueIndex(i)); + appendGroupToBlockBuilder(builder, getCellOrNull(groupId)); + } + default -> { + int firstValueIndex = selected.getFirstValueIndex(i); + var cell = getCellOrNull(selected.getInt(firstValueIndex)); + for (int p = 1; p < selectedCount; p++) { + int groupId = selected.getInt(firstValueIndex + p); + cell = combineCell(cell, getCellOrNull(groupId)); + } + appendGroupToBlockBuilder(builder, cell); + } } } return builder.build(); } } + private Block buildWithSelected(IntVector selected) { + try (BytesRefBlock.Builder builder = blockFactory.newBytesRefBlockBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + appendGroupToBlockBuilder(builder, getCellOrNull(selected.getInt(i))); + } + return builder.build(); + } + } + + @Override + Block build(IntBlock selected) { + var vector = selected.asVector(); + if (vector != null) { + return buildWithSelected(vector); + } else { + return buildWithSelected(selected); + } + } + @Override public void close() { Releasables.close(bytes, cells, super::close); diff --git a/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/enrich/EnrichResultBuilderForDouble.java b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/enrich/EnrichResultBuilderForDouble.java index 93c178d81632..e15b8f7d6d4b 100644 --- a/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/enrich/EnrichResultBuilderForDouble.java +++ b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/enrich/EnrichResultBuilderForDouble.java @@ -12,6 +12,7 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.core.Releasables; @@ -25,9 +26,9 @@ final class EnrichResultBuilderForDouble extends EnrichResultBuilder { private ObjectArray cells; - EnrichResultBuilderForDouble(BlockFactory blockFactory, int channel, int totalPositions) { - super(blockFactory, channel, totalPositions); - this.cells = blockFactory.bigArrays().newObjectArray(totalPositions); + EnrichResultBuilderForDouble(BlockFactory blockFactory, int channel) { + super(blockFactory, channel); + this.cells = blockFactory.bigArrays().newObjectArray(1); } @Override @@ -39,6 +40,7 @@ void addInputPage(IntVector positions, Page page) { continue; } int cellPosition = positions.getInt(i); + cells = blockFactory.bigArrays().grow(cells, cellPosition + 1); final var oldCell = cells.get(cellPosition); final var newCell = extendCell(oldCell, valueCount); cells.set(cellPosition, newCell); @@ -59,30 +61,82 @@ private double[] extendCell(double[] oldCell, int newValueCount) { } } - @Override - Block build() { - try (DoubleBlock.Builder builder = blockFactory.newDoubleBlockBuilder(totalPositions)) { - for (int i = 0; i < totalPositions; i++) { - final var cell = cells.get(i); - if (cell == null) { - builder.appendNull(); - continue; - } - if (cell.length > 1) { - builder.beginPositionEntry(); - } - // TODO: sort and dedup - for (var v : cell) { - builder.appendDouble(v); - } - if (cell.length > 1) { - builder.endPositionEntry(); + private double[] combineCell(double[] first, double[] second) { + if (first == null) { + return second; + } + if (second == null) { + return first; + } + var result = new double[first.length + second.length]; + System.arraycopy(first, 0, result, 0, first.length); + System.arraycopy(second, 0, result, first.length, second.length); + return result; + } + + private void appendGroupToBlockBuilder(DoubleBlock.Builder builder, double[] group) { + if (group == null) { + builder.appendNull(); + } else if (group.length == 1) { + builder.appendDouble(group[0]); + } else { + builder.beginPositionEntry(); + // TODO: sort and dedup and set MvOrdering + for (var v : group) { + builder.appendDouble(v); + } + builder.endPositionEntry(); + } + } + + private double[] getCellOrNull(int position) { + return position < cells.size() ? cells.get(position) : null; + } + + private Block buildWithSelected(IntBlock selected) { + try (DoubleBlock.Builder builder = blockFactory.newDoubleBlockBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int selectedCount = selected.getValueCount(i); + switch (selectedCount) { + case 0 -> builder.appendNull(); + case 1 -> { + int groupId = selected.getInt(selected.getFirstValueIndex(i)); + appendGroupToBlockBuilder(builder, getCellOrNull(groupId)); + } + default -> { + int firstValueIndex = selected.getFirstValueIndex(i); + var cell = getCellOrNull(selected.getInt(firstValueIndex)); + for (int p = 1; p < selectedCount; p++) { + int groupId = selected.getInt(firstValueIndex + p); + cell = combineCell(cell, getCellOrNull(groupId)); + } + appendGroupToBlockBuilder(builder, cell); + } } } return builder.build(); } } + private Block buildWithSelected(IntVector selected) { + try (DoubleBlock.Builder builder = blockFactory.newDoubleBlockBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + appendGroupToBlockBuilder(builder, getCellOrNull(selected.getInt(i))); + } + return builder.build(); + } + } + + @Override + Block build(IntBlock selected) { + var vector = selected.asVector(); + if (vector != null) { + return buildWithSelected(vector); + } else { + return buildWithSelected(selected); + } + } + @Override public void close() { Releasables.close(cells, super::close); diff --git a/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/enrich/EnrichResultBuilderForInt.java b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/enrich/EnrichResultBuilderForInt.java index 4dec877e0d1e..223a8eb88f0b 100644 --- a/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/enrich/EnrichResultBuilderForInt.java +++ b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/enrich/EnrichResultBuilderForInt.java @@ -25,9 +25,9 @@ final class EnrichResultBuilderForInt extends EnrichResultBuilder { private ObjectArray cells; - EnrichResultBuilderForInt(BlockFactory blockFactory, int channel, int totalPositions) { - super(blockFactory, channel, totalPositions); - this.cells = blockFactory.bigArrays().newObjectArray(totalPositions); + EnrichResultBuilderForInt(BlockFactory blockFactory, int channel) { + super(blockFactory, channel); + this.cells = blockFactory.bigArrays().newObjectArray(1); } @Override @@ -39,6 +39,7 @@ void addInputPage(IntVector positions, Page page) { continue; } int cellPosition = positions.getInt(i); + cells = blockFactory.bigArrays().grow(cells, cellPosition + 1); final var oldCell = cells.get(cellPosition); final var newCell = extendCell(oldCell, valueCount); cells.set(cellPosition, newCell); @@ -59,30 +60,82 @@ private int[] extendCell(int[] oldCell, int newValueCount) { } } - @Override - Block build() { - try (IntBlock.Builder builder = blockFactory.newIntBlockBuilder(totalPositions)) { - for (int i = 0; i < totalPositions; i++) { - final var cell = cells.get(i); - if (cell == null) { - builder.appendNull(); - continue; - } - if (cell.length > 1) { - builder.beginPositionEntry(); - } - // TODO: sort and dedup - for (var v : cell) { - builder.appendInt(v); - } - if (cell.length > 1) { - builder.endPositionEntry(); + private int[] combineCell(int[] first, int[] second) { + if (first == null) { + return second; + } + if (second == null) { + return first; + } + var result = new int[first.length + second.length]; + System.arraycopy(first, 0, result, 0, first.length); + System.arraycopy(second, 0, result, first.length, second.length); + return result; + } + + private void appendGroupToBlockBuilder(IntBlock.Builder builder, int[] group) { + if (group == null) { + builder.appendNull(); + } else if (group.length == 1) { + builder.appendInt(group[0]); + } else { + builder.beginPositionEntry(); + // TODO: sort and dedup and set MvOrdering + for (var v : group) { + builder.appendInt(v); + } + builder.endPositionEntry(); + } + } + + private int[] getCellOrNull(int position) { + return position < cells.size() ? cells.get(position) : null; + } + + private Block buildWithSelected(IntBlock selected) { + try (IntBlock.Builder builder = blockFactory.newIntBlockBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int selectedCount = selected.getValueCount(i); + switch (selectedCount) { + case 0 -> builder.appendNull(); + case 1 -> { + int groupId = selected.getInt(selected.getFirstValueIndex(i)); + appendGroupToBlockBuilder(builder, getCellOrNull(groupId)); + } + default -> { + int firstValueIndex = selected.getFirstValueIndex(i); + var cell = getCellOrNull(selected.getInt(firstValueIndex)); + for (int p = 1; p < selectedCount; p++) { + int groupId = selected.getInt(firstValueIndex + p); + cell = combineCell(cell, getCellOrNull(groupId)); + } + appendGroupToBlockBuilder(builder, cell); + } } } return builder.build(); } } + private Block buildWithSelected(IntVector selected) { + try (IntBlock.Builder builder = blockFactory.newIntBlockBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + appendGroupToBlockBuilder(builder, getCellOrNull(selected.getInt(i))); + } + return builder.build(); + } + } + + @Override + Block build(IntBlock selected) { + var vector = selected.asVector(); + if (vector != null) { + return buildWithSelected(vector); + } else { + return buildWithSelected(selected); + } + } + @Override public void close() { Releasables.close(cells, super::close); diff --git a/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/enrich/EnrichResultBuilderForLong.java b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/enrich/EnrichResultBuilderForLong.java index 0dd4d1d0a8a0..674b2e01c570 100644 --- a/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/enrich/EnrichResultBuilderForLong.java +++ b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/enrich/EnrichResultBuilderForLong.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.util.ObjectArray; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.Page; @@ -25,9 +26,9 @@ final class EnrichResultBuilderForLong extends EnrichResultBuilder { private ObjectArray cells; - EnrichResultBuilderForLong(BlockFactory blockFactory, int channel, int totalPositions) { - super(blockFactory, channel, totalPositions); - this.cells = blockFactory.bigArrays().newObjectArray(totalPositions); + EnrichResultBuilderForLong(BlockFactory blockFactory, int channel) { + super(blockFactory, channel); + this.cells = blockFactory.bigArrays().newObjectArray(1); } @Override @@ -39,6 +40,7 @@ void addInputPage(IntVector positions, Page page) { continue; } int cellPosition = positions.getInt(i); + cells = blockFactory.bigArrays().grow(cells, cellPosition + 1); final var oldCell = cells.get(cellPosition); final var newCell = extendCell(oldCell, valueCount); cells.set(cellPosition, newCell); @@ -59,30 +61,82 @@ private long[] extendCell(long[] oldCell, int newValueCount) { } } - @Override - Block build() { - try (LongBlock.Builder builder = blockFactory.newLongBlockBuilder(totalPositions)) { - for (int i = 0; i < totalPositions; i++) { - final var cell = cells.get(i); - if (cell == null) { - builder.appendNull(); - continue; - } - if (cell.length > 1) { - builder.beginPositionEntry(); - } - // TODO: sort and dedup - for (var v : cell) { - builder.appendLong(v); - } - if (cell.length > 1) { - builder.endPositionEntry(); + private long[] combineCell(long[] first, long[] second) { + if (first == null) { + return second; + } + if (second == null) { + return first; + } + var result = new long[first.length + second.length]; + System.arraycopy(first, 0, result, 0, first.length); + System.arraycopy(second, 0, result, first.length, second.length); + return result; + } + + private void appendGroupToBlockBuilder(LongBlock.Builder builder, long[] group) { + if (group == null) { + builder.appendNull(); + } else if (group.length == 1) { + builder.appendLong(group[0]); + } else { + builder.beginPositionEntry(); + // TODO: sort and dedup and set MvOrdering + for (var v : group) { + builder.appendLong(v); + } + builder.endPositionEntry(); + } + } + + private long[] getCellOrNull(int position) { + return position < cells.size() ? cells.get(position) : null; + } + + private Block buildWithSelected(IntBlock selected) { + try (LongBlock.Builder builder = blockFactory.newLongBlockBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int selectedCount = selected.getValueCount(i); + switch (selectedCount) { + case 0 -> builder.appendNull(); + case 1 -> { + int groupId = selected.getInt(selected.getFirstValueIndex(i)); + appendGroupToBlockBuilder(builder, getCellOrNull(groupId)); + } + default -> { + int firstValueIndex = selected.getFirstValueIndex(i); + var cell = getCellOrNull(selected.getInt(firstValueIndex)); + for (int p = 1; p < selectedCount; p++) { + int groupId = selected.getInt(firstValueIndex + p); + cell = combineCell(cell, getCellOrNull(groupId)); + } + appendGroupToBlockBuilder(builder, cell); + } } } return builder.build(); } } + private Block buildWithSelected(IntVector selected) { + try (LongBlock.Builder builder = blockFactory.newLongBlockBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + appendGroupToBlockBuilder(builder, getCellOrNull(selected.getInt(i))); + } + return builder.build(); + } + } + + @Override + Block build(IntBlock selected) { + var vector = selected.asVector(); + if (vector != null) { + return buildWithSelected(vector); + } else { + return buildWithSelected(selected); + } + } + @Override public void close() { Releasables.close(cells, super::close); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java index 366fb4ff55ba..17d189626d4e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichLookupService.java @@ -30,7 +30,10 @@ import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.BlockStreamInput; import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.LocalCircuitBreaker; +import org.elasticsearch.compute.data.OrdinalBytesRefBlock; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.lucene.ValuesSourceReaderOperator; import org.elasticsearch.compute.operator.Driver; @@ -246,6 +249,14 @@ private void doLookup( ActionListener listener ) { Block inputBlock = inputPage.getBlock(0); + final IntBlock selectedPositions; + if (inputBlock instanceof OrdinalBytesRefBlock ordinalBytesRefBlock) { + inputBlock = ordinalBytesRefBlock.getDictionaryVector().asBlock(); + selectedPositions = ordinalBytesRefBlock.getOrdinalsBlock(); + selectedPositions.mustIncRef(); + } else { + selectedPositions = IntVector.range(0, inputBlock.getPositionCount(), blockFactory).asBlock(); + } LocalCircuitBreaker localBreaker = null; try { if (inputBlock.areAllValuesNull()) { @@ -321,7 +332,7 @@ private void doLookup( // merging field-values by position final int[] mergingChannels = IntStream.range(0, extractFields.size()).map(i -> i + 2).toArray(); intermediateOperators.add( - new MergePositionsOperator(inputPage.getPositionCount(), 1, mergingChannels, mergingTypes, driverContext.blockFactory()) + new MergePositionsOperator(1, mergingChannels, mergingTypes, selectedPositions, driverContext.blockFactory()) ); AtomicReference result = new AtomicReference<>(); OutputOperator outputOperator = new OutputOperator(List.of(), Function.identity(), result::set); @@ -362,7 +373,7 @@ private void doLookup( } catch (Exception e) { listener.onFailure(e); } finally { - Releasables.close(localBreaker); + Releasables.close(selectedPositions, localBreaker); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichResultBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichResultBuilder.java index 5bb42f309069..062abb1917d8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichResultBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichResultBuilder.java @@ -10,6 +10,7 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.core.Releasable; @@ -21,13 +22,11 @@ abstract class EnrichResultBuilder implements Releasable { protected final BlockFactory blockFactory; protected final int channel; - protected final int totalPositions; private long usedBytes; - EnrichResultBuilder(BlockFactory blockFactory, int channel, int totalPositions) { + EnrichResultBuilder(BlockFactory blockFactory, int channel) { this.blockFactory = blockFactory; this.channel = channel; - this.totalPositions = totalPositions; } /** @@ -38,7 +37,7 @@ abstract class EnrichResultBuilder implements Releasable { */ abstract void addInputPage(IntVector positions, Page page); - abstract Block build(); + abstract Block build(IntBlock selected); final void adjustBreaker(long bytes) { blockFactory.breaker().addEstimateBytesAndMaybeBreak(bytes, "<>"); @@ -50,21 +49,21 @@ public void close() { blockFactory.breaker().addWithoutBreaking(-usedBytes); } - static EnrichResultBuilder enrichResultBuilder(ElementType elementType, BlockFactory blockFactory, int channel, int totalPositions) { + static EnrichResultBuilder enrichResultBuilder(ElementType elementType, BlockFactory blockFactory, int channel) { return switch (elementType) { - case NULL -> new EnrichResultBuilderForNull(blockFactory, channel, totalPositions); - case INT -> new EnrichResultBuilderForInt(blockFactory, channel, totalPositions); - case LONG -> new EnrichResultBuilderForLong(blockFactory, channel, totalPositions); - case DOUBLE -> new EnrichResultBuilderForDouble(blockFactory, channel, totalPositions); - case BOOLEAN -> new EnrichResultBuilderForBoolean(blockFactory, channel, totalPositions); - case BYTES_REF -> new EnrichResultBuilderForBytesRef(blockFactory, channel, totalPositions); + case NULL -> new EnrichResultBuilderForNull(blockFactory, channel); + case INT -> new EnrichResultBuilderForInt(blockFactory, channel); + case LONG -> new EnrichResultBuilderForLong(blockFactory, channel); + case DOUBLE -> new EnrichResultBuilderForDouble(blockFactory, channel); + case BOOLEAN -> new EnrichResultBuilderForBoolean(blockFactory, channel); + case BYTES_REF -> new EnrichResultBuilderForBytesRef(blockFactory, channel); default -> throw new IllegalArgumentException("no enrich result builder for [" + elementType + "]"); }; } private static class EnrichResultBuilderForNull extends EnrichResultBuilder { - EnrichResultBuilderForNull(BlockFactory blockFactory, int channel, int totalPositions) { - super(blockFactory, channel, totalPositions); + EnrichResultBuilderForNull(BlockFactory blockFactory, int channel) { + super(blockFactory, channel); } @Override @@ -73,8 +72,8 @@ void addInputPage(IntVector positions, Page page) { } @Override - Block build() { - return blockFactory.newConstantNullBlock(totalPositions); + Block build(IntBlock selected) { + return blockFactory.newConstantNullBlock(selected.getPositionCount()); } } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/MergePositionsOperator.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/MergePositionsOperator.java index a3b7a8be61e2..3e1f46100c4f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/MergePositionsOperator.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/MergePositionsOperator.java @@ -46,16 +46,16 @@ final class MergePositionsOperator implements Operator { private boolean finished = false; private final int positionChannel; - private final EnrichResultBuilder[] builders; + private final IntBlock selectedPositions; private Page outputPage; MergePositionsOperator( - int positionCount, int positionChannel, int[] mergingChannels, ElementType[] mergingTypes, + IntBlock selectedPositions, BlockFactory blockFactory ) { if (mergingChannels.length != mergingTypes.length) { @@ -70,13 +70,15 @@ final class MergePositionsOperator implements Operator { this.builders = new EnrichResultBuilder[mergingTypes.length]; try { for (int i = 0; i < mergingTypes.length; i++) { - builders[i] = EnrichResultBuilder.enrichResultBuilder(mergingTypes[i], blockFactory, mergingChannels[i], positionCount); + builders[i] = EnrichResultBuilder.enrichResultBuilder(mergingTypes[i], blockFactory, mergingChannels[i]); } } finally { if (builders[builders.length - 1] == null) { - Releasables.close(builders); + Releasables.close(Releasables.wrap(builders)); } } + selectedPositions.mustIncRef(); + this.selectedPositions = selectedPositions; } @Override @@ -102,7 +104,7 @@ public void finish() { final Block[] blocks = new Block[builders.length]; try { for (int i = 0; i < builders.length; i++) { - blocks[i] = builders[i].build(); + blocks[i] = builders[i].build(selectedPositions); } outputPage = new Page(blocks); } finally { @@ -127,7 +129,7 @@ public Page getOutput() { @Override public void close() { - Releasables.close(Releasables.wrap(builders), () -> { + Releasables.close(Releasables.wrap(builders), selectedPositions, () -> { if (outputPage != null) { outputPage.releaseBlocks(); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/X-EnrichResultBuilder.java.st b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/X-EnrichResultBuilder.java.st index 4c5c9fabfa79..7066b8b8f12a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/X-EnrichResultBuilder.java.st +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/X-EnrichResultBuilder.java.st @@ -18,10 +18,15 @@ import org.elasticsearch.common.util.ObjectArray; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; $if(long)$ +import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.$Type$Block; +$elseif(int)$ +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; $else$ import org.elasticsearch.compute.data.$Type$Block; +import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.IntVector; $endif$ import org.elasticsearch.compute.data.Page; @@ -36,16 +41,17 @@ import java.util.Arrays; final class EnrichResultBuilderFor$Type$ extends EnrichResultBuilder { $if(BytesRef)$ private final BytesRefArray bytes; // shared between all cells + private BytesRef scratch = new BytesRef(); $endif$ private ObjectArray<$if(BytesRef)$int$else$$type$$endif$[]> cells; - EnrichResultBuilderFor$Type$(BlockFactory blockFactory, int channel, int totalPositions) { - super(blockFactory, channel, totalPositions); - this.cells = blockFactory.bigArrays().newObjectArray(totalPositions); + EnrichResultBuilderFor$Type$(BlockFactory blockFactory, int channel) { + super(blockFactory, channel); + this.cells = blockFactory.bigArrays().newObjectArray(1); $if(BytesRef)$ BytesRefArray bytes = null; try { - bytes = new BytesRefArray(totalPositions * 3L, blockFactory.bigArrays()); + bytes = new BytesRefArray(1L, blockFactory.bigArrays()); this.bytes = bytes; } finally { if (bytes == null) { @@ -67,6 +73,7 @@ $endif$ continue; } int cellPosition = positions.getInt(i); + cells = blockFactory.bigArrays().grow(cells, cellPosition + 1); final var oldCell = cells.get(cellPosition); final var newCell = extendCell(oldCell, valueCount); cells.set(cellPosition, newCell); @@ -96,37 +103,90 @@ $endif$ } } - @Override - Block build() { - try ($Type$Block.Builder builder = blockFactory.new$Type$BlockBuilder(totalPositions)) { + private $if(BytesRef)$int$else$$type$$endif$[] combineCell($if(BytesRef)$int$else$$type$$endif$[] first, $if(BytesRef)$int$else$$type$$endif$[] second) { + if (first == null) { + return second; + } + if (second == null) { + return first; + } + var result = new $if(BytesRef)$int$else$$type$$endif$[first.length + second.length]; + System.arraycopy(first, 0, result, 0, first.length); + System.arraycopy(second, 0, result, first.length, second.length); + return result; + } + + private void appendGroupToBlockBuilder($Type$Block.Builder builder, $if(BytesRef)$int$else$$type$$endif$[] group) { + if (group == null) { + builder.appendNull(); + } else if (group.length == 1) { $if(BytesRef)$ - BytesRef scratch = new BytesRef(); + builder.appendBytesRef(bytes.get(group[0], scratch)); +$else$ + builder.append$Type$(group[0]); $endif$ - for (int i = 0; i < totalPositions; i++) { - final var cell = cells.get(i); - if (cell == null) { - builder.appendNull(); - continue; - } - if (cell.length > 1) { - builder.beginPositionEntry(); - } - // TODO: sort and dedup - for (var v : cell) { + } else { + builder.beginPositionEntry(); + // TODO: sort and dedup and set MvOrdering + for (var v : group) { $if(BytesRef)$ - builder.appendBytesRef(bytes.get(v, scratch)); + builder.appendBytesRef(bytes.get(v, scratch)); $else$ - builder.append$Type$(v); + builder.append$Type$(v); $endif$ - } - if (cell.length > 1) { - builder.endPositionEntry(); + } + builder.endPositionEntry(); + } + } + + private $if(BytesRef)$int$else$$type$$endif$[] getCellOrNull(int position) { + return position < cells.size() ? cells.get(position) : null; + } + + private Block buildWithSelected(IntBlock selected) { + try ($Type$Block.Builder builder = blockFactory.new$Type$BlockBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int selectedCount = selected.getValueCount(i); + switch (selectedCount) { + case 0 -> builder.appendNull(); + case 1 -> { + int groupId = selected.getInt(selected.getFirstValueIndex(i)); + appendGroupToBlockBuilder(builder, getCellOrNull(groupId)); + } + default -> { + int firstValueIndex = selected.getFirstValueIndex(i); + var cell = getCellOrNull(selected.getInt(firstValueIndex)); + for (int p = 1; p < selectedCount; p++) { + int groupId = selected.getInt(firstValueIndex + p); + cell = combineCell(cell, getCellOrNull(groupId)); + } + appendGroupToBlockBuilder(builder, cell); + } } } return builder.build(); } } + private Block buildWithSelected(IntVector selected) { + try ($Type$Block.Builder builder = blockFactory.new$Type$BlockBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + appendGroupToBlockBuilder(builder, getCellOrNull(selected.getInt(i))); + } + return builder.build(); + } + } + + @Override + Block build(IntBlock selected) { + var vector = selected.asVector(); + if (vector != null) { + return buildWithSelected(vector); + } else { + return buildWithSelected(selected); + } + } + @Override public void close() { Releasables.close($if(BytesRef)$bytes, $endif$cells, super::close); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichResultBuilderTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichResultBuilderTests.java index f6e8b9107504..24ca02a9d2e0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichResultBuilderTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichResultBuilderTests.java @@ -15,6 +15,8 @@ import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.Page; import org.elasticsearch.test.ESTestCase; @@ -30,10 +32,10 @@ public class EnrichResultBuilderTests extends ESTestCase { public void testBytesRef() { BlockFactory blockFactory = blockFactory(); - Map> expectedValues = new HashMap<>(); + Map> inputValues = new HashMap<>(); int numPages = between(0, 10); int maxPosition = between(0, 100); - var resultBuilder = EnrichResultBuilder.enrichResultBuilder(ElementType.BYTES_REF, blockFactory, 0, maxPosition + 1); + var resultBuilder = EnrichResultBuilder.enrichResultBuilder(ElementType.BYTES_REF, blockFactory, 0); for (int i = 0; i < numPages; i++) { int numRows = between(1, 100); try ( @@ -52,7 +54,7 @@ public void testBytesRef() { } for (int v = 0; v < numValues; v++) { BytesRef val = new BytesRef(randomByteArrayOfLength(10)); - expectedValues.computeIfAbsent(position, k -> new ArrayList<>()).add(val); + inputValues.computeIfAbsent(position, k -> new ArrayList<>()).add(val); valuesBuilder.appendBytesRef(val); } if (numValues > 1) { @@ -64,18 +66,60 @@ public void testBytesRef() { } } } - try (BytesRefBlock actualOutput = (BytesRefBlock) resultBuilder.build()) { - assertThat(actualOutput.getPositionCount(), equalTo(maxPosition + 1)); - for (int i = 0; i < actualOutput.getPositionCount(); i++) { - List values = expectedValues.get(i); - if (actualOutput.isNull(i)) { - assertNull(values); + try (IntVector selected = IntVector.range(0, maxPosition + 1, blockFactory)) { + try (BytesRefBlock actualOutput = (BytesRefBlock) resultBuilder.build(selected.asBlock())) { + assertThat(actualOutput.getPositionCount(), equalTo(maxPosition + 1)); + for (int i = 0; i < actualOutput.getPositionCount(); i++) { + List values = inputValues.get(i); + if (actualOutput.isNull(i)) { + assertNull(values); + } else { + int valueCount = actualOutput.getValueCount(i); + int first = actualOutput.getFirstValueIndex(i); + assertThat(valueCount, equalTo(values.size())); + for (int v = 0; v < valueCount; v++) { + assertThat(actualOutput.getBytesRef(first + v, new BytesRef()), equalTo(values.get(v))); + } + } + } + } + } + try (IntBlock.Builder selectedBuilder = blockFactory.newIntBlockBuilder(between(1, 10))) { + int selectedPositions = between(1, 100); + Map> expectedValues = new HashMap<>(); + for (int i = 0; i < selectedPositions; i++) { + int ps = randomIntBetween(0, 3); + List values = new ArrayList<>(); + if (ps == 0) { + selectedBuilder.appendNull(); } else { - int valueCount = actualOutput.getValueCount(i); - int first = actualOutput.getFirstValueIndex(i); - assertThat(valueCount, equalTo(values.size())); - for (int v = 0; v < valueCount; v++) { - assertThat(actualOutput.getBytesRef(first + v, new BytesRef()), equalTo(values.get(v))); + selectedBuilder.beginPositionEntry(); + for (int p = 0; p < ps; p++) { + int position = randomIntBetween(0, maxPosition); + selectedBuilder.appendInt(position); + values.addAll(inputValues.getOrDefault(position, List.of())); + } + selectedBuilder.endPositionEntry(); + } + if (values.isEmpty()) { + expectedValues.put(i, null); + } else { + expectedValues.put(i, values); + } + } + try (var selected = selectedBuilder.build(); BytesRefBlock actualOutput = (BytesRefBlock) resultBuilder.build(selected)) { + assertThat(actualOutput.getPositionCount(), equalTo(selected.getPositionCount())); + for (int i = 0; i < actualOutput.getPositionCount(); i++) { + List values = expectedValues.get(i); + if (actualOutput.isNull(i)) { + assertNull(values); + } else { + int valueCount = actualOutput.getValueCount(i); + int first = actualOutput.getFirstValueIndex(i); + assertThat(valueCount, equalTo(values.size())); + for (int v = 0; v < valueCount; v++) { + assertThat(actualOutput.getBytesRef(first + v, new BytesRef()), equalTo(values.get(v))); + } } } } @@ -89,7 +133,7 @@ public void testLong() { Map> expectedValues = new HashMap<>(); int numPages = between(0, 10); int maxPosition = between(0, 100); - var resultBuilder = EnrichResultBuilder.enrichResultBuilder(ElementType.LONG, blockFactory, 0, maxPosition + 1); + var resultBuilder = EnrichResultBuilder.enrichResultBuilder(ElementType.LONG, blockFactory, 0); for (int i = 0; i < numPages; i++) { int numRows = between(1, 100); try ( @@ -120,18 +164,20 @@ public void testLong() { } } } - try (LongBlock actualOutput = (LongBlock) resultBuilder.build()) { - assertThat(actualOutput.getPositionCount(), equalTo(maxPosition + 1)); - for (int i = 0; i < actualOutput.getPositionCount(); i++) { - List values = expectedValues.get(i); - if (actualOutput.isNull(i)) { - assertNull(values); - } else { - int valueCount = actualOutput.getValueCount(i); - int first = actualOutput.getFirstValueIndex(i); - assertThat(valueCount, equalTo(values.size())); - for (int v = 0; v < valueCount; v++) { - assertThat(actualOutput.getLong(first + v), equalTo(values.get(v))); + try (IntVector selected = IntVector.range(0, maxPosition + 1, blockFactory)) { + try (LongBlock actualOutput = (LongBlock) resultBuilder.build(selected.asBlock())) { + assertThat(actualOutput.getPositionCount(), equalTo(maxPosition + 1)); + for (int i = 0; i < actualOutput.getPositionCount(); i++) { + List values = expectedValues.get(i); + if (actualOutput.isNull(i)) { + assertNull(values); + } else { + int valueCount = actualOutput.getValueCount(i); + int first = actualOutput.getFirstValueIndex(i); + assertThat(valueCount, equalTo(values.size())); + for (int v = 0; v < valueCount; v++) { + assertThat(actualOutput.getLong(first + v), equalTo(values.get(v))); + } } } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/MergePositionsOperatorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/MergePositionsOperatorTests.java index 09bc36a5390a..df49fff5191b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/MergePositionsOperatorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/MergePositionsOperatorTests.java @@ -18,7 +18,9 @@ import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.ElementType; import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.Page; +import org.elasticsearch.core.Releasables; import org.elasticsearch.test.ESTestCase; import java.util.List; @@ -31,11 +33,12 @@ public void testSimple() throws Exception { BigArrays bigArrays = new MockBigArrays(PageCacheRecycler.NON_RECYCLING_INSTANCE, ByteSizeValue.ofGb(1)).withCircuitBreaking(); CircuitBreaker breaker = bigArrays.breakerService().getBreaker(CircuitBreaker.REQUEST); BlockFactory blockFactory = new BlockFactory(breaker, bigArrays); + IntVector selected = IntVector.range(0, 7, blockFactory); MergePositionsOperator mergeOperator = new MergePositionsOperator( - 7, 0, new int[] { 1, 2 }, new ElementType[] { ElementType.BYTES_REF, ElementType.INT }, + selected.asBlock(), blockFactory ); { @@ -123,8 +126,7 @@ public void testSimple() throws Exception { assertTrue(f2.isNull(4)); assertThat(BlockUtils.toJavaObject(f2, 5), equalTo(2023)); assertTrue(f2.isNull(6)); - mergeOperator.close(); - out.releaseBlocks(); + Releasables.close(mergeOperator, selected, out::releaseBlocks); MockBigArrays.ensureAllArraysAreReleased(); } } From f8fe610966e8f9aae222b961ff05c9d1b418de78 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Tue, 16 Apr 2024 17:15:59 +0100 Subject: [PATCH 08/43] [ML] Add GET _inference for all inference endpoints (#107517) --- docs/changelog/107517.yaml | 5 +++++ .../reference/inference/get-inference.asciidoc | 2 ++ .../rest-api-spec/api/inference.get_model.json | 6 ++++++ .../rest/RestGetInferenceModelAction.java | 7 ++++++- .../test/inference/inference_crud.yml | 18 ++++++++++++++++++ 5 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/107517.yaml diff --git a/docs/changelog/107517.yaml b/docs/changelog/107517.yaml new file mode 100644 index 000000000000..4d7830699ad4 --- /dev/null +++ b/docs/changelog/107517.yaml @@ -0,0 +1,5 @@ +pr: 107517 +summary: Add GET `_inference` for all inference endpoints +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/reference/inference/get-inference.asciidoc b/docs/reference/inference/get-inference.asciidoc index 2cfc17a3b620..74a430139d89 100644 --- a/docs/reference/inference/get-inference.asciidoc +++ b/docs/reference/inference/get-inference.asciidoc @@ -18,6 +18,8 @@ non-NLP models, use the <>. [[get-inference-api-request]] ==== {api-request-title} +`GET /_inference` + `GET /_inference/_all` `GET /_inference/` diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.get_model.json b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.get_model.json index f9340810e2e4..3749c2ec9577 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/inference.get_model.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/inference.get_model.json @@ -11,6 +11,12 @@ }, "url":{ "paths":[ + { + "path":"/_inference", + "methods":[ + "GET" + ] + }, { "path":"/_inference/{inference_id}", "methods":[ diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestGetInferenceModelAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestGetInferenceModelAction.java index 4de6ff7af1f1..34d0f2647b2d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestGetInferenceModelAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/RestGetInferenceModelAction.java @@ -34,7 +34,12 @@ public String getName() { @Override public List routes() { - return List.of(new Route(GET, "_inference/_all"), new Route(GET, INFERENCE_ID_PATH), new Route(GET, TASK_TYPE_INFERENCE_ID_PATH)); + return List.of( + new Route(GET, "_inference"), + new Route(GET, "_inference/_all"), + new Route(GET, INFERENCE_ID_PATH), + new Route(GET, TASK_TYPE_INFERENCE_ID_PATH) + ); } @Override diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/inference/inference_crud.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/inference/inference_crud.yml index 39a107373c8a..ec8ca43a44b2 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/inference/inference_crud.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/inference/inference_crud.yml @@ -38,3 +38,21 @@ "input": "important text" } - match: { error.reason: "Unknown task_type [bad]" } + +--- +"Test get all": + - do: + inference.get_model: + inference_id: "*" + - length: { models: 0} + + - do: + inference.get_model: + inference_id: _all + - length: { models: 0} + + - do: + inference.get_model: + inference_id: "" + - length: { models: 0} + From c2df24b2bafb106e745d35073608e547776d78ab Mon Sep 17 00:00:00 2001 From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com> Date: Tue, 16 Apr 2024 12:26:01 -0400 Subject: [PATCH 09/43] [ML] Refactor models to expose fields used for rate limiting (#107444) * Adding rate limit interface to openai and hugging face * Adding cohere rate limit service settings * Switching to request manager * Adjusting constructor scope * Removing uri from cohere * Adding rate limit fields for azure * Fixing spotless * Only storing hash code values --- .../AzureOpenAiEmbeddingsAction.java | 6 +-- .../action/cohere/CohereActionCreator.java | 4 +- .../action/cohere/CohereEmbeddingsAction.java | 9 +++-- .../action/cohere/CohereRerankAction.java | 9 +++-- .../action/huggingface/HuggingFaceAction.java | 11 +++-- .../openai/OpenAiChatCompletionAction.java | 6 +-- .../action/openai/OpenAiEmbeddingsAction.java | 6 +-- .../external/cohere/CohereAccount.java | 15 ++++++- .../external/http/RequestExecutor.java | 4 +- ... AzureOpenAiEmbeddingsRequestManager.java} | 21 ++++++---- .../sender/AzureOpenAiRequestManager.java | 30 ++++++++++++++ .../http/sender/BaseRequestManager.java | 40 +++++++++++++++++++ ...va => CohereEmbeddingsRequestManager.java} | 17 ++++---- .../http/sender/CohereRequestManager.java | 28 +++++++++++++ ...r.java => CohereRerankRequestManager.java} | 19 +++++---- .../http/sender/HttpRequestSender.java | 2 +- ...or.java => HuggingFaceRequestManager.java} | 39 +++++++++++++----- .../http/sender/InferenceRequest.java | 2 +- .../external/http/sender/NoopTask.java | 2 +- ...va => OpenAiCompletionRequestManager.java} | 22 +++++----- ...va => OpenAiEmbeddingsRequestManager.java} | 25 +++++++----- .../http/sender/OpenAiRequestManager.java | 35 ++++++++++++++++ .../http/sender/RequestExecutorService.java | 2 +- ...equestCreator.java => RequestManager.java} | 5 ++- .../external/http/sender/RequestTask.java | 6 +-- .../external/http/sender/Sender.java | 2 +- .../huggingface/HuggingFaceAccount.java | 9 ++++- .../external/openai/OpenAiAccount.java | 14 ++++++- .../external/ratelimit/RateLimitable.java | 25 ++++++++++++ .../AzureOpenAiEmbeddingsRequest.java | 11 ++--- .../cohere/CohereEmbeddingsRequest.java | 14 +++---- .../request/cohere/CohereRerankRequest.java | 14 +++---- .../HuggingFaceInferenceRequest.java | 15 +++---- .../openai/OpenAiChatCompletionRequest.java | 14 +++---- .../openai/OpenAiEmbeddingsRequest.java | 21 +++------- .../inference/services/ServiceUtils.java | 6 +++ .../azureopenai/AzureOpenAiModel.java | 18 ++++++++- .../AzureOpenAiRateLimitServiceSettings.java | 15 +++++++ .../AzureOpenAiEmbeddingsModel.java | 6 ++- .../AzureOpenAiEmbeddingsServiceSettings.java | 5 ++- .../services/cohere/CohereModel.java | 21 +++++++++- .../embeddings/CohereEmbeddingsModel.java | 12 +++++- .../cohere/rerank/CohereRerankModel.java | 12 +++++- .../huggingface/HuggingFaceModel.java | 30 +++++++++++--- .../HuggingFaceRateLimitServiceSettings.java | 17 ++++++++ .../HuggingFaceServiceSettings.java | 3 +- .../elser/HuggingFaceElserModel.java | 19 +++------ .../elser/HuggingFaceElserSecretSettings.java | 3 +- .../HuggingFaceElserServiceSettings.java | 6 ++- .../HuggingFaceEmbeddingsModel.java | 19 +++------ .../services/openai/OpenAiModel.java | 32 ++++++++++++++- .../OpenAiRateLimitServiceSettings.java | 21 ++++++++++ .../completion/OpenAiChatCompletionModel.java | 7 +++- .../OpenAiChatCompletionServiceSettings.java | 6 ++- .../embeddings/OpenAiEmbeddingsModel.java | 7 +++- .../OpenAiEmbeddingsServiceSettings.java | 6 ++- .../services/settings/ApiKeySecrets.java | 14 +++++++ .../settings/DefaultSecretSettings.java | 2 +- .../cohere/CohereEmbeddingsActionTests.java | 2 +- .../sender/ExecutableRequestCreatorTests.java | 10 ++--- .../http/sender/HttpRequestSenderTests.java | 2 +- ...beddingsExecutableRequestCreatorTests.java | 15 ++++--- .../sender/RequestExecutorServiceTests.java | 4 +- .../http/sender/RequestTaskTests.java | 10 ++--- .../sender/SingleRequestManagerTests.java | 2 +- .../AzureOpenAiEmbeddingsRequestTests.java | 4 -- .../cohere/CohereEmbeddingsRequestTests.java | 4 +- .../HuggingFaceInferenceRequestTests.java | 4 -- .../OpenAiChatCompletionRequestTests.java | 9 +---- .../openai/OpenAiEmbeddingsRequestTests.java | 3 -- 70 files changed, 618 insertions(+), 242 deletions(-) rename x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/{AzureOpenAiEmbeddingsExecutableRequestCreator.java => AzureOpenAiEmbeddingsRequestManager.java} (77%) create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureOpenAiRequestManager.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/BaseRequestManager.java rename x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/{CohereEmbeddingsExecutableRequestCreator.java => CohereEmbeddingsRequestManager.java} (79%) create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereRequestManager.java rename x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/{CohereRerankExecutableRequestCreator.java => CohereRerankRequestManager.java} (76%) rename x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/{HuggingFaceExecutableRequestCreator.java => HuggingFaceRequestManager.java} (62%) rename x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/{OpenAiCompletionExecutableRequestCreator.java => OpenAiCompletionRequestManager.java} (78%) rename x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/{OpenAiEmbeddingsExecutableRequestCreator.java => OpenAiEmbeddingsRequestManager.java} (77%) create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiRequestManager.java rename x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/{ExecutableRequestCreator.java => RequestManager.java} (86%) create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/ratelimit/RateLimitable.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiRateLimitServiceSettings.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceRateLimitServiceSettings.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiRateLimitServiceSettings.java create mode 100644 x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/ApiKeySecrets.java diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiEmbeddingsAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiEmbeddingsAction.java index a682ad2bb23d..1b2226dd3f9f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiEmbeddingsAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/azureopenai/AzureOpenAiEmbeddingsAction.java @@ -12,7 +12,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; -import org.elasticsearch.xpack.inference.external.http.sender.AzureOpenAiEmbeddingsExecutableRequestCreator; +import org.elasticsearch.xpack.inference.external.http.sender.AzureOpenAiEmbeddingsRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; import org.elasticsearch.xpack.inference.external.http.sender.Sender; import org.elasticsearch.xpack.inference.services.ServiceComponents; @@ -27,14 +27,14 @@ public class AzureOpenAiEmbeddingsAction implements ExecutableAction { private final String errorMessage; - private final AzureOpenAiEmbeddingsExecutableRequestCreator requestCreator; + private final AzureOpenAiEmbeddingsRequestManager requestCreator; private final Sender sender; public AzureOpenAiEmbeddingsAction(Sender sender, AzureOpenAiEmbeddingsModel model, ServiceComponents serviceComponents) { Objects.requireNonNull(serviceComponents); Objects.requireNonNull(model); this.sender = Objects.requireNonNull(sender); - requestCreator = new AzureOpenAiEmbeddingsExecutableRequestCreator(model, serviceComponents.truncator()); + requestCreator = new AzureOpenAiEmbeddingsRequestManager(model, serviceComponents.truncator(), serviceComponents.threadPool()); errorMessage = constructFailedToSendRequestMessage(model.getUri(), "Azure OpenAI embeddings"); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereActionCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereActionCreator.java index b8e1b34c11f2..9f54950dba2d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereActionCreator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereActionCreator.java @@ -33,13 +33,13 @@ public CohereActionCreator(Sender sender, ServiceComponents serviceComponents) { public ExecutableAction create(CohereEmbeddingsModel model, Map taskSettings, InputType inputType) { var overriddenModel = CohereEmbeddingsModel.of(model, taskSettings, inputType); - return new CohereEmbeddingsAction(sender, overriddenModel); + return new CohereEmbeddingsAction(sender, overriddenModel, serviceComponents.threadPool()); } @Override public ExecutableAction create(CohereRerankModel model, Map taskSettings) { var overriddenModel = CohereRerankModel.of(model, taskSettings); - return new CohereRerankAction(sender, overriddenModel); + return new CohereRerankAction(sender, overriddenModel, serviceComponents.threadPool()); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereEmbeddingsAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereEmbeddingsAction.java index f4fddf65ea21..63e51d99a8ce 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereEmbeddingsAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereEmbeddingsAction.java @@ -11,8 +11,9 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; -import org.elasticsearch.xpack.inference.external.http.sender.CohereEmbeddingsExecutableRequestCreator; +import org.elasticsearch.xpack.inference.external.http.sender.CohereEmbeddingsRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; import org.elasticsearch.xpack.inference.external.http.sender.Sender; import org.elasticsearch.xpack.inference.services.cohere.embeddings.CohereEmbeddingsModel; @@ -26,16 +27,16 @@ public class CohereEmbeddingsAction implements ExecutableAction { private final String failedToSendRequestErrorMessage; private final Sender sender; - private final CohereEmbeddingsExecutableRequestCreator requestCreator; + private final CohereEmbeddingsRequestManager requestCreator; - public CohereEmbeddingsAction(Sender sender, CohereEmbeddingsModel model) { + public CohereEmbeddingsAction(Sender sender, CohereEmbeddingsModel model, ThreadPool threadPool) { Objects.requireNonNull(model); this.sender = Objects.requireNonNull(sender); this.failedToSendRequestErrorMessage = constructFailedToSendRequestMessage( model.getServiceSettings().getCommonSettings().uri(), "Cohere embeddings" ); - requestCreator = new CohereEmbeddingsExecutableRequestCreator(model); + requestCreator = CohereEmbeddingsRequestManager.of(model, threadPool); } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereRerankAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereRerankAction.java index 5209781b0058..0613b8ef7645 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereRerankAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereRerankAction.java @@ -11,8 +11,9 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; -import org.elasticsearch.xpack.inference.external.http.sender.CohereRerankExecutableRequestCreator; +import org.elasticsearch.xpack.inference.external.http.sender.CohereRerankRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; import org.elasticsearch.xpack.inference.external.http.sender.Sender; import org.elasticsearch.xpack.inference.services.cohere.rerank.CohereRerankModel; @@ -26,16 +27,16 @@ public class CohereRerankAction implements ExecutableAction { private final String failedToSendRequestErrorMessage; private final Sender sender; - private final CohereRerankExecutableRequestCreator requestCreator; + private final CohereRerankRequestManager requestCreator; - public CohereRerankAction(Sender sender, CohereRerankModel model) { + public CohereRerankAction(Sender sender, CohereRerankModel model, ThreadPool threadPool) { Objects.requireNonNull(model); this.sender = Objects.requireNonNull(sender); this.failedToSendRequestErrorMessage = constructFailedToSendRequestMessage( model.getServiceSettings().getCommonSettings().uri(), "Cohere rerank" ); - requestCreator = new CohereRerankExecutableRequestCreator(model); + requestCreator = CohereRerankRequestManager.of(model, threadPool); } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/huggingface/HuggingFaceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/huggingface/HuggingFaceAction.java index 6d6580c391cf..1e5f01f801e1 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/huggingface/HuggingFaceAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/huggingface/HuggingFaceAction.java @@ -13,7 +13,7 @@ import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.external.http.retry.ResponseHandler; -import org.elasticsearch.xpack.inference.external.http.sender.HuggingFaceExecutableRequestCreator; +import org.elasticsearch.xpack.inference.external.http.sender.HuggingFaceRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; import org.elasticsearch.xpack.inference.external.http.sender.Sender; import org.elasticsearch.xpack.inference.services.ServiceComponents; @@ -28,7 +28,7 @@ public class HuggingFaceAction implements ExecutableAction { private final String errorMessage; private final Sender sender; - private final HuggingFaceExecutableRequestCreator requestCreator; + private final HuggingFaceRequestManager requestCreator; public HuggingFaceAction( Sender sender, @@ -40,7 +40,12 @@ public HuggingFaceAction( Objects.requireNonNull(serviceComponents); Objects.requireNonNull(requestType); this.sender = Objects.requireNonNull(sender); - requestCreator = new HuggingFaceExecutableRequestCreator(model, responseHandler, serviceComponents.truncator()); + requestCreator = HuggingFaceRequestManager.of( + model, + responseHandler, + serviceComponents.truncator(), + serviceComponents.threadPool() + ); errorMessage = format( "Failed to send Hugging Face %s request from inference entity id [%s]", requestType, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiChatCompletionAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiChatCompletionAction.java index c474f3bfb0ab..5d75adedddde 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiChatCompletionAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiChatCompletionAction.java @@ -16,7 +16,7 @@ import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; -import org.elasticsearch.xpack.inference.external.http.sender.OpenAiCompletionExecutableRequestCreator; +import org.elasticsearch.xpack.inference.external.http.sender.OpenAiCompletionRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.Sender; import org.elasticsearch.xpack.inference.services.ServiceComponents; import org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionModel; @@ -30,7 +30,7 @@ public class OpenAiChatCompletionAction implements ExecutableAction { private final String errorMessage; - private final OpenAiCompletionExecutableRequestCreator requestCreator; + private final OpenAiCompletionRequestManager requestCreator; private final Sender sender; @@ -38,7 +38,7 @@ public OpenAiChatCompletionAction(Sender sender, OpenAiChatCompletionModel model Objects.requireNonNull(serviceComponents); Objects.requireNonNull(model); this.sender = Objects.requireNonNull(sender); - this.requestCreator = new OpenAiCompletionExecutableRequestCreator(model); + this.requestCreator = OpenAiCompletionRequestManager.of(model, serviceComponents.threadPool()); this.errorMessage = constructFailedToSendRequestMessage(model.getServiceSettings().uri(), "OpenAI chat completions"); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiEmbeddingsAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiEmbeddingsAction.java index 8a64b9f922ac..3e92d206b425 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiEmbeddingsAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/action/openai/OpenAiEmbeddingsAction.java @@ -13,7 +13,7 @@ import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; -import org.elasticsearch.xpack.inference.external.http.sender.OpenAiEmbeddingsExecutableRequestCreator; +import org.elasticsearch.xpack.inference.external.http.sender.OpenAiEmbeddingsRequestManager; import org.elasticsearch.xpack.inference.external.http.sender.Sender; import org.elasticsearch.xpack.inference.services.ServiceComponents; import org.elasticsearch.xpack.inference.services.openai.embeddings.OpenAiEmbeddingsModel; @@ -27,14 +27,14 @@ public class OpenAiEmbeddingsAction implements ExecutableAction { private final String errorMessage; - private final OpenAiEmbeddingsExecutableRequestCreator requestCreator; + private final OpenAiEmbeddingsRequestManager requestCreator; private final Sender sender; public OpenAiEmbeddingsAction(Sender sender, OpenAiEmbeddingsModel model, ServiceComponents serviceComponents) { Objects.requireNonNull(serviceComponents); Objects.requireNonNull(model); this.sender = Objects.requireNonNull(sender); - requestCreator = new OpenAiEmbeddingsExecutableRequestCreator(model, serviceComponents.truncator()); + requestCreator = OpenAiEmbeddingsRequestManager.of(model, serviceComponents.truncator(), serviceComponents.threadPool()); errorMessage = constructFailedToSendRequestMessage(model.getServiceSettings().uri(), "OpenAI embeddings"); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/cohere/CohereAccount.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/cohere/CohereAccount.java index 9847d496d14e..9fc5640f8cf7 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/cohere/CohereAccount.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/cohere/CohereAccount.java @@ -7,15 +7,26 @@ package org.elasticsearch.xpack.inference.external.cohere; +import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.settings.SecureString; -import org.elasticsearch.core.Nullable; +import org.elasticsearch.xpack.inference.services.cohere.CohereModel; import java.net.URI; +import java.net.URISyntaxException; import java.util.Objects; -public record CohereAccount(@Nullable URI url, SecureString apiKey) { +import static org.elasticsearch.xpack.inference.external.request.RequestUtils.buildUri; + +public record CohereAccount(URI uri, SecureString apiKey) { + + public static CohereAccount of(CohereModel model, CheckedSupplier uriBuilder) { + var uri = buildUri(model.uri(), "Cohere", uriBuilder); + + return new CohereAccount(uri, model.apiKey()); + } public CohereAccount { + Objects.requireNonNull(uri); Objects.requireNonNull(apiKey); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/RequestExecutor.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/RequestExecutor.java index 7b0287e9652f..63c042ce8a62 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/RequestExecutor.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/RequestExecutor.java @@ -11,8 +11,8 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.InferenceServiceResults; -import org.elasticsearch.xpack.inference.external.http.sender.ExecutableRequestCreator; import org.elasticsearch.xpack.inference.external.http.sender.InferenceInputs; +import org.elasticsearch.xpack.inference.external.http.sender.RequestManager; import java.util.concurrent.TimeUnit; @@ -28,7 +28,7 @@ public interface RequestExecutor { boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException; void execute( - ExecutableRequestCreator requestCreator, + RequestManager requestCreator, InferenceInputs inferenceInputs, @Nullable TimeValue timeout, ActionListener listener diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureOpenAiEmbeddingsExecutableRequestCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureOpenAiEmbeddingsRequestManager.java similarity index 77% rename from x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureOpenAiEmbeddingsExecutableRequestCreator.java rename to x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureOpenAiEmbeddingsRequestManager.java index b3f53d5f3f23..06152b50822a 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureOpenAiEmbeddingsExecutableRequestCreator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureOpenAiEmbeddingsRequestManager.java @@ -12,8 +12,8 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.inference.common.Truncator; -import org.elasticsearch.xpack.inference.external.azureopenai.AzureOpenAiAccount; import org.elasticsearch.xpack.inference.external.azureopenai.AzureOpenAiResponseHandler; import org.elasticsearch.xpack.inference.external.http.retry.RequestSender; import org.elasticsearch.xpack.inference.external.http.retry.ResponseHandler; @@ -27,9 +27,9 @@ import static org.elasticsearch.xpack.inference.common.Truncator.truncate; -public class AzureOpenAiEmbeddingsExecutableRequestCreator implements ExecutableRequestCreator { +public class AzureOpenAiEmbeddingsRequestManager extends AzureOpenAiRequestManager { - private static final Logger logger = LogManager.getLogger(AzureOpenAiEmbeddingsExecutableRequestCreator.class); + private static final Logger logger = LogManager.getLogger(AzureOpenAiEmbeddingsRequestManager.class); private static final ResponseHandler HANDLER = createEmbeddingsHandler(); @@ -37,13 +37,20 @@ private static ResponseHandler createEmbeddingsHandler() { return new AzureOpenAiResponseHandler("azure openai text embedding", OpenAiEmbeddingsResponseEntity::fromResponse); } + public static AzureOpenAiEmbeddingsRequestManager of(AzureOpenAiEmbeddingsModel model, Truncator truncator, ThreadPool threadPool) { + return new AzureOpenAiEmbeddingsRequestManager( + Objects.requireNonNull(model), + Objects.requireNonNull(truncator), + Objects.requireNonNull(threadPool) + ); + } + private final Truncator truncator; private final AzureOpenAiEmbeddingsModel model; - private final AzureOpenAiAccount account; - public AzureOpenAiEmbeddingsExecutableRequestCreator(AzureOpenAiEmbeddingsModel model, Truncator truncator) { + public AzureOpenAiEmbeddingsRequestManager(AzureOpenAiEmbeddingsModel model, Truncator truncator, ThreadPool threadPool) { + super(threadPool, model); this.model = Objects.requireNonNull(model); - this.account = AzureOpenAiAccount.fromModel(model); this.truncator = Objects.requireNonNull(truncator); } @@ -57,7 +64,7 @@ public Runnable create( ActionListener listener ) { var truncatedInput = truncate(input, model.getServiceSettings().maxInputTokens()); - AzureOpenAiEmbeddingsRequest request = new AzureOpenAiEmbeddingsRequest(truncator, account, truncatedInput, model); + AzureOpenAiEmbeddingsRequest request = new AzureOpenAiEmbeddingsRequest(truncator, truncatedInput, model); return new ExecutableInferenceRequest(requestSender, logger, request, context, HANDLER, hasRequestCompletedFunction, listener); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureOpenAiRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureOpenAiRequestManager.java new file mode 100644 index 000000000000..312ac8de262a --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/AzureOpenAiRequestManager.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.http.sender; + +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.inference.services.azureopenai.AzureOpenAiModel; + +import java.util.Objects; + +public abstract class AzureOpenAiRequestManager extends BaseRequestManager { + protected AzureOpenAiRequestManager(ThreadPool threadPool, AzureOpenAiModel model) { + super(threadPool, model.getInferenceEntityId(), RateLimitGrouping.of(model)); + } + + record RateLimitGrouping(int resourceNameHash, int deploymentIdHash) { + public static RateLimitGrouping of(AzureOpenAiModel model) { + Objects.requireNonNull(model); + + return new RateLimitGrouping( + model.rateLimitServiceSettings().resourceName().hashCode(), + model.rateLimitServiceSettings().deploymentId().hashCode() + ); + } + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/BaseRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/BaseRequestManager.java new file mode 100644 index 000000000000..b26489765e07 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/BaseRequestManager.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.http.sender; + +import org.elasticsearch.threadpool.ThreadPool; + +import java.util.Objects; + +import static org.elasticsearch.xpack.inference.InferencePlugin.UTILITY_THREAD_POOL_NAME; + +abstract class BaseRequestManager implements RequestManager { + private final ThreadPool threadPool; + private final String inferenceEntityId; + private final Object rateLimitGroup; + + BaseRequestManager(ThreadPool threadPool, String inferenceEntityId, Object rateLimitGroup) { + this.threadPool = Objects.requireNonNull(threadPool); + this.inferenceEntityId = Objects.requireNonNull(inferenceEntityId); + this.rateLimitGroup = Objects.requireNonNull(rateLimitGroup); + } + + protected void execute(Runnable runnable) { + threadPool.executor(UTILITY_THREAD_POOL_NAME).execute(runnable); + } + + @Override + public String inferenceEntityId() { + return inferenceEntityId; + } + + @Override + public Object rateLimitGrouping() { + return rateLimitGroup; + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereEmbeddingsExecutableRequestCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereEmbeddingsRequestManager.java similarity index 79% rename from x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereEmbeddingsExecutableRequestCreator.java rename to x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereEmbeddingsRequestManager.java index 6488996d2edc..0bf1c11285ad 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereEmbeddingsExecutableRequestCreator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereEmbeddingsRequestManager.java @@ -12,7 +12,7 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.inference.InferenceServiceResults; -import org.elasticsearch.xpack.inference.external.cohere.CohereAccount; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.inference.external.cohere.CohereResponseHandler; import org.elasticsearch.xpack.inference.external.http.retry.RequestSender; import org.elasticsearch.xpack.inference.external.http.retry.ResponseHandler; @@ -24,20 +24,23 @@ import java.util.Objects; import java.util.function.Supplier; -public class CohereEmbeddingsExecutableRequestCreator implements ExecutableRequestCreator { - private static final Logger logger = LogManager.getLogger(CohereEmbeddingsExecutableRequestCreator.class); +public class CohereEmbeddingsRequestManager extends CohereRequestManager { + private static final Logger logger = LogManager.getLogger(CohereEmbeddingsRequestManager.class); private static final ResponseHandler HANDLER = createEmbeddingsHandler(); private static ResponseHandler createEmbeddingsHandler() { return new CohereResponseHandler("cohere text embedding", CohereEmbeddingsResponseEntity::fromResponse); } - private final CohereAccount account; + public static CohereEmbeddingsRequestManager of(CohereEmbeddingsModel model, ThreadPool threadPool) { + return new CohereEmbeddingsRequestManager(Objects.requireNonNull(model), Objects.requireNonNull(threadPool)); + } + private final CohereEmbeddingsModel model; - public CohereEmbeddingsExecutableRequestCreator(CohereEmbeddingsModel model) { + private CohereEmbeddingsRequestManager(CohereEmbeddingsModel model, ThreadPool threadPool) { + super(threadPool, model); this.model = Objects.requireNonNull(model); - account = new CohereAccount(this.model.getServiceSettings().getCommonSettings().uri(), this.model.getSecretSettings().apiKey()); } @Override @@ -49,7 +52,7 @@ public Runnable create( HttpClientContext context, ActionListener listener ) { - CohereEmbeddingsRequest request = new CohereEmbeddingsRequest(account, input, model); + CohereEmbeddingsRequest request = new CohereEmbeddingsRequest(input, model); return new ExecutableInferenceRequest(requestSender, logger, request, context, HANDLER, hasRequestCompletedFunction, listener); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereRequestManager.java new file mode 100644 index 000000000000..7ce71439f988 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereRequestManager.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.http.sender; + +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.inference.services.cohere.CohereModel; + +import java.util.Objects; + +abstract class CohereRequestManager extends BaseRequestManager { + + protected CohereRequestManager(ThreadPool threadPool, CohereModel model) { + super(threadPool, model.getInferenceEntityId(), RateLimitGrouping.of(model)); + } + + record RateLimitGrouping(int apiKeyHash) { + public static RateLimitGrouping of(CohereModel model) { + Objects.requireNonNull(model); + + return new RateLimitGrouping(model.apiKey().hashCode()); + } + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereRerankExecutableRequestCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereRerankRequestManager.java similarity index 76% rename from x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereRerankExecutableRequestCreator.java rename to x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereRerankRequestManager.java index 432a5334ac00..1778663a194e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereRerankExecutableRequestCreator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/CohereRerankRequestManager.java @@ -12,7 +12,7 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.inference.InferenceServiceResults; -import org.elasticsearch.xpack.inference.external.cohere.CohereAccount; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.inference.external.cohere.CohereResponseHandler; import org.elasticsearch.xpack.inference.external.http.retry.RequestSender; import org.elasticsearch.xpack.inference.external.http.retry.ResponseHandler; @@ -24,20 +24,23 @@ import java.util.Objects; import java.util.function.Supplier; -public class CohereRerankExecutableRequestCreator implements ExecutableRequestCreator { - private static final Logger logger = LogManager.getLogger(CohereRerankExecutableRequestCreator.class); +public class CohereRerankRequestManager extends CohereRequestManager { + private static final Logger logger = LogManager.getLogger(CohereRerankRequestManager.class); private static final ResponseHandler HANDLER = createCohereResponseHandler(); private static ResponseHandler createCohereResponseHandler() { return new CohereResponseHandler("cohere rerank", (request, response) -> CohereRankedResponseEntity.fromResponse(response)); } - private final CohereAccount account; + public static CohereRerankRequestManager of(CohereRerankModel model, ThreadPool threadPool) { + return new CohereRerankRequestManager(Objects.requireNonNull(model), Objects.requireNonNull(threadPool)); + } + private final CohereRerankModel model; - public CohereRerankExecutableRequestCreator(CohereRerankModel model) { - this.model = Objects.requireNonNull(model); - account = new CohereAccount(this.model.getServiceSettings().getCommonSettings().uri(), this.model.getSecretSettings().apiKey()); + private CohereRerankRequestManager(CohereRerankModel model, ThreadPool threadPool) { + super(threadPool, model); + this.model = model; } @Override @@ -49,7 +52,7 @@ public Runnable create( HttpClientContext context, ActionListener listener ) { - CohereRerankRequest request = new CohereRerankRequest(account, query, input, model); + CohereRerankRequest request = new CohereRerankRequest(query, input, model); return new ExecutableInferenceRequest(requestSender, logger, request, context, HANDLER, hasRequestCompletedFunction, listener); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSender.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSender.java index a98b172ccbd4..d33786084816 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSender.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSender.java @@ -124,7 +124,7 @@ public void close() throws IOException { */ @Override public void send( - ExecutableRequestCreator requestCreator, + RequestManager requestCreator, InferenceInputs inferenceInputs, @Nullable TimeValue timeout, ActionListener listener diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HuggingFaceExecutableRequestCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HuggingFaceRequestManager.java similarity index 62% rename from x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HuggingFaceExecutableRequestCreator.java rename to x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HuggingFaceRequestManager.java index 7c70f738105d..a06a6da2bbb1 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HuggingFaceExecutableRequestCreator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/HuggingFaceRequestManager.java @@ -12,6 +12,7 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.inference.common.Truncator; import org.elasticsearch.xpack.inference.external.http.retry.RequestSender; import org.elasticsearch.xpack.inference.external.http.retry.ResponseHandler; @@ -25,19 +26,32 @@ import static org.elasticsearch.xpack.inference.common.Truncator.truncate; -public class HuggingFaceExecutableRequestCreator implements ExecutableRequestCreator { - private static final Logger logger = LogManager.getLogger(HuggingFaceExecutableRequestCreator.class); +public class HuggingFaceRequestManager extends BaseRequestManager { + private static final Logger logger = LogManager.getLogger(HuggingFaceRequestManager.class); + + public static HuggingFaceRequestManager of( + HuggingFaceModel model, + ResponseHandler responseHandler, + Truncator truncator, + ThreadPool threadPool + ) { + return new HuggingFaceRequestManager( + Objects.requireNonNull(model), + Objects.requireNonNull(responseHandler), + Objects.requireNonNull(truncator), + Objects.requireNonNull(threadPool) + ); + } private final HuggingFaceModel model; - private final HuggingFaceAccount account; private final ResponseHandler responseHandler; private final Truncator truncator; - public HuggingFaceExecutableRequestCreator(HuggingFaceModel model, ResponseHandler responseHandler, Truncator truncator) { - this.model = Objects.requireNonNull(model); - account = new HuggingFaceAccount(model.getUri(), model.getApiKey()); - this.responseHandler = Objects.requireNonNull(responseHandler); - this.truncator = Objects.requireNonNull(truncator); + private HuggingFaceRequestManager(HuggingFaceModel model, ResponseHandler responseHandler, Truncator truncator, ThreadPool threadPool) { + super(threadPool, model.getInferenceEntityId(), RateLimitGrouping.of(model)); + this.model = model; + this.responseHandler = responseHandler; + this.truncator = truncator; } @Override @@ -50,7 +64,7 @@ public Runnable create( ActionListener listener ) { var truncatedInput = truncate(input, model.getTokenLimit()); - var request = new HuggingFaceInferenceRequest(truncator, account, truncatedInput, model); + var request = new HuggingFaceInferenceRequest(truncator, truncatedInput, model); return new ExecutableInferenceRequest( requestSender, @@ -62,4 +76,11 @@ public Runnable create( listener ); } + + record RateLimitGrouping(int accountHash) { + + public static RateLimitGrouping of(HuggingFaceModel model) { + return new RateLimitGrouping(new HuggingFaceAccount(model.rateLimitServiceSettings().uri(), model.apiKey()).hashCode()); + } + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/InferenceRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/InferenceRequest.java index 5d5e8df40c22..3c711bb79717 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/InferenceRequest.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/InferenceRequest.java @@ -21,7 +21,7 @@ public interface InferenceRequest { /** * Returns the creator that handles building an executable request based on the input provided. */ - ExecutableRequestCreator getRequestCreator(); + RequestManager getRequestCreator(); /** * Returns the query associated with this request. Used for Rerank tasks. diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/NoopTask.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/NoopTask.java index cca00b2e9bf5..0355880b3f71 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/NoopTask.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/NoopTask.java @@ -16,7 +16,7 @@ class NoopTask implements RejectableTask { @Override - public ExecutableRequestCreator getRequestCreator() { + public RequestManager getRequestCreator() { return null; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiCompletionExecutableRequestCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiCompletionRequestManager.java similarity index 78% rename from x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiCompletionExecutableRequestCreator.java rename to x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiCompletionRequestManager.java index 853038e1a7ca..9c6c216c6127 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiCompletionExecutableRequestCreator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiCompletionRequestManager.java @@ -13,9 +13,9 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.inference.external.http.retry.RequestSender; import org.elasticsearch.xpack.inference.external.http.retry.ResponseHandler; -import org.elasticsearch.xpack.inference.external.openai.OpenAiAccount; import org.elasticsearch.xpack.inference.external.openai.OpenAiChatCompletionResponseHandler; import org.elasticsearch.xpack.inference.external.request.openai.OpenAiChatCompletionRequest; import org.elasticsearch.xpack.inference.external.response.openai.OpenAiChatCompletionResponseEntity; @@ -25,23 +25,21 @@ import java.util.Objects; import java.util.function.Supplier; -public class OpenAiCompletionExecutableRequestCreator implements ExecutableRequestCreator { +public class OpenAiCompletionRequestManager extends OpenAiRequestManager { - private static final Logger logger = LogManager.getLogger(OpenAiCompletionExecutableRequestCreator.class); + private static final Logger logger = LogManager.getLogger(OpenAiCompletionRequestManager.class); private static final ResponseHandler HANDLER = createCompletionHandler(); - private final OpenAiChatCompletionModel model; + public static OpenAiCompletionRequestManager of(OpenAiChatCompletionModel model, ThreadPool threadPool) { + return new OpenAiCompletionRequestManager(Objects.requireNonNull(model), Objects.requireNonNull(threadPool)); + } - private final OpenAiAccount account; + private final OpenAiChatCompletionModel model; - public OpenAiCompletionExecutableRequestCreator(OpenAiChatCompletionModel model) { + private OpenAiCompletionRequestManager(OpenAiChatCompletionModel model, ThreadPool threadPool) { + super(threadPool, model, OpenAiChatCompletionRequest::buildDefaultUri); this.model = Objects.requireNonNull(model); - this.account = new OpenAiAccount( - this.model.getServiceSettings().uri(), - this.model.getServiceSettings().organizationId(), - this.model.getSecretSettings().apiKey() - ); } @Override @@ -53,7 +51,7 @@ public Runnable create( HttpClientContext context, ActionListener listener ) { - OpenAiChatCompletionRequest request = new OpenAiChatCompletionRequest(account, input, model); + OpenAiChatCompletionRequest request = new OpenAiChatCompletionRequest(input, model); return new ExecutableInferenceRequest(requestSender, logger, request, context, HANDLER, hasRequestCompletedFunction, listener); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiEmbeddingsExecutableRequestCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiEmbeddingsRequestManager.java similarity index 77% rename from x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiEmbeddingsExecutableRequestCreator.java rename to x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiEmbeddingsRequestManager.java index 8f867c374e2d..3a0a8fd64a65 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiEmbeddingsExecutableRequestCreator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiEmbeddingsRequestManager.java @@ -12,10 +12,10 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.inference.common.Truncator; import org.elasticsearch.xpack.inference.external.http.retry.RequestSender; import org.elasticsearch.xpack.inference.external.http.retry.ResponseHandler; -import org.elasticsearch.xpack.inference.external.openai.OpenAiAccount; import org.elasticsearch.xpack.inference.external.openai.OpenAiResponseHandler; import org.elasticsearch.xpack.inference.external.request.openai.OpenAiEmbeddingsRequest; import org.elasticsearch.xpack.inference.external.response.openai.OpenAiEmbeddingsResponseEntity; @@ -27,9 +27,9 @@ import static org.elasticsearch.xpack.inference.common.Truncator.truncate; -public class OpenAiEmbeddingsExecutableRequestCreator implements ExecutableRequestCreator { +public class OpenAiEmbeddingsRequestManager extends OpenAiRequestManager { - private static final Logger logger = LogManager.getLogger(OpenAiEmbeddingsExecutableRequestCreator.class); + private static final Logger logger = LogManager.getLogger(OpenAiEmbeddingsRequestManager.class); private static final ResponseHandler HANDLER = createEmbeddingsHandler(); @@ -37,17 +37,20 @@ private static ResponseHandler createEmbeddingsHandler() { return new OpenAiResponseHandler("openai text embedding", OpenAiEmbeddingsResponseEntity::fromResponse); } + public static OpenAiEmbeddingsRequestManager of(OpenAiEmbeddingsModel model, Truncator truncator, ThreadPool threadPool) { + return new OpenAiEmbeddingsRequestManager( + Objects.requireNonNull(model), + Objects.requireNonNull(truncator), + Objects.requireNonNull(threadPool) + ); + } + private final Truncator truncator; private final OpenAiEmbeddingsModel model; - private final OpenAiAccount account; - public OpenAiEmbeddingsExecutableRequestCreator(OpenAiEmbeddingsModel model, Truncator truncator) { + private OpenAiEmbeddingsRequestManager(OpenAiEmbeddingsModel model, Truncator truncator, ThreadPool threadPool) { + super(threadPool, model, OpenAiEmbeddingsRequest::buildDefaultUri); this.model = Objects.requireNonNull(model); - this.account = new OpenAiAccount( - this.model.getServiceSettings().uri(), - this.model.getServiceSettings().organizationId(), - this.model.getSecretSettings().apiKey() - ); this.truncator = Objects.requireNonNull(truncator); } @@ -61,7 +64,7 @@ public Runnable create( ActionListener listener ) { var truncatedInput = truncate(input, model.getServiceSettings().maxInputTokens()); - OpenAiEmbeddingsRequest request = new OpenAiEmbeddingsRequest(truncator, account, truncatedInput, model); + OpenAiEmbeddingsRequest request = new OpenAiEmbeddingsRequest(truncator, truncatedInput, model); return new ExecutableInferenceRequest(requestSender, logger, request, context, HANDLER, hasRequestCompletedFunction, listener); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiRequestManager.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiRequestManager.java new file mode 100644 index 000000000000..74c119971480 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiRequestManager.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.http.sender; + +import org.elasticsearch.common.CheckedSupplier; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.inference.external.openai.OpenAiAccount; +import org.elasticsearch.xpack.inference.services.openai.OpenAiModel; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Objects; + +abstract class OpenAiRequestManager extends BaseRequestManager { + + protected OpenAiRequestManager(ThreadPool threadPool, OpenAiModel model, CheckedSupplier uriBuilder) { + super(threadPool, model.getInferenceEntityId(), RateLimitGrouping.of(model, uriBuilder)); + } + + record RateLimitGrouping(int accountHash, int modelIdHash) { + public static RateLimitGrouping of(OpenAiModel model, CheckedSupplier uriBuilder) { + Objects.requireNonNull(model); + + return new RateLimitGrouping( + OpenAiAccount.of(model, uriBuilder).hashCode(), + model.rateLimitServiceSettings().modelId().hashCode() + ); + } + } +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorService.java index 0a5ab8f87ef1..d5a13c2e0771 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorService.java @@ -265,7 +265,7 @@ public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedE * @param listener an {@link ActionListener} for the response or failure */ public void execute( - ExecutableRequestCreator requestCreator, + RequestManager requestCreator, InferenceInputs inferenceInputs, @Nullable TimeValue timeout, ActionListener listener diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ExecutableRequestCreator.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestManager.java similarity index 86% rename from x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ExecutableRequestCreator.java rename to x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestManager.java index dc279573d5c9..7d3cca596f1d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/ExecutableRequestCreator.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestManager.java @@ -12,6 +12,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.xpack.inference.external.http.retry.RequestSender; +import org.elasticsearch.xpack.inference.external.ratelimit.RateLimitable; import java.util.List; import java.util.function.Supplier; @@ -19,7 +20,7 @@ /** * A contract for constructing a {@link Runnable} to handle sending an inference request to a 3rd party service. */ -public interface ExecutableRequestCreator { +public interface RequestManager extends RateLimitable { Runnable create( @Nullable String query, List input, @@ -28,4 +29,6 @@ Runnable create( HttpClientContext context, ActionListener listener ); + + String inferenceEntityId(); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTask.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTask.java index 6628b9ef425e..738592464232 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTask.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTask.java @@ -26,13 +26,13 @@ class RequestTask implements RejectableTask { private final AtomicBoolean finished = new AtomicBoolean(); - private final ExecutableRequestCreator requestCreator; + private final RequestManager requestCreator; private final String query; private final List input; private final ActionListener listener; RequestTask( - ExecutableRequestCreator requestCreator, + RequestManager requestCreator, InferenceInputs inferenceInputs, @Nullable TimeValue timeout, ThreadPool threadPool, @@ -111,7 +111,7 @@ public void onRejection(Exception e) { } @Override - public ExecutableRequestCreator getRequestCreator() { + public RequestManager getRequestCreator() { return requestCreator; } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/Sender.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/Sender.java index 40fd7836667d..5a3af3d4a377 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/Sender.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/Sender.java @@ -18,7 +18,7 @@ public interface Sender extends Closeable { void start(); void send( - ExecutableRequestCreator requestCreator, + RequestManager requestCreator, InferenceInputs inferenceInputs, @Nullable TimeValue timeout, ActionListener listener diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/huggingface/HuggingFaceAccount.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/huggingface/HuggingFaceAccount.java index 771c7b6adaea..a31a968a25a9 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/huggingface/HuggingFaceAccount.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/huggingface/HuggingFaceAccount.java @@ -8,14 +8,19 @@ package org.elasticsearch.xpack.inference.external.huggingface; import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.xpack.inference.services.huggingface.HuggingFaceModel; import java.net.URI; import java.util.Objects; -public record HuggingFaceAccount(URI url, SecureString apiKey) { +public record HuggingFaceAccount(URI uri, SecureString apiKey) { + + public static HuggingFaceAccount of(HuggingFaceModel model) { + return new HuggingFaceAccount(model.rateLimitServiceSettings().uri(), model.apiKey()); + } public HuggingFaceAccount { - Objects.requireNonNull(url); + Objects.requireNonNull(uri); Objects.requireNonNull(apiKey); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiAccount.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiAccount.java index a89032277ff8..07ccf298a0bd 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiAccount.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/openai/OpenAiAccount.java @@ -7,15 +7,27 @@ package org.elasticsearch.xpack.inference.external.openai; +import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.core.Nullable; +import org.elasticsearch.xpack.inference.services.openai.OpenAiModel; import java.net.URI; +import java.net.URISyntaxException; import java.util.Objects; -public record OpenAiAccount(@Nullable URI url, @Nullable String organizationId, SecureString apiKey) { +import static org.elasticsearch.xpack.inference.external.request.RequestUtils.buildUri; + +public record OpenAiAccount(URI uri, @Nullable String organizationId, SecureString apiKey) { + + public static OpenAiAccount of(OpenAiModel model, CheckedSupplier uriBuilder) { + var uri = buildUri(model.rateLimitServiceSettings().uri(), "OpenAI", uriBuilder); + + return new OpenAiAccount(uri, model.rateLimitServiceSettings().organizationId(), model.apiKey()); + } public OpenAiAccount { + Objects.requireNonNull(uri); Objects.requireNonNull(apiKey); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/ratelimit/RateLimitable.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/ratelimit/RateLimitable.java new file mode 100644 index 000000000000..e6f0d5371d03 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/ratelimit/RateLimitable.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.external.ratelimit; + +/** + * Defines the contract for the settings and grouping of requests for how they are rate limited. + */ +public interface RateLimitable { + + // TODO add a method for retrieving the rate limit settings that would be used to instantiate a RateLimiter + + /** + * Returns an object responsible for containing the all the fields that uniquely identify how a request will be rate limited. + * In practice the class should contain things like api key, url, model, or any headers that would impact rate limiting. + * The class must implement hashcode such that these fields are taken into account. + * + * The returned object defines the bucket that a request should be placed when determine how it is rate limited. + */ + Object rateLimitGrouping(); +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/azureopenai/AzureOpenAiEmbeddingsRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/azureopenai/AzureOpenAiEmbeddingsRequest.java index c943d5f54b4f..f60d0130a01b 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/azureopenai/AzureOpenAiEmbeddingsRequest.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/azureopenai/AzureOpenAiEmbeddingsRequest.java @@ -39,14 +39,9 @@ public class AzureOpenAiEmbeddingsRequest implements AzureOpenAiRequest { private final URI uri; private final AzureOpenAiEmbeddingsModel model; - public AzureOpenAiEmbeddingsRequest( - Truncator truncator, - AzureOpenAiAccount account, - Truncator.TruncationResult input, - AzureOpenAiEmbeddingsModel model - ) { + public AzureOpenAiEmbeddingsRequest(Truncator truncator, Truncator.TruncationResult input, AzureOpenAiEmbeddingsModel model) { this.truncator = Objects.requireNonNull(truncator); - this.account = Objects.requireNonNull(account); + this.account = AzureOpenAiAccount.fromModel(model); this.truncationResult = Objects.requireNonNull(input); this.model = Objects.requireNonNull(model); this.uri = model.getUri(); @@ -100,7 +95,7 @@ public String getInferenceEntityId() { public Request truncate() { var truncatedInput = truncator.truncate(truncationResult.input()); - return new AzureOpenAiEmbeddingsRequest(truncator, account, truncatedInput, model); + return new AzureOpenAiEmbeddingsRequest(truncator, truncatedInput, model); } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequest.java index 45f25a4dd35f..5f3278788b69 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequest.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequest.java @@ -26,25 +26,22 @@ import java.util.List; import java.util.Objects; -import static org.elasticsearch.xpack.inference.external.request.RequestUtils.buildUri; import static org.elasticsearch.xpack.inference.external.request.RequestUtils.createAuthBearerHeader; public class CohereEmbeddingsRequest implements Request { private final CohereAccount account; private final List input; - private final URI uri; private final CohereEmbeddingsTaskSettings taskSettings; private final String model; private final CohereEmbeddingType embeddingType; private final String inferenceEntityId; - public CohereEmbeddingsRequest(CohereAccount account, List input, CohereEmbeddingsModel embeddingsModel) { + public CohereEmbeddingsRequest(List input, CohereEmbeddingsModel embeddingsModel) { Objects.requireNonNull(embeddingsModel); - this.account = Objects.requireNonNull(account); + account = CohereAccount.of(embeddingsModel, CohereEmbeddingsRequest::buildDefaultUri); this.input = Objects.requireNonNull(input); - uri = buildUri(this.account.url(), "Cohere", CohereEmbeddingsRequest::buildDefaultUri); taskSettings = embeddingsModel.getTaskSettings(); model = embeddingsModel.getServiceSettings().getCommonSettings().modelId(); embeddingType = embeddingsModel.getServiceSettings().getEmbeddingType(); @@ -53,7 +50,7 @@ public CohereEmbeddingsRequest(CohereAccount account, List input, Cohere @Override public HttpRequest createHttpRequest() { - HttpPost httpPost = new HttpPost(uri); + HttpPost httpPost = new HttpPost(account.uri()); ByteArrayEntity byteEntity = new ByteArrayEntity( Strings.toString(new CohereEmbeddingsRequestEntity(input, taskSettings, model, embeddingType)).getBytes(StandardCharsets.UTF_8) @@ -74,7 +71,7 @@ public String getInferenceEntityId() { @Override public URI getURI() { - return uri; + return account.uri(); } @Override @@ -87,8 +84,7 @@ public boolean[] getTruncationInfo() { return null; } - // default for testing - static URI buildDefaultUri() throws URISyntaxException { + public static URI buildDefaultUri() throws URISyntaxException { return new URIBuilder().setScheme("https") .setHost(CohereUtils.HOST) .setPathSegments(CohereUtils.VERSION_1, CohereUtils.EMBEDDINGS_PATH) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereRerankRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereRerankRequest.java index b8f3916582bf..f87bdb9ab7d4 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereRerankRequest.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereRerankRequest.java @@ -25,7 +25,6 @@ import java.util.List; import java.util.Objects; -import static org.elasticsearch.xpack.inference.external.request.RequestUtils.buildUri; import static org.elasticsearch.xpack.inference.external.request.RequestUtils.createAuthBearerHeader; public class CohereRerankRequest implements Request { @@ -33,18 +32,16 @@ public class CohereRerankRequest implements Request { private final CohereAccount account; private final String query; private final List input; - private final URI uri; private final CohereRerankTaskSettings taskSettings; private final String model; private final String inferenceEntityId; - public CohereRerankRequest(CohereAccount account, String query, List input, CohereRerankModel model) { + public CohereRerankRequest(String query, List input, CohereRerankModel model) { Objects.requireNonNull(model); - this.account = Objects.requireNonNull(account); + this.account = CohereAccount.of(model, CohereRerankRequest::buildDefaultUri); this.input = Objects.requireNonNull(input); this.query = Objects.requireNonNull(query); - uri = buildUri(this.account.url(), "Cohere", CohereRerankRequest::buildDefaultUri); taskSettings = model.getTaskSettings(); this.model = model.getServiceSettings().getCommonSettings().modelId(); inferenceEntityId = model.getInferenceEntityId(); @@ -52,7 +49,7 @@ public CohereRerankRequest(CohereAccount account, String query, List inp @Override public HttpRequest createHttpRequest() { - HttpPost httpPost = new HttpPost(uri); + HttpPost httpPost = new HttpPost(account.uri()); ByteArrayEntity byteEntity = new ByteArrayEntity( Strings.toString(new CohereRerankRequestEntity(query, input, taskSettings, model)).getBytes(StandardCharsets.UTF_8) @@ -73,7 +70,7 @@ public String getInferenceEntityId() { @Override public URI getURI() { - return uri; + return account.uri(); } @Override @@ -86,8 +83,7 @@ public boolean[] getTruncationInfo() { return null; } - // default for testing - static URI buildDefaultUri() throws URISyntaxException { + public static URI buildDefaultUri() throws URISyntaxException { return new URIBuilder().setScheme("https") .setHost(CohereUtils.HOST) .setPathSegments(CohereUtils.VERSION_1, CohereUtils.RERANK_PATH) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/huggingface/HuggingFaceInferenceRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/huggingface/HuggingFaceInferenceRequest.java index cd4fef6f0e82..74427d7dbc21 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/huggingface/HuggingFaceInferenceRequest.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/huggingface/HuggingFaceInferenceRequest.java @@ -31,20 +31,15 @@ public class HuggingFaceInferenceRequest implements Request { private final Truncator.TruncationResult truncationResult; private final HuggingFaceModel model; - public HuggingFaceInferenceRequest( - Truncator truncator, - HuggingFaceAccount account, - Truncator.TruncationResult input, - HuggingFaceModel model - ) { + public HuggingFaceInferenceRequest(Truncator truncator, Truncator.TruncationResult input, HuggingFaceModel model) { this.truncator = Objects.requireNonNull(truncator); - this.account = Objects.requireNonNull(account); + this.account = HuggingFaceAccount.of(model); this.truncationResult = Objects.requireNonNull(input); this.model = Objects.requireNonNull(model); } public HttpRequest createHttpRequest() { - HttpPost httpPost = new HttpPost(account.url()); + HttpPost httpPost = new HttpPost(account.uri()); ByteArrayEntity byteEntity = new ByteArrayEntity( Strings.toString(new HuggingFaceInferenceRequestEntity(truncationResult.input())).getBytes(StandardCharsets.UTF_8) @@ -57,7 +52,7 @@ public HttpRequest createHttpRequest() { } public URI getURI() { - return account.url(); + return account.uri(); } @Override @@ -69,7 +64,7 @@ public String getInferenceEntityId() { public Request truncate() { var truncateResult = truncator.truncate(truncationResult.input()); - return new HuggingFaceInferenceRequest(truncator, account, truncateResult, model); + return new HuggingFaceInferenceRequest(truncator, truncateResult, model); } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequest.java index e53d4e736273..9fa653316174 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequest.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequest.java @@ -24,7 +24,6 @@ import java.util.List; import java.util.Objects; -import static org.elasticsearch.xpack.inference.external.request.RequestUtils.buildUri; import static org.elasticsearch.xpack.inference.external.request.RequestUtils.createAuthBearerHeader; import static org.elasticsearch.xpack.inference.external.request.openai.OpenAiUtils.createOrgHeader; @@ -32,19 +31,17 @@ public class OpenAiChatCompletionRequest implements OpenAiRequest { private final OpenAiAccount account; private final List input; - private final URI uri; private final OpenAiChatCompletionModel model; - public OpenAiChatCompletionRequest(OpenAiAccount account, List input, OpenAiChatCompletionModel model) { - this.account = Objects.requireNonNull(account); + public OpenAiChatCompletionRequest(List input, OpenAiChatCompletionModel model) { + this.account = OpenAiAccount.of(model, OpenAiChatCompletionRequest::buildDefaultUri); this.input = Objects.requireNonNull(input); - this.uri = buildUri(this.account.url(), "OpenAI", OpenAiChatCompletionRequest::buildDefaultUri); this.model = Objects.requireNonNull(model); } @Override public HttpRequest createHttpRequest() { - HttpPost httpPost = new HttpPost(uri); + HttpPost httpPost = new HttpPost(account.uri()); ByteArrayEntity byteEntity = new ByteArrayEntity( Strings.toString( @@ -66,7 +63,7 @@ public HttpRequest createHttpRequest() { @Override public URI getURI() { - return uri; + return account.uri(); } @Override @@ -86,8 +83,7 @@ public String getInferenceEntityId() { return model.getInferenceEntityId(); } - // default for testing - static URI buildDefaultUri() throws URISyntaxException { + public static URI buildDefaultUri() throws URISyntaxException { return new URIBuilder().setScheme("https") .setHost(OpenAiUtils.HOST) .setPathSegments(OpenAiUtils.VERSION_1, OpenAiUtils.CHAT_PATH, OpenAiUtils.COMPLETIONS_PATH) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiEmbeddingsRequest.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiEmbeddingsRequest.java index df5d3024fd48..f82e7ff3f526 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiEmbeddingsRequest.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiEmbeddingsRequest.java @@ -24,7 +24,6 @@ import java.nio.charset.StandardCharsets; import java.util.Objects; -import static org.elasticsearch.xpack.inference.external.request.RequestUtils.buildUri; import static org.elasticsearch.xpack.inference.external.request.RequestUtils.createAuthBearerHeader; import static org.elasticsearch.xpack.inference.external.request.openai.OpenAiUtils.createOrgHeader; @@ -33,24 +32,17 @@ public class OpenAiEmbeddingsRequest implements OpenAiRequest { private final Truncator truncator; private final OpenAiAccount account; private final Truncator.TruncationResult truncationResult; - private final URI uri; private final OpenAiEmbeddingsModel model; - public OpenAiEmbeddingsRequest( - Truncator truncator, - OpenAiAccount account, - Truncator.TruncationResult input, - OpenAiEmbeddingsModel model - ) { + public OpenAiEmbeddingsRequest(Truncator truncator, Truncator.TruncationResult input, OpenAiEmbeddingsModel model) { this.truncator = Objects.requireNonNull(truncator); - this.account = Objects.requireNonNull(account); + this.account = OpenAiAccount.of(model, OpenAiEmbeddingsRequest::buildDefaultUri); this.truncationResult = Objects.requireNonNull(input); - this.uri = buildUri(this.account.url(), "OpenAI", OpenAiEmbeddingsRequest::buildDefaultUri); this.model = Objects.requireNonNull(model); } public HttpRequest createHttpRequest() { - HttpPost httpPost = new HttpPost(uri); + HttpPost httpPost = new HttpPost(account.uri()); ByteArrayEntity byteEntity = new ByteArrayEntity( Strings.toString( @@ -83,14 +75,14 @@ public String getInferenceEntityId() { @Override public URI getURI() { - return uri; + return account.uri(); } @Override public Request truncate() { var truncatedInput = truncator.truncate(truncationResult.input()); - return new OpenAiEmbeddingsRequest(truncator, account, truncatedInput, model); + return new OpenAiEmbeddingsRequest(truncator, truncatedInput, model); } @Override @@ -98,8 +90,7 @@ public boolean[] getTruncationInfo() { return truncationResult.truncated().clone(); } - // default for testing - static URI buildDefaultUri() throws URISyntaxException { + public static URI buildDefaultUri() throws URISyntaxException { return new URIBuilder().setScheme("https") .setHost(OpenAiUtils.HOST) .setPathSegments(OpenAiUtils.VERSION_1, OpenAiUtils.EMBEDDINGS_PATH) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java index 163175514957..329e7664e5d4 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ServiceUtils.java @@ -22,6 +22,7 @@ import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.core.inference.results.TextEmbedding; import org.elasticsearch.xpack.core.inference.results.TextEmbeddingResults; +import org.elasticsearch.xpack.inference.services.settings.ApiKeySecrets; import java.net.URI; import java.net.URISyntaxException; @@ -402,4 +403,9 @@ public static void getEmbeddingSize(Model model, InferenceService service, Actio } private static final String TEST_EMBEDDING_INPUT = "how big"; + + public static SecureString apiKey(@Nullable ApiKeySecrets secrets) { + // To avoid a possible null pointer throughout the code we'll create a noop api key of an empty array + return secrets == null ? new SecureString(new char[0]) : secrets.apiKey(); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiModel.java index 66070cab0e51..5e50229e2564 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiModel.java @@ -17,23 +17,35 @@ import java.net.URI; import java.util.Map; +import java.util.Objects; public abstract class AzureOpenAiModel extends Model { protected URI uri; + private final AzureOpenAiRateLimitServiceSettings rateLimitServiceSettings; - public AzureOpenAiModel(ModelConfigurations configurations, ModelSecrets secrets) { + public AzureOpenAiModel( + ModelConfigurations configurations, + ModelSecrets secrets, + AzureOpenAiRateLimitServiceSettings rateLimitServiceSettings + ) { super(configurations, secrets); + + this.rateLimitServiceSettings = Objects.requireNonNull(rateLimitServiceSettings); } protected AzureOpenAiModel(AzureOpenAiModel model, TaskSettings taskSettings) { super(model, taskSettings); + this.uri = model.getUri(); + rateLimitServiceSettings = model.rateLimitServiceSettings(); } protected AzureOpenAiModel(AzureOpenAiModel model, ServiceSettings serviceSettings) { super(model, serviceSettings); + this.uri = model.getUri(); + rateLimitServiceSettings = model.rateLimitServiceSettings(); } public abstract ExecutableAction accept(AzureOpenAiActionVisitor creator, Map taskSettings); @@ -46,4 +58,8 @@ public URI getUri() { public void setUri(URI newUri) { this.uri = newUri; } + + public AzureOpenAiRateLimitServiceSettings rateLimitServiceSettings() { + return rateLimitServiceSettings; + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiRateLimitServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiRateLimitServiceSettings.java new file mode 100644 index 000000000000..9a474c9059f3 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiRateLimitServiceSettings.java @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.azureopenai; + +public interface AzureOpenAiRateLimitServiceSettings { + String resourceName(); + + String deploymentId(); + +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsModel.java index 4c3272013f0e..93d1e31a3bed 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsModel.java @@ -64,7 +64,11 @@ public AzureOpenAiEmbeddingsModel( AzureOpenAiEmbeddingsTaskSettings taskSettings, @Nullable AzureOpenAiSecretSettings secrets ) { - super(new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings, taskSettings), new ModelSecrets(secrets)); + super( + new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings, taskSettings), + new ModelSecrets(secrets), + serviceSettings + ); try { this.uri = getEmbeddingsUri(serviceSettings.resourceName(), serviceSettings.deploymentId(), serviceSettings.apiVersion()); } catch (URISyntaxException e) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsServiceSettings.java index c3d9e3eb69a5..4153aef9cd74 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/embeddings/AzureOpenAiEmbeddingsServiceSettings.java @@ -21,6 +21,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.ServiceUtils; +import org.elasticsearch.xpack.inference.services.azureopenai.AzureOpenAiRateLimitServiceSettings; import java.io.IOException; import java.util.Map; @@ -40,7 +41,7 @@ /** * Defines the service settings for interacting with OpenAI's text embedding models. */ -public class AzureOpenAiEmbeddingsServiceSettings implements ServiceSettings { +public class AzureOpenAiEmbeddingsServiceSettings implements ServiceSettings, AzureOpenAiRateLimitServiceSettings { public static final String NAME = "azure_openai_embeddings_service_settings"; @@ -164,10 +165,12 @@ private AzureOpenAiEmbeddingsServiceSettings(CommonFields fields) { ); } + @Override public String resourceName() { return resourceName; } + @Override public String deploymentId() { return deploymentId; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereModel.java index 81a27e1e536f..fdff730a83a3 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereModel.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.inference.services.cohere; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.Model; import org.elasticsearch.inference.ModelConfigurations; @@ -15,21 +17,38 @@ import org.elasticsearch.inference.TaskSettings; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.external.action.cohere.CohereActionVisitor; +import org.elasticsearch.xpack.inference.services.ServiceUtils; +import org.elasticsearch.xpack.inference.services.settings.ApiKeySecrets; +import java.net.URI; import java.util.Map; public abstract class CohereModel extends Model { - public CohereModel(ModelConfigurations configurations, ModelSecrets secrets) { + private final SecureString apiKey; + + public CohereModel(ModelConfigurations configurations, ModelSecrets secrets, @Nullable ApiKeySecrets apiKeySecrets) { super(configurations, secrets); + + apiKey = ServiceUtils.apiKey(apiKeySecrets); } protected CohereModel(CohereModel model, TaskSettings taskSettings) { super(model, taskSettings); + + apiKey = model.apiKey(); } protected CohereModel(CohereModel model, ServiceSettings serviceSettings) { super(model, serviceSettings); + + apiKey = model.apiKey(); + } + + public SecureString apiKey() { + return apiKey; } public abstract ExecutableAction accept(CohereActionVisitor creator, Map taskSettings, InputType inputType); + + public abstract URI uri(); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsModel.java index be25361724c1..c1c46787a60a 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/embeddings/CohereEmbeddingsModel.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.inference.services.cohere.CohereModel; import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; +import java.net.URI; import java.util.Map; public class CohereEmbeddingsModel extends CohereModel { @@ -54,7 +55,11 @@ public CohereEmbeddingsModel( CohereEmbeddingsTaskSettings taskSettings, @Nullable DefaultSecretSettings secretSettings ) { - super(new ModelConfigurations(modelId, taskType, service, serviceSettings, taskSettings), new ModelSecrets(secretSettings)); + super( + new ModelConfigurations(modelId, taskType, service, serviceSettings, taskSettings), + new ModelSecrets(secretSettings), + secretSettings + ); } private CohereEmbeddingsModel(CohereEmbeddingsModel model, CohereEmbeddingsTaskSettings taskSettings) { @@ -84,4 +89,9 @@ public DefaultSecretSettings getSecretSettings() { public ExecutableAction accept(CohereActionVisitor visitor, Map taskSettings, InputType inputType) { return visitor.create(this, taskSettings, inputType); } + + @Override + public URI uri() { + return getServiceSettings().getCommonSettings().uri(); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankModel.java index 78e0e419c418..2a02cab60688 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankModel.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.inference.services.cohere.CohereModel; import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; +import java.net.URI; import java.util.Map; public class CohereRerankModel extends CohereModel { @@ -54,7 +55,11 @@ public CohereRerankModel( CohereRerankTaskSettings taskSettings, @Nullable DefaultSecretSettings secretSettings ) { - super(new ModelConfigurations(modelId, taskType, service, serviceSettings, taskSettings), new ModelSecrets(secretSettings)); + super( + new ModelConfigurations(modelId, taskType, service, serviceSettings, taskSettings), + new ModelSecrets(secretSettings), + secretSettings + ); } private CohereRerankModel(CohereRerankModel model, CohereRerankTaskSettings taskSettings) { @@ -91,4 +96,9 @@ public DefaultSecretSettings getSecretSettings() { public ExecutableAction accept(CohereActionVisitor visitor, Map taskSettings, InputType inputType) { return visitor.create(this, taskSettings); } + + @Override + public URI uri() { + return getServiceSettings().getCommonSettings().uri(); + } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceModel.java index 82076c865fee..7e51688dcbc1 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceModel.java @@ -8,24 +8,42 @@ package org.elasticsearch.xpack.inference.services.huggingface; import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.Model; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.external.action.huggingface.HuggingFaceActionVisitor; +import org.elasticsearch.xpack.inference.services.ServiceUtils; +import org.elasticsearch.xpack.inference.services.settings.ApiKeySecrets; -import java.net.URI; +import java.util.Objects; public abstract class HuggingFaceModel extends Model { - public HuggingFaceModel(ModelConfigurations configurations, ModelSecrets secrets) { + private final HuggingFaceRateLimitServiceSettings rateLimitServiceSettings; + private final SecureString apiKey; + + public HuggingFaceModel( + ModelConfigurations configurations, + ModelSecrets secrets, + HuggingFaceRateLimitServiceSettings rateLimitServiceSettings, + @Nullable ApiKeySecrets apiKeySecrets + ) { super(configurations, secrets); + this.rateLimitServiceSettings = Objects.requireNonNull(rateLimitServiceSettings); + apiKey = ServiceUtils.apiKey(apiKeySecrets); } - public abstract ExecutableAction accept(HuggingFaceActionVisitor creator); - - public abstract URI getUri(); + public HuggingFaceRateLimitServiceSettings rateLimitServiceSettings() { + return rateLimitServiceSettings; + } - public abstract SecureString getApiKey(); + public SecureString apiKey() { + return apiKey; + } public abstract Integer getTokenLimit(); + + public abstract ExecutableAction accept(HuggingFaceActionVisitor creator); + } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceRateLimitServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceRateLimitServiceSettings.java new file mode 100644 index 000000000000..51f034275c13 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceRateLimitServiceSettings.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.huggingface; + +import java.net.URI; + +/** + * The service setting fields for hugging face that determine how to rate limit requests. + */ +public interface HuggingFaceRateLimitServiceSettings { + URI uri(); +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceSettings.java index b151e9c800a7..03a02cca44d7 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceSettings.java @@ -35,7 +35,7 @@ import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractSimilarity; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeAsType; -public class HuggingFaceServiceSettings implements ServiceSettings { +public class HuggingFaceServiceSettings implements ServiceSettings, HuggingFaceRateLimitServiceSettings { public static final String NAME = "hugging_face_service_settings"; public static HuggingFaceServiceSettings fromMap(Map map) { @@ -141,6 +141,7 @@ public void writeTo(StreamOutput out) throws IOException { } } + @Override public URI uri() { return uri; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserModel.java index 296856620862..8a947ce9a024 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserModel.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.inference.services.huggingface.elser; -import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; @@ -16,7 +15,6 @@ import org.elasticsearch.xpack.inference.external.action.huggingface.HuggingFaceActionVisitor; import org.elasticsearch.xpack.inference.services.huggingface.HuggingFaceModel; -import java.net.URI; import java.util.Map; public class HuggingFaceElserModel extends HuggingFaceModel { @@ -43,7 +41,12 @@ public HuggingFaceElserModel( HuggingFaceElserServiceSettings serviceSettings, @Nullable HuggingFaceElserSecretSettings secretSettings ) { - super(new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings), new ModelSecrets(secretSettings)); + super( + new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings), + new ModelSecrets(secretSettings), + serviceSettings, + secretSettings + ); } @Override @@ -61,16 +64,6 @@ public ExecutableAction accept(HuggingFaceActionVisitor creator) { return creator.create(this); } - @Override - public URI getUri() { - return getServiceSettings().uri(); - } - - @Override - public SecureString getApiKey() { - return getSecretSettings().apiKey(); - } - @Override public Integer getTokenLimit() { return getServiceSettings().maxInputTokens(); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserSecretSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserSecretSettings.java index e6560a9dd1af..48c8997f2a1b 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserSecretSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserSecretSettings.java @@ -17,6 +17,7 @@ import org.elasticsearch.inference.ModelSecrets; import org.elasticsearch.inference.SecretSettings; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.inference.services.settings.ApiKeySecrets; import java.io.IOException; import java.util.Map; @@ -24,7 +25,7 @@ import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractRequiredSecureString; -public record HuggingFaceElserSecretSettings(SecureString apiKey) implements SecretSettings { +public record HuggingFaceElserSecretSettings(SecureString apiKey) implements SecretSettings, ApiKeySecrets { public static final String NAME = "hugging_face_elser_secret_settings"; static final String API_KEY = "api_key"; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserServiceSettings.java index 6949c0917bce..5b382919fd00 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserServiceSettings.java @@ -15,6 +15,7 @@ import org.elasticsearch.inference.ServiceSettings; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.inference.services.huggingface.HuggingFaceRateLimitServiceSettings; import java.io.IOException; import java.net.URI; @@ -25,7 +26,10 @@ import static org.elasticsearch.xpack.inference.services.ServiceUtils.createUri; import static org.elasticsearch.xpack.inference.services.huggingface.HuggingFaceServiceSettings.extractUri; -public record HuggingFaceElserServiceSettings(URI uri, Integer maxInputTokens) implements ServiceSettings { +public record HuggingFaceElserServiceSettings(URI uri, Integer maxInputTokens) + implements + ServiceSettings, + HuggingFaceRateLimitServiceSettings { public static final String NAME = "hugging_face_elser_service_settings"; private static final Integer ELSER_TOKEN_LIMIT = 512; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/embeddings/HuggingFaceEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/embeddings/HuggingFaceEmbeddingsModel.java index 351173de95cc..1cee26558b49 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/embeddings/HuggingFaceEmbeddingsModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/embeddings/HuggingFaceEmbeddingsModel.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.inference.services.huggingface.embeddings; -import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; @@ -18,7 +17,6 @@ import org.elasticsearch.xpack.inference.services.huggingface.HuggingFaceServiceSettings; import org.elasticsearch.xpack.inference.services.settings.DefaultSecretSettings; -import java.net.URI; import java.util.Map; public class HuggingFaceEmbeddingsModel extends HuggingFaceModel { @@ -46,7 +44,12 @@ public HuggingFaceEmbeddingsModel( HuggingFaceServiceSettings serviceSettings, @Nullable DefaultSecretSettings secrets ) { - super(new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings), new ModelSecrets(secrets)); + super( + new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings), + new ModelSecrets(secrets), + serviceSettings, + secrets + ); } public HuggingFaceEmbeddingsModel(HuggingFaceEmbeddingsModel model, HuggingFaceServiceSettings serviceSettings) { @@ -69,16 +72,6 @@ public DefaultSecretSettings getSecretSettings() { return (DefaultSecretSettings) super.getSecretSettings(); } - @Override - public URI getUri() { - return getServiceSettings().uri(); - } - - @Override - public SecureString getApiKey() { - return getSecretSettings().apiKey(); - } - @Override public Integer getTokenLimit() { return getServiceSettings().maxInputTokens(); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiModel.java index 1e158725f531..caf09de31794 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiModel.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.inference.services.openai; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.core.Nullable; import org.elasticsearch.inference.Model; import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.ModelSecrets; @@ -14,21 +16,49 @@ import org.elasticsearch.inference.TaskSettings; import org.elasticsearch.xpack.inference.external.action.ExecutableAction; import org.elasticsearch.xpack.inference.external.action.openai.OpenAiActionVisitor; +import org.elasticsearch.xpack.inference.services.ServiceUtils; +import org.elasticsearch.xpack.inference.services.settings.ApiKeySecrets; import java.util.Map; +import java.util.Objects; public abstract class OpenAiModel extends Model { - public OpenAiModel(ModelConfigurations configurations, ModelSecrets secrets) { + private final OpenAiRateLimitServiceSettings rateLimitServiceSettings; + private final SecureString apiKey; + + public OpenAiModel( + ModelConfigurations configurations, + ModelSecrets secrets, + OpenAiRateLimitServiceSettings rateLimitServiceSettings, + @Nullable ApiKeySecrets apiKeySecrets + ) { super(configurations, secrets); + + this.rateLimitServiceSettings = Objects.requireNonNull(rateLimitServiceSettings); + apiKey = ServiceUtils.apiKey(apiKeySecrets); } protected OpenAiModel(OpenAiModel model, TaskSettings taskSettings) { super(model, taskSettings); + + rateLimitServiceSettings = model.rateLimitServiceSettings(); + apiKey = model.apiKey(); } protected OpenAiModel(OpenAiModel model, ServiceSettings serviceSettings) { super(model, serviceSettings); + + rateLimitServiceSettings = model.rateLimitServiceSettings(); + apiKey = model.apiKey(); + } + + public SecureString apiKey() { + return apiKey; + } + + public OpenAiRateLimitServiceSettings rateLimitServiceSettings() { + return rateLimitServiceSettings; } public abstract ExecutableAction accept(OpenAiActionVisitor creator, Map taskSettings); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiRateLimitServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiRateLimitServiceSettings.java new file mode 100644 index 000000000000..be7378fd83f5 --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiRateLimitServiceSettings.java @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.openai; + +import java.net.URI; + +/** + * The service setting fields for openai that determine how to rate limit requests. + */ +public interface OpenAiRateLimitServiceSettings { + String modelId(); + + URI uri(); + + String organizationId(); +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionModel.java index 467c4f44f34f..b1b670c0911f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionModel.java @@ -55,7 +55,12 @@ public OpenAiChatCompletionModel( OpenAiChatCompletionTaskSettings taskSettings, @Nullable DefaultSecretSettings secrets ) { - super(new ModelConfigurations(modelId, taskType, service, serviceSettings, taskSettings), new ModelSecrets(secrets)); + super( + new ModelConfigurations(modelId, taskType, service, serviceSettings, taskSettings), + new ModelSecrets(secrets), + serviceSettings, + secrets + ); } private OpenAiChatCompletionModel(OpenAiChatCompletionModel originalModel, OpenAiChatCompletionTaskSettings taskSettings) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionServiceSettings.java index 16b0ed5d4703..7a8bafb8b109 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/completion/OpenAiChatCompletionServiceSettings.java @@ -17,6 +17,7 @@ import org.elasticsearch.inference.ServiceSettings; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.inference.services.openai.OpenAiRateLimitServiceSettings; import java.io.IOException; import java.net.URI; @@ -36,7 +37,7 @@ /** * Defines the service settings for interacting with OpenAI's chat completion models. */ -public class OpenAiChatCompletionServiceSettings implements ServiceSettings { +public class OpenAiChatCompletionServiceSettings implements ServiceSettings, OpenAiRateLimitServiceSettings { public static final String NAME = "openai_completion_service_settings"; @@ -94,14 +95,17 @@ public OpenAiChatCompletionServiceSettings(StreamInput in) throws IOException { this.maxInputTokens = in.readOptionalVInt(); } + @Override public String modelId() { return modelId; } + @Override public URI uri() { return uri; } + @Override public String organizationId() { return organizationId; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsModel.java index e822fa069598..18a1d8a5b658 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsModel.java @@ -58,7 +58,12 @@ public OpenAiEmbeddingsModel( OpenAiEmbeddingsTaskSettings taskSettings, @Nullable DefaultSecretSettings secrets ) { - super(new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings, taskSettings), new ModelSecrets(secrets)); + super( + new ModelConfigurations(inferenceEntityId, taskType, service, serviceSettings, taskSettings), + new ModelSecrets(secrets), + serviceSettings, + secrets + ); } private OpenAiEmbeddingsModel(OpenAiEmbeddingsModel originalModel, OpenAiEmbeddingsTaskSettings taskSettings) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettings.java index 1e5c93ea9ae2..e3fc02580cf0 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/embeddings/OpenAiEmbeddingsServiceSettings.java @@ -21,6 +21,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.ServiceUtils; +import org.elasticsearch.xpack.inference.services.openai.OpenAiRateLimitServiceSettings; import java.io.IOException; import java.net.URI; @@ -43,7 +44,7 @@ /** * Defines the service settings for interacting with OpenAI's text embedding models. */ -public class OpenAiEmbeddingsServiceSettings implements ServiceSettings { +public class OpenAiEmbeddingsServiceSettings implements ServiceSettings, OpenAiRateLimitServiceSettings { public static final String NAME = "openai_service_settings"; @@ -184,10 +185,12 @@ private OpenAiEmbeddingsServiceSettings(CommonFields fields, Boolean dimensionsS ); } + @Override public URI uri() { return uri; } + @Override public String organizationId() { return organizationId; } @@ -210,6 +213,7 @@ public Integer maxInputTokens() { return maxInputTokens; } + @Override public String modelId() { return modelId; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/ApiKeySecrets.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/ApiKeySecrets.java new file mode 100644 index 000000000000..687a8b378b1f --- /dev/null +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/ApiKeySecrets.java @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.settings; + +import org.elasticsearch.common.settings.SecureString; + +public interface ApiKeySecrets { + SecureString apiKey(); +} diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/DefaultSecretSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/DefaultSecretSettings.java index 8587f254c0b4..6affa998c089 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/DefaultSecretSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/settings/DefaultSecretSettings.java @@ -28,7 +28,7 @@ * Contains secret settings that are common to all services. * @param apiKey the key used to authenticate with the 3rd party service */ -public record DefaultSecretSettings(SecureString apiKey) implements SecretSettings { +public record DefaultSecretSettings(SecureString apiKey) implements SecretSettings, ApiKeySecrets { public static final String NAME = "default_secret_settings"; static final String API_KEY = "api_key"; diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereEmbeddingsActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereEmbeddingsActionTests.java index da81007b7d2b..06cae11bc8d5 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereEmbeddingsActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/action/cohere/CohereEmbeddingsActionTests.java @@ -352,7 +352,7 @@ private CohereEmbeddingsAction createAction( ) { var model = CohereEmbeddingsModelTests.createModel(url, apiKey, taskSettings, 1024, 1024, modelName, embeddingType); - return new CohereEmbeddingsAction(sender, model); + return new CohereEmbeddingsAction(sender, model, threadPool); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/ExecutableRequestCreatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/ExecutableRequestCreatorTests.java index 9a85b00c0148..31297ed432ef 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/ExecutableRequestCreatorTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/ExecutableRequestCreatorTests.java @@ -22,19 +22,19 @@ import static org.mockito.Mockito.when; public class ExecutableRequestCreatorTests { - public static ExecutableRequestCreator createMock() { - var mockCreator = mock(ExecutableRequestCreator.class); + public static RequestManager createMock() { + var mockCreator = mock(RequestManager.class); when(mockCreator.create(any(), anyList(), any(), any(), any(), any())).thenReturn(() -> {}); return mockCreator; } - public static ExecutableRequestCreator createMock(RequestSender requestSender) { + public static RequestManager createMock(RequestSender requestSender) { return createMock(requestSender, "id"); } - public static ExecutableRequestCreator createMock(RequestSender requestSender, String modelId) { - var mockCreator = mock(ExecutableRequestCreator.class); + public static RequestManager createMock(RequestSender requestSender, String modelId) { + var mockCreator = mock(RequestManager.class); doAnswer(invocation -> { @SuppressWarnings("unchecked") diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSenderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSenderTests.java index b7a07242c6d8..395c04641350 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSenderTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSenderTests.java @@ -106,7 +106,7 @@ public void testCreateSender_SendsRequestAndReceivesResponse() throws Exception PlainActionFuture listener = new PlainActionFuture<>(); sender.send( - OpenAiEmbeddingsExecutableRequestCreatorTests.makeCreator(getUrl(webServer), null, "key", "model", null), + OpenAiEmbeddingsExecutableRequestCreatorTests.makeCreator(getUrl(webServer), null, "key", "model", null, threadPool), new DocumentsOnlyInput(List.of("abc")), null, listener diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiEmbeddingsExecutableRequestCreatorTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiEmbeddingsExecutableRequestCreatorTests.java index 53537a3ff77c..37fce8d3f3a7 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiEmbeddingsExecutableRequestCreatorTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/OpenAiEmbeddingsExecutableRequestCreatorTests.java @@ -8,33 +8,36 @@ package org.elasticsearch.xpack.inference.external.http.sender; import org.elasticsearch.core.Nullable; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.inference.common.TruncatorTests; import static org.elasticsearch.xpack.inference.services.openai.embeddings.OpenAiEmbeddingsModelTests.createModel; public class OpenAiEmbeddingsExecutableRequestCreatorTests { - public static OpenAiEmbeddingsExecutableRequestCreator makeCreator( + public static OpenAiEmbeddingsRequestManager makeCreator( String url, @Nullable String org, String apiKey, String modelName, - @Nullable String user + @Nullable String user, + ThreadPool threadPool ) { var model = createModel(url, org, apiKey, modelName, user); - return new OpenAiEmbeddingsExecutableRequestCreator(model, TruncatorTests.createTruncator()); + return OpenAiEmbeddingsRequestManager.of(model, TruncatorTests.createTruncator(), threadPool); } - public static OpenAiEmbeddingsExecutableRequestCreator makeCreator( + public static OpenAiEmbeddingsRequestManager makeCreator( String url, @Nullable String org, String apiKey, String modelName, @Nullable String user, - String inferenceEntityId + String inferenceEntityId, + ThreadPool threadPool ) { var model = createModel(url, org, apiKey, modelName, user, inferenceEntityId); - return new OpenAiEmbeddingsExecutableRequestCreator(model, TruncatorTests.createTruncator()); + return OpenAiEmbeddingsRequestManager.of(model, TruncatorTests.createTruncator(), threadPool); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceTests.java index 24a261dfe47c..ff88ba221d98 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceTests.java @@ -109,7 +109,7 @@ public void testIsTerminated_AfterStopFromSeparateThread() throws Exception { PlainActionFuture listener = new PlainActionFuture<>(); service.execute( - OpenAiEmbeddingsExecutableRequestCreatorTests.makeCreator("url", null, "key", "id", null), + OpenAiEmbeddingsExecutableRequestCreatorTests.makeCreator("url", null, "key", "id", null, threadPool), new DocumentsOnlyInput(List.of()), null, listener @@ -179,7 +179,7 @@ public void testTaskThrowsError_CallsOnFailure() { PlainActionFuture listener = new PlainActionFuture<>(); service.execute( - OpenAiEmbeddingsExecutableRequestCreatorTests.makeCreator("url", null, "key", "id", null), + OpenAiEmbeddingsExecutableRequestCreatorTests.makeCreator("url", null, "key", "id", null, threadPool), new DocumentsOnlyInput(List.of()), null, listener diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTaskTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTaskTests.java index 14a7e28eb84d..13c395180cd1 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTaskTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTaskTests.java @@ -59,7 +59,7 @@ public void testExecuting_DoesNotCallOnFailureForTimeout_AfterIllegalArgumentExc ActionListener listener = mock(ActionListener.class); var requestTask = new RequestTask( - OpenAiEmbeddingsExecutableRequestCreatorTests.makeCreator("url", null, "key", "model", null, "id"), + OpenAiEmbeddingsExecutableRequestCreatorTests.makeCreator("url", null, "key", "model", null, "id", threadPool), new DocumentsOnlyInput(List.of("abc")), TimeValue.timeValueMillis(1), mockThreadPool, @@ -79,7 +79,7 @@ public void testRequest_ReturnsTimeoutException() { PlainActionFuture listener = new PlainActionFuture<>(); var requestTask = new RequestTask( - OpenAiEmbeddingsExecutableRequestCreatorTests.makeCreator("url", null, "key", "model", null, "id"), + OpenAiEmbeddingsExecutableRequestCreatorTests.makeCreator("url", null, "key", "model", null, "id", threadPool), new DocumentsOnlyInput(List.of("abc")), TimeValue.timeValueMillis(1), threadPool, @@ -105,7 +105,7 @@ public void testRequest_DoesNotCallOnFailureTwiceWhenTimingOut() throws Exceptio }).when(listener).onFailure(any()); var requestTask = new RequestTask( - OpenAiEmbeddingsExecutableRequestCreatorTests.makeCreator("url", null, "key", "model", null, "id"), + OpenAiEmbeddingsExecutableRequestCreatorTests.makeCreator("url", null, "key", "model", null, "id", threadPool), new DocumentsOnlyInput(List.of("abc")), TimeValue.timeValueMillis(1), threadPool, @@ -137,7 +137,7 @@ public void testRequest_DoesNotCallOnResponseAfterTimingOut() throws Exception { }).when(listener).onFailure(any()); var requestTask = new RequestTask( - OpenAiEmbeddingsExecutableRequestCreatorTests.makeCreator("url", null, "key", "model", null, "id"), + OpenAiEmbeddingsExecutableRequestCreatorTests.makeCreator("url", null, "key", "model", null, "id", threadPool), new DocumentsOnlyInput(List.of("abc")), TimeValue.timeValueMillis(1), threadPool, @@ -167,7 +167,7 @@ public void testRequest_DoesNotCallOnFailureForTimeout_AfterAlreadyCallingOnResp ActionListener listener = mock(ActionListener.class); var requestTask = new RequestTask( - OpenAiEmbeddingsExecutableRequestCreatorTests.makeCreator("url", null, "key", "model", null, "id"), + OpenAiEmbeddingsExecutableRequestCreatorTests.makeCreator("url", null, "key", "model", null, "id", threadPool), new DocumentsOnlyInput(List.of("abc")), TimeValue.timeValueMillis(1), mockThreadPool, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/SingleRequestManagerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/SingleRequestManagerTests.java index ab8bf244a4d2..55965bc2354d 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/SingleRequestManagerTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/SingleRequestManagerTests.java @@ -17,7 +17,7 @@ public class SingleRequestManagerTests extends ESTestCase { public void testExecute_DoesNotCallRequestCreatorCreate_WhenInputIsNull() { - var requestCreator = mock(ExecutableRequestCreator.class); + var requestCreator = mock(RequestManager.class); var request = mock(InferenceRequest.class); when(request.getRequestCreator()).thenReturn(requestCreator); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/azureopenai/AzureOpenAiEmbeddingsRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/azureopenai/AzureOpenAiEmbeddingsRequestTests.java index 8e7c831a9820..88e6880b72f0 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/azureopenai/AzureOpenAiEmbeddingsRequestTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/azureopenai/AzureOpenAiEmbeddingsRequestTests.java @@ -14,7 +14,6 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.inference.common.Truncator; import org.elasticsearch.xpack.inference.common.TruncatorTests; -import org.elasticsearch.xpack.inference.external.azureopenai.AzureOpenAiAccount; import org.elasticsearch.xpack.inference.services.azureopenai.embeddings.AzureOpenAiEmbeddingsModel; import org.elasticsearch.xpack.inference.services.azureopenai.embeddings.AzureOpenAiEmbeddingsModelTests; @@ -106,11 +105,8 @@ public static AzureOpenAiEmbeddingsRequest createRequest( entraId, "id" ); - var account = AzureOpenAiAccount.fromModel(embeddingsModel); - return new AzureOpenAiEmbeddingsRequest( TruncatorTests.createTruncator(), - account, new Truncator.TruncationResult(List.of(input), new boolean[] { false }), embeddingsModel ); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestTests.java index 32911eeb44ad..d30b809603ee 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.inference.InputType; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.xpack.inference.external.cohere.CohereAccount; import org.elasticsearch.xpack.inference.services.cohere.CohereTruncation; import org.elasticsearch.xpack.inference.services.cohere.embeddings.CohereEmbeddingType; import org.elasticsearch.xpack.inference.services.cohere.embeddings.CohereEmbeddingsModel; @@ -178,7 +177,6 @@ public void testCreateRequest_TruncateNone() throws IOException { } public static CohereEmbeddingsRequest createRequest(List input, CohereEmbeddingsModel model) { - var account = new CohereAccount(model.getServiceSettings().getCommonSettings().uri(), model.getSecretSettings().apiKey()); - return new CohereEmbeddingsRequest(account, input, model); + return new CohereEmbeddingsRequest(input, model); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/huggingface/HuggingFaceInferenceRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/huggingface/HuggingFaceInferenceRequestTests.java index 469ea28d42e5..50fa769298f8 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/huggingface/HuggingFaceInferenceRequestTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/huggingface/HuggingFaceInferenceRequestTests.java @@ -9,12 +9,10 @@ import org.apache.http.HttpHeaders; import org.apache.http.client.methods.HttpPost; -import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.inference.common.Truncator; import org.elasticsearch.xpack.inference.common.TruncatorTests; -import org.elasticsearch.xpack.inference.external.huggingface.HuggingFaceAccount; import org.elasticsearch.xpack.inference.services.huggingface.embeddings.HuggingFaceEmbeddingsModelTests; import java.io.IOException; @@ -70,11 +68,9 @@ public void testIsTruncated_ReturnsTrue() throws URISyntaxException, IOException } public static HuggingFaceInferenceRequest createRequest(String url, String apiKey, String input) throws URISyntaxException { - var account = new HuggingFaceAccount(new URI(url), new SecureString(apiKey.toCharArray())); return new HuggingFaceInferenceRequest( TruncatorTests.createTruncator(), - account, new Truncator.TruncationResult(List.of(input), new boolean[] { false }), HuggingFaceEmbeddingsModelTests.createModel(url, apiKey) ); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequestTests.java index 7858bdf4d125..b71508021edd 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequestTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiChatCompletionRequestTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.xpack.inference.external.openai.OpenAiAccount; import org.elasticsearch.xpack.inference.services.openai.completion.OpenAiChatCompletionModelTests; import java.io.IOException; @@ -120,13 +119,7 @@ public static OpenAiChatCompletionRequest createRequest( @Nullable String user ) { var chatCompletionModel = OpenAiChatCompletionModelTests.createChatCompletionModel(url, org, apiKey, model, user); - - var account = new OpenAiAccount( - chatCompletionModel.getServiceSettings().uri(), - org, - chatCompletionModel.getSecretSettings().apiKey() - ); - return new OpenAiChatCompletionRequest(account, List.of(input), chatCompletionModel); + return new OpenAiChatCompletionRequest(List.of(input), chatCompletionModel); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiEmbeddingsRequestTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiEmbeddingsRequestTests.java index ebff1c5e096e..935b27cfb688 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiEmbeddingsRequestTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/openai/OpenAiEmbeddingsRequestTests.java @@ -14,7 +14,6 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.inference.common.Truncator; import org.elasticsearch.xpack.inference.common.TruncatorTests; -import org.elasticsearch.xpack.inference.external.openai.OpenAiAccount; import org.elasticsearch.xpack.inference.services.openai.embeddings.OpenAiEmbeddingsModelTests; import java.io.IOException; @@ -118,10 +117,8 @@ public static OpenAiEmbeddingsRequest createRequest( ) { var embeddingsModel = OpenAiEmbeddingsModelTests.createModel(url, org, apiKey, model, user, (Integer) null); - var account = new OpenAiAccount(embeddingsModel.getServiceSettings().uri(), org, embeddingsModel.getSecretSettings().apiKey()); return new OpenAiEmbeddingsRequest( TruncatorTests.createTruncator(), - account, new Truncator.TruncationResult(List.of(input), new boolean[] { false }), embeddingsModel ); From ecacb2effbc48f191c2ac9f61b599626ea6cb4e3 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Tue, 16 Apr 2024 18:46:20 +0200 Subject: [PATCH 10/43] Consider total number of aggregations not number of non-empty aggregations (#107536) Test are failing because we are computing incorrectly the doc count error for terms aggregation. The only difference with the previous versions is that we are considering the number of empty aggregations instead of the total number of aggregations when computing this value. Making this change it makes the test happy. --- .../aggregations/bucket/terms/AbstractInternalTerms.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractInternalTerms.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractInternalTerms.java index 11bd63bcdaa8..af1cabdc2738 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractInternalTerms.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/AbstractInternalTerms.java @@ -221,6 +221,7 @@ public final AggregatorReducer termsAggregationReducer(AggregationReduceContext private class TermsAggregationReducer implements AggregatorReducer { private final List> bucketsList; private final AggregationReduceContext reduceContext; + private final int size; private long sumDocCountError = 0; private final long[] otherDocCount = new long[] { 0 }; @@ -236,6 +237,7 @@ private class TermsAggregationReducer implements AggregatorReducer { private TermsAggregationReducer(AggregationReduceContext reduceContext, int size) { bucketsList = new ArrayList<>(size); this.reduceContext = reduceContext; + this.size = size; } @Override @@ -326,7 +328,7 @@ public InternalAggregation get() { if (sumDocCountError == -1) { docCountError = -1; } else { - docCountError = bucketsList.size() == 1 ? 0 : sumDocCountError; + docCountError = size == 1 ? 0 : sumDocCountError; } return create(name, result, reduceContext.isFinalReduce() ? getOrder() : thisReduceOrder, docCountError, otherDocCount[0]); } From e6d421c830b44caadffa6f2e4436e453cbdfb6df Mon Sep 17 00:00:00 2001 From: Chris Hegarty <62058229+ChrisHegarty@users.noreply.github.com> Date: Tue, 16 Apr 2024 18:25:49 +0100 Subject: [PATCH 11/43] Add release highlight for #106133 (#107487) This commit adds a release highlight for #106133. --- docs/changelog/106133.yaml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/changelog/106133.yaml b/docs/changelog/106133.yaml index fe71992a0f4f..6dd7bf6cea08 100644 --- a/docs/changelog/106133.yaml +++ b/docs/changelog/106133.yaml @@ -1,5 +1,19 @@ pr: 106133 -summary: Add an optimised vector distance function for aarch64 +summary: Add a SIMD (Neon) optimised vector distance function for int8 area: Search type: enhancement issues: [] +highlight: + title: A SIMD (Neon) optimised vector distance function for merging int8 Scalar Quantized vectors has been added + body: |- + An optimised int8 vector distance implementation for aarch64 has been added. + This implementation is currently only used during merging. + The vector distance implementation outperforms Lucene's Pamana Vector + implementation for binary comparisons by approx 5x (depending on the number + of dimensions). It does so by means of SIMD (Neon) intrinsics compiled into a + separate native library and link by Panama's FFI. Comparisons are performed on + off-heap mmap'ed vector data. + Macro benchmarks, SO_Dense_Vector with scalar quantization enabled, shows + significant improvements in merge times, approximately 3 times faster. + notable: true + From 73a17a17fe1cc80a6f436c207e3a8cc45bff9b60 Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Tue, 16 Apr 2024 14:25:21 -0400 Subject: [PATCH 12/43] [ESQL] Moving argument compatibility checking for Equals (#107546) Fixed the conflicts, and re-submitting. Please see #105217 for full details, history, and discussion. I'll use the commit message from that PR as well. Continuing my work from #104490, this PR moves the parameter compatibility checking for Equals into the type resolution check. This is a somewhat bigger change than for Add, as there was no ES|QL base class for binary comparison operators before this. I've added EsqlBinaryComparison as that base class, and migrated all of the binary comparisons to be based off of that (except for NullEquals, see note below). In order to maintain compatibility with the current behavior, I've kept it so that unsigned longs are only inter-operable with other unsigned longs. We've talked a lot about changing that, and I consider this work a prerequisite for that. I've also added a bunch of test cases to Equals and NotEquals, which should have the side effect of filling out the type support table in the equals docs. As noted in the comments, I'll have follow up PRs for the other binary comparisons to add tests, but this PR is already too long. Note about NullEquals: There is an ES|QL NullEquals class, which inherits from the QL version, but I don't think it works. I didn't see any tests or docs for it, and trying it out in the demo instance gave me a syntax error. I think we need to delve into what's going on there, but this PR isn't the right place for it. This reverts commit 225edaf6076770385b4d091af89a546020ec5c79. --- docs/changelog/107537.yaml | 5 - .../src/main/resources/conditional.csv-spec | 7 + .../predicate/operator/comparison/Equals.java | 47 +++-- .../comparison/EsqlBinaryComparison.java | 164 +++++++++++++++ .../operator/comparison/GreaterThan.java | 33 ++- .../comparison/GreaterThanOrEqual.java | 34 ++- .../operator/comparison/LessThan.java | 36 ++-- .../operator/comparison/LessThanOrEqual.java | 31 ++- .../operator/comparison/NotEquals.java | 82 +++++--- .../DateTimeArithmeticOperation.java | 8 +- .../arithmetic/EsqlArithmeticOperation.java | 23 +- .../function/AbstractFunctionTestCase.java | 49 +++++ .../expression/function/TestCaseSupplier.java | 92 +++++++- .../operator/arithmetic/AddTests.java | 8 +- .../operator/arithmetic/DivTests.java | 18 +- .../operator/arithmetic/ModTests.java | 18 +- .../operator/comparison/EqualsTests.java | 198 +++++++++++++++--- .../comparison/GreaterThanOrEqualTests.java | 21 +- .../operator/comparison/GreaterThanTests.java | 21 +- .../comparison/LessThanOrEqualTests.java | 20 +- .../operator/comparison/LessThanTests.java | 20 +- .../operator/comparison/NotEqualsTests.java | 197 ++++++++++++++--- .../esql/optimizer/OptimizerRulesTests.java | 2 +- 23 files changed, 872 insertions(+), 262 deletions(-) delete mode 100644 docs/changelog/107537.yaml create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EsqlBinaryComparison.java diff --git a/docs/changelog/107537.yaml b/docs/changelog/107537.yaml deleted file mode 100644 index d6d502b394c3..000000000000 --- a/docs/changelog/107537.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107537 -summary: "Revert \"[ES|QL] Moving argument compatibility checking for Equals\"" -area: ES|QL -type: bug -issues: [] diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/conditional.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/conditional.csv-spec index f574722f691e..64a8c1d9da31 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/conditional.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/conditional.csv-spec @@ -156,6 +156,9 @@ nullOnMultivaluesComparisonOperation required_feature: esql.disable_nullable_opts ROW a = 5, b = [ 1, 2 ]| EVAL same = a == b| LIMIT 1 | WHERE same IS NULL; +warning:Line 1:38: evaluation of [a == b] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:38: java.lang.IllegalArgumentException: single-value function encountered multi-value + a:integer | b:integer | same:boolean 5 | [1, 2] | null @@ -166,6 +169,8 @@ notNullOnMultivaluesComparisonOperation required_feature: esql.disable_nullable_opts ROW a = 5, b = [ 1, 2 ]| EVAL same = a == b| LIMIT 1 | WHERE same IS NOT NULL; +warning:Line 1:38: evaluation of [a == b] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:38: java.lang.IllegalArgumentException: single-value function encountered multi-value a:integer | b:integer | same:boolean ; @@ -175,6 +180,8 @@ notNullOnMultivaluesComparisonOperationWithPartialMatch required_feature: esql.disable_nullable_opts ROW a = 5, b = [ 5, 2 ]| EVAL same = a == b| LIMIT 1 | WHERE same IS NOT NULL; +warning:Line 1:38: evaluation of [a == b] failed, treating result as null. Only first 20 failures recorded. +warning:Line 1:38: java.lang.IllegalArgumentException: single-value function encountered multi-value a:integer | b:integer | same:boolean ; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/Equals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/Equals.java index 9fb899b8e36d..62eec13af008 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/Equals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/Equals.java @@ -8,33 +8,48 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Evaluator; -import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; +import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; +import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.TypeResolutions; +import org.elasticsearch.xpack.ql.expression.predicate.Negatable; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; import java.time.ZoneId; +import java.util.Map; + +public class Equals extends EsqlBinaryComparison implements Negatable { + private static final Map evaluatorMap = Map.ofEntries( + Map.entry(DataTypes.BOOLEAN, EqualsBoolsEvaluator.Factory::new), + Map.entry(DataTypes.INTEGER, EqualsIntsEvaluator.Factory::new), + Map.entry(DataTypes.DOUBLE, EqualsDoublesEvaluator.Factory::new), + Map.entry(DataTypes.LONG, EqualsLongsEvaluator.Factory::new), + Map.entry(DataTypes.UNSIGNED_LONG, EqualsLongsEvaluator.Factory::new), + Map.entry(DataTypes.DATETIME, EqualsLongsEvaluator.Factory::new), + Map.entry(EsqlDataTypes.GEO_POINT, EqualsGeometriesEvaluator.Factory::new), + Map.entry(EsqlDataTypes.CARTESIAN_POINT, EqualsGeometriesEvaluator.Factory::new), + Map.entry(EsqlDataTypes.GEO_SHAPE, EqualsGeometriesEvaluator.Factory::new), + Map.entry(EsqlDataTypes.CARTESIAN_SHAPE, EqualsGeometriesEvaluator.Factory::new), + Map.entry(DataTypes.KEYWORD, EqualsKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.TEXT, EqualsKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.VERSION, EqualsKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.IP, EqualsKeywordsEvaluator.Factory::new) + ); -import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; - -public class Equals extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals { public Equals(Source source, Expression left, Expression right) { - super(source, left, right); + super(source, left, right, BinaryComparisonProcessor.BinaryComparisonOperation.EQ, evaluatorMap); } public Equals(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, zoneId); - } - - @Override - protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { - return EsqlTypeResolutions.isExact(e, sourceText(), DEFAULT); + super(source, left, right, BinaryComparisonProcessor.BinaryComparisonOperation.EQ, zoneId, evaluatorMap); } @Override - protected NodeInfo info() { + protected NodeInfo info() { return NodeInfo.create(this, Equals::new, left(), right(), zoneId()); } @@ -48,6 +63,11 @@ public Equals swapLeftAndRight() { return new Equals(source(), right(), left(), zoneId()); } + @Override + public BinaryComparison reverse() { + return this; + } + @Override public BinaryComparison negate() { return new NotEquals(source(), left(), right(), zoneId()); @@ -82,4 +102,5 @@ static boolean processBools(boolean lhs, boolean rhs) { static boolean processGeometries(BytesRef lhs, BytesRef rhs) { return lhs.equals(rhs); } + } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EsqlBinaryComparison.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EsqlBinaryComparison.java new file mode 100644 index 000000000000..58a808893c4c --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/EsqlBinaryComparison.java @@ -0,0 +1,164 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison; + +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; +import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; +import org.elasticsearch.xpack.esql.expression.function.scalar.math.Cast; +import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; +import org.elasticsearch.xpack.esql.type.EsqlDataTypeRegistry; +import org.elasticsearch.xpack.ql.expression.Expression; +import org.elasticsearch.xpack.ql.expression.TypeResolutions; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor; +import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; + +import java.time.ZoneId; +import java.util.Map; +import java.util.function.Function; + +import static org.elasticsearch.common.logging.LoggerMessageFormat.format; +import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG; + +public abstract class EsqlBinaryComparison extends BinaryComparison implements EvaluatorMapper { + + private final Map evaluatorMap; + + protected EsqlBinaryComparison( + Source source, + Expression left, + Expression right, + /* TODO: BinaryComparisonOperator is an enum with a bunch of functionality we don't really want. We should extract an interface and + create a symbol only version like we did for BinaryArithmeticOperation. Ideally, they could be the same class. + */ + BinaryComparisonProcessor.BinaryComparisonOperation operation, + Map evaluatorMap + ) { + this(source, left, right, operation, null, evaluatorMap); + } + + protected EsqlBinaryComparison( + Source source, + Expression left, + Expression right, + BinaryComparisonProcessor.BinaryComparisonOperation operation, + // TODO: We are definitely not doing the right thing with this zoneId + ZoneId zoneId, + Map evaluatorMap + ) { + super(source, left, right, operation, zoneId); + this.evaluatorMap = evaluatorMap; + } + + @Override + public EvalOperator.ExpressionEvaluator.Factory toEvaluator( + Function toEvaluator + ) { + // Our type is always boolean, so figure out the evaluator type from the inputs + DataType commonType = EsqlDataTypeRegistry.INSTANCE.commonType(left().dataType(), right().dataType()); + EvalOperator.ExpressionEvaluator.Factory lhs; + EvalOperator.ExpressionEvaluator.Factory rhs; + + if (commonType.isNumeric()) { + lhs = Cast.cast(source(), left().dataType(), commonType, toEvaluator.apply(left())); + rhs = Cast.cast(source(), right().dataType(), commonType, toEvaluator.apply(right())); + } else { + lhs = toEvaluator.apply(left()); + rhs = toEvaluator.apply(right()); + } + + if (evaluatorMap.containsKey(commonType) == false) { + throw new EsqlIllegalArgumentException("Unsupported type " + left().dataType()); + } + return evaluatorMap.get(commonType).apply(source(), lhs, rhs); + } + + @Override + public Boolean fold() { + return (Boolean) EvaluatorMapper.super.fold(); + } + + @Override + protected TypeResolution resolveType() { + TypeResolution typeResolution = super.resolveType(); + if (typeResolution.unresolved()) { + return typeResolution; + } + + return checkCompatibility(); + } + + @Override + protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { + return TypeResolutions.isType( + e, + evaluatorMap::containsKey, + sourceText(), + paramOrdinal, + evaluatorMap.keySet().stream().map(DataType::typeName).toArray(String[]::new) + ); + } + + /** + * Check if the two input types are compatible for this operation + * + * @return TypeResolution.TYPE_RESOLVED iff the types are compatible. Otherwise, an appropriate type resolution error. + */ + protected TypeResolution checkCompatibility() { + DataType leftType = left().dataType(); + DataType rightType = right().dataType(); + + // Unsigned long is only interoperable with other unsigned longs + if ((rightType == UNSIGNED_LONG && (false == (leftType == UNSIGNED_LONG || leftType == DataTypes.NULL))) + || (leftType == UNSIGNED_LONG && (false == (rightType == UNSIGNED_LONG || rightType == DataTypes.NULL)))) { + return new TypeResolution(formatIncompatibleTypesMessage()); + } + + if ((leftType.isNumeric() && rightType.isNumeric()) + || (DataTypes.isString(leftType) && DataTypes.isString(rightType)) + || leftType.equals(rightType) + || DataTypes.isNull(leftType) + || DataTypes.isNull(rightType)) { + return TypeResolution.TYPE_RESOLVED; + } + return new TypeResolution(formatIncompatibleTypesMessage()); + } + + public String formatIncompatibleTypesMessage() { + if (left().dataType().equals(UNSIGNED_LONG)) { + return format( + null, + "first argument of [{}] is [unsigned_long] and second is [{}]. " + + "[unsigned_long] can only be operated on together with another [unsigned_long]", + sourceText(), + right().dataType().typeName() + ); + } + if (right().dataType().equals(UNSIGNED_LONG)) { + return format( + null, + "first argument of [{}] is [{}] and second is [unsigned_long]. " + + "[unsigned_long] can only be operated on together with another [unsigned_long]", + sourceText(), + left().dataType().typeName() + ); + } + return format( + null, + "first argument of [{}] is [{}] so second argument must also be [{}] but was [{}]", + sourceText(), + left().dataType().isNumeric() ? "numeric" : left().dataType().typeName(), + left().dataType().isNumeric() ? "numeric" : left().dataType().typeName(), + right().dataType().typeName() + ); + } + +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThan.java index 5683a9d0d7e8..3eca0e858acb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThan.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThan.java @@ -8,29 +8,42 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Evaluator; -import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; +import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.TypeResolutions; +import org.elasticsearch.xpack.ql.expression.predicate.Negatable; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; import java.time.ZoneId; +import java.util.Map; -import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; +public class GreaterThan extends EsqlBinaryComparison implements Negatable { + private static final Map evaluatorMap = Map.ofEntries( + Map.entry(DataTypes.INTEGER, GreaterThanIntsEvaluator.Factory::new), + Map.entry(DataTypes.DOUBLE, GreaterThanDoublesEvaluator.Factory::new), + Map.entry(DataTypes.LONG, GreaterThanLongsEvaluator.Factory::new), + Map.entry(DataTypes.UNSIGNED_LONG, GreaterThanLongsEvaluator.Factory::new), + Map.entry(DataTypes.DATETIME, GreaterThanLongsEvaluator.Factory::new), + Map.entry(DataTypes.KEYWORD, GreaterThanKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.TEXT, GreaterThanKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.VERSION, GreaterThanKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.IP, GreaterThanKeywordsEvaluator.Factory::new) + ); -public class GreaterThan extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan { - public GreaterThan(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, zoneId); + public GreaterThan(Source source, Expression left, Expression right) { + super(source, left, right, BinaryComparisonProcessor.BinaryComparisonOperation.GT, evaluatorMap); } - @Override - protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { - return EsqlTypeResolutions.isExact(e, sourceText(), DEFAULT); + public GreaterThan(Source source, Expression left, Expression right, ZoneId zoneId) { + super(source, left, right, BinaryComparisonProcessor.BinaryComparisonOperation.GT, zoneId, evaluatorMap); } @Override - protected NodeInfo info() { + protected NodeInfo info() { return NodeInfo.create(this, GreaterThan::new, left(), right(), zoneId()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqual.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqual.java index ebb29998fb99..f99a85420870 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqual.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/GreaterThanOrEqual.java @@ -8,30 +8,42 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Evaluator; -import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; +import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.TypeResolutions; +import org.elasticsearch.xpack.ql.expression.predicate.Negatable; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; import java.time.ZoneId; +import java.util.Map; -import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; +public class GreaterThanOrEqual extends EsqlBinaryComparison implements Negatable { + private static final Map evaluatorMap = Map.ofEntries( + Map.entry(DataTypes.INTEGER, GreaterThanOrEqualIntsEvaluator.Factory::new), + Map.entry(DataTypes.DOUBLE, GreaterThanOrEqualDoublesEvaluator.Factory::new), + Map.entry(DataTypes.LONG, GreaterThanOrEqualLongsEvaluator.Factory::new), + Map.entry(DataTypes.UNSIGNED_LONG, GreaterThanOrEqualLongsEvaluator.Factory::new), + Map.entry(DataTypes.DATETIME, GreaterThanOrEqualLongsEvaluator.Factory::new), + Map.entry(DataTypes.KEYWORD, GreaterThanOrEqualKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.TEXT, GreaterThanOrEqualKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.VERSION, GreaterThanOrEqualKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.IP, GreaterThanOrEqualKeywordsEvaluator.Factory::new) + ); -public class GreaterThanOrEqual extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThanOrEqual { - - public GreaterThanOrEqual(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, zoneId); + public GreaterThanOrEqual(Source source, Expression left, Expression right) { + super(source, left, right, BinaryComparisonProcessor.BinaryComparisonOperation.GTE, evaluatorMap); } - @Override - protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { - return EsqlTypeResolutions.isExact(e, sourceText(), DEFAULT); + public GreaterThanOrEqual(Source source, Expression left, Expression right, ZoneId zoneId) { + super(source, left, right, BinaryComparisonProcessor.BinaryComparisonOperation.GTE, zoneId, evaluatorMap); } @Override - protected NodeInfo info() { + protected NodeInfo info() { return NodeInfo.create(this, GreaterThanOrEqual::new, left(), right(), zoneId()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThan.java index 12f54270b65d..6b82df1d67da 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThan.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThan.java @@ -8,38 +8,44 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Evaluator; -import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; +import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.TypeResolutions; +import org.elasticsearch.xpack.ql.expression.predicate.Negatable; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; import java.time.ZoneId; +import java.util.Map; -import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; +public class LessThan extends EsqlBinaryComparison implements Negatable { -public class LessThan extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan { + private static final Map evaluatorMap = Map.ofEntries( + Map.entry(DataTypes.INTEGER, LessThanIntsEvaluator.Factory::new), + Map.entry(DataTypes.DOUBLE, LessThanDoublesEvaluator.Factory::new), + Map.entry(DataTypes.LONG, LessThanLongsEvaluator.Factory::new), + Map.entry(DataTypes.UNSIGNED_LONG, LessThanLongsEvaluator.Factory::new), + Map.entry(DataTypes.DATETIME, LessThanLongsEvaluator.Factory::new), + Map.entry(DataTypes.KEYWORD, LessThanKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.TEXT, LessThanKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.VERSION, LessThanKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.IP, LessThanKeywordsEvaluator.Factory::new) + ); public LessThan(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, zoneId); + super(source, left, right, BinaryComparisonProcessor.BinaryComparisonOperation.LT, zoneId, evaluatorMap); } @Override - protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { - return EsqlTypeResolutions.isExact(e, sourceText(), DEFAULT); - } - - @Override - protected NodeInfo info() { + protected NodeInfo info() { return NodeInfo.create(this, LessThan::new, left(), right(), zoneId()); } @Override - protected org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan replaceChildren( - Expression newLeft, - Expression newRight - ) { + protected LessThan replaceChildren(Expression newLeft, Expression newRight) { return new LessThan(source(), newLeft, newRight, zoneId()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqual.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqual.java index e75733a9e234..ac6a92aaf097 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqual.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/LessThanOrEqual.java @@ -8,29 +8,38 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Evaluator; -import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; +import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.TypeResolutions; +import org.elasticsearch.xpack.ql.expression.predicate.Negatable; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; import java.time.ZoneId; +import java.util.Map; -import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; +public class LessThanOrEqual extends EsqlBinaryComparison implements Negatable { + private static final Map evaluatorMap = Map.ofEntries( + Map.entry(DataTypes.INTEGER, LessThanOrEqualIntsEvaluator.Factory::new), + Map.entry(DataTypes.DOUBLE, LessThanOrEqualDoublesEvaluator.Factory::new), + Map.entry(DataTypes.LONG, LessThanOrEqualLongsEvaluator.Factory::new), + Map.entry(DataTypes.UNSIGNED_LONG, LessThanOrEqualLongsEvaluator.Factory::new), + Map.entry(DataTypes.DATETIME, LessThanOrEqualLongsEvaluator.Factory::new), + Map.entry(DataTypes.KEYWORD, LessThanOrEqualKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.TEXT, LessThanOrEqualKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.VERSION, LessThanOrEqualKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.IP, LessThanOrEqualKeywordsEvaluator.Factory::new) + ); -public class LessThanOrEqual extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThanOrEqual { public LessThanOrEqual(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, zoneId); + super(source, left, right, BinaryComparisonProcessor.BinaryComparisonOperation.LTE, zoneId, evaluatorMap); } @Override - protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { - return EsqlTypeResolutions.isExact(e, sourceText(), DEFAULT); - } - - @Override - protected NodeInfo info() { + protected NodeInfo info() { return NodeInfo.create(this, LessThanOrEqual::new, left(), right(), zoneId()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEquals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEquals.java index 6fbed572cdc0..9c931ec7433e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEquals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/predicate/operator/comparison/NotEquals.java @@ -8,45 +8,44 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Evaluator; -import org.elasticsearch.xpack.esql.expression.EsqlTypeResolutions; +import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; +import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.TypeResolutions; +import org.elasticsearch.xpack.ql.expression.predicate.Negatable; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; +import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparisonProcessor; import org.elasticsearch.xpack.ql.tree.NodeInfo; import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.type.DataType; +import org.elasticsearch.xpack.ql.type.DataTypes; import java.time.ZoneId; +import java.util.Map; -import static org.elasticsearch.xpack.ql.expression.TypeResolutions.ParamOrdinal.DEFAULT; +public class NotEquals extends EsqlBinaryComparison implements Negatable { + private static final Map evaluatorMap = Map.ofEntries( + Map.entry(DataTypes.BOOLEAN, NotEqualsBoolsEvaluator.Factory::new), + Map.entry(DataTypes.INTEGER, NotEqualsIntsEvaluator.Factory::new), + Map.entry(DataTypes.DOUBLE, NotEqualsDoublesEvaluator.Factory::new), + Map.entry(DataTypes.LONG, NotEqualsLongsEvaluator.Factory::new), + Map.entry(DataTypes.UNSIGNED_LONG, NotEqualsLongsEvaluator.Factory::new), + Map.entry(DataTypes.DATETIME, NotEqualsLongsEvaluator.Factory::new), + Map.entry(EsqlDataTypes.GEO_POINT, NotEqualsGeometriesEvaluator.Factory::new), + Map.entry(EsqlDataTypes.CARTESIAN_POINT, NotEqualsGeometriesEvaluator.Factory::new), + Map.entry(EsqlDataTypes.GEO_SHAPE, NotEqualsGeometriesEvaluator.Factory::new), + Map.entry(EsqlDataTypes.CARTESIAN_SHAPE, NotEqualsGeometriesEvaluator.Factory::new), + Map.entry(DataTypes.KEYWORD, NotEqualsKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.TEXT, NotEqualsKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.VERSION, NotEqualsKeywordsEvaluator.Factory::new), + Map.entry(DataTypes.IP, NotEqualsKeywordsEvaluator.Factory::new) + ); -public class NotEquals extends org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals { - public NotEquals(Source source, Expression left, Expression right, ZoneId zoneId) { - super(source, left, right, zoneId); - } - - @Override - protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { - return EsqlTypeResolutions.isExact(e, sourceText(), DEFAULT); - } - - @Override - protected NodeInfo info() { - return NodeInfo.create(this, NotEquals::new, left(), right(), zoneId()); - } - - @Override - protected NotEquals replaceChildren(Expression newLeft, Expression newRight) { - return new NotEquals(source(), newLeft, newRight, zoneId()); + public NotEquals(Source source, Expression left, Expression right) { + super(source, left, right, BinaryComparisonProcessor.BinaryComparisonOperation.NEQ, evaluatorMap); } - @Override - public NotEquals swapLeftAndRight() { - return new NotEquals(source(), right(), left(), zoneId()); - } - - @Override - public BinaryComparison negate() { - return new Equals(source(), left(), right(), zoneId()); + public NotEquals(Source source, Expression left, Expression right, ZoneId zoneId) { + super(source, left, right, BinaryComparisonProcessor.BinaryComparisonOperation.NEQ, zoneId, evaluatorMap); } @Evaluator(extraName = "Ints") @@ -78,4 +77,29 @@ static boolean processBools(boolean lhs, boolean rhs) { static boolean processGeometries(BytesRef lhs, BytesRef rhs) { return false == lhs.equals(rhs); } + + @Override + public BinaryComparison reverse() { + return this; + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, NotEquals::new, left(), right(), zoneId()); + } + + @Override + protected NotEquals replaceChildren(Expression newLeft, Expression newRight) { + return new NotEquals(source(), newLeft, newRight, zoneId()); + } + + @Override + public NotEquals swapLeftAndRight() { + return new NotEquals(source(), right(), left(), zoneId()); + } + + @Override + public BinaryComparison negate() { + return new Equals(source(), left(), right(), zoneId()); + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java index 0f550862ed1f..a45707a0197d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java @@ -43,10 +43,10 @@ interface DatetimeArithmeticEvaluator { Expression left, Expression right, OperationSymbol op, - ArithmeticEvaluator ints, - ArithmeticEvaluator longs, - ArithmeticEvaluator ulongs, - ArithmeticEvaluator doubles, + BinaryEvaluator ints, + BinaryEvaluator longs, + BinaryEvaluator ulongs, + BinaryEvaluator doubles, DatetimeArithmeticEvaluator datetimes ) { super(source, left, right, op, ints, longs, ulongs, doubles); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java index 22f5798e5b1c..ba283bc4d877 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java @@ -71,14 +71,15 @@ public String symbol() { } /** Arithmetic (quad) function. */ - interface ArithmeticEvaluator { + @FunctionalInterface + public interface BinaryEvaluator { ExpressionEvaluator.Factory apply(Source source, ExpressionEvaluator.Factory lhs, ExpressionEvaluator.Factory rhs); } - private final ArithmeticEvaluator ints; - private final ArithmeticEvaluator longs; - private final ArithmeticEvaluator ulongs; - private final ArithmeticEvaluator doubles; + private final BinaryEvaluator ints; + private final BinaryEvaluator longs; + private final BinaryEvaluator ulongs; + private final BinaryEvaluator doubles; private DataType dataType; @@ -87,10 +88,10 @@ interface ArithmeticEvaluator { Expression left, Expression right, OperationSymbol op, - ArithmeticEvaluator ints, - ArithmeticEvaluator longs, - ArithmeticEvaluator ulongs, - ArithmeticEvaluator doubles + BinaryEvaluator ints, + BinaryEvaluator longs, + BinaryEvaluator ulongs, + BinaryEvaluator doubles ) { super(source, left, right, op); this.ints = ints; @@ -139,7 +140,7 @@ protected TypeResolution checkCompatibility() { return TypeResolution.TYPE_RESOLVED; } - static String formatIncompatibleTypesMessage(String symbol, DataType leftType, DataType rightType) { + public static String formatIncompatibleTypesMessage(String symbol, DataType leftType, DataType rightType) { return format(null, "[{}] has arguments with incompatible types [{}] and [{}]", symbol, leftType.typeName(), rightType.typeName()); } @@ -152,7 +153,7 @@ public ExpressionEvaluator.Factory toEvaluator(Function errorsForCasesWithoutExamples( return suppliers; } + public static String errorMessageStringForBinaryOperators( + boolean includeOrdinal, + List> validPerPosition, + List types + ) { + try { + return typeErrorMessage(includeOrdinal, validPerPosition, types); + } catch (IllegalStateException e) { + // This means all the positional args were okay, so the expected error is from the combination + if (types.get(0).equals(DataTypes.UNSIGNED_LONG)) { + return "first argument of [] is [unsigned_long] and second is [" + + types.get(1).typeName() + + "]. [unsigned_long] can only be operated on together with another [unsigned_long]"; + + } + if (types.get(1).equals(DataTypes.UNSIGNED_LONG)) { + return "first argument of [] is [" + + types.get(0).typeName() + + "] and second is [unsigned_long]. [unsigned_long] can only be operated on together with another [unsigned_long]"; + } + return "first argument of [] is [" + + (types.get(0).isNumeric() ? "numeric" : types.get(0).typeName()) + + "] so second argument must also be [" + + (types.get(0).isNumeric() ? "numeric" : types.get(0).typeName()) + + "] but was [" + + types.get(1).typeName() + + "]"; + + } + } + /** * Adds test cases containing unsupported parameter types that immediately fail. */ @@ -931,6 +962,24 @@ protected static String typeErrorMessage(boolean includeOrdinal, List types) { return types.stream().map(t -> "<" + t.typeName() + ">").collect(Collectors.joining(", ")); } + public static List stringCases( + BinaryOperator expected, + BiFunction evaluatorToString, + List warnings, + DataType expectedType + ) { + List lhsSuppliers = new ArrayList<>(); + List rhsSuppliers = new ArrayList<>(); + List suppliers = new ArrayList<>(); + for (DataType type : AbstractConvertFunction.STRING_TYPES) { + lhsSuppliers.addAll(stringCases(type)); + rhsSuppliers.addAll(stringCases(type)); + casesCrossProduct( + expected, + lhsSuppliers, + rhsSuppliers, + evaluatorToString, + (lhs, rhs) -> warnings, + suppliers, + expectedType, + true + ); + } + return suppliers; + } + @Override public TestCase get() { TestCase supplied = supplier.get(); @@ -258,14 +284,14 @@ public static List castToDoubleSuppliersFromRange(Double Min, return suppliers; } - public record NumericTypeTestConfig(Number min, Number max, BinaryOperator expected, String evaluatorName) {} + public record NumericTypeTestConfig(Number min, Number max, BiFunction expected, String evaluatorName) {} - public record NumericTypeTestConfigs( - NumericTypeTestConfig intStuff, - NumericTypeTestConfig longStuff, - NumericTypeTestConfig doubleStuff + public record NumericTypeTestConfigs( + NumericTypeTestConfig intStuff, + NumericTypeTestConfig longStuff, + NumericTypeTestConfig doubleStuff ) { - public NumericTypeTestConfig get(DataType type) { + public NumericTypeTestConfig get(DataType type) { if (type == DataTypes.INTEGER) { return intStuff; } @@ -312,8 +338,47 @@ public static List getSuppliersForNumericType(DataType type, throw new IllegalArgumentException("bogus numeric type [" + type + "]"); } + public static List forBinaryComparisonWithWidening( + NumericTypeTestConfigs typeStuff, + String lhsName, + String rhsName, + BiFunction> warnings, + boolean allowRhsZero + ) { + List suppliers = new ArrayList<>(); + List numericTypes = List.of(DataTypes.INTEGER, DataTypes.LONG, DataTypes.DOUBLE); + + for (DataType lhsType : numericTypes) { + for (DataType rhsType : numericTypes) { + DataType expected = widen(lhsType, rhsType); + NumericTypeTestConfig expectedTypeStuff = typeStuff.get(expected); + BiFunction evaluatorToString = (lhs, rhs) -> expectedTypeStuff.evaluatorName() + + "[" + + lhsName + + "=" + + getCastEvaluator("Attribute[channel=0]", lhs, expected) + + ", " + + rhsName + + "=" + + getCastEvaluator("Attribute[channel=1]", rhs, expected) + + "]"; + casesCrossProduct( + (l, r) -> expectedTypeStuff.expected().apply((Number) l, (Number) r), + getSuppliersForNumericType(lhsType, expectedTypeStuff.min(), expectedTypeStuff.max(), allowRhsZero), + getSuppliersForNumericType(rhsType, expectedTypeStuff.min(), expectedTypeStuff.max(), allowRhsZero), + evaluatorToString, + warnings, + suppliers, + DataTypes.BOOLEAN, + true + ); + } + } + return suppliers; + } + public static List forBinaryWithWidening( - NumericTypeTestConfigs typeStuff, + NumericTypeTestConfigs typeStuff, String lhsName, String rhsName, BiFunction> warnings, @@ -325,7 +390,7 @@ public static List forBinaryWithWidening( for (DataType lhsType : numericTypes) { for (DataType rhsType : numericTypes) { DataType expected = widen(lhsType, rhsType); - NumericTypeTestConfig expectedTypeStuff = typeStuff.get(expected); + NumericTypeTestConfig expectedTypeStuff = typeStuff.get(expected); BiFunction evaluatorToString = (lhs, rhs) -> expectedTypeStuff.evaluatorName() + "[" + lhsName @@ -885,7 +950,7 @@ public static List doubleCases(double min, double max, boolea return cases; } - private static List booleanCases() { + public static List booleanCases() { return List.of( new TypedDataSupplier("", () -> true, DataTypes.BOOLEAN), new TypedDataSupplier("", () -> false, DataTypes.BOOLEAN) @@ -1267,9 +1332,14 @@ public Matcher evaluatorToString() { * exists because we can't generate random values from the test parameter generation functions, and instead need to return * suppliers which generate the random values at test execution time. */ - public record TypedDataSupplier(String name, Supplier supplier, DataType type) { + public record TypedDataSupplier(String name, Supplier supplier, DataType type, boolean forceLiteral) { + + public TypedDataSupplier(String name, Supplier supplier, DataType type) { + this(name, supplier, type, false); + } + public TypedData get() { - return new TypedData(supplier.get(), type, name); + return new TypedData(supplier.get(), type, name, forceLiteral); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java index c40d037890d5..2596959c449d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java @@ -43,20 +43,20 @@ public static Iterable parameters() { List suppliers = new ArrayList<>(); suppliers.addAll( TestCaseSupplier.forBinaryWithWidening( - new TestCaseSupplier.NumericTypeTestConfigs( - new TestCaseSupplier.NumericTypeTestConfig( + new TestCaseSupplier.NumericTypeTestConfigs( + new TestCaseSupplier.NumericTypeTestConfig<>( (Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1, (l, r) -> l.intValue() + r.intValue(), "AddIntsEvaluator" ), - new TestCaseSupplier.NumericTypeTestConfig( + new TestCaseSupplier.NumericTypeTestConfig<>( (Long.MIN_VALUE >> 1) - 1, (Long.MAX_VALUE >> 1) - 1, (l, r) -> l.longValue() + r.longValue(), "AddLongsEvaluator" ), - new TestCaseSupplier.NumericTypeTestConfig( + new TestCaseSupplier.NumericTypeTestConfig<>( Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, (l, r) -> l.doubleValue() + r.doubleValue(), diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivTests.java index 528324f07a08..f3348ab2dcba 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivTests.java @@ -34,20 +34,20 @@ public static Iterable parameters() { List suppliers = new ArrayList<>(); suppliers.addAll( TestCaseSupplier.forBinaryWithWidening( - new TestCaseSupplier.NumericTypeTestConfigs( - new TestCaseSupplier.NumericTypeTestConfig( + new TestCaseSupplier.NumericTypeTestConfigs( + new TestCaseSupplier.NumericTypeTestConfig<>( (Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1, (l, r) -> l.intValue() / r.intValue(), "DivIntsEvaluator" ), - new TestCaseSupplier.NumericTypeTestConfig( + new TestCaseSupplier.NumericTypeTestConfig<>( (Long.MIN_VALUE >> 1) - 1, (Long.MAX_VALUE >> 1) - 1, (l, r) -> l.longValue() / r.longValue(), "DivLongsEvaluator" ), - new TestCaseSupplier.NumericTypeTestConfig(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, (l, r) -> { + new TestCaseSupplier.NumericTypeTestConfig<>(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, (l, r) -> { double v = l.doubleValue() / r.doubleValue(); if (Double.isFinite(v)) { return v; @@ -90,20 +90,20 @@ public static Iterable parameters() { suppliers = errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), DivTests::divErrorMessageString); // Divide by zero cases - all of these should warn and return null - TestCaseSupplier.NumericTypeTestConfigs typeStuff = new TestCaseSupplier.NumericTypeTestConfigs( - new TestCaseSupplier.NumericTypeTestConfig( + TestCaseSupplier.NumericTypeTestConfigs typeStuff = new TestCaseSupplier.NumericTypeTestConfigs<>( + new TestCaseSupplier.NumericTypeTestConfig<>( (Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1, (l, r) -> null, "DivIntsEvaluator" ), - new TestCaseSupplier.NumericTypeTestConfig( + new TestCaseSupplier.NumericTypeTestConfig<>( (Long.MIN_VALUE >> 1) - 1, (Long.MAX_VALUE >> 1) - 1, (l, r) -> null, "DivLongsEvaluator" ), - new TestCaseSupplier.NumericTypeTestConfig( + new TestCaseSupplier.NumericTypeTestConfig<>( Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, (l, r) -> null, @@ -115,7 +115,7 @@ public static Iterable parameters() { for (DataType lhsType : numericTypes) { for (DataType rhsType : numericTypes) { DataType expected = TestCaseSupplier.widen(lhsType, rhsType); - TestCaseSupplier.NumericTypeTestConfig expectedTypeStuff = typeStuff.get(expected); + TestCaseSupplier.NumericTypeTestConfig expectedTypeStuff = typeStuff.get(expected); BiFunction evaluatorToString = (lhs, rhs) -> expectedTypeStuff.evaluatorName() + "[" + "lhs" diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModTests.java index d2af83e91ec6..a70f2c788525 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModTests.java @@ -34,20 +34,20 @@ public static Iterable parameters() { List suppliers = new ArrayList<>(); suppliers.addAll( TestCaseSupplier.forBinaryWithWidening( - new TestCaseSupplier.NumericTypeTestConfigs( - new TestCaseSupplier.NumericTypeTestConfig( + new TestCaseSupplier.NumericTypeTestConfigs( + new TestCaseSupplier.NumericTypeTestConfig<>( (Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1, (l, r) -> l.intValue() % r.intValue(), "ModIntsEvaluator" ), - new TestCaseSupplier.NumericTypeTestConfig( + new TestCaseSupplier.NumericTypeTestConfig<>( (Long.MIN_VALUE >> 1) - 1, (Long.MAX_VALUE >> 1) - 1, (l, r) -> l.longValue() % r.longValue(), "ModLongsEvaluator" ), - new TestCaseSupplier.NumericTypeTestConfig( + new TestCaseSupplier.NumericTypeTestConfig<>( Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, (l, r) -> l.doubleValue() % r.doubleValue(), @@ -77,20 +77,20 @@ public static Iterable parameters() { suppliers = errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), ModTests::modErrorMessageString); // Divide by zero cases - all of these should warn and return null - TestCaseSupplier.NumericTypeTestConfigs typeStuff = new TestCaseSupplier.NumericTypeTestConfigs( - new TestCaseSupplier.NumericTypeTestConfig( + TestCaseSupplier.NumericTypeTestConfigs typeStuff = new TestCaseSupplier.NumericTypeTestConfigs<>( + new TestCaseSupplier.NumericTypeTestConfig<>( (Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1, (l, r) -> null, "ModIntsEvaluator" ), - new TestCaseSupplier.NumericTypeTestConfig( + new TestCaseSupplier.NumericTypeTestConfig<>( (Long.MIN_VALUE >> 1) - 1, (Long.MAX_VALUE >> 1) - 1, (l, r) -> null, "ModLongsEvaluator" ), - new TestCaseSupplier.NumericTypeTestConfig( + new TestCaseSupplier.NumericTypeTestConfig<>( Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, (l, r) -> null, @@ -102,7 +102,7 @@ public static Iterable parameters() { for (DataType lhsType : numericTypes) { for (DataType rhsType : numericTypes) { DataType expected = TestCaseSupplier.widen(lhsType, rhsType); - TestCaseSupplier.NumericTypeTestConfig expectedTypeStuff = typeStuff.get(expected); + TestCaseSupplier.NumericTypeTestConfig expectedTypeStuff = typeStuff.get(expected); BiFunction evaluatorToString = (lhs, rhs) -> expectedTypeStuff.evaluatorName() + "[" + "lhs" diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java index 0a1e9bdfaf34..0739cd4670c0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java @@ -11,52 +11,198 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.Equals; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; -import org.hamcrest.Matcher; +import org.elasticsearch.xpack.ql.util.NumericUtils; +import java.math.BigInteger; +import java.util.ArrayList; import java.util.List; import java.util.function.Supplier; -import static org.hamcrest.Matchers.equalTo; - -public class EqualsTests extends AbstractBinaryComparisonTestCase { +public class EqualsTests extends AbstractFunctionTestCase { public EqualsTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); } @ParametersFactory public static Iterable parameters() { - return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("Int == Int", () -> { - int rhs = randomInt(); - int lhs = randomInt(); - return new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(lhs, DataTypes.INTEGER, "lhs"), - new TestCaseSupplier.TypedData(rhs, DataTypes.INTEGER, "rhs") + List suppliers = new ArrayList<>(); + suppliers.addAll( + TestCaseSupplier.forBinaryComparisonWithWidening( + new TestCaseSupplier.NumericTypeTestConfigs<>( + new TestCaseSupplier.NumericTypeTestConfig<>( + (Integer.MIN_VALUE >> 1) - 1, + (Integer.MAX_VALUE >> 1) - 1, + (l, r) -> l.intValue() == r.intValue(), + "EqualsIntsEvaluator" + ), + new TestCaseSupplier.NumericTypeTestConfig<>( + (Long.MIN_VALUE >> 1) - 1, + (Long.MAX_VALUE >> 1) - 1, + (l, r) -> l.longValue() == r.longValue(), + "EqualsLongsEvaluator" + ), + new TestCaseSupplier.NumericTypeTestConfig<>( + Double.NEGATIVE_INFINITY, + Double.POSITIVE_INFINITY, + // NB: this has different behavior than Double::equals + (l, r) -> l.doubleValue() == r.doubleValue(), + "EqualsDoublesEvaluator" + ) ), - "EqualsIntsEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]", + "lhs", + "rhs", + (lhs, rhs) -> List.of(), + false + ) + ); + + // Unsigned Long cases + // TODO: These should be integrated into the type cross product above, but are currently broken + // see https://github.com/elastic/elasticsearch/issues/102935 + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "EqualsLongsEvaluator", + "lhs", + "rhs", + Object::equals, DataTypes.BOOLEAN, - equalTo(lhs == rhs) - ); - }))); - } + TestCaseSupplier.ulongCases(BigInteger.ZERO, NumericUtils.UNSIGNED_LONG_MAX, true), + TestCaseSupplier.ulongCases(BigInteger.ZERO, NumericUtils.UNSIGNED_LONG_MAX, true), + List.of(), + false + ) + ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "EqualsBoolsEvaluator", + "lhs", + "rhs", + Object::equals, + DataTypes.BOOLEAN, + TestCaseSupplier.booleanCases(), + TestCaseSupplier.booleanCases(), + List.of(), + false + ) + ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "EqualsKeywordsEvaluator", + "lhs", + "rhs", + Object::equals, + DataTypes.BOOLEAN, + TestCaseSupplier.ipCases(), + TestCaseSupplier.ipCases(), + List.of(), + false + ) + ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "EqualsKeywordsEvaluator", + "lhs", + "rhs", + Object::equals, + DataTypes.BOOLEAN, + TestCaseSupplier.versionCases(""), + TestCaseSupplier.versionCases(""), + List.of(), + false + ) + ); + // Datetime + // TODO: I'm surprised this passes. Shouldn't there be a cast from DateTime to Long? + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "EqualsLongsEvaluator", + "lhs", + "rhs", + Object::equals, + DataTypes.BOOLEAN, + TestCaseSupplier.dateCases(), + TestCaseSupplier.dateCases(), + List.of(), + false + ) + ); - @Override - protected > Matcher resultMatcher(T lhs, T rhs) { - return equalTo(lhs.equals(rhs)); - } + suppliers.addAll( + TestCaseSupplier.stringCases( + Object::equals, + (lhsType, rhsType) -> "EqualsKeywordsEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]", + List.of(), + DataTypes.BOOLEAN + ) + ); - @Override - protected BinaryComparison build(Source source, Expression lhs, Expression rhs) { - return new Equals(source, lhs, rhs); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "EqualsGeometriesEvaluator", + "lhs", + "rhs", + Object::equals, + DataTypes.BOOLEAN, + TestCaseSupplier.geoPointCases(), + TestCaseSupplier.geoPointCases(), + List.of(), + false + ) + ); + + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "EqualsGeometriesEvaluator", + "lhs", + "rhs", + Object::equals, + DataTypes.BOOLEAN, + TestCaseSupplier.geoShapeCases(), + TestCaseSupplier.geoShapeCases(), + List.of(), + false + ) + ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "EqualsGeometriesEvaluator", + "lhs", + "rhs", + Object::equals, + DataTypes.BOOLEAN, + TestCaseSupplier.cartesianPointCases(), + TestCaseSupplier.cartesianPointCases(), + List.of(), + false + ) + ); + + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "EqualsGeometriesEvaluator", + "lhs", + "rhs", + Object::equals, + DataTypes.BOOLEAN, + TestCaseSupplier.cartesianShapeCases(), + TestCaseSupplier.cartesianShapeCases(), + List.of(), + false + ) + ); + + return parameterSuppliersFromTypedData( + errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), AbstractFunctionTestCase::errorMessageStringForBinaryOperators) + ); } @Override - protected boolean isEquality() { - return true; + protected Expression build(Source source, List args) { + return new Equals(source, args.get(0), args.get(1)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java index ad8dba7d6306..f45dedff837c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java @@ -11,26 +11,25 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.GreaterThanOrEqual; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; -import org.hamcrest.Matcher; -import java.time.ZoneOffset; import java.util.List; import java.util.function.Supplier; import static org.hamcrest.Matchers.equalTo; -public class GreaterThanOrEqualTests extends AbstractBinaryComparisonTestCase { +public class GreaterThanOrEqualTests extends AbstractFunctionTestCase { public GreaterThanOrEqualTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); } @ParametersFactory public static Iterable parameters() { + // ToDo: Add the full set of typed test cases here return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("Int >= Int", () -> { int rhs = randomInt(); int lhs = randomInt(); @@ -47,17 +46,7 @@ public static Iterable parameters() { } @Override - protected > Matcher resultMatcher(T lhs, T rhs) { - return equalTo(lhs.compareTo(rhs) >= 0); - } - - @Override - protected BinaryComparison build(Source source, Expression lhs, Expression rhs) { - return new GreaterThanOrEqual(source, lhs, rhs, ZoneOffset.UTC); - } - - @Override - protected boolean isEquality() { - return false; + protected Expression build(Source source, List args) { + return new GreaterThanOrEqual(source, args.get(0), args.get(1)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java index b631a742f788..e872af5b7c77 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java @@ -11,26 +11,25 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.GreaterThan; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; -import org.hamcrest.Matcher; -import java.time.ZoneOffset; import java.util.List; import java.util.function.Supplier; import static org.hamcrest.Matchers.equalTo; -public class GreaterThanTests extends AbstractBinaryComparisonTestCase { +public class GreaterThanTests extends AbstractFunctionTestCase { public GreaterThanTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); } @ParametersFactory public static Iterable parameters() { + // ToDo: Add the full set of typed test cases here return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("Int > Int", () -> { int rhs = randomInt(); int lhs = randomInt(); @@ -47,17 +46,7 @@ public static Iterable parameters() { } @Override - protected > Matcher resultMatcher(T lhs, T rhs) { - return equalTo(lhs.compareTo(rhs) > 0); - } - - @Override - protected BinaryComparison build(Source source, Expression lhs, Expression rhs) { - return new GreaterThan(source, lhs, rhs, ZoneOffset.UTC); - } - - @Override - protected boolean isEquality() { - return false; + protected Expression build(Source source, List args) { + return new GreaterThan(source, args.get(0), args.get(1)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java index 7864a0dda9fe..8bba0c4a5afb 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java @@ -11,20 +11,18 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.LessThanOrEqual; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; -import org.hamcrest.Matcher; -import java.time.ZoneOffset; import java.util.List; import java.util.function.Supplier; import static org.hamcrest.Matchers.equalTo; -public class LessThanOrEqualTests extends AbstractBinaryComparisonTestCase { +public class LessThanOrEqualTests extends AbstractFunctionTestCase { public LessThanOrEqualTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); } @@ -47,17 +45,7 @@ public static Iterable parameters() { } @Override - protected > Matcher resultMatcher(T lhs, T rhs) { - return equalTo(lhs.compareTo(rhs) <= 0); - } - - @Override - protected BinaryComparison build(Source source, Expression lhs, Expression rhs) { - return new LessThanOrEqual(source, lhs, rhs, ZoneOffset.UTC); - } - - @Override - protected boolean isEquality() { - return false; + protected Expression build(Source source, List args) { + return new LessThanOrEqual(source, args.get(0), args.get(1), null); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java index 826e88551077..ab726dc51fbe 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java @@ -11,20 +11,18 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.LessThan; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; -import org.hamcrest.Matcher; -import java.time.ZoneOffset; import java.util.List; import java.util.function.Supplier; import static org.hamcrest.Matchers.equalTo; -public class LessThanTests extends AbstractBinaryComparisonTestCase { +public class LessThanTests extends AbstractFunctionTestCase { public LessThanTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); } @@ -47,17 +45,7 @@ public static Iterable parameters() { } @Override - protected > Matcher resultMatcher(T lhs, T rhs) { - return equalTo(lhs.compareTo(rhs) < 0); - } - - @Override - protected BinaryComparison build(Source source, Expression lhs, Expression rhs) { - return new LessThan(source, lhs, rhs, ZoneOffset.UTC); - } - - @Override - protected boolean isEquality() { - return false; + protected Expression build(Source source, List args) { + return new LessThan(source, args.get(0), args.get(1), null); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java index 0d6bb32fe248..ec5d2338adae 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java @@ -11,53 +11,192 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.xpack.esql.evaluator.predicate.operator.comparison.NotEquals; +import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.ql.expression.Expression; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; -import org.hamcrest.Matcher; -import java.time.ZoneOffset; +import java.math.BigInteger; +import java.util.ArrayList; import java.util.List; import java.util.function.Supplier; -import static org.hamcrest.Matchers.equalTo; - -public class NotEqualsTests extends AbstractBinaryComparisonTestCase { +public class NotEqualsTests extends AbstractFunctionTestCase { public NotEqualsTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); } @ParametersFactory public static Iterable parameters() { - return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("Int != Int", () -> { - int rhs = randomInt(); - int lhs = randomInt(); - return new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(lhs, DataTypes.INTEGER, "lhs"), - new TestCaseSupplier.TypedData(rhs, DataTypes.INTEGER, "rhs") + List suppliers = new ArrayList<>(); + suppliers.addAll( + + TestCaseSupplier.forBinaryComparisonWithWidening( + new TestCaseSupplier.NumericTypeTestConfigs<>( + new TestCaseSupplier.NumericTypeTestConfig<>( + (Integer.MIN_VALUE >> 1) - 1, + (Integer.MAX_VALUE >> 1) - 1, + (l, r) -> l.intValue() != r.intValue(), + "NotEqualsIntsEvaluator" + ), + new TestCaseSupplier.NumericTypeTestConfig<>( + (Long.MIN_VALUE >> 1) - 1, + (Long.MAX_VALUE >> 1) - 1, + (l, r) -> l.longValue() != r.longValue(), + "NotEqualsLongsEvaluator" + ), + new TestCaseSupplier.NumericTypeTestConfig<>( + Double.NEGATIVE_INFINITY, + Double.POSITIVE_INFINITY, + // NB: this has different behavior than Double::equals + (l, r) -> l.doubleValue() != r.doubleValue(), + "NotEqualsDoublesEvaluator" + ) ), - "NotEqualsIntsEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]", + "lhs", + "rhs", + (lhs, rhs) -> List.of(), + false + ) + ); + // Unsigned Long cases + // TODO: These should be integrated into the type cross product above, but are currently broken + // see https://github.com/elastic/elasticsearch/issues/102935 + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "NotEqualsLongsEvaluator", + "lhs", + "rhs", + (l, r) -> false == l.equals(r), DataTypes.BOOLEAN, - equalTo(lhs != rhs) - ); - }))); - } - - @Override - protected > Matcher resultMatcher(T lhs, T rhs) { - return equalTo(false == lhs.equals(rhs)); - } - - @Override - protected BinaryComparison build(Source source, Expression lhs, Expression rhs) { - return new NotEquals(source, lhs, rhs, ZoneOffset.UTC); + TestCaseSupplier.ulongCases(BigInteger.ZERO, BigInteger.valueOf(Long.MAX_VALUE), true), + TestCaseSupplier.ulongCases(BigInteger.ZERO, BigInteger.valueOf(Long.MAX_VALUE), true), + List.of(), + true + ) + ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "NotEqualsBoolsEvaluator", + "lhs", + "rhs", + (l, r) -> false == l.equals(r), + DataTypes.BOOLEAN, + TestCaseSupplier.booleanCases(), + TestCaseSupplier.booleanCases(), + List.of(), + false + ) + ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "NotEqualsKeywordsEvaluator", + "lhs", + "rhs", + (l, r) -> false == l.equals(r), + DataTypes.BOOLEAN, + TestCaseSupplier.ipCases(), + TestCaseSupplier.ipCases(), + List.of(), + false + ) + ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "NotEqualsKeywordsEvaluator", + "lhs", + "rhs", + (l, r) -> false == l.equals(r), + DataTypes.BOOLEAN, + TestCaseSupplier.versionCases(""), + TestCaseSupplier.versionCases(""), + List.of(), + false + ) + ); + // Datetime + // TODO: I'm surprised this passes. Shouldn't there be a cast from DateTime to Long? + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "NotEqualsLongsEvaluator", + "lhs", + "rhs", + (l, r) -> false == l.equals(r), + DataTypes.BOOLEAN, + TestCaseSupplier.dateCases(), + TestCaseSupplier.dateCases(), + List.of(), + false + ) + ); + suppliers.addAll( + TestCaseSupplier.stringCases( + (l, r) -> false == l.equals(r), + (lhsType, rhsType) -> "NotEqualsKeywordsEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]", + List.of(), + DataTypes.BOOLEAN + ) + ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "NotEqualsGeometriesEvaluator", + "lhs", + "rhs", + (l, r) -> false == l.equals(r), + DataTypes.BOOLEAN, + TestCaseSupplier.geoPointCases(), + TestCaseSupplier.geoPointCases(), + List.of(), + false + ) + ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "NotEqualsGeometriesEvaluator", + "lhs", + "rhs", + (l, r) -> false == l.equals(r), + DataTypes.BOOLEAN, + TestCaseSupplier.geoShapeCases(), + TestCaseSupplier.geoShapeCases(), + List.of(), + false + ) + ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "NotEqualsGeometriesEvaluator", + "lhs", + "rhs", + (l, r) -> false == l.equals(r), + DataTypes.BOOLEAN, + TestCaseSupplier.cartesianPointCases(), + TestCaseSupplier.cartesianPointCases(), + List.of(), + false + ) + ); + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + "NotEqualsGeometriesEvaluator", + "lhs", + "rhs", + (l, r) -> false == l.equals(r), + DataTypes.BOOLEAN, + TestCaseSupplier.cartesianShapeCases(), + TestCaseSupplier.cartesianShapeCases(), + List.of(), + false + ) + ); + return parameterSuppliersFromTypedData( + errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), AbstractFunctionTestCase::errorMessageStringForBinaryOperators) + ); } @Override - protected boolean isEquality() { - return true; + protected Expression build(Source source, List args) { + return new NotEquals(source, args.get(0), args.get(1)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java index 01fcd222a514..28944252191b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java @@ -495,7 +495,7 @@ public void testPropagateEquals_VarEq2OrVarNeq5() { // a = 2 OR 3 < a < 4 OR a > 2 OR a!= 2 -> TRUE public void testPropagateEquals_VarEq2OrVarRangeGt3Lt4OrVarGt2OrVarNe2() { FieldAttribute fa = getFieldAttribute(); - org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals eq = equalsOf(fa, TWO); + Equals eq = equalsOf(fa, TWO); Range range = rangeOf(fa, THREE, false, FOUR, false); GreaterThan gt = greaterThanOf(fa, TWO); NotEquals neq = notEqualsOf(fa, TWO); From 05a25bfd3c6021a603173d941fa6997259889de2 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Tue, 16 Apr 2024 20:41:59 +0200 Subject: [PATCH 13/43] Avoid allocating 1k buffers for tiny slices/files in BlobCacheBufferedIndexInput (#107461) Looking through some real world heap dumps, there's at times a lot of instances of these things that have a 1k buffer allocated but are only a couple of bytes in length. We can save tens of MB in examined cases by just sizing the buffer smarter here. --- .../common/BlobCacheBufferedIndexInput.java | 19 ++++++++++++++----- .../input/DirectBlobContainerIndexInput.java | 9 +-------- .../input/MetadataCachingIndexInput.java | 11 ++--------- 3 files changed, 17 insertions(+), 22 deletions(-) diff --git a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/common/BlobCacheBufferedIndexInput.java b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/common/BlobCacheBufferedIndexInput.java index 71ee6da24cdc..cc193e8e2cfe 100644 --- a/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/common/BlobCacheBufferedIndexInput.java +++ b/x-pack/plugin/blob-cache/src/main/java/org/elasticsearch/blobcache/common/BlobCacheBufferedIndexInput.java @@ -42,6 +42,8 @@ public abstract class BlobCacheBufferedIndexInput extends IndexInput implements // LUCENE-888 for details. public static final int MERGE_BUFFER_SIZE = 4096; + private final long length; + private final int bufferSize; protected ByteBuffer buffer = EMPTY_BYTEBUFFER; @@ -56,15 +58,22 @@ public final byte readByte() throws IOException { return buffer.get(); } - public BlobCacheBufferedIndexInput(String resourceDesc, IOContext context) { - this(resourceDesc, bufferSize(context)); + public BlobCacheBufferedIndexInput(String resourceDesc, IOContext context, long length) { + this(resourceDesc, bufferSize(context), length); } /** Inits BufferedIndexInput with a specific bufferSize */ - public BlobCacheBufferedIndexInput(String resourceDesc, int bufferSize) { + public BlobCacheBufferedIndexInput(String resourceDesc, int bufferSize, long length) { super(resourceDesc); - checkBufferSize(bufferSize); - this.bufferSize = bufferSize; + int bufSize = Math.max(MIN_BUFFER_SIZE, (int) Math.min(bufferSize, length)); + checkBufferSize(bufSize); + this.bufferSize = bufSize; + this.length = length; + } + + @Override + public final long length() { + return length; } public int getBufferSize() { diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInput.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInput.java index 37b5fd5c14a9..ecc6588ffdf5 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInput.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/DirectBlobContainerIndexInput.java @@ -72,7 +72,6 @@ public final class DirectBlobContainerIndexInput extends BlobCacheBufferedIndexI private final FileInfo fileInfo; private final IndexInputStats stats; private final long offset; - private final long length; // the following are only mutable so they can be adjusted after cloning/slicing private volatile boolean isClone; @@ -101,7 +100,7 @@ private DirectBlobContainerIndexInput( long length, long sequentialReadSize ) { - super(name, bufferSize); // TODO should use blob cache + super(name, bufferSize, length); // TODO should use blob cache this.position = position; assert sequentialReadSize >= 0; this.sequentialReadSize = sequentialReadSize; @@ -111,7 +110,6 @@ private DirectBlobContainerIndexInput( : "this method should only be used with blobs that are NOT stored in metadata's hash field " + "(fileInfo: " + fileInfo + ')'; this.stats = Objects.requireNonNull(stats); this.offset = offset; - this.length = length; this.closed = new AtomicBoolean(false); this.isClone = false; } @@ -332,11 +330,6 @@ public void close() throws IOException { } } - @Override - public long length() { - return length; - } - @Override public String toString() { return super.toString() + "[read seq=" + (streamForSequentialReads != null ? "yes" : "no") + ']'; diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/MetadataCachingIndexInput.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/MetadataCachingIndexInput.java index ff8633bdaad1..cd9aa5aec74c 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/MetadataCachingIndexInput.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/MetadataCachingIndexInput.java @@ -100,7 +100,6 @@ public abstract class MetadataCachingIndexInput extends BlobCacheBufferedIndexIn protected final IOContext context; protected final IndexInputStats stats; private final long offset; - private final long length; // the following are only mutable so they can be adjusted after cloning/slicing private volatile boolean isClone; @@ -122,7 +121,7 @@ protected MetadataCachingIndexInput( ByteRange headerBlobCacheByteRange, ByteRange footerBlobCacheByteRange ) { - super(name, context); + super(name, context, length); this.isCfs = IndexFileNames.matchesExtension(name, "cfs"); this.logger = Objects.requireNonNull(logger); this.fileInfo = Objects.requireNonNull(fileInfo); @@ -131,7 +130,6 @@ protected MetadataCachingIndexInput( : "this method should only be used with blobs that are NOT stored in metadata's hash field " + "(fileInfo: " + fileInfo + ')'; this.stats = Objects.requireNonNull(stats); this.offset = offset; - this.length = length; this.closed = new AtomicBoolean(false); this.isClone = false; this.directory = Objects.requireNonNull(directory); @@ -160,7 +158,7 @@ protected MetadataCachingIndexInput(MetadataCachingIndexInput input) { input.stats, input.offset, input.compoundFileOffset, - input.length, + input.length(), input.cacheFileReference, input.defaultRangeSize, input.recoveryRangeSize, @@ -645,11 +643,6 @@ public final void close() throws IOException { } } - @Override - public final long length() { - return length; - } - @Override public MetadataCachingIndexInput clone() { final MetadataCachingIndexInput clone = (MetadataCachingIndexInput) super.clone(); From 75badc2f2f40d29da0ac15d1ebb1eb10a4851b89 Mon Sep 17 00:00:00 2001 From: Carlos Delgado <6339205+carlosdelest@users.noreply.github.com> Date: Tue, 16 Apr 2024 20:48:53 +0200 Subject: [PATCH 14/43] Fix IpScriptFieldType CI - #106900 (#107440) I've been tracing the problems with these tests (as the [first attempt I made](https://github.com/elastic/elasticsearch/pull/107066) was unrelated to the actual bug). I discovered that the actual problem was that the `BytesRefHash` for `terms` in the `IpScriptFieldTermsQuery` was not finding terms that were actually there. The seed that was used to reproduce this failure was triggering multiple slices for performing the search. As `BytesRefHash` is not a threadsafe class, that made me think about some kind of synchronization issue with the underlying `BytesRefHash` structure for the `IpScriptFieldTermsQuery` Adding a `synchronized` block to the `terms` on access removed the problem. I've tried to reproduce the issue with > 90k iterations of the tests and have been unable to reproduce it. Closes #106900 --- .../common/util/BytesRefHash.java | 30 +++++++++++++++- .../runtime/AbstractIpScriptFieldQuery.java | 2 +- .../runtime/IpScriptFieldTermsQuery.java | 34 +++++++++++++++++-- .../index/mapper/IpScriptFieldTypeTests.java | 1 - .../runtime/IpScriptFieldTermsQueryTests.java | 30 ++++++++++++---- 5 files changed, 84 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/util/BytesRefHash.java b/server/src/main/java/org/elasticsearch/common/util/BytesRefHash.java index 8ebc920af961..9e0d14d6bfb2 100644 --- a/server/src/main/java/org/elasticsearch/common/util/BytesRefHash.java +++ b/server/src/main/java/org/elasticsearch/common/util/BytesRefHash.java @@ -133,10 +133,14 @@ public BytesRef get(long id, BytesRef dest) { * Get the id associated with key */ public long find(BytesRef key, int code) { + return find(key, code, spare); + } + + private long find(BytesRef key, int code, BytesRef intermediate) { final long slot = slot(rehash(code), mask); for (long index = slot;; index = nextSlot(index, mask)) { final long id = id(index); - if (id == -1L || key.bytesEquals(get(id, spare))) { + if (id == -1L || key.bytesEquals(get(id, intermediate))) { return id; } } @@ -147,6 +151,15 @@ public long find(BytesRef key) { return find(key, key.hashCode()); } + /** + * Allows finding a key in the hash in a thread safe manner, by providing an intermediate + * BytesRef reference to storing intermediate results. As long as each thread provides + * its own intermediate instance, this method is thread safe. + */ + private long threadSafeFind(BytesRef key, BytesRef intermediate) { + return find(key, key.hashCode(), intermediate); + } + private long set(BytesRef key, int code, long id) { assert rehash(key.hashCode()) == code; assert size < maxSize; @@ -236,4 +249,19 @@ public long ramBytesUsed() { return BASE_RAM_BYTES_USED + bytesRefs.ramBytesUsed() + ids.ramBytesUsed() + hashes.ramBytesUsed() + spare.bytes.length; } + /** + * Returns a finder class that can be used to find keys in the hash in a thread-safe manner + */ + public Finder newFinder() { + return new Finder(); + } + + public class Finder { + private final BytesRef intermediate = new BytesRef(); + + public long find(BytesRef key) { + return threadSafeFind(key, intermediate); + } + } + } diff --git a/server/src/main/java/org/elasticsearch/search/runtime/AbstractIpScriptFieldQuery.java b/server/src/main/java/org/elasticsearch/search/runtime/AbstractIpScriptFieldQuery.java index 52b81a3e613c..2d4df68cd83d 100644 --- a/server/src/main/java/org/elasticsearch/search/runtime/AbstractIpScriptFieldQuery.java +++ b/server/src/main/java/org/elasticsearch/search/runtime/AbstractIpScriptFieldQuery.java @@ -36,7 +36,7 @@ protected final boolean matches(IpFieldScript scriptContext, int docId) { /** * Does the value match this query? */ - protected abstract boolean matches(BytesRef[] values, int conut); + protected abstract boolean matches(BytesRef[] values, int count); protected static InetAddress decode(BytesRef ref) { return InetAddressPoint.decode(BytesReference.toBytes(new BytesArray(ref))); diff --git a/server/src/main/java/org/elasticsearch/search/runtime/IpScriptFieldTermsQuery.java b/server/src/main/java/org/elasticsearch/search/runtime/IpScriptFieldTermsQuery.java index b1f5d99b1ca4..06bce89cea02 100644 --- a/server/src/main/java/org/elasticsearch/search/runtime/IpScriptFieldTermsQuery.java +++ b/server/src/main/java/org/elasticsearch/search/runtime/IpScriptFieldTermsQuery.java @@ -8,6 +8,8 @@ package org.elasticsearch.search.runtime; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.TwoPhaseIterator; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.common.util.BytesRefHash; @@ -24,16 +26,42 @@ public IpScriptFieldTermsQuery(Script script, IpFieldScript.LeafFactory leafFact this.terms = terms; } - @Override - protected boolean matches(BytesRef[] values, int count) { + boolean matches(BytesRef[] values, int count, BytesRefHash.Finder finder) { for (int i = 0; i < count; i++) { - if (terms.find(values[i]) >= 0) { + if (finder.find(values[i]) >= 0) { return true; } } return false; } + @Override + protected final boolean matches(BytesRef[] values, int count) { + throw new UnsupportedOperationException("This leads to non-thread safe usage of BytesRefHash; use createTwoPhaseIterator instead"); + } + + boolean matches(IpFieldScript scriptContext, int docId, BytesRefHash.Finder finder) { + scriptContext.runForDoc(docId); + return matches(scriptContext.values(), scriptContext.count(), finder); + } + + protected final TwoPhaseIterator createTwoPhaseIterator(IpFieldScript scriptContext, DocIdSetIterator approximation) { + return new TwoPhaseIterator(approximation) { + private final BytesRefHash.Finder finder = terms.newFinder(); + + @Override + public boolean matches() { + // We need to use a thread safe finder, as this can be called from multiple threads + return IpScriptFieldTermsQuery.this.matches(scriptContext, approximation.docID(), finder); + } + + @Override + public float matchCost() { + return MATCH_COST; + } + }; + } + @Override public final String toString(String field) { StringBuilder b = new StringBuilder(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java index bf16456c7476..4726424ada5f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java @@ -228,7 +228,6 @@ protected Query randomTermQuery(MappedFieldType ft, SearchExecutionContext ctx) } @Override - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/106900") public void testTermsQuery() throws IOException { try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { addDocument(iw, List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"192.168.0.1\"]}")))); diff --git a/server/src/test/java/org/elasticsearch/search/runtime/IpScriptFieldTermsQueryTests.java b/server/src/test/java/org/elasticsearch/search/runtime/IpScriptFieldTermsQueryTests.java index 369672d6aabd..3ba704411b5a 100644 --- a/server/src/test/java/org/elasticsearch/search/runtime/IpScriptFieldTermsQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/runtime/IpScriptFieldTermsQueryTests.java @@ -15,19 +15,34 @@ import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.BytesRefHash; +import org.elasticsearch.script.IpFieldScript; import org.elasticsearch.script.Script; +import java.net.InetAddress; + import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.startsWith; +import static org.mockito.Mockito.mock; -public class IpScriptFieldTermsQueryTests extends AbstractIpScriptFieldQueryTestCase { +public class IpScriptFieldTermsQueryTests extends AbstractScriptFieldQueryTestCase { @Override protected IpScriptFieldTermsQuery createTestInstance() { return createTestInstance(between(1, 100)); } + protected final IpFieldScript.LeafFactory leafFactory = mock(IpFieldScript.LeafFactory.class); + + @Override + public final void testVisit() { + assertEmptyVisit(); + } + + protected static BytesRef encode(InetAddress addr) { + return new BytesRef(InetAddressPoint.encode(addr)); + } + private IpScriptFieldTermsQuery createTestInstance(int size) { BytesRefHash terms = new BytesRefHash(size, BigArrays.NON_RECYCLING_INSTANCE); while (terms.size() < size) { @@ -80,12 +95,13 @@ public void testMatches() { terms.add(ip1); terms.add(ip2); IpScriptFieldTermsQuery query = new IpScriptFieldTermsQuery(randomScript(), leafFactory, "test", terms); - assertTrue(query.matches(new BytesRef[] { ip1 }, 1)); - assertTrue(query.matches(new BytesRef[] { ip2 }, 1)); - assertTrue(query.matches(new BytesRef[] { ip1, notIp }, 2)); - assertTrue(query.matches(new BytesRef[] { notIp, ip1 }, 2)); - assertFalse(query.matches(new BytesRef[] { notIp }, 1)); - assertFalse(query.matches(new BytesRef[] { notIp, ip1 }, 1)); + BytesRefHash.Finder finder = terms.newFinder(); + assertTrue(query.matches(new BytesRef[] { ip1 }, 1, finder)); + assertTrue(query.matches(new BytesRef[] { ip2 }, 1, finder)); + assertTrue(query.matches(new BytesRef[] { ip1, notIp }, 2, finder)); + assertTrue(query.matches(new BytesRef[] { notIp, ip1 }, 2, finder)); + assertFalse(query.matches(new BytesRef[] { notIp }, 1, finder)); + assertFalse(query.matches(new BytesRef[] { notIp, ip1 }, 1, finder)); } @Override From ac5f5640e54f1188d26cabc5d291f81b145d4d4d Mon Sep 17 00:00:00 2001 From: shainaraskas <58563081+shainaraskas@users.noreply.github.com> Date: Tue, 16 Apr 2024 14:55:55 -0400 Subject: [PATCH 15/43] [DOCS] Update tutorial to discourage editing managed ILM policies (#107074) --- .../example-index-lifecycle-policy.asciidoc | 70 +++++++++++++------ 1 file changed, 49 insertions(+), 21 deletions(-) diff --git a/docs/reference/ilm/example-index-lifecycle-policy.asciidoc b/docs/reference/ilm/example-index-lifecycle-policy.asciidoc index 91a5283a9b63..6ec261fabc44 100644 --- a/docs/reference/ilm/example-index-lifecycle-policy.asciidoc +++ b/docs/reference/ilm/example-index-lifecycle-policy.asciidoc @@ -1,5 +1,3 @@ -[role="xpack"] - [[example-using-index-lifecycle-policy]] === Tutorial: Customize built-in {ilm-init} policies @@ -9,9 +7,9 @@ {es} includes the following built-in {ilm-init} policies: -- `logs` -- `metrics` -- `synthetics` +- `logs@lifecycle` +- `metrics@lifecycle` +- `synthetics@lifecycle` {agent} uses these policies to manage backing indices for its data streams. This tutorial shows you how to use {kib}’s **Index Lifecycle Policies** to @@ -67,20 +65,25 @@ node.roles: [ data_warm ] cluster. [discrete] -[[example-using-index-lifecycle-policy-view-ilm-policy]] -==== View the policy +[[example-using-index-lifecycle-policy-duplicate-ilm-policy]] +==== Duplicate the policy {agent} uses data streams with an index pattern of `logs-*-*` to store log -monitoring data. The built-in `logs` {ilm-init} policy automatically manages -backing indices for these data streams. +monitoring data. The managed `logs@lifecycle` {ilm-init} policy automatically manages +backing indices for these data streams. + +If you don't want to use the policy defaults, then you can customize the managed policy and then save it as a new policy. You can then use the new policy in related component templates and index templates. + +CAUTION: You should never edit managed policies directly. Changes to managed policies might be rolled back or overwritten. -To view the `logs` policy in {kib}: +To save the `logs@lifecycle` policy as a new policy in {kib}: -. Open the menu and go to **Stack Management > Index Lifecycle Policies**. -. Select **Include managed system policies**. -. Select the `logs` policy. +. Open the menu and go to **Stack Management** > **Index Lifecycle Policies**. +. Toggle **Include managed system policies**. +. Select the `logs@lifecycle` policy. +. On the **Edit policy logs** page, toggle **Save as new policy**, and then provide a new name for the policy, for example, `logs-custom`. -The `logs` policy uses the recommended rollover defaults: Start writing to a new +The `logs@lifecycle` policy uses the recommended rollover defaults: Start writing to a new index when the current write index reaches 50GB or becomes 30 days old. To view or change the rollover settings, click **Advanced settings** for the hot @@ -90,16 +93,12 @@ settings. [role="screenshot"] image::images/ilm/tutorial-ilm-hotphaserollover-default.png[View rollover defaults] -Note that {kib} displays a warning that editing a managed policy can break -Kibana. For this tutorial, you can ignore that warning and proceed with -modifying the policy. - [discrete] [[ilm-ex-modify-policy]] ==== Modify the policy -The default `logs` policy is designed to prevent the creation of many tiny daily -indices. You can modify the policy to meet your performance requirements and +The default `logs@lifecycle` policy is designed to prevent the creation of many tiny daily +indices. You can modify your copy of the policy to meet your performance requirements and manage resource usage. . Activate the warm phase and click **Advanced settings**. @@ -127,4 +126,33 @@ deletes indices 90 days after rollover. [role="screenshot"] image::images/ilm/tutorial-ilm-delete-rollover.png[Add a delete phase] -. Click **Save Policy**. +. Click **Save as new policy**. + +TIP: Copies of managed {ilm-init} policies are also marked as **Managed**. You can use the <> to update the `_meta.managed` parameter to `false`. + +[discrete] +[[example-using-index-lifecycle-policy-apply-policy]] +==== Apply the policy + +To apply your new {ilm-init} policy to the `logs` index template, create or edit the `logs@custom` component template. + +A `*@custom` component template allows you to customize the mappings and settings of managed index templates, without having to override managed index templates or component templates. This type of component template is automatically picked up by the index template. <>. + +. Click on the **Component Template** tab and click **Create component template**. +. Under **Logistics**, name the component template `logs@custom`. +. Under **Index settings**, set the {ilm-init} policy name created in the previous step: ++ +[source,JSON] +-------------------------------------------------- +{ + "index": { + "lifecycle": { + "name": "logs-custom" + } + } +} +-------------------------------------------------- ++ +. Continue to **Review**, and then click **Save component template**. +. Click the **Index Templates**, tab, and then select the `logs` index template. +. In the summary, view the **Component templates** list. `logs@custom` should be listed. \ No newline at end of file From 5a2dc6dc00e18176f01e59fbad38ffee7b39274a Mon Sep 17 00:00:00 2001 From: William Brafford Date: Tue, 16 Apr 2024 17:11:45 -0400 Subject: [PATCH 16/43] Fix system index thread pool tests (#107443) * Allow rejected executions when filling up a thread pool queue * Move test to integration tests * Avoid setting maxConcurrentShardRequests to 1 * Test all index descriptors defined in the Kibana plugin --- modules/kibana/build.gradle | 1 + .../kibana/KibanaThreadPoolIT.java | 64 +++++++++++++++++++ .../kibana/KibanaThreadPoolTests.java | 52 --------------- ...ava => SystemIndexThreadPoolTestCase.java} | 20 ++++-- 4 files changed, 80 insertions(+), 57 deletions(-) create mode 100644 modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java delete mode 100644 modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaThreadPoolTests.java rename test/framework/src/main/java/org/elasticsearch/indices/{SystemIndexThreadPoolTests.java => SystemIndexThreadPoolTestCase.java} (81%) diff --git a/modules/kibana/build.gradle b/modules/kibana/build.gradle index b38b446c67fd..a4ff51e99069 100644 --- a/modules/kibana/build.gradle +++ b/modules/kibana/build.gradle @@ -5,6 +5,7 @@ * in compliance with, at your election, the Elastic License 2.0 or the Server * Side Public License, v 1. */ +apply plugin: 'elasticsearch.internal-cluster-test' apply plugin: 'elasticsearch.internal-java-rest-test' esplugin { diff --git a/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java b/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java new file mode 100644 index 000000000000..317bfa9edd1c --- /dev/null +++ b/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.kibana; + +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.indices.SystemIndexThreadPoolTestCase; +import org.elasticsearch.plugins.Plugin; + +import java.util.Collection; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; + +public class KibanaThreadPoolIT extends SystemIndexThreadPoolTestCase { + + @Override + protected Collection> nodePlugins() { + return Set.of(KibanaPlugin.class); + } + + public void testKibanaThreadPool() { + List kibanaSystemIndices = Stream.of( + KibanaPlugin.KIBANA_INDEX_DESCRIPTOR.getIndexPattern(), + KibanaPlugin.REPORTING_INDEX_DESCRIPTOR.getIndexPattern(), + KibanaPlugin.APM_AGENT_CONFIG_INDEX_DESCRIPTOR.getIndexPattern(), + KibanaPlugin.APM_CUSTOM_LINK_INDEX_DESCRIPTOR.getIndexPattern() + ).map(s -> s.replace("*", randomAlphaOfLength(8).toLowerCase(Locale.ROOT))).toList(); + + runWithBlockedThreadPools(() -> { + for (String index : kibanaSystemIndices) { + // index documents + String idToDelete = client().prepareIndex(index).setSource(Map.of("foo", "delete me!")).get().getId(); + String idToUpdate = client().prepareIndex(index).setSource(Map.of("foo", "update me!")).get().getId(); + + // bulk index, delete, and update + Client bulkClient = client(); + BulkResponse response = bulkClient.prepareBulk(index) + .add(bulkClient.prepareIndex(index).setSource(Map.of("foo", "search me!"))) + .add(bulkClient.prepareDelete(index, idToDelete)) + .add(bulkClient.prepareUpdate().setId(idToUpdate).setDoc(Map.of("foo", "I'm updated!"))) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .get(); + assertNoFailures(response); + + // match-all search + assertHitCount(client().prepareSearch(index).setQuery(QueryBuilders.matchAllQuery()), 2); + } + }); + } +} diff --git a/modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaThreadPoolTests.java b/modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaThreadPoolTests.java deleted file mode 100644 index 0974fd6d36b1..000000000000 --- a/modules/kibana/src/test/java/org/elasticsearch/kibana/KibanaThreadPoolTests.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.kibana; - -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.support.WriteRequest; -import org.elasticsearch.client.internal.Client; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.indices.SystemIndexThreadPoolTests; -import org.elasticsearch.plugins.Plugin; - -import java.util.Collection; -import java.util.Map; -import java.util.Set; - -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; - -public class KibanaThreadPoolTests extends SystemIndexThreadPoolTests { - - @Override - protected Collection> nodePlugins() { - return Set.of(KibanaPlugin.class); - } - - public void testKibanaThreadPool() { - runWithBlockedThreadPools(() -> { - // index documents - String idToDelete = client().prepareIndex(".kibana").setSource(Map.of("foo", "delete me!")).get().getId(); - String idToUpdate = client().prepareIndex(".kibana").setSource(Map.of("foo", "update me!")).get().getId(); - - // bulk index, delete, and update - Client bulkClient = client(); - BulkResponse response = bulkClient.prepareBulk(".kibana") - .add(bulkClient.prepareIndex(".kibana").setSource(Map.of("foo", "search me!"))) - .add(bulkClient.prepareDelete(".kibana", idToDelete)) - .add(bulkClient.prepareUpdate().setId(idToUpdate).setDoc(Map.of("foo", "I'm updated!"))) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .get(); - assertNoFailures(response); - - // match-all search - assertHitCount(client().prepareSearch(".kibana").setQuery(QueryBuilders.matchAllQuery()), 2); - }); - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/indices/SystemIndexThreadPoolTests.java b/test/framework/src/main/java/org/elasticsearch/indices/SystemIndexThreadPoolTestCase.java similarity index 81% rename from test/framework/src/main/java/org/elasticsearch/indices/SystemIndexThreadPoolTests.java rename to test/framework/src/main/java/org/elasticsearch/indices/SystemIndexThreadPoolTestCase.java index 7db5d10c5fcf..b33320539b57 100644 --- a/test/framework/src/main/java/org/elasticsearch/indices/SystemIndexThreadPoolTests.java +++ b/test/framework/src/main/java/org/elasticsearch/indices/SystemIndexThreadPoolTestCase.java @@ -9,6 +9,7 @@ package org.elasticsearch.indices; import org.elasticsearch.action.search.SearchPhaseExecutionException; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESIntegTestCase; @@ -19,6 +20,7 @@ import java.util.concurrent.Phaser; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.startsWith; /** @@ -31,7 +33,7 @@ *

When implementing this class, don't forget to override {@link ESIntegTestCase#nodePlugins()} if * the relevant system index is defined in a plugin.

*/ -public abstract class SystemIndexThreadPoolTests extends ESIntegTestCase { +public abstract class SystemIndexThreadPoolTestCase extends ESIntegTestCase { private static final String USER_INDEX = "user_index"; @@ -67,7 +69,6 @@ protected void runWithBlockedThreadPools(Runnable runnable) { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/106957") public void testUserThreadPoolsAreBlocked() { assertAcked(client().admin().indices().prepareCreate(USER_INDEX)); @@ -88,9 +89,13 @@ private void assertThreadPoolsBlocked() { assertThat(e2.getMessage(), startsWith("rejected execution of ActionRunnable")); var e3 = expectThrows( SearchPhaseExecutionException.class, - () -> client().prepareSearch(USER_INDEX).setQuery(QueryBuilders.matchAllQuery()).get() + () -> client().prepareSearch(USER_INDEX) + .setQuery(QueryBuilders.matchAllQuery()) + // Request times out if max concurrent shard requests is set to 1 + .setMaxConcurrentShardRequests(usually() ? SearchRequest.DEFAULT_MAX_CONCURRENT_SHARD_REQUESTS : randomIntBetween(2, 10)) + .get() ); - assertThat(e3.getMessage(), startsWith("all shards failed")); + assertThat(e3.getMessage(), containsString("all shards failed")); } private void fillThreadPoolQueues() { @@ -101,7 +106,12 @@ private void fillThreadPoolQueues() { // fill up the queue for (int i = 0; i < info.getQueueSize().singles(); i++) { - threadPool.executor(threadPoolName).submit(() -> {}); + try { + threadPool.executor(threadPoolName).submit(() -> {}); + } catch (EsRejectedExecutionException e) { + // we can't be sure that some other task won't get queued in a test cluster + // but we should put all the tasks in there anyway + } } } } From f1f271f6336ff3c692ff44639e246f0d3a03b209 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Tue, 16 Apr 2024 14:56:15 -0700 Subject: [PATCH 17/43] [TEST] Unmute TextFieldMapperTests#testBlockLoaderParentFromRowStrideReader (#107548) Randomly stumbled upon this - #104158 which is referenced is closed so this can be unmuted. --- .../org/elasticsearch/index/mapper/TextFieldMapperTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 1c5ae3baca82..5d0c1c01ecdc 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -1454,7 +1454,6 @@ public void testBlockLoaderFromParentColumnReader() throws IOException { testBlockLoaderFromParent(true, randomBoolean()); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/104158") public void testBlockLoaderParentFromRowStrideReader() throws IOException { testBlockLoaderFromParent(false, randomBoolean()); } From f5c7938ab8043383aebeac0998a40967da55f924 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 16 Apr 2024 16:57:14 -0500 Subject: [PATCH 18/43] Adding cache_stats to geoip stats API (#107334) --- docs/changelog/107334.yaml | 5 + .../ingest/apis/geoip-stats-api.asciidoc | 37 +++++++- .../ingest/geoip/DatabaseNodeService.java | 5 + .../ingest/geoip/GeoIpCache.java | 38 +++++++- .../ingest/geoip/stats/CacheStats.java | 41 +++++++++ .../ingest/geoip/stats/GeoIpStatsAction.java | 26 +++++- .../stats/GeoIpStatsTransportAction.java | 2 + .../ingest/geoip/GeoIpCacheTests.java | 33 +++++++ .../stats/CacheStatsSerializingTests.java | 92 +++++++++++++++++++ ...atsActionNodeResponseSerializingTests.java | 1 + .../GeoIpStatsActionNodeResponseTests.java | 1 + .../test/ingest_geoip/30_geoip_stats.yml | 8 ++ .../org/elasticsearch/TransportVersions.java | 1 + 13 files changed, 287 insertions(+), 3 deletions(-) create mode 100644 docs/changelog/107334.yaml create mode 100644 modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/CacheStats.java create mode 100644 modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/CacheStatsSerializingTests.java diff --git a/docs/changelog/107334.yaml b/docs/changelog/107334.yaml new file mode 100644 index 000000000000..d1e8df2fa9c4 --- /dev/null +++ b/docs/changelog/107334.yaml @@ -0,0 +1,5 @@ +pr: 107334 +summary: Adding `cache_stats` to geoip stats API +area: Ingest Node +type: enhancement +issues: [] diff --git a/docs/reference/ingest/apis/geoip-stats-api.asciidoc b/docs/reference/ingest/apis/geoip-stats-api.asciidoc index 84a2b00737e5..4917441c0020 100644 --- a/docs/reference/ingest/apis/geoip-stats-api.asciidoc +++ b/docs/reference/ingest/apis/geoip-stats-api.asciidoc @@ -24,7 +24,7 @@ GET _ingest/geoip/stats `manage` <> to use this API. * If <> is -disabled, this API returns zero values and an empty `nodes` object. +disabled and no custom databases are configured, this API returns zero values and an empty `nodes` object. [role="child_attributes"] [[geoip-stats-api-response-body]] @@ -83,6 +83,41 @@ Downloaded databases for the node. (string) Name of the database. ====== +`cache_stats`:: +(object) +GeoIP cache stats for the node. ++ +.Properties of `cache_stats` +[%collapsible%open] +====== +`count`:: +(Long) +Number of cached entries. + +`hits`:: +(Long) +The number of enrich lookups served from cache. + +`misses`:: +(Long) +The number of times geoIP lookups couldn't be +served from cache. + +`evictions`:: +(Long) +The number cache entries evicted from the cache. + +`hits_time_in_millis`:: +(Long) +The amount of time in milliseconds spent fetching data from the cache on succesful cache hits only. + +`misses_time_in_millis`:: +(Long) +The amount of time in milliseconds spent fetching data from the cache and the backing GeoIP2 database and updating the +cache, on cache misses only. + +====== + `files_in_temp`:: (array of strings) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java index 540f463be469..266d40f2f9d5 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java @@ -29,6 +29,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.ingest.IngestService; +import org.elasticsearch.ingest.geoip.stats.CacheStats; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.search.SearchHit; import org.elasticsearch.watcher.ResourceWatcherService; @@ -506,4 +507,8 @@ public Set getFilesInTemp() { throw new UncheckedIOException(e); } } + + public CacheStats getCacheStats() { + return cache.getCacheStats(); + } } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java index ff7532562441..fc70e2fbb594 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java @@ -12,10 +12,14 @@ import org.elasticsearch.common.cache.Cache; import org.elasticsearch.common.cache.CacheBuilder; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.ingest.geoip.stats.CacheStats; import java.net.InetAddress; import java.nio.file.Path; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; +import java.util.function.LongSupplier; /** * The in-memory cache for the geoip data. There should only be 1 instance of this class. @@ -38,16 +42,24 @@ public String toString() { } }; + private final LongSupplier relativeNanoTimeProvider; private final Cache cache; + private final AtomicLong hitsTimeInNanos = new AtomicLong(0); + private final AtomicLong missesTimeInNanos = new AtomicLong(0); // package private for testing - GeoIpCache(long maxSize) { + GeoIpCache(long maxSize, LongSupplier relativeNanoTimeProvider) { if (maxSize < 0) { throw new IllegalArgumentException("geoip max cache size must be 0 or greater"); } + this.relativeNanoTimeProvider = relativeNanoTimeProvider; this.cache = CacheBuilder.builder().setMaximumWeight(maxSize).build(); } + GeoIpCache(long maxSize) { + this(maxSize, System::nanoTime); + } + @SuppressWarnings("unchecked") T putIfAbsent( InetAddress ip, @@ -56,11 +68,14 @@ T putIfAbsent( ) { // can't use cache.computeIfAbsent due to the elevated permissions for the jackson (run via the cache loader) CacheKey cacheKey = new CacheKey(ip, databasePath); + long cacheStart = relativeNanoTimeProvider.getAsLong(); // intentionally non-locking for simplicity...it's OK if we re-put the same key/value in the cache during a race condition. AbstractResponse response = cache.get(cacheKey); + long cacheRequestTime = relativeNanoTimeProvider.getAsLong() - cacheStart; // populate the cache for this key, if necessary if (response == null) { + long retrieveStart = relativeNanoTimeProvider.getAsLong(); response = retrieveFunction.apply(ip); // if the response from the database was null, then use the no-result sentinel value if (response == null) { @@ -68,6 +83,10 @@ T putIfAbsent( } // store the result or no-result in the cache cache.put(cacheKey, response); + long databaseRequestAndCachePutTime = relativeNanoTimeProvider.getAsLong() - retrieveStart; + missesTimeInNanos.addAndGet(cacheRequestTime + databaseRequestAndCachePutTime); + } else { + hitsTimeInNanos.addAndGet(cacheRequestTime); } if (response == NO_RESULT) { @@ -99,6 +118,23 @@ public int count() { return cache.count(); } + /** + * Returns stats about this cache as of this moment. There is no guarantee that the counts reconcile (for example hits + misses = count) + * because no locking is performed when requesting these stats. + * @return Current stats about this cache + */ + public CacheStats getCacheStats() { + Cache.CacheStats stats = cache.stats(); + return new CacheStats( + cache.count(), + stats.getHits(), + stats.getMisses(), + stats.getEvictions(), + TimeValue.nsecToMSec(hitsTimeInNanos.get()), + TimeValue.nsecToMSec(missesTimeInNanos.get()) + ); + } + /** * The key to use for the cache. Since this cache can span multiple geoip processors that all use different databases, the database * path is needed to be included in the cache key. For example, if we only used the IP address as the key the City and ASN the same diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/CacheStats.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/CacheStats.java new file mode 100644 index 000000000000..16a234e37e99 --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/CacheStats.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip.stats; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; + +import java.io.IOException; + +public record CacheStats(long count, long hits, long misses, long evictions, long hitsTimeInMillis, long missesTimeInMillis) + implements + Writeable { + + public CacheStats(StreamInput streamInput) throws IOException { + this( + streamInput.readLong(), + streamInput.readLong(), + streamInput.readLong(), + streamInput.readLong(), + streamInput.readLong(), + streamInput.readLong() + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeLong(count); + out.writeLong(hits); + out.writeLong(misses); + out.writeLong(evictions); + out.writeLong(hitsTimeInMillis); + out.writeLong(missesTimeInMillis); + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsAction.java index db1242888ca8..e2aec479a674 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsAction.java @@ -130,6 +130,15 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (response.configDatabases.isEmpty() == false) { builder.array("config_databases", response.configDatabases.toArray(String[]::new)); } + builder.startObject("cache_stats"); + CacheStats cacheStats = response.cacheStats; + builder.field("count", cacheStats.count()); + builder.field("hits", cacheStats.hits()); + builder.field("misses", cacheStats.misses()); + builder.field("evictions", cacheStats.evictions()); + builder.field("hits_time_in_millis", cacheStats.hitsTimeInMillis()); + builder.field("misses_time_in_millis", cacheStats.missesTimeInMillis()); + builder.endObject(); builder.endObject(); } builder.endObject(); @@ -154,6 +163,7 @@ public int hashCode() { public static class NodeResponse extends BaseNodeResponse { private final GeoIpDownloaderStats downloaderStats; + private final CacheStats cacheStats; private final Set databases; private final Set filesInTemp; private final Set configDatabases; @@ -161,6 +171,11 @@ public static class NodeResponse extends BaseNodeResponse { protected NodeResponse(StreamInput in) throws IOException { super(in); downloaderStats = in.readBoolean() ? new GeoIpDownloaderStats(in) : null; + if (in.getTransportVersion().onOrAfter(TransportVersions.GEOIP_CACHE_STATS)) { + cacheStats = in.readBoolean() ? new CacheStats(in) : null; + } else { + cacheStats = null; + } databases = in.readCollectionAsImmutableSet(StreamInput::readString); filesInTemp = in.readCollectionAsImmutableSet(StreamInput::readString); configDatabases = in.getTransportVersion().onOrAfter(TransportVersions.V_8_0_0) @@ -171,12 +186,14 @@ protected NodeResponse(StreamInput in) throws IOException { protected NodeResponse( DiscoveryNode node, GeoIpDownloaderStats downloaderStats, + CacheStats cacheStats, Set databases, Set filesInTemp, Set configDatabases ) { super(node); this.downloaderStats = downloaderStats; + this.cacheStats = cacheStats; this.databases = Set.copyOf(databases); this.filesInTemp = Set.copyOf(filesInTemp); this.configDatabases = Set.copyOf(configDatabases); @@ -205,6 +222,12 @@ public void writeTo(StreamOutput out) throws IOException { if (downloaderStats != null) { downloaderStats.writeTo(out); } + if (out.getTransportVersion().onOrAfter(TransportVersions.GEOIP_CACHE_STATS)) { + out.writeBoolean(cacheStats != null); + if (cacheStats != null) { + cacheStats.writeTo(out); + } + } out.writeStringCollection(databases); out.writeStringCollection(filesInTemp); if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_0_0)) { @@ -218,6 +241,7 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; NodeResponse that = (NodeResponse) o; return downloaderStats.equals(that.downloaderStats) + && Objects.equals(cacheStats, that.cacheStats) && databases.equals(that.databases) && filesInTemp.equals(that.filesInTemp) && Objects.equals(configDatabases, that.configDatabases); @@ -225,7 +249,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(downloaderStats, databases, filesInTemp, configDatabases); + return Objects.hash(downloaderStats, cacheStats, databases, filesInTemp, configDatabases); } } } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsTransportAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsTransportAction.java index 13f9544e1b9e..1e1778a81c4a 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsTransportAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsTransportAction.java @@ -76,9 +76,11 @@ protected NodeResponse newNodeResponse(StreamInput in, DiscoveryNode node) throw protected NodeResponse nodeOperation(NodeRequest request, Task task) { GeoIpDownloader geoIpTask = geoIpDownloaderTaskExecutor.getCurrentTask(); GeoIpDownloaderStats downloaderStats = geoIpTask == null || geoIpTask.getStatus() == null ? null : geoIpTask.getStatus(); + CacheStats cacheStats = registry.getCacheStats(); return new NodeResponse( transportService.getLocalNode(), downloaderStats, + cacheStats, registry.getAvailableDatabases(), registry.getFilesInTemp(), registry.getConfigDatabases() diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpCacheTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpCacheTests.java index d049ca3f9bcd..03ab51d51017 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpCacheTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpCacheTests.java @@ -11,12 +11,16 @@ import com.maxmind.geoip2.model.AbstractResponse; import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.ingest.geoip.stats.CacheStats; import org.elasticsearch.test.ESTestCase; import java.net.InetAddress; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; +import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.mock; public class GeoIpCacheTests extends ESTestCase { @@ -83,4 +87,33 @@ public void testInvalidInit() { IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> new GeoIpCache(-1)); assertEquals("geoip max cache size must be 0 or greater", ex.getMessage()); } + + public void testGetCacheStats() { + final long maxCacheSize = 2; + final AtomicLong testNanoTime = new AtomicLong(0); + // We use a relative time provider that increments 1ms every time it is called. So each operation appears to take 1ms + GeoIpCache cache = new GeoIpCache(maxCacheSize, () -> testNanoTime.addAndGet(TimeValue.timeValueMillis(1).getNanos())); + AbstractResponse response = mock(AbstractResponse.class); + String databasePath = "path/to/db1"; + InetAddress key1 = InetAddresses.forString("127.0.0.1"); + InetAddress key2 = InetAddresses.forString("127.0.0.2"); + InetAddress key3 = InetAddresses.forString("127.0.0.3"); + + cache.putIfAbsent(key1, databasePath, ip -> response); // cache miss + cache.putIfAbsent(key2, databasePath, ip -> response); // cache miss + cache.putIfAbsent(key1, databasePath, ip -> response); // cache hit + cache.putIfAbsent(key1, databasePath, ip -> response); // cache hit + cache.putIfAbsent(key1, databasePath, ip -> response); // cache hit + cache.putIfAbsent(key3, databasePath, ip -> response); // cache miss, key2 will be evicted + cache.putIfAbsent(key2, databasePath, ip -> response); // cache miss, key1 will be evicted + CacheStats cacheStats = cache.getCacheStats(); + assertThat(cacheStats.count(), equalTo(maxCacheSize)); + assertThat(cacheStats.hits(), equalTo(3L)); + assertThat(cacheStats.misses(), equalTo(4L)); + assertThat(cacheStats.evictions(), equalTo(2L)); + // There are 3 hits, each taking 1ms: + assertThat(cacheStats.hitsTimeInMillis(), equalTo(3L)); + // There are 4 misses. Each is made up of a cache query, and a database query, each being 1ms: + assertThat(cacheStats.missesTimeInMillis(), equalTo(8L)); + } } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/CacheStatsSerializingTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/CacheStatsSerializingTests.java new file mode 100644 index 000000000000..fa6b0c366d2d --- /dev/null +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/CacheStatsSerializingTests.java @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.ingest.geoip.stats; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; + +public class CacheStatsSerializingTests extends AbstractWireSerializingTestCase { + @Override + protected Writeable.Reader instanceReader() { + return CacheStats::new; + } + + @Override + protected CacheStats createTestInstance() { + return createRandomInstance(); + } + + @Override + protected CacheStats mutateInstance(CacheStats instance) throws IOException { + long count = instance.count(); + long hits = instance.hits(); + long misses = instance.misses(); + long evictions = instance.evictions(); + long hitsTimeInMillis = instance.hitsTimeInMillis(); + long missesTimeInMillis = instance.missesTimeInMillis(); + return switch (between(0, 5)) { + case 0 -> new CacheStats( + randomValueOtherThan(count, ESTestCase::randomLong), + hits, + misses, + evictions, + hitsTimeInMillis, + missesTimeInMillis + ); + case 1 -> new CacheStats( + count, + randomValueOtherThan(hits, ESTestCase::randomLong), + misses, + evictions, + hitsTimeInMillis, + missesTimeInMillis + ); + case 2 -> new CacheStats( + count, + hits, + randomValueOtherThan(misses, ESTestCase::randomLong), + evictions, + hitsTimeInMillis, + missesTimeInMillis + ); + case 3 -> new CacheStats( + count, + hits, + misses, + randomValueOtherThan(evictions, ESTestCase::randomLong), + hitsTimeInMillis, + missesTimeInMillis + ); + case 4 -> new CacheStats( + count, + hits, + misses, + evictions, + randomValueOtherThan(hitsTimeInMillis, ESTestCase::randomLong), + missesTimeInMillis + ); + case 5 -> new CacheStats( + count, + hits, + misses, + evictions, + hitsTimeInMillis, + randomValueOtherThan(missesTimeInMillis, ESTestCase::randomLong) + ); + default -> throw new IllegalStateException("Unexpected value"); + }; + } + + static CacheStats createRandomInstance() { + return new CacheStats(randomLong(), randomLong(), randomLong(), randomLong(), randomLong(), randomLong()); + } +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsActionNodeResponseSerializingTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsActionNodeResponseSerializingTests.java index 1008dcf56c4f..6ff2e589270e 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsActionNodeResponseSerializingTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsActionNodeResponseSerializingTests.java @@ -40,6 +40,7 @@ static GeoIpStatsAction.NodeResponse createRandomInstance() { return new GeoIpStatsAction.NodeResponse( node, GeoIpDownloaderStatsSerializingTests.createRandomInstance(), + randomBoolean() ? null : CacheStatsSerializingTests.createRandomInstance(), databases, files, configDatabases diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsActionNodeResponseTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsActionNodeResponseTests.java index 27a332c3b42f..919f82a4cbc1 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsActionNodeResponseTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsActionNodeResponseTests.java @@ -28,6 +28,7 @@ public void testInputsAreDefensivelyCopied() { GeoIpStatsAction.NodeResponse nodeResponse = new GeoIpStatsAction.NodeResponse( node, GeoIpDownloaderStatsSerializingTests.createRandomInstance(), + randomBoolean() ? null : CacheStatsSerializingTests.createRandomInstance(), databases, files, configDatabases diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/30_geoip_stats.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/30_geoip_stats.yml index 852b2047a47e..3d28802aeaf6 100644 --- a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/30_geoip_stats.yml +++ b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/30_geoip_stats.yml @@ -8,3 +8,11 @@ - gte: { stats.databases_count: 0 } - gte: { stats.total_download_time: 0 } - is_true: nodes + - set: + nodes._arbitrary_key_: node_id + - gte: { nodes.$node_id.cache_stats.count: 0 } + - gte: { nodes.$node_id.cache_stats.hits: 0 } + - gte: { nodes.$node_id.cache_stats.misses: 0 } + - gte: { nodes.$node_id.cache_stats.evictions: 0 } + - gte: { nodes.$node_id.cache_stats.hits_time_in_millis: 0 } + - gte: { nodes.$node_id.cache_stats.misses_time_in_millis: 0 } diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 72404e405edd..5c5a53603450 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -174,6 +174,7 @@ static TransportVersion def(int id) { public static final TransportVersion TRACK_FLUSH_TIME_EXCLUDING_WAITING_ON_LOCKS = def(8_633_00_0); public static final TransportVersion ML_INFERENCE_AZURE_OPENAI_EMBEDDINGS = def(8_634_00_0); public static final TransportVersion ILM_SHRINK_ENABLE_WRITE = def(8_635_00_0); + public static final TransportVersion GEOIP_CACHE_STATS = def(8_636_00_0); /* * STOP! READ THIS FIRST! No, really, From 84f1b2695327a418d19622019e1f3b55048b375e Mon Sep 17 00:00:00 2001 From: Liam Thompson <32779855+leemthompo@users.noreply.github.com> Date: Wed, 17 Apr 2024 09:54:56 +0200 Subject: [PATCH 19/43] Update esql-version.asciidoc (#107547) Fix heading misalignment and typo. --- docs/reference/esql/esql-version.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/esql/esql-version.asciidoc b/docs/reference/esql/esql-version.asciidoc index 16bf1f66e316..daeb796ecc5b 100644 --- a/docs/reference/esql/esql-version.asciidoc +++ b/docs/reference/esql/esql-version.asciidoc @@ -13,7 +13,7 @@ [discrete] [[esql-versions-explanation]] -=== How versions work +==== How versions work {esql} language versions are independent of {es} versions. Versioning the language ensures that your queries will always @@ -45,5 +45,5 @@ POST /_query?format=txt We won't make breaking changes to released {esql} versions and versions will remain supported until they are deprecated. New features, bug fixes, and performance improvements -will be continue to be added to released {esql} versions, +will continue to be added to released {esql} versions, provided they do not involve breaking changes. From 1d0c470de0a93e8667a7e02780faf24b1c4d5a79 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Wed, 17 Apr 2024 10:55:32 +0200 Subject: [PATCH 20/43] Stop using ReleasableLock in o.e.c.cache.Cache to save O(10M) in heap (#107555) I have a couple heap dumps that show the lock wrapper alone waste O(10M) in heap for these things. Also, I suspect the indirection does cost non-trivial performance here in some cases. => lets spend a couple more lines of code to save that overhead --- .../org/elasticsearch/common/cache/Cache.java | 86 ++++++++++++++----- 1 file changed, 64 insertions(+), 22 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/cache/Cache.java b/server/src/main/java/org/elasticsearch/common/cache/Cache.java index 98a4b90da73d..e380375a29de 100644 --- a/server/src/main/java/org/elasticsearch/common/cache/Cache.java +++ b/server/src/main/java/org/elasticsearch/common/cache/Cache.java @@ -8,7 +8,6 @@ package org.elasticsearch.common.cache; -import org.elasticsearch.common.util.concurrent.ReleasableLock; import org.elasticsearch.core.Tuple; import java.lang.reflect.Array; @@ -19,6 +18,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.LongAdder; +import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -178,10 +178,10 @@ private static final class Entry { */ private final class CacheSegment { // read/write lock protecting mutations to the segment - ReadWriteLock segmentLock = new ReentrantReadWriteLock(); + final ReadWriteLock segmentLock = new ReentrantReadWriteLock(); - ReleasableLock readLock = new ReleasableLock(segmentLock.readLock()); - ReleasableLock writeLock = new ReleasableLock(segmentLock.writeLock()); + final Lock readLock = segmentLock.readLock(); + final Lock writeLock = segmentLock.writeLock(); Map>> map; @@ -196,8 +196,11 @@ private final class CacheSegment { */ Entry get(K key, long now, boolean eagerEvict) { CompletableFuture> future; - try (ReleasableLock ignored = readLock.acquire()) { + readLock.lock(); + try { future = map == null ? null : map.get(key); + } finally { + readLock.unlock(); } if (future != null) { Entry entry; @@ -213,8 +216,11 @@ Entry get(K key, long now, boolean eagerEvict) { if (isExpired(entry, now)) { misses.increment(); if (eagerEvict) { - try (ReleasableLock ignored = lruLock.acquire()) { + lruLock.lock(); + try { evictEntry(entry); + } finally { + lruLock.unlock(); } } return null; @@ -240,7 +246,8 @@ Entry get(K key, long now, boolean eagerEvict) { Tuple, Entry> put(K key, V value, long now) { Entry entry = new Entry<>(key, value, now); Entry existing = null; - try (ReleasableLock ignored = writeLock.acquire()) { + writeLock.lock(); + try { try { if (map == null) { map = new HashMap<>(); @@ -252,6 +259,8 @@ Tuple, Entry> put(K key, V value, long now) { } catch (ExecutionException | InterruptedException e) { throw new IllegalStateException(e); } + } finally { + writeLock.unlock(); } return Tuple.tuple(entry, existing); } @@ -263,7 +272,8 @@ Tuple, Entry> put(K key, V value, long now) { */ void remove(K key) { CompletableFuture> future; - try (ReleasableLock ignored = writeLock.acquire()) { + writeLock.lock(); + try { if (map == null) { future = null; } else { @@ -272,6 +282,8 @@ void remove(K key) { map = null; } } + } finally { + writeLock.unlock(); } if (future != null) { evictions.increment(); @@ -290,7 +302,8 @@ void remove(K key) { void remove(K key, V value, boolean notify) { CompletableFuture> future; boolean removed = false; - try (ReleasableLock ignored = writeLock.acquire()) { + writeLock.lock(); + try { future = map == null ? null : map.get(key); try { if (future != null) { @@ -307,6 +320,8 @@ void remove(K key, V value, boolean notify) { } catch (ExecutionException | InterruptedException e) { throw new IllegalStateException(e); } + } finally { + writeLock.unlock(); } if (future != null && removed) { @@ -333,7 +348,7 @@ void remove(K key, V value, boolean notify) { Entry tail; // lock protecting mutations to the LRU list - private final ReleasableLock lruLock = new ReleasableLock(new ReentrantLock()); + private final ReentrantLock lruLock = new ReentrantLock(); /** * Returns the value to which the specified key is mapped, or null if this map contains no mapping for the key. @@ -380,15 +395,18 @@ public V computeIfAbsent(K key, CacheLoader loader) throws ExecutionExcept // need a mechanism to ensure that load is invoked at most once, but we are not invoking load while holding // the segment lock; to do this, we atomically put a future in the map that can load the value, and then // get the value from this future on the thread that won the race to place the future into the segment map - CacheSegment segment = getCacheSegment(key); + final CacheSegment segment = getCacheSegment(key); CompletableFuture> future; CompletableFuture> completableFuture = new CompletableFuture<>(); - try (ReleasableLock ignored = segment.writeLock.acquire()) { + segment.writeLock.lock(); + try { if (segment.map == null) { segment.map = new HashMap<>(); } future = segment.map.putIfAbsent(key, completableFuture); + } finally { + segment.writeLock.unlock(); } BiFunction, Throwable, ? extends V> handler = (ok, ex) -> { @@ -396,7 +414,8 @@ public V computeIfAbsent(K key, CacheLoader loader) throws ExecutionExcept promote(ok, now); return ok.value; } else { - try (ReleasableLock ignored = segment.writeLock.acquire()) { + segment.writeLock.lock(); + try { CompletableFuture> sanity = segment.map == null ? null : segment.map.get(key); if (sanity != null && sanity.isCompletedExceptionally()) { segment.map.remove(key); @@ -404,6 +423,8 @@ public V computeIfAbsent(K key, CacheLoader loader) throws ExecutionExcept segment.map = null; } } + } finally { + segment.writeLock.unlock(); } return null; } @@ -461,13 +482,16 @@ private void put(K key, V value, long now) { CacheSegment segment = getCacheSegment(key); Tuple, Entry> tuple = segment.put(key, value, now); boolean replaced = false; - try (ReleasableLock ignored = lruLock.acquire()) { + lruLock.lock(); + try { if (tuple.v2() != null && tuple.v2().state == State.EXISTING) { if (unlink(tuple.v2())) { replaced = true; } } promote(tuple.v1(), now); + } finally { + lruLock.unlock(); } if (replaced) { removalListener.onRemoval( @@ -479,8 +503,11 @@ private void put(K key, V value, long now) { private void notifyWithInvalidated(CompletableFuture> f) { try { Entry entry = f.get(); - try (ReleasableLock ignored = lruLock.acquire()) { + lruLock.lock(); + try { delete(entry, RemovalNotification.RemovalReason.INVALIDATED); + } finally { + lruLock.unlock(); } } catch (ExecutionException e) { // ok @@ -521,7 +548,8 @@ public void invalidateAll() { Entry h; boolean[] haveSegmentLock = new boolean[NUMBER_OF_SEGMENTS]; - try (ReleasableLock ignored = lruLock.acquire()) { + lruLock.lock(); + try { try { for (int i = 0; i < NUMBER_OF_SEGMENTS; i++) { segments[i].segmentLock.writeLock().lock(); @@ -546,6 +574,8 @@ public void invalidateAll() { } } } + } finally { + lruLock.unlock(); } while (h != null) { removalListener.onRemoval(new RemovalNotification<>(h.key, h.value, RemovalNotification.RemovalReason.INVALIDATED)); @@ -558,8 +588,11 @@ public void invalidateAll() { */ public void refresh() { long now = now(); - try (ReleasableLock ignored = lruLock.acquire()) { + lruLock.lock(); + try { evict(now); + } finally { + lruLock.unlock(); } } @@ -589,7 +622,7 @@ public long weight() { * @return an LRU-ordered {@link Iterable} over the keys in the cache */ public Iterable keys() { - return () -> new Iterator() { + return () -> new Iterator<>() { private final CacheIterator iterator = new CacheIterator(head); @Override @@ -617,7 +650,7 @@ public void remove() { * @return an LRU-ordered {@link Iterable} over the values in the cache */ public Iterable values() { - return () -> new Iterator() { + return () -> new Iterator<>() { private final CacheIterator iterator = new CacheIterator(head); @Override @@ -647,7 +680,8 @@ public void remove() { */ public void forEach(BiConsumer consumer) { for (CacheSegment segment : segments) { - try (ReleasableLock ignored = segment.readLock.acquire()) { + segment.readLock.lock(); + try { if (segment.map == null) { continue; } @@ -661,6 +695,8 @@ public void forEach(BiConsumer consumer) { throw new IllegalStateException(e); } } + } finally { + segment.readLock.unlock(); } } } @@ -692,9 +728,12 @@ public void remove() { if (entry != null) { CacheSegment segment = getCacheSegment(entry.key); segment.remove(entry.key, entry.value, false); - try (ReleasableLock ignored = lruLock.acquire()) { + lruLock.lock(); + try { current = null; delete(entry, RemovalNotification.RemovalReason.INVALIDATED); + } finally { + lruLock.unlock(); } } } @@ -736,7 +775,8 @@ public long getEvictions() { private void promote(Entry entry, long now) { boolean promoted = true; - try (ReleasableLock ignored = lruLock.acquire()) { + lruLock.lock(); + try { switch (entry.state) { case DELETED -> promoted = false; case EXISTING -> relinkAtHead(entry); @@ -745,6 +785,8 @@ private void promote(Entry entry, long now) { if (promoted) { evict(now); } + } finally { + lruLock.unlock(); } } From 6c4e01e331f537eafe82ed576ca3cbe39f175ae8 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Wed, 17 Apr 2024 12:09:41 +0300 Subject: [PATCH 21/43] Revert "Muting (#107484)" (#107500) This reverts commit ae23a6f85a29840aa104a99d9fc5ef073e649d07. --- .../org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java index fc76ae2c6791..5ca5da555718 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java @@ -395,7 +395,6 @@ public void testILMWaitsForTimeSeriesEndTimeToLapse() throws Exception { }, 30, TimeUnit.SECONDS); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/103981") @TestLogging(value = "org.elasticsearch.xpack.ilm:TRACE", reason = "https://github.com/elastic/elasticsearch/issues/103981") public void testRollupNonTSIndex() throws Exception { createIndex(index, alias, false); From a94f2b056a44dba3d08a8afdbf28cc5baf785314 Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 17 Apr 2024 10:41:26 +0100 Subject: [PATCH 22/43] Always validate node ID on relocation (#107420) Follow-up to complete the change started in #107407, removing the temporary compatibility shim. --- .../elasticsearch/index/shard/IndexShard.java | 27 +++++-------------- .../recovery/RecoverySourceHandler.java | 7 ++++- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index a52b289493cd..66427ddcee29 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -757,16 +757,6 @@ public IndexShardState markAsRecovering(String reason, RecoveryState recoverySta private final AtomicBoolean primaryReplicaResyncInProgress = new AtomicBoolean(); - // temporary compatibility shim while adding targetNodeId parameter to dependencies - @Deprecated(forRemoval = true) - public void relocated( - final String targetAllocationId, - final BiConsumer> consumer, - final ActionListener listener - ) throws IllegalIndexShardStateException, IllegalStateException { - relocated(null, targetAllocationId, consumer, listener); - } - /** * Completes the relocation. Operations are blocked and current operations are drained before changing state to relocated. The provided * {@link BiConsumer} is executed after all operations are successfully blocked. @@ -868,8 +858,7 @@ public void onFailure(Exception e) { } } - // TODO only nullable temporarily, remove once deprecated relocated() override is removed, see ES-6725 - private void verifyRelocatingState(@Nullable String targetNodeId) { + private void verifyRelocatingState(String targetNodeId) { if (state != IndexShardState.STARTED) { throw new IndexShardNotStartedException(shardId, state); } @@ -883,14 +872,12 @@ private void verifyRelocatingState(@Nullable String targetNodeId) { throw new IllegalIndexShardStateException(shardId, IndexShardState.STARTED, ": shard is no longer relocating " + shardRouting); } - if (targetNodeId != null) { - if (targetNodeId.equals(shardRouting.relocatingNodeId()) == false) { - throw new IllegalIndexShardStateException( - shardId, - IndexShardState.STARTED, - ": shard is no longer relocating to node [" + targetNodeId + "]: " + shardRouting - ); - } + if (Objects.equals(targetNodeId, shardRouting.relocatingNodeId()) == false) { + throw new IllegalIndexShardStateException( + shardId, + IndexShardState.STARTED, + ": shard is no longer relocating to node [" + targetNodeId + "]: " + shardRouting + ); } if (primaryReplicaResyncInProgress.get()) { diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java index 618bc847e3a7..538cfdabef32 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java @@ -1287,7 +1287,12 @@ void finalizeRecovery(long targetLocalCheckpoint, long trimAboveSeqNo, ActionLis logger.trace("performing relocation hand-off"); cancellableThreads.execute( // this acquires all IndexShard operation permits and will thus delay new recoveries until it is done - () -> shard.relocated(request.targetAllocationId(), recoveryTarget::handoffPrimaryContext, finalStep) + () -> shard.relocated( + request.targetNode().getId(), + request.targetAllocationId(), + recoveryTarget::handoffPrimaryContext, + finalStep + ) ); /* * if the recovery process fails after disabling primary mode on the source shard, both relocation source and From 2f9e753463909f820493fd191ecc8f8bffb30b29 Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 17 Apr 2024 11:49:02 +0100 Subject: [PATCH 23/43] Add setting for max connections to S3 (#107533) The S3 SDK permits changing the maximum number of concurrent connections that it will open, but today there's no way to adjust this setting within Elasticsearch. This commit adds a setting for this parameter. --- docs/changelog/107533.yaml | 5 +++++ .../snapshot-restore/repository-s3.asciidoc | 4 ++++ .../repositories/s3/S3ClientSettings.java | 18 ++++++++++++++++++ .../repositories/s3/S3RepositoryPlugin.java | 1 + .../repositories/s3/S3Service.java | 1 + .../repositories/s3/S3ClientSettingsTests.java | 17 +++++++++++++++++ 6 files changed, 46 insertions(+) create mode 100644 docs/changelog/107533.yaml diff --git a/docs/changelog/107533.yaml b/docs/changelog/107533.yaml new file mode 100644 index 000000000000..da95cfd5b312 --- /dev/null +++ b/docs/changelog/107533.yaml @@ -0,0 +1,5 @@ +pr: 107533 +summary: Add setting for max connections to S3 +area: Snapshot/Restore +type: enhancement +issues: [] diff --git a/docs/reference/snapshot-restore/repository-s3.asciidoc b/docs/reference/snapshot-restore/repository-s3.asciidoc index 9ee630c37eee..9fd1724bb054 100644 --- a/docs/reference/snapshot-restore/repository-s3.asciidoc +++ b/docs/reference/snapshot-restore/repository-s3.asciidoc @@ -159,6 +159,10 @@ settings belong in the `elasticsearch.yml` file. of data over an established, open connection to the repository before it closes the connection. The default value is 50 seconds. +`max_connections`:: + + The maximum number of concurrent connections to S3. The default value is `50`. + `max_retries`:: The number of retries to use when an S3 request fails. The default value is diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java index ab322786fcd4..3d36a08473b5 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3ClientSettings.java @@ -124,6 +124,13 @@ final class S3ClientSettings { key -> Setting.timeSetting(key, TimeValue.timeValueMillis(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT), Property.NodeScope) ); + /** The maximum number of concurrent connections to use. */ + static final Setting.AffixSetting MAX_CONNECTIONS_SETTING = Setting.affixKeySetting( + PREFIX, + "max_connections", + key -> Setting.intSetting(key, ClientConfiguration.DEFAULT_MAX_CONNECTIONS, 1, Property.NodeScope) + ); + /** The number of retries to use when an s3 request fails. */ static final Setting.AffixSetting MAX_RETRIES_SETTING = Setting.affixKeySetting( PREFIX, @@ -195,6 +202,9 @@ final class S3ClientSettings { /** The read timeout for the s3 client. */ final int readTimeoutMillis; + /** The maximum number of concurrent connections to use. */ + final int maxConnections; + /** The number of retries to use for the s3 client. */ final int maxRetries; @@ -223,6 +233,7 @@ private S3ClientSettings( String proxyUsername, String proxyPassword, int readTimeoutMillis, + int maxConnections, int maxRetries, boolean throttleRetries, boolean pathStyleAccess, @@ -239,6 +250,7 @@ private S3ClientSettings( this.proxyUsername = proxyUsername; this.proxyPassword = proxyPassword; this.readTimeoutMillis = readTimeoutMillis; + this.maxConnections = maxConnections; this.maxRetries = maxRetries; this.throttleRetries = throttleRetries; this.pathStyleAccess = pathStyleAccess; @@ -268,6 +280,7 @@ S3ClientSettings refine(Settings repositorySettings) { final int newReadTimeoutMillis = Math.toIntExact( getRepoSettingOrDefault(READ_TIMEOUT_SETTING, normalizedSettings, TimeValue.timeValueMillis(readTimeoutMillis)).millis() ); + final int newMaxConnections = getRepoSettingOrDefault(MAX_CONNECTIONS_SETTING, normalizedSettings, maxConnections); final int newMaxRetries = getRepoSettingOrDefault(MAX_RETRIES_SETTING, normalizedSettings, maxRetries); final boolean newThrottleRetries = getRepoSettingOrDefault(USE_THROTTLE_RETRIES_SETTING, normalizedSettings, throttleRetries); final boolean newPathStyleAccess = getRepoSettingOrDefault(USE_PATH_STYLE_ACCESS, normalizedSettings, pathStyleAccess); @@ -290,6 +303,7 @@ S3ClientSettings refine(Settings repositorySettings) { && proxyPort == newProxyPort && proxyScheme == newProxyScheme && newReadTimeoutMillis == readTimeoutMillis + && maxConnections == newMaxConnections && maxRetries == newMaxRetries && newThrottleRetries == throttleRetries && Objects.equals(credentials, newCredentials) @@ -309,6 +323,7 @@ S3ClientSettings refine(Settings repositorySettings) { proxyUsername, proxyPassword, newReadTimeoutMillis, + newMaxConnections, newMaxRetries, newThrottleRetries, newPathStyleAccess, @@ -417,6 +432,7 @@ static S3ClientSettings getClientSettings(final Settings settings, final String proxyUsername.toString(), proxyPassword.toString(), Math.toIntExact(getConfigValue(settings, clientName, READ_TIMEOUT_SETTING).millis()), + getConfigValue(settings, clientName, MAX_CONNECTIONS_SETTING), getConfigValue(settings, clientName, MAX_RETRIES_SETTING), getConfigValue(settings, clientName, USE_THROTTLE_RETRIES_SETTING), getConfigValue(settings, clientName, USE_PATH_STYLE_ACCESS), @@ -438,6 +454,7 @@ public boolean equals(final Object o) { final S3ClientSettings that = (S3ClientSettings) o; return proxyPort == that.proxyPort && readTimeoutMillis == that.readTimeoutMillis + && maxConnections == that.maxConnections && maxRetries == that.maxRetries && throttleRetries == that.throttleRetries && Objects.equals(credentials, that.credentials) @@ -465,6 +482,7 @@ public int hashCode() { proxyPassword, readTimeoutMillis, maxRetries, + maxConnections, throttleRetries, disableChunkedEncoding, region, diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java index 26047c3b416a..1092b76001cb 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java @@ -125,6 +125,7 @@ public List> getSettings() { S3ClientSettings.PROXY_USERNAME_SETTING, S3ClientSettings.PROXY_PASSWORD_SETTING, S3ClientSettings.READ_TIMEOUT_SETTING, + S3ClientSettings.MAX_CONNECTIONS_SETTING, S3ClientSettings.MAX_RETRIES_SETTING, S3ClientSettings.USE_THROTTLE_RETRIES_SETTING, S3ClientSettings.USE_PATH_STYLE_ACCESS, diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java index c8a7cc12a90f..4a1308fa5941 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java @@ -242,6 +242,7 @@ static ClientConfiguration buildConfiguration(S3ClientSettings clientSettings) { clientConfiguration.setSignerOverride(clientSettings.signerOverride); } + clientConfiguration.setMaxConnections(clientSettings.maxConnections); clientConfiguration.setMaxErrorRetry(clientSettings.maxRetries); clientConfiguration.setUseThrottleRetries(clientSettings.throttleRetries); clientConfiguration.setSocketTimeout(clientSettings.readTimeoutMillis); diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java index 31bfd3a5e157..c8aaa2a5e0a4 100644 --- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java +++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ClientSettingsTests.java @@ -42,6 +42,7 @@ public void testThereIsADefaultClientByDefault() { assertThat(defaultSettings.proxyUsername, is(emptyString())); assertThat(defaultSettings.proxyPassword, is(emptyString())); assertThat(defaultSettings.readTimeoutMillis, is(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT)); + assertThat(defaultSettings.maxConnections, is(ClientConfiguration.DEFAULT_MAX_CONNECTIONS)); assertThat(defaultSettings.maxRetries, is(ClientConfiguration.DEFAULT_RETRY_POLICY.getMaxErrorRetry())); assertThat(defaultSettings.throttleRetries, is(ClientConfiguration.DEFAULT_THROTTLE_RETRIES)); } @@ -197,4 +198,20 @@ public void testSignerOverrideCanBeSet() { ClientConfiguration configuration = S3Service.buildConfiguration(settings.get("other")); assertThat(configuration.getSignerOverride(), is(signerOverride)); } + + public void testMaxConnectionsCanBeSet() { + final int maxConnections = between(1, 100); + final Map settings = S3ClientSettings.load( + Settings.builder().put("s3.client.other.max_connections", maxConnections).build() + ); + assertThat(settings.get("default").maxConnections, is(ClientConfiguration.DEFAULT_MAX_CONNECTIONS)); + assertThat(settings.get("other").maxConnections, is(maxConnections)); + ClientConfiguration defaultConfiguration = S3Service.buildConfiguration(settings.get("default")); + assertThat(defaultConfiguration.getMaxConnections(), is(ClientConfiguration.DEFAULT_MAX_CONNECTIONS)); + ClientConfiguration configuration = S3Service.buildConfiguration(settings.get("other")); + assertThat(configuration.getMaxConnections(), is(maxConnections)); + + // the default appears in the docs so let's make sure it doesn't change: + assertEquals(50, ClientConfiguration.DEFAULT_MAX_CONNECTIONS); + } } From 9b91d9c072c9fb6acc7772239117808cba1c7682 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 17 Apr 2024 14:15:48 +0200 Subject: [PATCH 24/43] Adjust exception checking for geoip downloader error logging. (#107550) Sometimes an exception is wrapped multiple times and then these logs are being emitted: ``` org.elasticsearch.transport.RemoteTransportException: [es-es-index-64c4d7dcd-4d7qb][10.2.58.152:9300][cluster:admin/persistent/start] Caused by: org.elasticsearch.transport.RemoteTransportException: [es-es-index-64c4d7dcd-j7s7v][10.2.9.216:9300][cluster:admin/persistent/start] Caused by: org.elasticsearch.ResourceAlreadyExistsException: task with id {geoip-downloader} already exist at org.elasticsearch.persistent.PersistentTasksClusterService$1.execute(PersistentTasksClusterService.java:120) at org.elasticsearch.cluster.service.MasterService$UnbatchedExecutor.execute(MasterService.java:550) at org.elasticsearch.cluster.service.MasterService.innerExecuteTasks(MasterService.java:1039) at org.elasticsearch.cluster.service.MasterService.executeTasks(MasterService.java:1004) at org.elasticsearch.cluster.service.MasterService.executeAndPublishBatch(MasterService.java:232) at org.elasticsearch.cluster.service.MasterService$BatchingTaskQueue$Processor.lambda$run$2(MasterService.java:1645) at org.elasticsearch.action.ActionListener.run(ActionListener.java:356) at org.elasticsearch.cluster.service.MasterService$BatchingTaskQueue$Processor.run(MasterService.java:1642) at org.elasticsearch.cluster.service.MasterService$5.lambda$doRun$0(MasterService.java:1237) at org.elasticsearch.action.ActionListener.run(ActionListener.java:356) at org.elasticsearch.cluster.service.MasterService$5.doRun(MasterService.java:1216) at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:984) at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:26) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.lang.Thread.run(Thread.java:1583) ``` In this case the real cause is `ResourceAlreadyExistsException`, which should't be logged as an error. Adjusted the exception cause checking to take into account that an exeception maybe wrapped twice by a `RemoteTransportException`. --- .../ingest/geoip/GeoIpDownloaderTaskExecutor.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java index 299e55d4d60a..c04dffe82b3c 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; @@ -347,7 +348,7 @@ private void startTask(Runnable onFailure) { new GeoIpTaskParams(), null, ActionListener.wrap(r -> logger.debug("Started geoip downloader task"), e -> { - Throwable t = e instanceof RemoteTransportException ? e.getCause() : e; + Throwable t = e instanceof RemoteTransportException ? ExceptionsHelper.unwrapCause(e) : e; if (t instanceof ResourceAlreadyExistsException == false) { logger.error("failed to create geoip downloader task", e); onFailure.run(); @@ -360,7 +361,7 @@ private void stopTask(Runnable onFailure) { ActionListener> listener = ActionListener.wrap( r -> logger.debug("Stopped geoip downloader task"), e -> { - Throwable t = e instanceof RemoteTransportException ? e.getCause() : e; + Throwable t = e instanceof RemoteTransportException ? ExceptionsHelper.unwrapCause(e) : e; if (t instanceof ResourceNotFoundException == false) { logger.error("failed to remove geoip downloader task", e); onFailure.run(); @@ -373,7 +374,7 @@ private void stopTask(Runnable onFailure) { // regardless of whether DATABASES_INDEX is an alias, resolve it to a concrete index Index databasesIndex = databasesAbstraction.getWriteIndex(); client.admin().indices().prepareDelete(databasesIndex.getName()).execute(ActionListener.wrap(rr -> {}, e -> { - Throwable t = e instanceof RemoteTransportException ? e.getCause() : e; + Throwable t = e instanceof RemoteTransportException ? ExceptionsHelper.unwrapCause(e) : e; if (t instanceof ResourceNotFoundException == false) { logger.warn("failed to remove " + databasesIndex, e); } From 33a71e3289cbbc4984d42d5c2f26cfccf65cc44e Mon Sep 17 00:00:00 2001 From: Liam Thompson <32779855+leemthompo@users.noreply.github.com> Date: Wed, 17 Apr 2024 14:37:07 +0200 Subject: [PATCH 25/43] [DOCS] Refactor book-scoped variables in `docs/reference/index.asciidoc` (#107413) * Remove `es-test-dir` book-scoped variable * Remove `plugins-examples-dir` book-scoped variable * Remove `:dependencies-dir:` and `:xes-repo-dir:` book-scoped variables - In `index.asciidoc`, two variables (`:dependencies-dir:` and `:xes-repo-dir:`) were removed. - In `sql/index.asciidoc`, the `:sql-tests:` path was updated to fuller path - In `esql/index.asciidoc`, the `:esql-tests:` path was updated idem * Replace `es-repo-dir` with `es-ref-dir` * Move `:include-xpack: true` to few files that use it, remove from index.asciidoc --- .../RestTestsFromDocSnippetTaskSpec.groovy | 6 +- .../categorize-text-aggregation.asciidoc | 6 +- .../inference-bucket-aggregation.asciidoc | 8 +- docs/reference/alias.asciidoc | 2 +- .../stemmer-override-tokenfilter.asciidoc | 2 +- docs/reference/api-conventions.asciidoc | 8 +- docs/reference/cat/alias.asciidoc | 14 +- docs/reference/cat/allocation.asciidoc | 18 +- docs/reference/cat/anomaly-detectors.asciidoc | 112 ++--- .../cat/component-templates.asciidoc | 14 +- docs/reference/cat/count.asciidoc | 10 +- docs/reference/cat/datafeeds.asciidoc | 40 +- .../reference/cat/dataframeanalytics.asciidoc | 20 +- docs/reference/cat/fielddata.asciidoc | 12 +- docs/reference/cat/health.asciidoc | 12 +- docs/reference/cat/indices.asciidoc | 20 +- docs/reference/cat/master.asciidoc | 14 +- docs/reference/cat/nodeattrs.asciidoc | 14 +- docs/reference/cat/nodes.asciidoc | 18 +- docs/reference/cat/pending_tasks.asciidoc | 16 +- docs/reference/cat/plugins.asciidoc | 14 +- docs/reference/cat/recovery.asciidoc | 22 +- docs/reference/cat/repositories.asciidoc | 14 +- docs/reference/cat/segments.asciidoc | 30 +- docs/reference/cat/shards.asciidoc | 16 +- docs/reference/cat/snapshots.asciidoc | 14 +- docs/reference/cat/tasks.asciidoc | 16 +- docs/reference/cat/templates.asciidoc | 14 +- docs/reference/cat/thread_pool.asciidoc | 16 +- docs/reference/cat/trainedmodel.asciidoc | 14 +- docs/reference/cat/transforms.asciidoc | 74 +-- .../delete-auto-follow-pattern.asciidoc | 2 +- .../get-auto-follow-pattern.asciidoc | 2 +- .../pause-auto-follow-pattern.asciidoc | 2 +- .../put-auto-follow-pattern.asciidoc | 2 +- .../resume-auto-follow-pattern.asciidoc | 2 +- .../ccr/apis/follow/get-follow-info.asciidoc | 2 +- .../apis/follow/post-pause-follow.asciidoc | 2 +- .../apis/follow/post-resume-follow.asciidoc | 2 +- .../ccr/apis/follow/post-unfollow.asciidoc | 2 +- .../ccr/apis/follow/put-follow.asciidoc | 2 +- .../reference/ccr/apis/get-ccr-stats.asciidoc | 2 +- docs/reference/ccr/getting-started.asciidoc | 2 +- docs/reference/cluster/get-settings.asciidoc | 4 +- docs/reference/cluster/health.asciidoc | 6 +- .../cluster/nodes-hot-threads.asciidoc | 4 +- docs/reference/cluster/nodes-info.asciidoc | 6 +- docs/reference/cluster/nodes-stats.asciidoc | 22 +- docs/reference/cluster/nodes-usage.asciidoc | 4 +- docs/reference/cluster/pending.asciidoc | 4 +- .../cluster/prevalidate-node-removal.asciidoc | 2 +- docs/reference/cluster/reroute.asciidoc | 2 +- docs/reference/cluster/state.asciidoc | 6 +- docs/reference/cluster/stats.asciidoc | 4 +- docs/reference/cluster/tasks.asciidoc | 14 +- .../cluster/update-desired-nodes.asciidoc | 2 +- .../cluster/update-settings.asciidoc | 6 +- .../data-streams/data-stream-apis.asciidoc | 32 +- .../lifecycle/apis/delete-lifecycle.asciidoc | 2 +- .../lifecycle/apis/explain-lifecycle.asciidoc | 2 +- .../apis/get-global-retention.asciidoc | 4 +- .../lifecycle/apis/get-lifecycle.asciidoc | 2 +- .../lifecycle/apis/put-lifecycle.asciidoc | 2 +- .../set-up-a-data-stream.asciidoc | 2 +- docs/reference/docs/bulk.asciidoc | 42 +- docs/reference/docs/delete-by-query.asciidoc | 44 +- docs/reference/docs/delete.asciidoc | 14 +- docs/reference/docs/get.asciidoc | 18 +- docs/reference/docs/index_.asciidoc | 18 +- docs/reference/docs/multi-get.asciidoc | 14 +- .../reference/docs/multi-termvectors.asciidoc | 22 +- docs/reference/docs/reindex.asciidoc | 12 +- docs/reference/docs/termvectors.asciidoc | 22 +- docs/reference/docs/update-by-query.asciidoc | 46 +- docs/reference/docs/update.asciidoc | 12 +- docs/reference/eql/eql-search-api.asciidoc | 6 +- docs/reference/eql/eql.asciidoc | 4 +- .../esql/esql-across-clusters.asciidoc | 10 +- docs/reference/esql/esql-enrich-data.asciidoc | 4 +- docs/reference/esql/esql-get-started.asciidoc | 6 +- .../esql/esql-index-options.asciidoc | 4 +- docs/reference/esql/index.asciidoc | 2 +- docs/reference/getting-started.asciidoc | 6 +- docs/reference/how-to/disk-usage.asciidoc | 2 +- docs/reference/how-to/indexing-speed.asciidoc | 2 +- docs/reference/how-to/knn-search.asciidoc | 2 +- ...lasticsearch-for-time-series-data.asciidoc | 12 +- .../ilm/apis/delete-lifecycle.asciidoc | 2 +- docs/reference/ilm/apis/explain.asciidoc | 2 +- .../reference/ilm/apis/get-lifecycle.asciidoc | 2 +- docs/reference/ilm/apis/get-status.asciidoc | 2 +- docs/reference/ilm/apis/move-to-step.asciidoc | 2 +- .../reference/ilm/apis/put-lifecycle.asciidoc | 2 +- .../apis/remove-policy-from-index.asciidoc | 2 +- docs/reference/ilm/apis/retry-policy.asciidoc | 2 +- docs/reference/ilm/apis/start.asciidoc | 2 +- docs/reference/ilm/apis/stop.asciidoc | 2 +- docs/reference/index-modules/blocks.asciidoc | 10 +- docs/reference/index.asciidoc | 7 - docs/reference/indices/add-alias.asciidoc | 2 +- docs/reference/indices/alias-exists.asciidoc | 4 +- docs/reference/indices/aliases.asciidoc | 2 +- .../indices/apis/reload-analyzers.asciidoc | 6 +- docs/reference/indices/clearcache.asciidoc | 6 +- docs/reference/indices/clone-index.asciidoc | 10 +- docs/reference/indices/close.asciidoc | 14 +- docs/reference/indices/create-index.asciidoc | 8 +- .../indices/dangling-index-delete.asciidoc | 4 +- .../indices/dangling-index-import.asciidoc | 4 +- .../indices/data-stream-stats.asciidoc | 2 +- docs/reference/indices/delete-alias.asciidoc | 2 +- .../delete-component-template.asciidoc | 4 +- .../indices/delete-data-stream.asciidoc | 2 +- .../indices/delete-index-template-v1.asciidoc | 2 +- .../indices/delete-index-template.asciidoc | 4 +- docs/reference/indices/delete-index.asciidoc | 8 +- docs/reference/indices/diskusage.asciidoc | 8 +- .../indices/downsample-data-stream.asciidoc | 2 +- .../indices/field-usage-stats.asciidoc | 12 +- docs/reference/indices/flush.asciidoc | 6 +- docs/reference/indices/forcemerge.asciidoc | 6 +- docs/reference/indices/get-alias.asciidoc | 6 +- .../indices/get-data-stream.asciidoc | 2 +- .../indices/get-field-mapping.asciidoc | 6 +- .../indices/get-index-template-v1.asciidoc | 8 +- docs/reference/indices/get-index.asciidoc | 12 +- docs/reference/indices/get-mapping.asciidoc | 10 +- docs/reference/indices/get-settings.asciidoc | 14 +- .../indices/index-template-exists-v1.asciidoc | 8 +- .../reference/indices/indices-exists.asciidoc | 12 +- docs/reference/indices/open-close.asciidoc | 10 +- .../indices/put-component-template.asciidoc | 10 +- .../indices/put-index-template-v1.asciidoc | 8 +- .../indices/put-index-template.asciidoc | 2 +- docs/reference/indices/put-mapping.asciidoc | 8 +- docs/reference/indices/recovery.asciidoc | 6 +- docs/reference/indices/refresh.asciidoc | 6 +- .../indices/resolve-cluster.asciidoc | 6 +- docs/reference/indices/resolve.asciidoc | 6 +- .../reference/indices/rollover-index.asciidoc | 16 +- docs/reference/indices/segments.asciidoc | 22 +- docs/reference/indices/shard-stores.asciidoc | 6 +- docs/reference/indices/shrink-index.asciidoc | 10 +- .../indices/simulate-template.asciidoc | 2 +- docs/reference/indices/split-index.asciidoc | 10 +- docs/reference/indices/stats.asciidoc | 18 +- .../indices/update-settings.asciidoc | 10 +- .../ingest/apis/delete-pipeline.asciidoc | 2 +- .../ingest/apis/get-pipeline.asciidoc | 2 +- .../ingest/apis/put-pipeline.asciidoc | 2 +- docs/reference/ingest/enrich.asciidoc | 4 +- .../ingest/processors/inference.asciidoc | 126 ++--- .../mapping/explicit-mapping.asciidoc | 4 +- .../mapping/mapping-settings-limit.asciidoc | 2 +- .../types/aggregate-metric-double.asciidoc | 4 +- .../mapping/types/dense-vector.asciidoc | 2 +- docs/reference/mapping/types/nested.asciidoc | 4 +- docs/reference/mapping/types/numeric.asciidoc | 6 +- .../migration/apis/deprecation.asciidoc | 2 +- .../migration/apis/feature-migration.asciidoc | 2 +- .../migrate_8_0/rest-api-changes.asciidoc | 2 +- .../anomaly-detection/apis/close-job.asciidoc | 4 +- .../apis/delete-calendar-event.asciidoc | 2 +- .../apis/delete-calendar-job.asciidoc | 4 +- .../apis/delete-calendar.asciidoc | 2 +- .../apis/delete-datafeed.asciidoc | 2 +- .../apis/delete-filter.asciidoc | 2 +- .../apis/delete-forecast.asciidoc | 2 +- .../apis/delete-job.asciidoc | 2 +- .../apis/delete-snapshot.asciidoc | 4 +- .../anomaly-detection/apis/flush-job.asciidoc | 2 +- .../anomaly-detection/apis/forecast.asciidoc | 2 +- .../apis/get-bucket.asciidoc | 14 +- .../apis/get-calendar-event.asciidoc | 4 +- .../apis/get-calendar.asciidoc | 4 +- .../apis/get-category.asciidoc | 4 +- .../apis/get-datafeed-stats.asciidoc | 32 +- .../apis/get-datafeed.asciidoc | 6 +- .../apis/get-filter.asciidoc | 4 +- .../apis/get-influencer.asciidoc | 14 +- ...-job-model-snapshot-upgrade-stats.asciidoc | 18 +- .../apis/get-job-stats.asciidoc | 106 ++-- .../anomaly-detection/apis/get-job.asciidoc | 42 +- .../apis/get-overall-buckets.asciidoc | 8 +- .../apis/get-record.asciidoc | 20 +- .../apis/get-snapshot.asciidoc | 12 +- .../anomaly-detection/apis/open-job.asciidoc | 2 +- .../anomaly-detection/apis/post-data.asciidoc | 2 +- .../apis/preview-datafeed.asciidoc | 2 +- .../apis/put-calendar-job.asciidoc | 4 +- .../apis/put-calendar.asciidoc | 2 +- .../apis/put-datafeed.asciidoc | 40 +- .../apis/put-filter.asciidoc | 2 +- .../anomaly-detection/apis/put-job.asciidoc | 132 ++--- .../anomaly-detection/apis/reset-job.asciidoc | 2 +- .../apis/revert-snapshot.asciidoc | 4 +- .../apis/start-datafeed.asciidoc | 2 +- .../apis/stop-datafeed.asciidoc | 2 +- .../apis/update-datafeed.asciidoc | 38 +- .../apis/update-filter.asciidoc | 2 +- .../apis/update-job.asciidoc | 60 +-- .../apis/update-snapshot.asciidoc | 6 +- .../apis/upgrade-job-model-snapshot.asciidoc | 4 +- .../apis/validate-detector.asciidoc | 36 +- .../ml/common/apis/get-ml-memory.asciidoc | 8 +- .../apis/delete-dfanalytics.asciidoc | 2 +- .../apis/explain-dfanalytics.asciidoc | 2 +- .../apis/get-dfanalytics-stats.asciidoc | 126 ++--- .../apis/get-dfanalytics.asciidoc | 10 +- .../apis/preview-dfanalytics.asciidoc | 2 +- .../apis/put-dfanalytics.asciidoc | 132 ++--- .../apis/start-dfanalytics.asciidoc | 4 +- .../apis/stop-dfanalytics.asciidoc | 6 +- .../apis/update-dfanalytics.asciidoc | 8 +- docs/reference/ml/ml-shared.asciidoc | 8 +- ...ar-trained-model-deployment-cache.asciidoc | 2 +- .../apis/delete-trained-models.asciidoc | 2 +- .../apis/get-trained-models-stats.asciidoc | 20 +- .../apis/get-trained-models.asciidoc | 456 +++++++++--------- .../infer-trained-model-deployment.asciidoc | 2 +- .../apis/infer-trained-model.asciidoc | 262 +++++----- ...put-trained-model-definition-part.asciidoc | 2 +- .../put-trained-model-vocabulary.asciidoc | 2 +- .../apis/put-trained-models.asciidoc | 138 +++--- .../start-trained-model-deployment.asciidoc | 4 +- .../stop-trained-model-deployment.asciidoc | 4 +- .../update-trained-model-deployment.asciidoc | 2 +- docs/reference/modules/cluster/misc.asciidoc | 2 +- .../cluster/remote-clusters-api-key.asciidoc | 2 +- .../cluster/remote-clusters-cert.asciidoc | 2 +- .../modules/indices/index_management.asciidoc | 4 +- docs/reference/modules/node.asciidoc | 10 +- docs/reference/query-dsl/knn-query.asciidoc | 2 +- .../query-dsl/script-score-query.asciidoc | 2 +- ...ear-repositories-metering-archive.asciidoc | 4 +- .../apis/get-repositories-metering.asciidoc | 4 +- docs/reference/rest-api/common-parms.asciidoc | 2 +- docs/reference/rest-api/defs.asciidoc | 2 +- docs/reference/rest-api/index.asciidoc | 92 ++-- .../security/disable-user-profile.asciidoc | 2 +- .../security/enable-user-profile.asciidoc | 2 +- .../rest-api/security/query-api-key.asciidoc | 2 +- .../rest-api/security/query-user.asciidoc | 2 +- .../update-user-profile-data.asciidoc | 6 +- docs/reference/rest-api/usage.asciidoc | 2 +- .../reference/rest-api/watcher/start.asciidoc | 2 +- docs/reference/rest-api/watcher/stop.asciidoc | 2 +- .../apis/create-stored-script-api.asciidoc | 2 +- .../apis/delete-stored-script-api.asciidoc | 2 +- .../apis/get-stored-script-api.asciidoc | 2 +- docs/reference/scripting/engine.asciidoc | 2 +- docs/reference/search/count.asciidoc | 28 +- docs/reference/search/explain.asciidoc | 24 +- docs/reference/search/field-caps.asciidoc | 6 +- docs/reference/search/knn-search.asciidoc | 22 +- .../search/multi-search-template-api.asciidoc | 2 +- docs/reference/search/multi-search.asciidoc | 10 +- docs/reference/search/rank-eval.asciidoc | 6 +- docs/reference/search/retriever.asciidoc | 18 +- docs/reference/search/rrf.asciidoc | 6 +- docs/reference/search/search-shards.asciidoc | 12 +- .../search/search-template-api.asciidoc | 16 +- .../search/search-vector-tile-api.asciidoc | 2 +- .../search-your-data/knn-search.asciidoc | 2 +- .../search-across-clusters.asciidoc | 2 +- .../search-with-synonyms.asciidoc | 2 +- .../semantic-search-inference.asciidoc | 12 +- .../search-your-data/semantic-search.asciidoc | 10 +- docs/reference/search/search.asciidoc | 48 +- docs/reference/search/validate.asciidoc | 20 +- .../apis/mount-snapshot.asciidoc | 2 +- .../apis/node-cache-stats.asciidoc | 2 +- .../settings/monitoring-settings.asciidoc | 50 +- .../settings/security-settings.asciidoc | 184 +++---- docs/reference/settings/ssl-settings.asciidoc | 50 +- docs/reference/setup/add-nodes.asciidoc | 2 +- docs/reference/setup/configuration.asciidoc | 2 +- .../important-settings/path-settings.asciidoc | 6 +- docs/reference/setup/install/deb.asciidoc | 2 + docs/reference/setup/install/rpm.asciidoc | 4 + .../setup/install/targz-start.asciidoc | 2 +- docs/reference/setup/install/targz.asciidoc | 2 + .../setup/install/zip-windows-start.asciidoc | 2 +- .../setup/install/zip-windows.asciidoc | 2 + docs/reference/setup/logging-config.asciidoc | 2 +- docs/reference/setup/restart-cluster.asciidoc | 12 +- .../shutdown/apis/shutdown-delete.asciidoc | 2 +- .../shutdown/apis/shutdown-get.asciidoc | 2 +- .../shutdown/apis/shutdown-put.asciidoc | 2 +- .../slm/apis/slm-get-status.asciidoc | 2 +- docs/reference/slm/apis/slm-put.asciidoc | 4 +- docs/reference/slm/apis/slm-start.asciidoc | 2 +- docs/reference/slm/apis/slm-stop.asciidoc | 2 +- .../apis/clean-up-repo-api.asciidoc | 2 +- .../apis/clone-snapshot-api.asciidoc | 2 +- .../apis/create-snapshot-api.asciidoc | 2 +- .../apis/delete-repo-api.asciidoc | 2 +- .../apis/delete-snapshot-api.asciidoc | 2 +- .../apis/get-repo-api.asciidoc | 2 +- .../apis/get-snapshot-api.asciidoc | 2 +- .../apis/get-snapshot-status-api.asciidoc | 2 +- .../apis/put-repo-api.asciidoc | 2 +- .../apis/restore-snapshot-api.asciidoc | 2 +- .../apis/verify-repo-api.asciidoc | 2 +- .../repository-read-only-url.asciidoc | 2 +- .../repository-shared-file-system.asciidoc | 4 +- .../sql/apis/sql-search-api.asciidoc | 2 +- docs/reference/sql/index.asciidoc | 2 +- docs/reference/sql/security.asciidoc | 2 +- docs/reference/tab-widgets/ilm.asciidoc | 2 +- .../tab-widgets/register-fs-repo.asciidoc | 4 +- .../tab-widgets/snapshot-repo.asciidoc | 2 +- .../apis/find-field-structure.asciidoc | 22 +- .../apis/find-message-structure.asciidoc | 22 +- .../apis/find-structure.asciidoc | 30 +- .../transform/apis/delete-transform.asciidoc | 2 +- .../apis/get-transform-stats.asciidoc | 52 +- .../transform/apis/get-transform.asciidoc | 8 +- .../transform/apis/preview-transform.asciidoc | 58 +-- .../transform/apis/put-transform.asciidoc | 68 +-- .../transform/apis/reset-transform.asciidoc | 2 +- .../apis/schedule-now-transform.asciidoc | 2 +- .../transform/apis/stop-transform.asciidoc | 2 +- .../transform/apis/update-transform.asciidoc | 56 +-- .../diagnose-unassigned-shards.asciidoc | 2 +- .../common-issues/high-cpu-usage.asciidoc | 2 +- .../high-jvm-memory-pressure.asciidoc | 2 +- .../troubleshooting/data/add-tier.asciidoc | 2 +- .../data-tiers-mixed-with-node-attr.asciidoc | 2 +- .../data/enable-cluster-allocation.asciidoc | 2 +- .../data/enable-index-allocation.asciidoc | 2 +- .../increase-cluster-shard-limit.asciidoc | 2 +- .../data/increase-shard-limit.asciidoc | 2 +- .../data/increase-tier-capacity.asciidoc | 2 +- .../data/restore-from-snapshot.asciidoc | 2 +- .../troubleshooting/data/start-ilm.asciidoc | 2 +- .../troubleshooting/data/start-slm.asciidoc | 2 +- .../disk/fix-data-node-out-of-disk.asciidoc | 4 +- .../disk/fix-master-node-out-of-disk.asciidoc | 2 +- .../disk/fix-other-node-out-of-disk.asciidoc | 2 +- .../snapshot/add-repository.asciidoc | 2 +- .../repeated-snapshot-failures.asciidoc | 2 +- .../troubleshooting-shards-capacity.asciidoc | 2 +- docs/reference/upgrade.asciidoc | 2 +- 344 files changed, 2331 insertions(+), 2328 deletions(-) diff --git a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/RestTestsFromDocSnippetTaskSpec.groovy b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/RestTestsFromDocSnippetTaskSpec.groovy index 6e86cba23588..45d389212195 100644 --- a/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/RestTestsFromDocSnippetTaskSpec.groovy +++ b/build-tools-internal/src/test/groovy/org/elasticsearch/gradle/internal/doc/RestTestsFromDocSnippetTaskSpec.groovy @@ -454,11 +454,11 @@ Requires the `manage_ml` cluster privilege. This privilege is included in the ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=snapshot-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=snapshot-id] [[ml-update-snapshot-request-body]] == {api-request-body-title} @@ -470,7 +470,7 @@ The following properties can be updated after the model snapshot is created: `retain`:: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=retain] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=retain] [[ml-update-snapshot-example]] diff --git a/docs/reference/aggregations/bucket/categorize-text-aggregation.asciidoc b/docs/reference/aggregations/bucket/categorize-text-aggregation.asciidoc index 8ecd66d6ae7a..399e6ed87581 100644 --- a/docs/reference/aggregations/bucket/categorize-text-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/categorize-text-aggregation.asciidoc @@ -40,15 +40,15 @@ is an object it has the following properties: ===== `char_filter`:::: (array of strings or objects) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=char-filter] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=char-filter] `tokenizer`:::: (string or object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=tokenizer] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=tokenizer] `filter`:::: (array of strings or objects) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=filter] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=filter] ===== `categorization_filters`:: diff --git a/docs/reference/aggregations/pipeline/inference-bucket-aggregation.asciidoc b/docs/reference/aggregations/pipeline/inference-bucket-aggregation.asciidoc index 911a123ded28..064881925e8d 100644 --- a/docs/reference/aggregations/pipeline/inference-bucket-aggregation.asciidoc +++ b/docs/reference/aggregations/pipeline/inference-bucket-aggregation.asciidoc @@ -65,7 +65,7 @@ aggregations some options can be overridden for each of the two types of model. `num_top_feature_importance_values`:: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-regression-num-top-feature-importance-values] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-regression-num-top-feature-importance-values] [discrete] [[inference-agg-classification-opt]] @@ -73,15 +73,15 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-regression-num `num_top_classes`:: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-num-top-classes] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-num-top-classes] `num_top_feature_importance_values`:: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-num-top-feature-importance-values] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-num-top-feature-importance-values] `prediction_field_type`:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-prediction-field-type] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-prediction-field-type] [[inference-bucket-agg-example]] diff --git a/docs/reference/alias.asciidoc b/docs/reference/alias.asciidoc index 5b30501ed7c9..e5c2db65778d 100644 --- a/docs/reference/alias.asciidoc +++ b/docs/reference/alias.asciidoc @@ -308,7 +308,7 @@ POST _aliases ---- // TEST[s/^/PUT _data_stream\/logs-nginx.access-prod\nPUT _data_stream\/logs-my_app-default\n/] -include::{es-repo-dir}/indices/aliases.asciidoc[tag=write-index-defaults] +include::{es-ref-dir}/indices/aliases.asciidoc[tag=write-index-defaults] TIP: We recommend using data streams to store append-only time series data. If you need to update or delete existing time series data, you can perform update or delete operations diff --git a/docs/reference/analysis/tokenfilters/stemmer-override-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/stemmer-override-tokenfilter.asciidoc index 99cbe695f8f9..7d070a94cb0d 100644 --- a/docs/reference/analysis/tokenfilters/stemmer-override-tokenfilter.asciidoc +++ b/docs/reference/analysis/tokenfilters/stemmer-override-tokenfilter.asciidoc @@ -48,7 +48,7 @@ Where the file looks like: [source,stemmer_override] -------------------------------------------------- -include::{es-test-dir}/cluster/config/analysis/stemmer_override.txt[] +include::{elasticsearch-root}/docs/src/test/cluster/config/analysis/stemmer_override.txt[] -------------------------------------------------- You can also define the overrides rules inline: diff --git a/docs/reference/api-conventions.asciidoc b/docs/reference/api-conventions.asciidoc index 1a63af19b0a3..25881b707d72 100644 --- a/docs/reference/api-conventions.asciidoc +++ b/docs/reference/api-conventions.asciidoc @@ -205,18 +205,18 @@ on a remote cluster is currently not supported. For example, this will throw an Multi-target APIs that can target indices support the following query string parameters: -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] The defaults settings for the above parameters depend on the API being used. Some multi-target APIs that can target indices also support the following query string parameter: -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=ignore_throttled] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=ignore_throttled] NOTE: APIs with a single target, such as the <>, do not support multi-target syntax. diff --git a/docs/reference/cat/alias.asciidoc b/docs/reference/cat/alias.asciidoc index e7329d05534f..b6459eaa93b8 100644 --- a/docs/reference/cat/alias.asciidoc +++ b/docs/reference/cat/alias.asciidoc @@ -39,19 +39,19 @@ wildcards (`*`). To retrieve all aliases, omit this parameter or use `*` or [[cat-alias-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=http-format] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=http-format] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-h] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-h] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=help] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=help] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=local] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-s] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-s] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-v] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] [[cat-alias-api-example]] ==== {api-examples-title} diff --git a/docs/reference/cat/allocation.asciidoc b/docs/reference/cat/allocation.asciidoc index 7bab1926cff0..0891406d1be4 100644 --- a/docs/reference/cat/allocation.asciidoc +++ b/docs/reference/cat/allocation.asciidoc @@ -30,26 +30,26 @@ and their disk space. [[cat-allocation-api-path-params]] ==== {api-path-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=node-id] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=node-id] [[cat-allocation-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bytes] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=bytes] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=http-format] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=http-format] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=local] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-h] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-h] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=help] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=help] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-s] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-s] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-v] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v] [[cat-allocation-api-response-body]] ==== {api-response-body-title} diff --git a/docs/reference/cat/anomaly-detectors.asciidoc b/docs/reference/cat/anomaly-detectors.asciidoc index 30272e67286c..607a88d1e1a5 100644 --- a/docs/reference/cat/anomaly-detectors.asciidoc +++ b/docs/reference/cat/anomaly-detectors.asciidoc @@ -43,20 +43,20 @@ For more information about {anomaly-detect}, see ``:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] [[cat-anomaly-detectors-query-params]] ==== {api-query-parms-title} `allow_no_match`:: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-jobs] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-jobs] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bytes] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=bytes] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=http-format] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=http-format] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-h] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-h] + If you do not specify which columns to include, the API returns the default columns. If you explicitly specify one or more columns, it returns only the @@ -65,75 +65,75 @@ specified columns. Valid columns are: `assignment_explanation`, `ae`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=assignment-explanation-anomaly-jobs] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=assignment-explanation-anomaly-jobs] `buckets.count`, `bc`, `bucketsCount`::: (Default) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=bucket-count-anomaly-jobs] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=bucket-count-anomaly-jobs] `buckets.time.exp_avg`, `btea`, `bucketsTimeExpAvg`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=bucket-time-exponential-average] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=bucket-time-exponential-average] `buckets.time.exp_avg_hour`, `bteah`, `bucketsTimeExpAvgHour`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=bucket-time-exponential-average-hour] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=bucket-time-exponential-average-hour] `buckets.time.max`, `btmax`, `bucketsTimeMax`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=bucket-time-maximum] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=bucket-time-maximum] `buckets.time.min`, `btmin`, `bucketsTimeMin`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=bucket-time-minimum] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=bucket-time-minimum] `buckets.time.total`, `btt`, `bucketsTimeTotal`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=bucket-time-total] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=bucket-time-total] `data.buckets`, `db`, `dataBuckets`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=bucket-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=bucket-count] `data.earliest_record`, `der`, `dataEarliestRecord`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=earliest-record-timestamp] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=earliest-record-timestamp] `data.empty_buckets`, `deb`, `dataEmptyBuckets`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=empty-bucket-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=empty-bucket-count] `data.input_bytes`, `dib`, `dataInputBytes`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=input-bytes] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=input-bytes] `data.input_fields`, `dif`, `dataInputFields`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=input-field-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=input-field-count] `data.input_records`, `dir`, `dataInputRecords`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=input-record-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=input-record-count] `data.invalid_dates`, `did`, `dataInvalidDates`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=invalid-date-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=invalid-date-count] `data.last`, `dl`, `dataLast`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=last-data-time] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=last-data-time] `data.last_empty_bucket`, `dleb`, `dataLastEmptyBucket`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=latest-empty-bucket-timestamp] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=latest-empty-bucket-timestamp] `data.last_sparse_bucket`, `dlsb`, `dataLastSparseBucket`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=latest-sparse-record-timestamp] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=latest-sparse-record-timestamp] `data.latest_record`, `dlr`, `dataLatestRecord`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=latest-record-timestamp] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=latest-record-timestamp] `data.missing_fields`, `dmf`, `dataMissingFields`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=missing-field-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=missing-field-count] `data.out_of_order_timestamps`, `doot`, `dataOutOfOrderTimestamps`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=out-of-order-timestamp-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=out-of-order-timestamp-count] `data.processed_fields`, `dpf`, `dataProcessedFields`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=processed-field-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=processed-field-count] `data.processed_records`, `dpr`, `dataProcessedRecords`::: (Default) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=processed-record-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=processed-record-count] `data.sparse_buckets`, `dsb`, `dataSparseBuckets`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=sparse-bucket-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=sparse-bucket-count] `forecasts.memory.avg`, `fmavg`, `forecastsMemoryAvg`::: The average memory usage in bytes for forecasts related to the {anomaly-job}. @@ -177,99 +177,99 @@ The total runtime in milliseconds for forecasts related to the {anomaly-job}. `forecasts.total`, `ft`, `forecastsTotal`::: (Default) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=forecast-total] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=forecast-total] `id`::: (Default) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] `model.bucket_allocation_failures`, `mbaf`, `modelBucketAllocationFailures`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=bucket-allocation-failures-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=bucket-allocation-failures-count] `model.by_fields`, `mbf`, `modelByFields`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=total-by-field-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=total-by-field-count] `model.bytes`, `mb`, `modelBytes`::: (Default) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-bytes] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-bytes] `model.bytes_exceeded`, `mbe`, `modelBytesExceeded`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-bytes-exceeded] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-bytes-exceeded] `model.categorization_status`, `mcs`, `modelCategorizationStatus`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=categorization-status] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=categorization-status] `model.categorized_doc_count`, `mcdc`, `modelCategorizedDocCount`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=categorized-doc-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=categorized-doc-count] `model.dead_category_count`, `mdcc`, `modelDeadCategoryCount`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dead-category-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dead-category-count] `model.failed_category_count`, `mdcc`, `modelFailedCategoryCount`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=failed-category-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=failed-category-count] `model.frequent_category_count`, `mfcc`, `modelFrequentCategoryCount`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=frequent-category-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=frequent-category-count] `model.log_time`, `mlt`, `modelLogTime`::: The timestamp when the model stats were gathered, according to server time. `model.memory_limit`, `mml`, `modelMemoryLimit`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-memory-limit-anomaly-jobs] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-memory-limit-anomaly-jobs] `model.memory_status`, `mms`, `modelMemoryStatus`::: (Default) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-memory-status] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-memory-status] `model.over_fields`, `mof`, `modelOverFields`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=total-over-field-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=total-over-field-count] `model.partition_fields`, `mpf`, `modelPartitionFields`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=total-partition-field-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=total-partition-field-count] `model.rare_category_count`, `mrcc`, `modelRareCategoryCount`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=rare-category-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=rare-category-count] `model.timestamp`, `mt`, `modelTimestamp`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-timestamp] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-timestamp] `model.total_category_count`, `mtcc`, `modelTotalCategoryCount`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=total-category-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=total-category-count] `node.address`, `na`, `nodeAddress`::: The network address of the node. + -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-jobs] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-jobs] `node.ephemeral_id`, `ne`, `nodeEphemeralId`::: The ephemeral ID of the node. + -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-jobs] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-jobs] `node.id`, `ni`, `nodeId`::: The unique identifier of the node. + -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-jobs] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-jobs] `node.name`, `nn`, `nodeName`::: The node name. + -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-jobs] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-jobs] `opened_time`, `ot`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=open-time] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=open-time] `state`, `s`::: (Default) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=state-anomaly-job] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=state-anomaly-job] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=help] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=help] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-s] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-s] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=time] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=time] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-v] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v] [[cat-anomaly-detectors-example]] ==== {api-examples-title} diff --git a/docs/reference/cat/component-templates.asciidoc b/docs/reference/cat/component-templates.asciidoc index e642899e7045..596c86befd1b 100644 --- a/docs/reference/cat/component-templates.asciidoc +++ b/docs/reference/cat/component-templates.asciidoc @@ -40,19 +40,19 @@ wildcard expressions. If omitted, all component templates are returned. [[cat-component-templates-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=http-format] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=http-format] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-h] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-h] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=help] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=help] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=local] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-s] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-s] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-v] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v] [[cat-component-templates-api-example]] diff --git a/docs/reference/cat/count.asciidoc b/docs/reference/cat/count.asciidoc index 5fdae8768e1e..37e602c75902 100644 --- a/docs/reference/cat/count.asciidoc +++ b/docs/reference/cat/count.asciidoc @@ -43,15 +43,15 @@ and indices, omit this parameter or use `*` or `_all`. [[cat-count-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=http-format] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=http-format] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-h] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-h] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=help] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=help] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-s] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-s] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-v] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v] [[cat-count-api-example]] diff --git a/docs/reference/cat/datafeeds.asciidoc b/docs/reference/cat/datafeeds.asciidoc index 82a7853a569c..9b6481191e59 100644 --- a/docs/reference/cat/datafeeds.asciidoc +++ b/docs/reference/cat/datafeeds.asciidoc @@ -42,18 +42,18 @@ NOTE: This API returns a maximum of 10,000 jobs. ``:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=datafeed-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=datafeed-id] [[cat-datafeeds-query-params]] ==== {api-query-parms-title} `allow_no_match`:: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-datafeeds] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-datafeeds] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=http-format] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=http-format] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-h] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-h] + If you do not specify which columns to include, the API returns the default columns. If you explicitly specify one or more columns, it returns only the @@ -62,60 +62,60 @@ specified columns. Valid columns are: `assignment_explanation`, `ae`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=assignment-explanation-datafeeds] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=assignment-explanation-datafeeds] `buckets.count`, `bc`, `bucketsCount`::: (Default) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=bucket-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=bucket-count] `id`::: (Default) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=datafeed-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=datafeed-id] `node.address`, `na`, `nodeAddress`::: The network address of the node. + -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-datafeeds] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-datafeeds] `node.ephemeral_id`, `ne`, `nodeEphemeralId`::: The ephemeral ID of the node. + -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-datafeeds] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-datafeeds] `node.id`, `ni`, `nodeId`::: The unique identifier of the node. + -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-datafeeds] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-datafeeds] `node.name`, `nn`, `nodeName`::: The node name. + -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-datafeeds] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-datafeeds] `search.bucket_avg`, `sba`, `searchBucketAvg`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=search-bucket-avg] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=search-bucket-avg] `search.count`, `sc`, `searchCount`::: (Default) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=search-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=search-count] `search.exp_avg_hour`, `seah`, `searchExpAvgHour`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=search-exp-avg-hour] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=search-exp-avg-hour] `search.time`, `st`, `searchTime`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=search-time] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=search-time] `state`, `s`::: (Default) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=state-datafeed] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=state-datafeed] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=help] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=help] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-s] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-s] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=time] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=time] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-v] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v] [[cat-datafeeds-example]] ==== {api-examples-title} diff --git a/docs/reference/cat/dataframeanalytics.asciidoc b/docs/reference/cat/dataframeanalytics.asciidoc index c39146ab2266..4c236ecf61ff 100644 --- a/docs/reference/cat/dataframeanalytics.asciidoc +++ b/docs/reference/cat/dataframeanalytics.asciidoc @@ -46,15 +46,15 @@ For more information, see <> and {ml-docs-setup-privileges} ``:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-data-frame-analytics-default] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-data-frame-analytics-default] [[cat-dfanalytics-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=http-format] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=http-format] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-h] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-h] + If you do not specify which columns to include, the API returns the default columns. If you explicitly specify one or more columns, it returns only the @@ -63,14 +63,14 @@ specified columns. Valid columns are: `assignment_explanation`, `ae`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=assignment-explanation-dfanalytics] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=assignment-explanation-dfanalytics] `create_time`, `ct`, `createTime`::: (Default) The time when the {dfanalytics-job} was created. `description`, `d`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=description-dfa] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=description-dfa] `dest_index`, `di`, `destIndex`::: Name of the destination index. @@ -80,7 +80,7 @@ Contains messages about the reason why a {dfanalytics-job} failed. `id`::: (Default) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-data-frame-analytics] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-data-frame-analytics] `model_memory_limit`, `mml`, `modelMemoryLimit`::: The approximate maximum amount of memory resources that are permitted for the @@ -115,13 +115,13 @@ The type of analysis that the {dfanalytics-job} performs. `version`, `v`::: The {es} version number in which the {dfanalytics-job} was created. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=help] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=help] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-s] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-s] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=time] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=time] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-v] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v] [[cat-dfanalytics-example]] diff --git a/docs/reference/cat/fielddata.asciidoc b/docs/reference/cat/fielddata.asciidoc index 55b9e1f56277..376ef1d97057 100644 --- a/docs/reference/cat/fielddata.asciidoc +++ b/docs/reference/cat/fielddata.asciidoc @@ -39,17 +39,17 @@ information. [[cat-fielddata-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bytes] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=bytes] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=http-format] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=http-format] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-h] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-h] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=help] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=help] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-s] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-s] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-v] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v] [[cat-fielddata-api-example]] diff --git a/docs/reference/cat/health.asciidoc b/docs/reference/cat/health.asciidoc index 84661ae8ef32..04a11699d3ec 100644 --- a/docs/reference/cat/health.asciidoc +++ b/docs/reference/cat/health.asciidoc @@ -51,22 +51,22 @@ over a longer period of time. See <>. [[cat-health-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=http-format] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=http-format] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-h] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-h] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=help] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=help] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-s] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-s] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=time] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=time] `ts` (timestamps):: (Optional, Boolean) If `true`, returns `HH:MM:SS` and {wikipedia}/Unix_time[Unix `epoch`] timestamps. Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-v] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v] [[cat-health-api-example]] diff --git a/docs/reference/cat/indices.asciidoc b/docs/reference/cat/indices.asciidoc index 43b10c83264c..64b90c4f8e35 100644 --- a/docs/reference/cat/indices.asciidoc +++ b/docs/reference/cat/indices.asciidoc @@ -62,11 +62,11 @@ and indices, omit this parameter or use `*` or `_all`. [[cat-indices-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bytes] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=bytes] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=http-format] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=http-format] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-h] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-h] `health`:: + @@ -81,24 +81,24 @@ are: By default, the response includes indices of any health status. -- -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=help] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=help] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=include-unloaded-segments] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=include-unloaded-segments] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] [[pri-flag]] `pri` (primary shards):: (Optional, Boolean) If `true`, the response only includes information from primary shards. Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-s] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-s] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=time] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=time] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-v] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] [[cat-indices-api-example]] diff --git a/docs/reference/cat/master.asciidoc b/docs/reference/cat/master.asciidoc index 151e0ac6516a..42348fc4939d 100644 --- a/docs/reference/cat/master.asciidoc +++ b/docs/reference/cat/master.asciidoc @@ -29,19 +29,19 @@ and name. [[cat-master-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=http-format] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=http-format] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-h] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-h] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=help] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=help] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=local] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-s] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-s] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-v] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v] [[cat-master-api-example]] diff --git a/docs/reference/cat/nodeattrs.asciidoc b/docs/reference/cat/nodeattrs.asciidoc index d4e07f153da2..2db0c3fc2027 100644 --- a/docs/reference/cat/nodeattrs.asciidoc +++ b/docs/reference/cat/nodeattrs.asciidoc @@ -27,9 +27,9 @@ Returns information about custom node attributes. [[cat-nodeattrs-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=http-format] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=http-format] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-h] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-h] + -- If you do not specify which columns to include, the API returns the default columns in the order listed below. If you explicitly specify one or more columns, it only returns the specified columns. @@ -61,15 +61,15 @@ Process ID, such as `13061`. Bound transport port, such as `9300`. -- -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=help] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=help] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=local] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-s] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-s] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-v] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v] [[cat-nodeattrs-api-example]] diff --git a/docs/reference/cat/nodes.asciidoc b/docs/reference/cat/nodes.asciidoc index da1ed532e41f..bfee57d1daad 100644 --- a/docs/reference/cat/nodes.asciidoc +++ b/docs/reference/cat/nodes.asciidoc @@ -27,15 +27,15 @@ Returns information about a cluster's nodes. [[cat-nodes-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bytes] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=bytes] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=http-format] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=http-format] `full_id`:: (Optional, Boolean) If `true`, return the full node ID. If `false`, return the shortened node ID. Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-h] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-h] + -- If you do not specify which columns to include, the API returns the default columns in the order listed below. If you explicitly specify one or more columns, it only returns the specified columns. @@ -328,17 +328,17 @@ Number of mappings, including <> and <> fields. Estimated heap overhead, in bytes, of mappings on this node, which allows for 1KiB of heap for every mapped field. -- -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=help] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=help] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-s] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-s] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=time] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=time] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-v] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=include-unloaded-segments] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=include-unloaded-segments] [[cat-nodes-api-example]] ==== {api-examples-title} diff --git a/docs/reference/cat/pending_tasks.asciidoc b/docs/reference/cat/pending_tasks.asciidoc index c576df17ced2..5dd6cb068814 100644 --- a/docs/reference/cat/pending_tasks.asciidoc +++ b/docs/reference/cat/pending_tasks.asciidoc @@ -28,21 +28,21 @@ Returns cluster-level changes that have not yet been executed, similar to the [[cat-pending-tasks-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=http-format] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=http-format] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-h] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-h] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=help] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=help] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=local] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-s] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-s] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=time] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=time] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-v] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v] [[cat-pending-tasks-api-example]] diff --git a/docs/reference/cat/plugins.asciidoc b/docs/reference/cat/plugins.asciidoc index 9a83887c484a..a812556887b7 100644 --- a/docs/reference/cat/plugins.asciidoc +++ b/docs/reference/cat/plugins.asciidoc @@ -29,19 +29,19 @@ Returns a list of plugins running on each node of a cluster. [[cat-plugins-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=http-format] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=http-format] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-h] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-h] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=help] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=help] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=local] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-s] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-s] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-v] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v] [[cat-plugins-api-example]] diff --git a/docs/reference/cat/recovery.asciidoc b/docs/reference/cat/recovery.asciidoc index 76fb66c6703d..058f4e69ae8e 100644 --- a/docs/reference/cat/recovery.asciidoc +++ b/docs/reference/cat/recovery.asciidoc @@ -39,7 +39,7 @@ The cat recovery API returns information about shard recoveries, both ongoing and completed. It is a more compact view of the JSON <> API. -include::{es-repo-dir}/indices/recovery.asciidoc[tag=shard-recovery-desc] +include::{es-ref-dir}/indices/recovery.asciidoc[tag=shard-recovery-desc] [[cat-recovery-path-params]] @@ -53,25 +53,25 @@ and indices, omit this parameter or use `*` or `_all`. [[cat-recovery-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=active-only] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=active-only] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bytes] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=bytes] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=detailed] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=detailed] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=http-format] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=http-format] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-h] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-h] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=help] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=help] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-query-parm] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-query-parm] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-s] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-s] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=time] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=time] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-v] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v] [[cat-recovery-api-example]] diff --git a/docs/reference/cat/repositories.asciidoc b/docs/reference/cat/repositories.asciidoc index edeb66b95423..ec2f243c27be 100644 --- a/docs/reference/cat/repositories.asciidoc +++ b/docs/reference/cat/repositories.asciidoc @@ -29,19 +29,19 @@ Returns the <> for a cluste [[cat-repositories-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=http-format] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=http-format] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-h] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-h] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=help] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=help] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=local] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-s] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-s] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-v] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v] [[cat-repositories-api-example]] diff --git a/docs/reference/cat/segments.asciidoc b/docs/reference/cat/segments.asciidoc index 900bebdb8168..872af679642d 100644 --- a/docs/reference/cat/segments.asciidoc +++ b/docs/reference/cat/segments.asciidoc @@ -44,11 +44,11 @@ and indices, omit this parameter or use `*` or `_all`. [[cat-segments-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bytes] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=bytes] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=http-format] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=http-format] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-h] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-h] + -- If you do not specify which columns to include, the API returns the default @@ -71,39 +71,39 @@ Valid columns are: `segment`:: (Default) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=segment] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=segment] `generation`:: (Default) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=generation] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=generation] `docs.count`:: (Default) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=docs-count] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=docs-count] `docs.deleted`:: (Default) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=docs-deleted] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=docs-deleted] `size`:: (Default) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=segment-size] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=segment-size] `size.memory`:: (Default) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=memory] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=memory] `committed`:: (Default) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=committed] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=committed] `searchable`:: (Default) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=segment-search] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=segment-search] `version`:: (Default) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=segment-version] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=segment-version] `compound`:: (Default) If `true`, the segment is stored in a compound file. This means Lucene @@ -113,11 +113,11 @@ merged all files from the segment in a single file to save file descriptors. ID of the node, such as `k0zy`. -- -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=help] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=help] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-s] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-s] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-v] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v] [[cat-segments-api-example]] diff --git a/docs/reference/cat/shards.asciidoc b/docs/reference/cat/shards.asciidoc index c3d21f29bfaf..74c017d86d8e 100644 --- a/docs/reference/cat/shards.asciidoc +++ b/docs/reference/cat/shards.asciidoc @@ -44,11 +44,11 @@ and indices, omit this parameter or use `*` or `_all`. [[cat-shards-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bytes] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=bytes] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=http-format] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=http-format] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-h] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-h] + -- If you do not specify which columns to include, the API returns the default @@ -297,15 +297,15 @@ values include: -- -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=help] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=help] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-s] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-s] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=time] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=time] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-v] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v] [[cat-shards-api-example]] diff --git a/docs/reference/cat/snapshots.asciidoc b/docs/reference/cat/snapshots.asciidoc index 059fd58630bc..820c4b56c783 100644 --- a/docs/reference/cat/snapshots.asciidoc +++ b/docs/reference/cat/snapshots.asciidoc @@ -46,9 +46,9 @@ If any repository fails during the request, {es} returns an error. [[cat-snapshots-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=http-format] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=http-format] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-h] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-h] + -- If you do not specify which columns to include, the API returns the default @@ -107,19 +107,19 @@ units>>. Reason for any snapshot failures. -- -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=help] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=help] `ignore_unavailable`:: (Optional, Boolean) If `true`, the response does not include information from unavailable snapshots. Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-s] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-s] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=time] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=time] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-v] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v] [[cat-snapshots-api-example]] diff --git a/docs/reference/cat/tasks.asciidoc b/docs/reference/cat/tasks.asciidoc index 1258c746bfb1..91d67baa72d7 100644 --- a/docs/reference/cat/tasks.asciidoc +++ b/docs/reference/cat/tasks.asciidoc @@ -41,13 +41,13 @@ of the JSON <> API. [[cat-tasks-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=detailed] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=detailed] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=http-format] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=http-format] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-h] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-h] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=help] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=help] `nodes`:: (Optional, string) @@ -58,17 +58,17 @@ wildcard (`*`) expressions. (Optional, string) Parent task ID used to limit the response. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-s] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-s] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=time] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=time] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-v] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v] [[cat-tasks-api-response-codes]] ==== {api-response-codes-title} -include::{es-repo-dir}/cluster/tasks.asciidoc[tag=tasks-api-404] +include::{es-ref-dir}/cluster/tasks.asciidoc[tag=tasks-api-404] [[cat-tasks-api-examples]] diff --git a/docs/reference/cat/templates.asciidoc b/docs/reference/cat/templates.asciidoc index 61dce67615e1..bcc8e9e4f5dc 100644 --- a/docs/reference/cat/templates.asciidoc +++ b/docs/reference/cat/templates.asciidoc @@ -39,19 +39,19 @@ expressions. If omitted, all templates are returned. [[cat-templates-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=http-format] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=http-format] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-h] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-h] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=help] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=help] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=local] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-s] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-s] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-v] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v] [[cat-templates-api-example]] diff --git a/docs/reference/cat/thread_pool.asciidoc b/docs/reference/cat/thread_pool.asciidoc index 244cf643798a..948ed9a1a7a3 100644 --- a/docs/reference/cat/thread_pool.asciidoc +++ b/docs/reference/cat/thread_pool.asciidoc @@ -40,9 +40,9 @@ request. Accepts wildcard expressions. [[cat-thread-pool-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=http-format] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=http-format] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-h] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-h] + -- If you do not specify which columns to include, the API returns the default @@ -113,17 +113,17 @@ Type of thread pool. Returned values are `fixed`, `fixed_auto_queue_size`, `dire -- -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=help] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=help] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=local] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-s] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-s] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=time] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=time] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-v] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v] [[cat-thread-pool-api-example]] diff --git a/docs/reference/cat/trainedmodel.asciidoc b/docs/reference/cat/trainedmodel.asciidoc index 6f305be845fc..74e83525ddfe 100644 --- a/docs/reference/cat/trainedmodel.asciidoc +++ b/docs/reference/cat/trainedmodel.asciidoc @@ -41,11 +41,11 @@ For more information, see <> and {ml-docs-setup-privileges} [[cat-trained-model-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bytes] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=bytes] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=http-format] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=http-format] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-h] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-h] + If you do not specify which columns to include, the API returns the default columns. If you explicitly specify one or more columns, it returns only the @@ -102,13 +102,13 @@ measuring the computational complexity of the model. `version`, `v`::: The {es} version number in which the trained model was created. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=help] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=help] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-s] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-s] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=time] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=time] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-v] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v] [[cat-trained-model-example]] diff --git a/docs/reference/cat/transforms.asciidoc b/docs/reference/cat/transforms.asciidoc index 1723d4ab73b0..0d4e9b691ac5 100644 --- a/docs/reference/cat/transforms.asciidoc +++ b/docs/reference/cat/transforms.asciidoc @@ -41,22 +41,22 @@ privileges. For more information, see <> and ``:: (Optional, string) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-id-wildcard] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-id-wildcard] [[cat-transforms-api-query-params]] ==== {api-query-parms-title} `allow_no_match`:: (Optional, Boolean) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-match-transforms1] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-match-transforms1] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=http-format] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=http-format] `from`:: (Optional, integer) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=from-transforms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=from-transforms] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-h] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-h] + If you do not specify which columns to include, the API returns the default columns. If you explicitly specify one or more columns, it returns only the @@ -66,14 +66,14 @@ Valid columns are: `changes_last_detection_time`, `cldt`::: (Default) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=checkpointing-changes-last-detected-at] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=checkpointing-changes-last-detected-at] `checkpoint`, `cp`::: (Default) The sequence number for the checkpoint. `checkpoint_duration_time_exp_avg`, `cdtea`, `checkpointTimeExpAvg`::: -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=exponential-avg-checkpoint-duration-ms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=exponential-avg-checkpoint-duration-ms] `checkpoint_progress`, `c`, `checkpointProgress`::: (Default) @@ -83,106 +83,106 @@ The progress of the next checkpoint that is currently in progress. The time the {transform} was created. `delete_time`, `dtime`::: -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=delete-time-ms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=delete-time-ms] `description`, `d`::: The description of the {transform}. `dest_index`, `di`, `destIndex`::: -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-index] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=dest-index] `documents_deleted`, `docd`::: -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=docs-deleted-transform] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=docs-deleted-transform] `documents_indexed`, `doci`::: -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=docs-indexed] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=docs-indexed] `docs_per_second`, `dps`::: -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-docs-per-second] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-docs-per-second] `documents_processed`, `docp`::: (Default) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=docs-processed] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=docs-processed] `frequency`, `f`::: -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=frequency] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=frequency] `id`::: (Default) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-id] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-id] `index_failure`, `if`::: -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-failures] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-failures] `index_time`, `itime`::: -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-time-ms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-time-ms] `index_total`, `it`::: -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-total] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-total] `indexed_documents_exp_avg`, `idea`::: -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=exponential-avg-documents-indexed] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=exponential-avg-documents-indexed] `last_search_time`, `lst`, `lastSearchTime`::: (Default) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=checkpointing-last-search-time] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=checkpointing-last-search-time] `max_page_search_size`, `mpsz`::: -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-max-page-search-size] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-max-page-search-size] `pages_processed`, `pp`::: -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=pages-processed] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=pages-processed] `pipeline`, `p`::: -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-pipeline] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=dest-pipeline] `processed_documents_exp_avg`, `pdea`::: -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=exponential-avg-documents-processed] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=exponential-avg-documents-processed] `processing_time`, `pt`::: -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=processing-time-ms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=processing-time-ms] `reason`, `r`::: -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=state-transform-reason] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=state-transform-reason] `search_failure`, `sf`::: -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=search-failures] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=search-failures] `search_time`, `stime`::: -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=search-time-ms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=search-time-ms] `search_total`, `st`::: -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=search-total] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=search-total] `source_index`, `si`, `sourceIndex`::: (Default) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-index-transforms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=source-index-transforms] `state`, `s`::: (Default) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=state-transform] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=state-transform] `transform_type`, `tt`::: Indicates the type of {transform}: `batch` or `continuous`. `trigger_count`, `tc`::: -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=trigger-count] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=trigger-count] `version`, `v`::: The version of {es} that existed on the node when the {transform} was created. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=help] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=help] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-s] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-s] `size`:: (Optional, integer) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=size-transforms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=size-transforms] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=time] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=time] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cat-v] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v] [[cat-transforms-api-examples]] ==== {api-examples-title} diff --git a/docs/reference/ccr/apis/auto-follow/delete-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/delete-auto-follow-pattern.asciidoc index a78148388a93..1c72fb8742b9 100644 --- a/docs/reference/ccr/apis/auto-follow/delete-auto-follow-pattern.asciidoc +++ b/docs/reference/ccr/apis/auto-follow/delete-auto-follow-pattern.asciidoc @@ -57,7 +57,7 @@ This API deletes a configured collection of [[ccr-delete-auto-follow-pattern-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] [[ccr-delete-auto-follow-pattern-examples]] ==== {api-examples-title} diff --git a/docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc index 4fa85d6ee638..46ef288b0508 100644 --- a/docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc +++ b/docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc @@ -75,7 +75,7 @@ This API will return the specified auto-follow pattern collection. [[ccr-get-auto-follow-pattern-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] [[ccr-get-auto-follow-pattern-examples]] ==== {api-examples-title} diff --git a/docs/reference/ccr/apis/auto-follow/pause-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/pause-auto-follow-pattern.asciidoc index ed0f24264069..1e64ab813e2a 100644 --- a/docs/reference/ccr/apis/auto-follow/pause-auto-follow-pattern.asciidoc +++ b/docs/reference/ccr/apis/auto-follow/pause-auto-follow-pattern.asciidoc @@ -43,7 +43,7 @@ meantime. [[ccr-pause-auto-follow-pattern-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] [[ccr-pause-auto-follow-pattern-examples]] ==== {api-examples-title} diff --git a/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc index 4b5ff5a5eb93..d08997068f70 100644 --- a/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc +++ b/docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc @@ -74,7 +74,7 @@ the new patterns. [[ccr-put-auto-follow-pattern-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] [[ccr-put-auto-follow-pattern-request-body]] ==== {api-request-body-title} diff --git a/docs/reference/ccr/apis/auto-follow/resume-auto-follow-pattern.asciidoc b/docs/reference/ccr/apis/auto-follow/resume-auto-follow-pattern.asciidoc index 5028b0f3d477..04da9b4a35ba 100644 --- a/docs/reference/ccr/apis/auto-follow/resume-auto-follow-pattern.asciidoc +++ b/docs/reference/ccr/apis/auto-follow/resume-auto-follow-pattern.asciidoc @@ -38,7 +38,7 @@ have been deleted or closed in the meantime. [[ccr-resume-auto-follow-pattern-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] [[ccr-resume-auto-follow-pattern-examples]] ==== {api-examples-title} diff --git a/docs/reference/ccr/apis/follow/get-follow-info.asciidoc b/docs/reference/ccr/apis/follow/get-follow-info.asciidoc index fd3d24e41be5..68fd6e210f88 100644 --- a/docs/reference/ccr/apis/follow/get-follow-info.asciidoc +++ b/docs/reference/ccr/apis/follow/get-follow-info.asciidoc @@ -52,7 +52,7 @@ replication options and whether the follower indices are active or paused. [[ccr-get-follow-info-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] [role="child_attributes"] [[ccr-get-follow-info-response-body]] diff --git a/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc b/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc index 58d5fbb03fa0..a4ab69aba8d8 100644 --- a/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc +++ b/docs/reference/ccr/apis/follow/post-pause-follow.asciidoc @@ -56,7 +56,7 @@ following task. [[ccr-post-pause-follow-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] [[ccr-post-pause-follow-examples]] ==== {api-examples-title} diff --git a/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc b/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc index b762f049bde6..47ba51a3fb8a 100644 --- a/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc +++ b/docs/reference/ccr/apis/follow/post-resume-follow.asciidoc @@ -69,7 +69,7 @@ returns, the follower index will resume fetching operations from the leader inde [[ccr-post-resume-follow-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] [[ccr-post-resume-follow-request-body]] ==== {api-request-body-title} diff --git a/docs/reference/ccr/apis/follow/post-unfollow.asciidoc b/docs/reference/ccr/apis/follow/post-unfollow.asciidoc index e8ca3526bbc8..b96777b455d3 100644 --- a/docs/reference/ccr/apis/follow/post-unfollow.asciidoc +++ b/docs/reference/ccr/apis/follow/post-unfollow.asciidoc @@ -63,7 +63,7 @@ irreversible operation. [[ccr-post-unfollow-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] [[ccr-post-unfollow-examples]] ==== {api-examples-title} diff --git a/docs/reference/ccr/apis/follow/put-follow.asciidoc b/docs/reference/ccr/apis/follow/put-follow.asciidoc index 11711432437e..eb83e2a13dcf 100644 --- a/docs/reference/ccr/apis/follow/put-follow.asciidoc +++ b/docs/reference/ccr/apis/follow/put-follow.asciidoc @@ -65,7 +65,7 @@ referenced leader index. When this API returns, the follower index exists, and follower shard requires transferring all the remote Lucene segment files to the follower index. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] [[ccr-put-follow-request-body]] ==== {api-request-body-title} diff --git a/docs/reference/ccr/apis/get-ccr-stats.asciidoc b/docs/reference/ccr/apis/get-ccr-stats.asciidoc index 2917e3f86372..128df5e47c77 100644 --- a/docs/reference/ccr/apis/get-ccr-stats.asciidoc +++ b/docs/reference/ccr/apis/get-ccr-stats.asciidoc @@ -56,7 +56,7 @@ shard-level stats as in the <>. `timeout`:: (Optional, time) Controls the amount of time to wait for results. Defaults to unlimited. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] [role="child_attributes"] [[ccr-get-stats-response-body]] diff --git a/docs/reference/ccr/getting-started.asciidoc b/docs/reference/ccr/getting-started.asciidoc index d6c455b510da..d30cd43a4db5 100644 --- a/docs/reference/ccr/getting-started.asciidoc +++ b/docs/reference/ccr/getting-started.asciidoc @@ -159,7 +159,7 @@ cluster with cluster alias `leader`. connected to. ==== -include::{es-repo-dir}/security/authentication/remote-clusters-privileges-cert.asciidoc[tag=configure-ccr-privileges] +include::{es-ref-dir}/security/authentication/remote-clusters-privileges-cert.asciidoc[tag=configure-ccr-privileges] [[ccr-getting-started-follower-index]] ==== Create a follower index to replicate a specific index diff --git a/docs/reference/cluster/get-settings.asciidoc b/docs/reference/cluster/get-settings.asciidoc index 931ebc9759a8..5a9fe81df61c 100644 --- a/docs/reference/cluster/get-settings.asciidoc +++ b/docs/reference/cluster/get-settings.asciidoc @@ -34,10 +34,10 @@ defined, but can also include the default settings by calling the ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=flat-settings] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=flat-settings] `include_defaults`:: (Optional, Boolean) If `true`, returns default cluster settings from the local node. Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] diff --git a/docs/reference/cluster/health.asciidoc b/docs/reference/cluster/health.asciidoc index 561c092f66a6..3a4058a55ce1 100644 --- a/docs/reference/cluster/health.asciidoc +++ b/docs/reference/cluster/health.asciidoc @@ -61,9 +61,9 @@ To target all data streams and indices in a cluster, omit this parameter or use (Optional, string) Can be one of `cluster`, `indices` or `shards`. Controls the details level of the health information returned. Defaults to `cluster`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=local] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] `wait_for_active_shards`:: (Optional, string) A number controlling to how many active shards to wait @@ -104,7 +104,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] (string) The name of the cluster. `status`:: -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cluster-health-status] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cluster-health-status] `timed_out`:: (Boolean) If `false` the response returned within the period of diff --git a/docs/reference/cluster/nodes-hot-threads.asciidoc b/docs/reference/cluster/nodes-hot-threads.asciidoc index 552d41629e1e..a7ee9604250d 100644 --- a/docs/reference/cluster/nodes-hot-threads.asciidoc +++ b/docs/reference/cluster/nodes-hot-threads.asciidoc @@ -30,7 +30,7 @@ threads. [[cluster-nodes-hot-threads-api-path-params]] ==== {api-path-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=node-id] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=node-id] [[cluster-nodes-hot-threads-api-query-params]] @@ -56,7 +56,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=node-id] troubleshooting, set this parameter to a large number (e.g. `9999`) to get information about all the threads in the system. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] `type`:: (Optional, string) The type to sample. Available options are `block`, `cpu`, and diff --git a/docs/reference/cluster/nodes-info.asciidoc b/docs/reference/cluster/nodes-info.asciidoc index 16da5a25d1fb..8ff7da3a16ad 100644 --- a/docs/reference/cluster/nodes-info.asciidoc +++ b/docs/reference/cluster/nodes-info.asciidoc @@ -101,7 +101,7 @@ can also request the metric `_all` to retrieve all metrics, or you can request the metric `_none` to suppress all metrics and retrieve only the identity of the node. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=node-id] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=node-id] [[cluster-nodes-info-api-response-body]] ==== {api-response-body-title} @@ -182,9 +182,9 @@ running process: [[cluster-nodes-info-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=flat-settings] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=flat-settings] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [[cluster-nodes-info-api-example]] diff --git a/docs/reference/cluster/nodes-stats.asciidoc b/docs/reference/cluster/nodes-stats.asciidoc index 07328ba98bce..a40d1f98cbd5 100644 --- a/docs/reference/cluster/nodes-stats.asciidoc +++ b/docs/reference/cluster/nodes-stats.asciidoc @@ -127,31 +127,31 @@ using metrics. * `dense_vector` -- -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=node-id] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=node-id] [[cluster-nodes-stats-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=completion-fields] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=completion-fields] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=fielddata-fields] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=fielddata-fields] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=fields] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=fields] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=groups] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=groups] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=level] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=level] `types`:: (Optional, string) A comma-separated list of document types for the `indexing` index metric. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=include-segment-file-sizes] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=include-segment-file-sizes] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=include-unloaded-segments] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=include-unloaded-segments] [role="child_attributes"] [[cluster-nodes-stats-api-response-body]] @@ -244,11 +244,11 @@ node. ======= `count`:: (integer) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=docs-count] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=docs-count] `deleted`:: (integer) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=docs-deleted] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=docs-deleted] ======= `store`:: diff --git a/docs/reference/cluster/nodes-usage.asciidoc b/docs/reference/cluster/nodes-usage.asciidoc index c62c42a57228..6c53919bcfbb 100644 --- a/docs/reference/cluster/nodes-usage.asciidoc +++ b/docs/reference/cluster/nodes-usage.asciidoc @@ -48,13 +48,13 @@ of features for each node. All the nodes selective options are explained that action has been called on the node. -- -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=node-id] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=node-id] [[cluster-nodes-usage-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [[cluster-nodes-usage-api-example]] diff --git a/docs/reference/cluster/pending.asciidoc b/docs/reference/cluster/pending.asciidoc index b82bdd8e022f..3e87234c7d26 100644 --- a/docs/reference/cluster/pending.asciidoc +++ b/docs/reference/cluster/pending.asciidoc @@ -34,9 +34,9 @@ might be reported by both task api and pending cluster tasks API. [[cluster-pending-api-path-params]] ==== {api-path-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=local] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] [[cluster-pending-api-response-body]] diff --git a/docs/reference/cluster/prevalidate-node-removal.asciidoc b/docs/reference/cluster/prevalidate-node-removal.asciidoc index d7f0ed64d6c0..16bf28c58668 100644 --- a/docs/reference/cluster/prevalidate-node-removal.asciidoc +++ b/docs/reference/cluster/prevalidate-node-removal.asciidoc @@ -34,7 +34,7 @@ Note that if the prevalidation result for a set of nodes returns `true` (i.e. it [[prevalidate-node-removal-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] `names`:: (Optional, string) Comma-separated list of node names. diff --git a/docs/reference/cluster/reroute.asciidoc b/docs/reference/cluster/reroute.asciidoc index d88df96b5000..b4e4809ae73b 100644 --- a/docs/reference/cluster/reroute.asciidoc +++ b/docs/reference/cluster/reroute.asciidoc @@ -112,7 +112,7 @@ query parameter, which will attempt a single retry round for these shards. (Optional, Boolean) If `true`, then retries allocation of shards that are blocked due to too many subsequent allocation failures. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [role="child_attributes"] [[cluster-reroute-api-request-body]] diff --git a/docs/reference/cluster/state.asciidoc b/docs/reference/cluster/state.asciidoc index 2836757a94fb..fcb2f5f2f5dc 100644 --- a/docs/reference/cluster/state.asciidoc +++ b/docs/reference/cluster/state.asciidoc @@ -115,15 +115,15 @@ Defaults to `true`. that are open, closed or both. Available options: `open`, `closed`, `none`, `all`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=flat-settings] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=flat-settings] `ignore_unavailable`:: (Optional, Boolean) If `true`, unavailable indices (missing or closed) will be ignored. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=local] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] `wait_for_metadata_version`:: (Optional, integer) Waits for the metadata version to be equal or greater diff --git a/docs/reference/cluster/stats.asciidoc b/docs/reference/cluster/stats.asciidoc index bdd3e166c22d..26b3553c3c17 100644 --- a/docs/reference/cluster/stats.asciidoc +++ b/docs/reference/cluster/stats.asciidoc @@ -33,7 +33,7 @@ memory usage) and information about the current nodes that form the cluster ==== {api-path-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=node-filter] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=node-filter] [[cluster-stats-api-query-params]] @@ -86,7 +86,7 @@ Unique identifier for the cluster. the last time the cluster statistics were refreshed. `status`:: -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=cluster-health-status] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cluster-health-status] + See <>. diff --git a/docs/reference/cluster/tasks.asciidoc b/docs/reference/cluster/tasks.asciidoc index 75a6352196b2..0ffd70095750 100644 --- a/docs/reference/cluster/tasks.asciidoc +++ b/docs/reference/cluster/tasks.asciidoc @@ -32,23 +32,23 @@ on one or more nodes in the cluster. [[tasks-api-path-params]] ==== {api-path-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=task-id] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=task-id] [[tasks-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=actions] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=actions] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=detailed] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=detailed] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=group-by] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=group-by] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=nodes] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=nodes] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=parent-task-id] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=parent-task-id] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] `wait_for_completion`:: (Optional, Boolean) If `true`, the request blocks until all found tasks are complete. diff --git a/docs/reference/cluster/update-desired-nodes.asciidoc b/docs/reference/cluster/update-desired-nodes.asciidoc index b7bbb8b3b7f4..c72a2b53208e 100644 --- a/docs/reference/cluster/update-desired-nodes.asciidoc +++ b/docs/reference/cluster/update-desired-nodes.asciidoc @@ -47,7 +47,7 @@ DELETE /_internal/desired_nodes [[update-desired-nodes-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] `dry_run`:: (Optional, Boolean) If `true`, then the request simulates the update and diff --git a/docs/reference/cluster/update-settings.asciidoc b/docs/reference/cluster/update-settings.asciidoc index c1a4397ee369..ca3d100e31e0 100644 --- a/docs/reference/cluster/update-settings.asciidoc +++ b/docs/reference/cluster/update-settings.asciidoc @@ -22,18 +22,18 @@ Configures <>. ==== {api-description-title} :strip-api-link: true -include::{es-repo-dir}/setup/configuration.asciidoc[tag=cluster-setting-precedence] +include::{es-ref-dir}/setup/configuration.asciidoc[tag=cluster-setting-precedence] [[cluster-update-settings-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=flat-settings] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=flat-settings] `include_defaults`:: (Optional, Boolean) If `true`, returns all default cluster settings. Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [[cluster-update-settings-api-example]] diff --git a/docs/reference/data-streams/data-stream-apis.asciidoc b/docs/reference/data-streams/data-stream-apis.asciidoc index d5a81a485af0..d525f0d8a788 100644 --- a/docs/reference/data-streams/data-stream-apis.asciidoc +++ b/docs/reference/data-streams/data-stream-apis.asciidoc @@ -41,34 +41,34 @@ The following API is available for <>: For concepts and tutorials, see <>. -include::{es-repo-dir}/indices/create-data-stream.asciidoc[] +include::{es-ref-dir}/indices/create-data-stream.asciidoc[] -include::{es-repo-dir}/indices/delete-data-stream.asciidoc[] +include::{es-ref-dir}/indices/delete-data-stream.asciidoc[] -include::{es-repo-dir}/indices/get-data-stream.asciidoc[] +include::{es-ref-dir}/indices/get-data-stream.asciidoc[] -include::{es-repo-dir}/indices/migrate-to-data-stream.asciidoc[] +include::{es-ref-dir}/indices/migrate-to-data-stream.asciidoc[] -include::{es-repo-dir}/indices/data-stream-stats.asciidoc[] +include::{es-ref-dir}/indices/data-stream-stats.asciidoc[] -include::{es-repo-dir}/data-streams/promote-data-stream-api.asciidoc[] +include::{es-ref-dir}/data-streams/promote-data-stream-api.asciidoc[] -include::{es-repo-dir}/data-streams/modify-data-streams-api.asciidoc[] +include::{es-ref-dir}/data-streams/modify-data-streams-api.asciidoc[] -include::{es-repo-dir}/data-streams/lifecycle/apis/put-lifecycle.asciidoc[] +include::{es-ref-dir}/data-streams/lifecycle/apis/put-lifecycle.asciidoc[] -include::{es-repo-dir}/data-streams/lifecycle/apis/get-lifecycle.asciidoc[] +include::{es-ref-dir}/data-streams/lifecycle/apis/get-lifecycle.asciidoc[] -include::{es-repo-dir}/data-streams/lifecycle/apis/delete-lifecycle.asciidoc[] +include::{es-ref-dir}/data-streams/lifecycle/apis/delete-lifecycle.asciidoc[] -include::{es-repo-dir}/data-streams/lifecycle/apis/explain-lifecycle.asciidoc[] +include::{es-ref-dir}/data-streams/lifecycle/apis/explain-lifecycle.asciidoc[] -include::{es-repo-dir}/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc[] +include::{es-ref-dir}/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc[] -include::{es-repo-dir}/data-streams/lifecycle/apis/put-global-retention.asciidoc[] +include::{es-ref-dir}/data-streams/lifecycle/apis/put-global-retention.asciidoc[] -include::{es-repo-dir}/data-streams/lifecycle/apis/get-global-retention.asciidoc[] +include::{es-ref-dir}/data-streams/lifecycle/apis/get-global-retention.asciidoc[] -include::{es-repo-dir}/data-streams/lifecycle/apis/delete-global-retention.asciidoc[] +include::{es-ref-dir}/data-streams/lifecycle/apis/delete-global-retention.asciidoc[] -include::{es-repo-dir}/indices/downsample-data-stream.asciidoc[] +include::{es-ref-dir}/indices/downsample-data-stream.asciidoc[] diff --git a/docs/reference/data-streams/lifecycle/apis/delete-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/delete-lifecycle.asciidoc index fd481d7ca481..0cf6ad395fb4 100644 --- a/docs/reference/data-streams/lifecycle/apis/delete-lifecycle.asciidoc +++ b/docs/reference/data-streams/lifecycle/apis/delete-lifecycle.asciidoc @@ -37,7 +37,7 @@ To target all data streams use `*` or `_all`. [[delete-data-lifecycle-api-query-parms]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=ds-expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=ds-expand-wildcards] + Defaults to `open`. diff --git a/docs/reference/data-streams/lifecycle/apis/explain-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/explain-lifecycle.asciidoc index a2609dcb78ec..e0e2df217335 100644 --- a/docs/reference/data-streams/lifecycle/apis/explain-lifecycle.asciidoc +++ b/docs/reference/data-streams/lifecycle/apis/explain-lifecycle.asciidoc @@ -40,7 +40,7 @@ execution. (Optional, Boolean) Includes default configurations related to the lifecycle of the target. Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [[data-streams-explain-lifecycle-example]] ==== {api-examples-title} diff --git a/docs/reference/data-streams/lifecycle/apis/get-global-retention.asciidoc b/docs/reference/data-streams/lifecycle/apis/get-global-retention.asciidoc index 03e485f3e7eb..0997c2d84ece 100644 --- a/docs/reference/data-streams/lifecycle/apis/get-global-retention.asciidoc +++ b/docs/reference/data-streams/lifecycle/apis/get-global-retention.asciidoc @@ -28,9 +28,9 @@ Gets the global retention configuration that is applied on data streams managed [[get-global-retention-api-query-parms]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=local] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] [[get-global-retention-api-response-body]] ==== {api-response-body-title} diff --git a/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc index 1bda7d8959be..83955417abd0 100644 --- a/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc +++ b/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc @@ -39,7 +39,7 @@ To target all data streams use `*` or `_all`. [[get-data-lifecycle-api-query-parms]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=ds-expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=ds-expand-wildcards] + Defaults to `open`. diff --git a/docs/reference/data-streams/lifecycle/apis/put-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/put-lifecycle.asciidoc index 53bd3c2b96f0..e68dc24f11a5 100644 --- a/docs/reference/data-streams/lifecycle/apis/put-lifecycle.asciidoc +++ b/docs/reference/data-streams/lifecycle/apis/put-lifecycle.asciidoc @@ -36,7 +36,7 @@ To target all data streams use `*` or `_all`. [[put-data-lifecycle-api-query-parms]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=ds-expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=ds-expand-wildcards] + Defaults to `open`. diff --git a/docs/reference/data-streams/set-up-a-data-stream.asciidoc b/docs/reference/data-streams/set-up-a-data-stream.asciidoc index 144146b897ef..57388a1199f5 100644 --- a/docs/reference/data-streams/set-up-a-data-stream.asciidoc +++ b/docs/reference/data-streams/set-up-a-data-stream.asciidoc @@ -261,7 +261,7 @@ PUT _data_stream/my-data-stream [[secure-data-stream]] === Secure the data stream -include::{es-repo-dir}/security/authorization/alias-privileges.asciidoc[tag=data-stream-security] +include::{es-ref-dir}/security/authorization/alias-privileges.asciidoc[tag=data-stream-security] For an example, see <>. diff --git a/docs/reference/docs/bulk.asciidoc b/docs/reference/docs/bulk.asciidoc index 1a32e64cedb1..02f7d7e941fe 100644 --- a/docs/reference/docs/bulk.asciidoc +++ b/docs/reference/docs/bulk.asciidoc @@ -249,21 +249,21 @@ on. were executed for each `index` or `create`. Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=pipeline] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=pipeline] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=refresh] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=refresh] `require_alias`:: (Optional, Boolean) If `true`, the request's actions must target an index alias. Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=routing] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=routing] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=source] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source_excludes] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=source_excludes] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source_includes] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=source_includes] `timeout`:: + @@ -280,7 +280,7 @@ timeout before failing. The actual wait time could be longer, particularly when multiple waits occur. -- -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] [[bulk-api-request-body]] ==== {api-request-body-title} @@ -293,15 +293,15 @@ Indexes the specified document if it does not already exist. The following line must contain the source data to be indexed. + -- -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bulk-index-ds] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=bulk-index-ds] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bulk-id] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=bulk-id] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bulk-list-executed-pipelines] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=bulk-list-executed-pipelines] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bulk-require-alias] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=bulk-require-alias] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bulk-dynamic-templates] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=bulk-dynamic-templates] -- `delete`:: @@ -309,12 +309,12 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bulk-dynamic-templates Removes the specified document from the index. + -- -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bulk-index] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=bulk-index] `_id`:: (Required, string) The document ID. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bulk-require-alias] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=bulk-require-alias] -- `index`:: @@ -324,15 +324,15 @@ If the document exists, replaces the document and increments the version. The following line must contain the source data to be indexed. + -- -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bulk-index] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=bulk-index] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bulk-id] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=bulk-id] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bulk-list-executed-pipelines] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=bulk-list-executed-pipelines] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bulk-require-alias] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=bulk-require-alias] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bulk-dynamic-templates] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=bulk-dynamic-templates] -- `update`:: @@ -341,12 +341,12 @@ Performs a partial document update. The following line must contain the partial document and update options. + -- -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bulk-index] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=bulk-index] `_id`:: (Required, string) The document ID. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=bulk-require-alias] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=bulk-require-alias] -- `doc`:: diff --git a/docs/reference/docs/delete-by-query.asciidoc b/docs/reference/docs/delete-by-query.asciidoc index d671cb9b5a63..8cde1da91121 100644 --- a/docs/reference/docs/delete-by-query.asciidoc +++ b/docs/reference/docs/delete-by-query.asciidoc @@ -175,76 +175,76 @@ this parameter or use `* or `_all`. [[docs-delete-by-query-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=analyzer] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=analyzer] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=analyze_wildcard] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=analyze_wildcard] `conflicts`:: (Optional, string) What to do if delete by query hits version conflicts: `abort` or `proceed`. Defaults to `abort`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=default_operator] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=default_operator] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=df] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=df] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `open`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=lenient] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=lenient] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=max_docs] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=max_docs] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=preference] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=preference] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=search-q] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=search-q] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=request_cache] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=request_cache] `refresh`:: (Optional, Boolean) If `true`, {es} refreshes all shards involved in the delete by query after the request completes. Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=requests_per_second] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=requests_per_second] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=routing] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=routing] `scroll`:: (Optional, <>) Period to retain the <> for scrolling. See <>. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=scroll_size] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=scroll_size] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=search_type] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=search_type] `search_timeout`:: (Optional, <>) Explicit timeout for each search request. Defaults to no timeout. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=slices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=slices] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sort] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=sort] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=stats] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=stats] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=terminate_after] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=terminate_after] `timeout`:: (Optional, <>) Period each deletion request <>. Defaults to `1m` (one minute). -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=version] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=version] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] [[docs-delete-by-query-api-request-body]] ==== {api-request-body-title} diff --git a/docs/reference/docs/delete.asciidoc b/docs/reference/docs/delete.asciidoc index 1d8ff699271b..452d7f7758bf 100644 --- a/docs/reference/docs/delete.asciidoc +++ b/docs/reference/docs/delete.asciidoc @@ -152,24 +152,24 @@ DELETE /my-index-000001/_doc/1?timeout=5m [[docs-delete-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=if_seq_no] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=if_seq_no] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=if_primary_term] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=if_primary_term] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=refresh] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=refresh] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=routing] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=routing] `timeout`:: (Optional, <>) Period to <>. Defaults to `1m` (one minute). -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=doc-version] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=doc-version] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=version_type] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=version_type] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] [[docs-delete-api-example]] ==== {api-examples-title} diff --git a/docs/reference/docs/get.asciidoc b/docs/reference/docs/get.asciidoc index d0d0db332eee..c71215fff8d7 100644 --- a/docs/reference/docs/get.asciidoc +++ b/docs/reference/docs/get.asciidoc @@ -162,27 +162,27 @@ deleted documents in the background as you continue to index more data. [[docs-get-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=preference] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=preference] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=realtime] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=realtime] `refresh`:: (Optional, Boolean) If `true`, the request refreshes the relevant shard before retrieving the document. Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=routing] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=routing] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=stored_fields] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=stored_fields] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=source] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source_excludes] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=source_excludes] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source_includes] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=source_includes] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=doc-version] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=doc-version] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=version_type] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=version_type] [[docs-get-api-response-body]] ==== {api-response-body-title} diff --git a/docs/reference/docs/index_.asciidoc b/docs/reference/docs/index_.asciidoc index 4577e0202480..9d359fd7d7f0 100644 --- a/docs/reference/docs/index_.asciidoc +++ b/docs/reference/docs/index_.asciidoc @@ -86,9 +86,9 @@ format and omit this parameter. [[docs-index-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=if_seq_no] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=if_seq_no] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=if_primary_term] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=if_primary_term] [[docs-index-api-op_type]] `op_type`:: @@ -101,11 +101,11 @@ If document id is specified, it defaults to `index`. Otherwise, it defaults to ` NOTE: If the request targets a data stream, an `op_type` of `create` is required. See <>. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=pipeline] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=pipeline] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=refresh] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=refresh] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=routing] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=routing] `timeout`:: + @@ -122,13 +122,13 @@ timeout before failing. The actual wait time could be longer, particularly when multiple waits occur. -- -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=doc-version] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=doc-version] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=version_type] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=version_type] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=require-alias] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=require-alias] [[docs-index-api-request-body]] ==== {api-request-body-title} diff --git a/docs/reference/docs/multi-get.asciidoc b/docs/reference/docs/multi-get.asciidoc index 065e22469611..7c3eafa9c79f 100644 --- a/docs/reference/docs/multi-get.asciidoc +++ b/docs/reference/docs/multi-get.asciidoc @@ -64,23 +64,23 @@ or when a document in the `docs` array does not specify an index. [[docs-multi-get-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=preference] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=preference] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=realtime] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=realtime] `refresh`:: (Optional, Boolean) If `true`, the request refreshes relevant shards before retrieving documents. Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=routing] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=routing] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=stored_fields] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=stored_fields] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=source] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source_excludes] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=source_excludes] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source_includes] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=source_includes] [[docs-multi-get-api-request-body]] ==== {api-request-body-title} diff --git a/docs/reference/docs/multi-termvectors.asciidoc b/docs/reference/docs/multi-termvectors.asciidoc index 90b31238a5c6..5a27e0b9b3a3 100644 --- a/docs/reference/docs/multi-termvectors.asciidoc +++ b/docs/reference/docs/multi-termvectors.asciidoc @@ -64,27 +64,27 @@ that can be included in the response. [[docs-multi-termvectors-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=fields] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=fields] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=field_statistics] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=field_statistics] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=offsets] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=offsets] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=payloads] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=payloads] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=positions] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=positions] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=preference] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=preference] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=routing] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=routing] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=realtime] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=realtime] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=term_statistics] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=term_statistics] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=version] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=version] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=version_type] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=version_type] [discrete] [[docs-multi-termvectors-api-example]] diff --git a/docs/reference/docs/reindex.asciidoc b/docs/reference/docs/reindex.asciidoc index 7c3cd8716dfe..146b519b05e8 100644 --- a/docs/reference/docs/reindex.asciidoc +++ b/docs/reference/docs/reindex.asciidoc @@ -483,21 +483,21 @@ timeout before failing. The actual wait time could be longer, particularly when multiple waits occur. -- -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] `wait_for_completion`:: (Optional, Boolean) If `true`, the request blocks until the operation is complete. Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=requests_per_second] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=requests_per_second] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=require-alias] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=require-alias] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=scroll] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=scroll] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=slices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=slices] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=max_docs] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=max_docs] [[docs-reindex-api-request-body]] ==== {api-request-body-title} diff --git a/docs/reference/docs/termvectors.asciidoc b/docs/reference/docs/termvectors.asciidoc index 8fa6392e08d5..31dfba1ac266 100644 --- a/docs/reference/docs/termvectors.asciidoc +++ b/docs/reference/docs/termvectors.asciidoc @@ -143,27 +143,27 @@ from is randomly selected. Use `routing` only to hit a particular shard. [[docs-termvectors-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=fields] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=fields] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=field_statistics] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=field_statistics] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=offsets] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=offsets] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=payloads] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=payloads] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=positions] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=positions] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=preference] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=preference] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=routing] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=routing] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=realtime] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=realtime] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=term_statistics] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=term_statistics] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=version] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=version] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=version_type] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=version_type] [[docs-termvectors-api-example]] ==== {api-examples-title} diff --git a/docs/reference/docs/update-by-query.asciidoc b/docs/reference/docs/update-by-query.asciidoc index 5ef9d288623c..bc63fa4e33d0 100644 --- a/docs/reference/docs/update-by-query.asciidoc +++ b/docs/reference/docs/update-by-query.asciidoc @@ -167,70 +167,70 @@ this parameter or use `*` or `_all`. [[docs-update-by-query-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=analyzer] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=analyzer] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=analyze_wildcard] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=analyze_wildcard] `conflicts`:: (Optional, string) What to do if update by query hits version conflicts: `abort` or `proceed`. Defaults to `abort`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=default_operator] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=default_operator] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=df] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=df] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `open`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=lenient] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=lenient] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=max_docs] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=max_docs] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=pipeline] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=pipeline] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=preference] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=preference] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=search-q] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=search-q] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=request_cache] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=request_cache] `refresh`:: (Optional, Boolean) If `true`, {es} refreshes affected shards to make the operation visible to search. Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=requests_per_second] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=requests_per_second] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=routing] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=routing] `scroll`:: (Optional, <>) Period to retain the <> for scrolling. See <>. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=scroll_size] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=scroll_size] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=search_type] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=search_type] `search_timeout`:: (Optional, <>) Explicit timeout for each search request. Defaults to no timeout. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=slices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=slices] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sort] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=sort] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=stats] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=stats] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=terminate_after] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=terminate_after] `timeout`:: + @@ -246,9 +246,9 @@ timeout before failing. The actual wait time could be longer, particularly when multiple waits occur. -- -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=version] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=version] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] [[docs-update-by-query-api-request-body]] ==== {api-request-body-title} diff --git a/docs/reference/docs/update.asciidoc b/docs/reference/docs/update.asciidoc index 989335eb702f..ca6a7e489449 100644 --- a/docs/reference/docs/update.asciidoc +++ b/docs/reference/docs/update.asciidoc @@ -53,22 +53,22 @@ automatically if it doesn't exist. For more information, see <>. [[docs-update-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=if_seq_no] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=if_seq_no] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=if_primary_term] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=if_primary_term] `lang`:: (Optional, string) The script language. Default: `painless`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=require-alias] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=require-alias] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=refresh] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=refresh] `retry_on_conflict`:: (Optional, integer) Specify how many times should the operation be retried when a conflict occurs. Default: 0. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=routing] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=routing] `_source`:: (Optional, list) Set to `false` to disable source retrieval (default: `true`). @@ -94,7 +94,7 @@ timeout before failing. The actual wait time could be longer, particularly when multiple waits occur. -- -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] [[update-api-example]] ==== {api-examples-title} diff --git a/docs/reference/eql/eql-search-api.asciidoc b/docs/reference/eql/eql-search-api.asciidoc index d4ea3f3c7499..d7f10f4627f6 100644 --- a/docs/reference/eql/eql-search-api.asciidoc +++ b/docs/reference/eql/eql-search-api.asciidoc @@ -99,7 +99,7 @@ ignored. Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `open`. @@ -206,7 +206,7 @@ returned. + A greater `fetch_size` value often increases search speed but uses more memory. -include::{es-repo-dir}/search/search.asciidoc[tag=fields-param-def] +include::{es-ref-dir}/search/search.asciidoc[tag=fields-param-def] `filter`:: (Optional, <>) @@ -280,7 +280,7 @@ command]. NOTE: This parameter may change the set of returned hits. However, it does not change the sort order of hits in the response. -include::{es-repo-dir}/search/search.asciidoc[tag=runtime-mappings-def] +include::{es-ref-dir}/search/search.asciidoc[tag=runtime-mappings-def] [[eql-search-api-params-size]] `size`:: diff --git a/docs/reference/eql/eql.asciidoc b/docs/reference/eql/eql.asciidoc index 2ede5e0fc737..8f3b5b893ea5 100644 --- a/docs/reference/eql/eql.asciidoc +++ b/docs/reference/eql/eql.asciidoc @@ -1013,7 +1013,7 @@ You can also use the `fields` parameter to retrieve and format specific fields in the response. This field is identical to the search API's <>. -include::{es-repo-dir}/search/search-your-data/retrieve-selected-fields.asciidoc[tag=fields-param-desc] +include::{es-ref-dir}/search/search-your-data/retrieve-selected-fields.asciidoc[tag=fields-param-desc] The following search request uses the `fields` parameter to retrieve values for the `event.type` field, all fields starting with `process.`, and the @@ -1039,7 +1039,7 @@ GET /my-data-stream/_eql/search?filter_path=-hits.events._source ---- // TEST[setup:sec_logs] -include::{es-repo-dir}/search/search-your-data/retrieve-selected-fields.asciidoc[tag=fields-param-callouts] +include::{es-ref-dir}/search/search-your-data/retrieve-selected-fields.asciidoc[tag=fields-param-callouts] The response includes values as a flat list in the `fields` section for each hit. diff --git a/docs/reference/esql/esql-across-clusters.asciidoc b/docs/reference/esql/esql-across-clusters.asciidoc index f35a62c49aca..95278314b025 100644 --- a/docs/reference/esql/esql-across-clusters.asciidoc +++ b/docs/reference/esql/esql-across-clusters.asciidoc @@ -13,16 +13,16 @@ With {esql}, you can execute a single query across multiple clusters. ==== Prerequisites -include::{es-repo-dir}/search/search-your-data/search-across-clusters.asciidoc[tag=ccs-prereqs] +include::{es-ref-dir}/search/search-your-data/search-across-clusters.asciidoc[tag=ccs-prereqs] -include::{es-repo-dir}/search/search-your-data/search-across-clusters.asciidoc[tag=ccs-gateway-seed-nodes] +include::{es-ref-dir}/search/search-your-data/search-across-clusters.asciidoc[tag=ccs-gateway-seed-nodes] -include::{es-repo-dir}/search/search-your-data/search-across-clusters.asciidoc[tag=ccs-proxy-mode] +include::{es-ref-dir}/search/search-your-data/search-across-clusters.asciidoc[tag=ccs-proxy-mode] [discrete] [[ccq-remote-cluster-setup]] ==== Remote cluster setup -include::{es-repo-dir}/search/search-your-data/search-across-clusters.asciidoc[tag=ccs-remote-cluster-setup] +include::{es-ref-dir}/search/search-your-data/search-across-clusters.asciidoc[tag=ccs-remote-cluster-setup] <1> Since `skip_unavailable` was not set on `cluster_three`, it uses the default of `false`. See the <> @@ -221,4 +221,4 @@ in the response, such as execution time, selected target indices, and shards. [[ccq-during-upgrade]] ==== Query across clusters during an upgrade -include::{es-repo-dir}/search/search-your-data/search-across-clusters.asciidoc[tag=ccs-during-upgrade] +include::{es-ref-dir}/search/search-your-data/search-across-clusters.asciidoc[tag=ccs-during-upgrade] diff --git a/docs/reference/esql/esql-enrich-data.asciidoc b/docs/reference/esql/esql-enrich-data.asciidoc index e465d7daae12..c48118d1c367 100644 --- a/docs/reference/esql/esql-enrich-data.asciidoc +++ b/docs/reference/esql/esql-enrich-data.asciidoc @@ -91,7 +91,7 @@ your query. [[esql-enrich-prereqs]] ==== Prerequisites -include::{es-repo-dir}/ingest/apis/enrich/put-enrich-policy.asciidoc[tag=enrich-policy-api-prereqs] +include::{es-ref-dir}/ingest/apis/enrich/put-enrich-policy.asciidoc[tag=enrich-policy-api-prereqs] [discrete] [[esql-create-enrich-source-index]] @@ -130,7 +130,7 @@ include::processing-commands/enrich.asciidoc[tag=examples] [[esql-update-enrich-data]] ==== Update an enrich index -include::{es-repo-dir}/ingest/apis/enrich/execute-enrich-policy.asciidoc[tag=update-enrich-index] +include::{es-ref-dir}/ingest/apis/enrich/execute-enrich-policy.asciidoc[tag=update-enrich-index] [discrete] [[esql-update-enrich-policies]] diff --git a/docs/reference/esql/esql-get-started.asciidoc b/docs/reference/esql/esql-get-started.asciidoc index 351a8efdc8ae..0e23c0d97e61 100644 --- a/docs/reference/esql/esql-get-started.asciidoc +++ b/docs/reference/esql/esql-get-started.asciidoc @@ -21,7 +21,7 @@ This getting started is also available as an https://github.com/elastic/elastics To follow along with the queries in this guide, you can either set up your own deployment, or use Elastic's public {esql} demo environment. -include::{es-repo-dir}/tab-widgets/esql/esql-getting-started-widget-sample-data.asciidoc[] +include::{es-ref-dir}/tab-widgets/esql/esql-getting-started-widget-sample-data.asciidoc[] [discrete] [[esql-getting-started-running-queries]] @@ -29,7 +29,7 @@ include::{es-repo-dir}/tab-widgets/esql/esql-getting-started-widget-sample-data. In {kib}, you can use Console or Discover to run {esql} queries: -include::{es-repo-dir}/tab-widgets/esql/esql-getting-started-widget-discover-console.asciidoc[] +include::{es-ref-dir}/tab-widgets/esql/esql-getting-started-widget-discover-console.asciidoc[] [discrete] [[esql-getting-started-first-query]] @@ -279,7 +279,7 @@ Before you can use `ENRICH`, you first need to <> and <> an <>. -include::{es-repo-dir}/tab-widgets/esql/esql-getting-started-widget-enrich-policy.asciidoc[] +include::{es-ref-dir}/tab-widgets/esql/esql-getting-started-widget-enrich-policy.asciidoc[] After creating and executing a policy, you can use it with the `ENRICH` command: diff --git a/docs/reference/esql/esql-index-options.asciidoc b/docs/reference/esql/esql-index-options.asciidoc index ba2307f611d4..721461bd9671 100644 --- a/docs/reference/esql/esql-index-options.asciidoc +++ b/docs/reference/esql/esql-index-options.asciidoc @@ -24,7 +24,7 @@ values may differ. The currently supported options are: -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `true`. @@ -37,7 +37,7 @@ missing or closed index. + Defaults to `true`. -include::{es-repo-dir}/search/search.asciidoc[tag=search-preference] +include::{es-ref-dir}/search/search.asciidoc[tag=search-preference] *Examples* diff --git a/docs/reference/esql/index.asciidoc b/docs/reference/esql/index.asciidoc index 531336277ba6..5cb02064dc79 100644 --- a/docs/reference/esql/index.asciidoc +++ b/docs/reference/esql/index.asciidoc @@ -1,7 +1,7 @@ [[esql]] = {esql} -:esql-tests: {xes-repo-dir}/../../plugin/esql/qa +:esql-tests: {elasticsearch-root}/x-pack/docs/{lang}/../../plugin/esql/qa :esql-specs: {esql-tests}/testFixtures/src/main/resources [partintro] diff --git a/docs/reference/getting-started.asciidoc b/docs/reference/getting-started.asciidoc index 3e474953a72f..2a5dbc2f0d03 100644 --- a/docs/reference/getting-started.asciidoc +++ b/docs/reference/getting-started.asciidoc @@ -24,8 +24,8 @@ The simplest way to set up {es} is to create a managed deployment with {ess} on {ecloud}. If you prefer to manage your own test environment, install and run {es} using Docker. -include::{es-repo-dir}/tab-widgets/code.asciidoc[] -include::{es-repo-dir}/tab-widgets/quick-start-install-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/code.asciidoc[] +include::{es-ref-dir}/tab-widgets/quick-start-install-widget.asciidoc[] [discrete] [[send-requests-to-elasticsearch]] @@ -36,7 +36,7 @@ with {es} using any client that sends HTTP requests, such as https://curl.se[curl]. You can also use {kib}'s Console to send requests to {es}. -include::{es-repo-dir}/tab-widgets/api-call-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/api-call-widget.asciidoc[] [discrete] [[add-data]] diff --git a/docs/reference/how-to/disk-usage.asciidoc b/docs/reference/how-to/disk-usage.asciidoc index 55a7b33f53cb..975067fd576b 100644 --- a/docs/reference/how-to/disk-usage.asciidoc +++ b/docs/reference/how-to/disk-usage.asciidoc @@ -94,7 +94,7 @@ Indices in Elasticsearch are stored in one or more shards. Each shard is a Lucen The <> can be used to reduce the number of segments per shard. In many cases, the number of segments can be reduced to one per shard by setting `max_num_segments=1`. -include::{es-repo-dir}/indices/forcemerge.asciidoc[tag=force-merge-read-only-warn] +include::{es-ref-dir}/indices/forcemerge.asciidoc[tag=force-merge-read-only-warn] [discrete] === Shrink index diff --git a/docs/reference/how-to/indexing-speed.asciidoc b/docs/reference/how-to/indexing-speed.asciidoc index 924207bca470..2bff5f82bf73 100644 --- a/docs/reference/how-to/indexing-speed.asciidoc +++ b/docs/reference/how-to/indexing-speed.asciidoc @@ -43,7 +43,7 @@ The operation that consists of making changes visible to search - called a <> - is costly, and calling it often while there is ongoing indexing activity can hurt indexing speed. -include::{es-repo-dir}/indices/refresh.asciidoc[tag=refresh-interval-default] +include::{es-ref-dir}/indices/refresh.asciidoc[tag=refresh-interval-default] This is the optimal configuration if you have no or very little search traffic (e.g. less than one search request every 5 minutes) and want to optimize for indexing speed. This behavior aims to automatically optimize bulk indexing in diff --git a/docs/reference/how-to/knn-search.asciidoc b/docs/reference/how-to/knn-search.asciidoc index bfe99ad615c4..194d122cef15 100644 --- a/docs/reference/how-to/knn-search.asciidoc +++ b/docs/reference/how-to/knn-search.asciidoc @@ -109,7 +109,7 @@ force merge to one segment, the kNN search only need to check a single, all-inclusive HNSW graph. Force merging `dense_vector` fields is an expensive operation that can take significant time to complete. -include::{es-repo-dir}/indices/forcemerge.asciidoc[tag=force-merge-read-only-warn] +include::{es-ref-dir}/indices/forcemerge.asciidoc[tag=force-merge-read-only-warn] [discrete] ==== Create large segments during bulk indexing diff --git a/docs/reference/how-to/use-elasticsearch-for-time-series-data.asciidoc b/docs/reference/how-to/use-elasticsearch-for-time-series-data.asciidoc index 18de2497760c..21a516aeb53f 100644 --- a/docs/reference/how-to/use-elasticsearch-for-time-series-data.asciidoc +++ b/docs/reference/how-to/use-elasticsearch-for-time-series-data.asciidoc @@ -26,7 +26,7 @@ stream. The steps for setting up data tiers vary based on your deployment type: -include::{es-repo-dir}/tab-widgets/data-tiers-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/data-tiers-widget.asciidoc[] [discrete] [[register-snapshot-repository]] @@ -39,7 +39,7 @@ To use {search-snaps}, you must register a supported snapshot repository. The steps for registering this repository vary based on your deployment type and storage provider: -include::{es-repo-dir}/tab-widgets/snapshot-repo-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/snapshot-repo-widget.asciidoc[] [discrete] [[create-edit-index-lifecycle-policy]] @@ -58,7 +58,7 @@ ensure your policy: * Uses {search-snaps} in the cold and frozen phases, if wanted. * Includes a delete phase, if needed. -include::{es-repo-dir}/tab-widgets/ilm-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/ilm-widget.asciidoc[] [discrete] [[create-ts-component-templates]] @@ -68,19 +68,19 @@ TIP: If you use {fleet} or {agent}, skip to <>. {fleet} and {agent} use built-in templates to create data streams for you. If you use a custom application, you need to set up your own data stream. -include::{es-repo-dir}/data-streams/set-up-a-data-stream.asciidoc[tag=ds-create-component-templates] +include::{es-ref-dir}/data-streams/set-up-a-data-stream.asciidoc[tag=ds-create-component-templates] [discrete] [[create-ts-index-template]] === Create an index template -include::{es-repo-dir}/data-streams/set-up-a-data-stream.asciidoc[tag=ds-create-index-template] +include::{es-ref-dir}/data-streams/set-up-a-data-stream.asciidoc[tag=ds-create-index-template] [discrete] [[add-data-to-data-stream]] === Add data to a data stream -include::{es-repo-dir}/data-streams/set-up-a-data-stream.asciidoc[tag=ds-create-data-stream] +include::{es-ref-dir}/data-streams/set-up-a-data-stream.asciidoc[tag=ds-create-data-stream] [discrete] [[search-visualize-your-data]] diff --git a/docs/reference/ilm/apis/delete-lifecycle.asciidoc b/docs/reference/ilm/apis/delete-lifecycle.asciidoc index b8e4c7a0bd21..632cb982b396 100644 --- a/docs/reference/ilm/apis/delete-lifecycle.asciidoc +++ b/docs/reference/ilm/apis/delete-lifecycle.asciidoc @@ -35,7 +35,7 @@ the request fails and returns an error. [[ilm-delete-lifecycle-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [[ilm-delete-lifecycle-example]] ==== {api-examples-title} diff --git a/docs/reference/ilm/apis/explain.asciidoc b/docs/reference/ilm/apis/explain.asciidoc index 64e9ec9d5241..fbe017619048 100644 --- a/docs/reference/ilm/apis/explain.asciidoc +++ b/docs/reference/ilm/apis/explain.asciidoc @@ -49,7 +49,7 @@ or `_all`. {ilm-init} and are in an error state, either due to an encountering an error while executing the policy, or attempting to use a policy that does not exist. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [[ilm-explain-lifecycle-example]] ==== {api-examples-title} diff --git a/docs/reference/ilm/apis/get-lifecycle.asciidoc b/docs/reference/ilm/apis/get-lifecycle.asciidoc index f736de20ed43..744361006548 100644 --- a/docs/reference/ilm/apis/get-lifecycle.asciidoc +++ b/docs/reference/ilm/apis/get-lifecycle.asciidoc @@ -36,7 +36,7 @@ modified date. If no policy is specified, returns all defined policies. [[ilm-get-lifecycle-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [[ilm-get-lifecycle-example]] ==== {api-examples-title} diff --git a/docs/reference/ilm/apis/get-status.asciidoc b/docs/reference/ilm/apis/get-status.asciidoc index 384f6e9e55ef..7e9e963f6f36 100644 --- a/docs/reference/ilm/apis/get-status.asciidoc +++ b/docs/reference/ilm/apis/get-status.asciidoc @@ -27,7 +27,7 @@ You can start or stop {ilm-init} with the <> and [[ilm-get-status-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [role="child_attributes"] [[ilm-get-status-response-body]] diff --git a/docs/reference/ilm/apis/move-to-step.asciidoc b/docs/reference/ilm/apis/move-to-step.asciidoc index f901b2402440..19cc9f708886 100644 --- a/docs/reference/ilm/apis/move-to-step.asciidoc +++ b/docs/reference/ilm/apis/move-to-step.asciidoc @@ -50,7 +50,7 @@ policy are considered valid, an index cannot move to a step that is not part of [[ilm-move-to-step-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [role="child_attributes"] [[ilm-move-to-step-request-body]] diff --git a/docs/reference/ilm/apis/put-lifecycle.asciidoc b/docs/reference/ilm/apis/put-lifecycle.asciidoc index 6d7c8ea5e297..ffd59a14d8c2 100644 --- a/docs/reference/ilm/apis/put-lifecycle.asciidoc +++ b/docs/reference/ilm/apis/put-lifecycle.asciidoc @@ -46,7 +46,7 @@ To avoid naming collisions with built-in and Fleet-managed ILM policies, avoid u [[ilm-put-lifecycle-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [[ilm-put-lifecycle-example]] ==== {api-examples-title} diff --git a/docs/reference/ilm/apis/remove-policy-from-index.asciidoc b/docs/reference/ilm/apis/remove-policy-from-index.asciidoc index 0bc501093697..20e0df9f3cb9 100644 --- a/docs/reference/ilm/apis/remove-policy-from-index.asciidoc +++ b/docs/reference/ilm/apis/remove-policy-from-index.asciidoc @@ -40,7 +40,7 @@ target. Supports wildcards (`*`). To target all data streams and indices, use [[ilm-remove-policy-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [[ilm-remove-policy-example]] ==== {api-examples-title} diff --git a/docs/reference/ilm/apis/retry-policy.asciidoc b/docs/reference/ilm/apis/retry-policy.asciidoc index c1c78bdb8824..cb2587fbb151 100644 --- a/docs/reference/ilm/apis/retry-policy.asciidoc +++ b/docs/reference/ilm/apis/retry-policy.asciidoc @@ -35,7 +35,7 @@ step. [[ilm-retry-policy-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [[ilm-retry-policy-example]] ==== {api-examples-title} diff --git a/docs/reference/ilm/apis/start.asciidoc b/docs/reference/ilm/apis/start.asciidoc index 1b5a4b9b2561..32db585c6b14 100644 --- a/docs/reference/ilm/apis/start.asciidoc +++ b/docs/reference/ilm/apis/start.asciidoc @@ -31,7 +31,7 @@ necessary if it has been stopped using the <>. [[ilm-start-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [[ilm-start-example]] ==== {api-examples-title} diff --git a/docs/reference/ilm/apis/stop.asciidoc b/docs/reference/ilm/apis/stop.asciidoc index 0fa7875b0efd..1e9cfb94d0b1 100644 --- a/docs/reference/ilm/apis/stop.asciidoc +++ b/docs/reference/ilm/apis/stop.asciidoc @@ -36,7 +36,7 @@ if {ilm-init} is running. [[ilm-stop-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [[ilm-stop-example]] ==== {api-examples-title} diff --git a/docs/reference/index-modules/blocks.asciidoc b/docs/reference/index-modules/blocks.asciidoc index 2d89676a8af2..5fc89dedf024 100644 --- a/docs/reference/index-modules/blocks.asciidoc +++ b/docs/reference/index-modules/blocks.asciidoc @@ -86,7 +86,7 @@ PUT /my-index-000001/_block/write [[add-index-block-api-path-params]] ==== {api-path-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index] + By default, you must explicitly name the indices you are adding blocks to. To allow the adding of blocks to indices with `_all`, `*`, or other wildcard @@ -116,17 +116,17 @@ Disable write operations. However, metadata changes are still allowed. [[add-index-block-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `open`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [discrete] [[add-index-block-api-example]] diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc index b09d67e99063..e47304f1e133 100644 --- a/docs/reference/index.asciidoc +++ b/docs/reference/index.asciidoc @@ -1,13 +1,6 @@ [[elasticsearch-reference]] = Elasticsearch Guide -:include-xpack: true -:es-test-dir: {elasticsearch-root}/docs/src/test -:plugins-examples-dir: {elasticsearch-root}/plugins/examples -:dependencies-dir: {elasticsearch-root}/build-tools-internal -:xes-repo-dir: {elasticsearch-root}/x-pack/docs/{lang} -:es-repo-dir: {elasticsearch-root}/docs/reference - include::../Versions.asciidoc[] include::links.asciidoc[] diff --git a/docs/reference/indices/add-alias.asciidoc b/docs/reference/indices/add-alias.asciidoc index 860e9ca46f79..e14af6a64a2e 100644 --- a/docs/reference/indices/add-alias.asciidoc +++ b/docs/reference/indices/add-alias.asciidoc @@ -46,7 +46,7 @@ indices return an error. [[add-alias-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [[add-alias-api-request-body]] ==== {api-request-body-title} diff --git a/docs/reference/indices/alias-exists.asciidoc b/docs/reference/indices/alias-exists.asciidoc index 997dbad5bd6c..f820a95028a0 100644 --- a/docs/reference/indices/alias-exists.asciidoc +++ b/docs/reference/indices/alias-exists.asciidoc @@ -44,7 +44,7 @@ omit this parameter or use `*` or `_all`. [[alias-exists-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `all`. @@ -52,7 +52,7 @@ Defaults to `all`. (Optional, Boolean) If `false`, requests that include a missing data stream or index in the `` return an error. Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=local] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local] [[alias-exists-api-response-codes]] ==== {api-response-codes-title} diff --git a/docs/reference/indices/aliases.asciidoc b/docs/reference/indices/aliases.asciidoc index 34248cc5f98d..1df9e0a4883b 100644 --- a/docs/reference/indices/aliases.asciidoc +++ b/docs/reference/indices/aliases.asciidoc @@ -43,7 +43,7 @@ for the index. [[indices-aliases-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [role="child_attributes"] [[indices-aliases-api-request-body]] diff --git a/docs/reference/indices/apis/reload-analyzers.asciidoc b/docs/reference/indices/apis/reload-analyzers.asciidoc index dc6ed7d916a1..ca5f540564f8 100644 --- a/docs/reference/indices/apis/reload-analyzers.asciidoc +++ b/docs/reference/indices/apis/reload-analyzers.asciidoc @@ -85,15 +85,15 @@ and indices, use `*` or `_all`. [[indices-reload-analyzers-api-query-params]] === {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `open`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] [discrete] diff --git a/docs/reference/indices/clearcache.asciidoc b/docs/reference/indices/clearcache.asciidoc index 10b590fce097..a3150ec6f72e 100644 --- a/docs/reference/indices/clearcache.asciidoc +++ b/docs/reference/indices/clearcache.asciidoc @@ -39,11 +39,11 @@ and indices, omit this parameter or use `*` or `_all`. [[clear-cache-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `open`. @@ -77,7 +77,7 @@ or field aliases. Comma-separated list of index names used to limit the request. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] `query`:: (Optional, Boolean) diff --git a/docs/reference/indices/clone-index.asciidoc b/docs/reference/indices/clone-index.asciidoc index ef8ed28c6ac0..c8e5d2e200f2 100644 --- a/docs/reference/indices/clone-index.asciidoc +++ b/docs/reference/indices/clone-index.asciidoc @@ -168,15 +168,15 @@ on index creation applies to the clone index action as well. (Required, string) Name of the source index to clone. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=target-index] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=target-index] [[clone-index-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [role="child_attributes"] [[clone-index-api-request-body]] @@ -185,6 +185,6 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] `aliases`:: (Optional, object of objects) Aliases for the resulting index. + -include::{es-repo-dir}/indices/create-index.asciidoc[tag=aliases-props] +include::{es-ref-dir}/indices/create-index.asciidoc[tag=aliases-props] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=target-index-settings] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=target-index-settings] diff --git a/docs/reference/indices/close.asciidoc b/docs/reference/indices/close.asciidoc index 5caa6e892289..a4bf1742fdea 100644 --- a/docs/reference/indices/close.asciidoc +++ b/docs/reference/indices/close.asciidoc @@ -29,13 +29,13 @@ POST /my-index-000001/_close You use the close index API to close open indices. -include::{es-repo-dir}/indices/open-close.asciidoc[tag=closed-index] +include::{es-ref-dir}/indices/open-close.asciidoc[tag=closed-index] [[close-index-api-path-params]] ==== {api-path-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index] + To close all indices, use `_all` or `*`. By default, you must explicitly name the indices you are closing. @@ -48,19 +48,19 @@ or using the <> API. [[close-index-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `open`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [[close-index-api-example]] diff --git a/docs/reference/indices/create-index.asciidoc b/docs/reference/indices/create-index.asciidoc index d39591a37df5..2e66f3d6030c 100644 --- a/docs/reference/indices/create-index.asciidoc +++ b/docs/reference/indices/create-index.asciidoc @@ -60,9 +60,9 @@ Index names must meet the following criteria: [[indices-create-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [role="child_attributes"] [[indices-create-api-request-body]] @@ -113,9 +113,9 @@ specified, this overwrites the `routing` value for search operations. // end::aliases-props[] -- -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=mappings] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=mappings] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=settings] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=settings] [[indices-create-api-example]] ==== {api-examples-title} diff --git a/docs/reference/indices/dangling-index-delete.asciidoc b/docs/reference/indices/dangling-index-delete.asciidoc index 53be30ef7628..6af35031e9e6 100644 --- a/docs/reference/indices/dangling-index-delete.asciidoc +++ b/docs/reference/indices/dangling-index-delete.asciidoc @@ -24,7 +24,7 @@ DELETE /_dangling/?accept_data_loss=true [[dangling-index-delete-api-desc]] ==== {api-description-title} -include::{es-repo-dir}/indices/dangling-indices-list.asciidoc[tag=dangling-index-description] +include::{es-ref-dir}/indices/dangling-indices-list.asciidoc[tag=dangling-index-description] Deletes a dangling index by referencing its UUID. Use the @@ -47,4 +47,4 @@ UUID of the index to delete. You can find this using the This field must be set to `true` in order to carry out the import, since it will no longer be possible to recover the data from the dangling index. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] diff --git a/docs/reference/indices/dangling-index-import.asciidoc b/docs/reference/indices/dangling-index-import.asciidoc index c022c51d4091..44cde56de8c9 100644 --- a/docs/reference/indices/dangling-index-import.asciidoc +++ b/docs/reference/indices/dangling-index-import.asciidoc @@ -24,7 +24,7 @@ POST /_dangling/?accept_data_loss=true [[dangling-index-import-api-desc]] ==== {api-description-title} -include::{es-repo-dir}/indices/dangling-indices-list.asciidoc[tag=dangling-index-description] +include::{es-ref-dir}/indices/dangling-indices-list.asciidoc[tag=dangling-index-description] Import a single index into the cluster by referencing its UUID. Use the <> to locate the UUID of an index. @@ -48,7 +48,7 @@ cannot know where the dangling index data came from or determine which shard copies are fresh and which are stale, it cannot guarantee that the imported data represents the latest state of the index when it was last in the cluster. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [[dangling-index-import-api-example]] ==== {api-examples-title} diff --git a/docs/reference/indices/data-stream-stats.asciidoc b/docs/reference/indices/data-stream-stats.asciidoc index a1f83701bb52..3ed285abc035 100644 --- a/docs/reference/indices/data-stream-stats.asciidoc +++ b/docs/reference/indices/data-stream-stats.asciidoc @@ -78,7 +78,7 @@ To target all data streams in a cluster, omit this parameter or use `*`. [[data-stream-stats-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=ds-expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=ds-expand-wildcards] + Defaults to `open`. diff --git a/docs/reference/indices/delete-alias.asciidoc b/docs/reference/indices/delete-alias.asciidoc index 680e255a9728..748862df0610 100644 --- a/docs/reference/indices/delete-alias.asciidoc +++ b/docs/reference/indices/delete-alias.asciidoc @@ -42,4 +42,4 @@ the request. Supports wildcards (`*`). [[delete-alias-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] diff --git a/docs/reference/indices/delete-component-template.asciidoc b/docs/reference/indices/delete-component-template.asciidoc index 21693c44aadc..0ca6560f17cc 100644 --- a/docs/reference/indices/delete-component-template.asciidoc +++ b/docs/reference/indices/delete-component-template.asciidoc @@ -52,10 +52,10 @@ that specify index mappings, settings, and aliases. [[delete-component-template-api-path-params]] ==== {api-path-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=component-template] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=component-template] [[delete-component-template-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] diff --git a/docs/reference/indices/delete-data-stream.asciidoc b/docs/reference/indices/delete-data-stream.asciidoc index eb970f0f29d4..38e7a00d451d 100644 --- a/docs/reference/indices/delete-data-stream.asciidoc +++ b/docs/reference/indices/delete-data-stream.asciidoc @@ -59,6 +59,6 @@ Wildcard (`*`) expressions are supported. [[delete-data-stream-api-query-parms]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=ds-expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=ds-expand-wildcards] + Defaults to `open`. diff --git a/docs/reference/indices/delete-index-template-v1.asciidoc b/docs/reference/indices/delete-index-template-v1.asciidoc index 9caf6935fe4f..ca0b5a0e726b 100644 --- a/docs/reference/indices/delete-index-template-v1.asciidoc +++ b/docs/reference/indices/delete-index-template-v1.asciidoc @@ -55,4 +55,4 @@ expressions are supported. [[delete-template-api-v1-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] diff --git a/docs/reference/indices/delete-index-template.asciidoc b/docs/reference/indices/delete-index-template.asciidoc index f410f091f686..02396310daff 100644 --- a/docs/reference/indices/delete-index-template.asciidoc +++ b/docs/reference/indices/delete-index-template.asciidoc @@ -55,10 +55,10 @@ and <> that can be applied automatically to new indices. [[delete-template-api-path-params]] ==== {api-path-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-template] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-template] [[delete-template-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] diff --git a/docs/reference/indices/delete-index.asciidoc b/docs/reference/indices/delete-index.asciidoc index 92f403c22ae3..d5d168154e44 100644 --- a/docs/reference/indices/delete-index.asciidoc +++ b/docs/reference/indices/delete-index.asciidoc @@ -50,14 +50,14 @@ setting to `false`. [[delete-index-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `open,closed`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] diff --git a/docs/reference/indices/diskusage.asciidoc b/docs/reference/indices/diskusage.asciidoc index 98c2c799cf05..3510ba346e5a 100644 --- a/docs/reference/indices/diskusage.asciidoc +++ b/docs/reference/indices/diskusage.asciidoc @@ -41,11 +41,11 @@ resources significantly. [[analyze-index-disk-usage-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `open`. @@ -53,13 +53,13 @@ Defaults to `open`. (Optional, Boolean) If `true`, the API performs a flush before analysis. If `false`, the response may not include uncommitted data. Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] `run_expensive_tasks`:: (Required, Boolean) Analyzing field disk usage is resource-intensive. To use the API, this parameter must be set to `true`. Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] [[analyze-index-disk-usage-api-example]] diff --git a/docs/reference/indices/downsample-data-stream.asciidoc b/docs/reference/indices/downsample-data-stream.asciidoc index 8226c365dd50..5ace4e03dfb6 100644 --- a/docs/reference/indices/downsample-data-stream.asciidoc +++ b/docs/reference/indices/downsample-data-stream.asciidoc @@ -103,7 +103,7 @@ or `manage` <> for the data stream. -- (Required, string) Name of the index to create. -include::{es-repo-dir}/indices/create-index.asciidoc[tag=index-name-reqs] +include::{es-ref-dir}/indices/create-index.asciidoc[tag=index-name-reqs] -- [role="child_attributes"] diff --git a/docs/reference/indices/field-usage-stats.asciidoc b/docs/reference/indices/field-usage-stats.asciidoc index 9ff3143cc389..9fd1d9e59eb3 100644 --- a/docs/reference/indices/field-usage-stats.asciidoc +++ b/docs/reference/indices/field-usage-stats.asciidoc @@ -33,20 +33,20 @@ GET /my-index-000001/_field_usage_stats [[field-usage-stats-api-path-params]] ==== {api-path-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index] [[field-usage-stats-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] `fields`:: + diff --git a/docs/reference/indices/flush.asciidoc b/docs/reference/indices/flush.asciidoc index 25d39a17af30..61c44f157da9 100644 --- a/docs/reference/indices/flush.asciidoc +++ b/docs/reference/indices/flush.asciidoc @@ -66,11 +66,11 @@ this parameter or use `*` or `_all`. [[flush-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `open`. @@ -90,7 +90,7 @@ This parameter is considered internal. -- -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] `wait_if_ongoing`:: + diff --git a/docs/reference/indices/forcemerge.asciidoc b/docs/reference/indices/forcemerge.asciidoc index e316e0c5e1ae..1d473acbd5d4 100644 --- a/docs/reference/indices/forcemerge.asciidoc +++ b/docs/reference/indices/forcemerge.asciidoc @@ -103,11 +103,11 @@ and indices, omit this parameter or use `*` or `_all`. [[forcemerge-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `open`. @@ -118,7 +118,7 @@ If `true`, after the force merge. Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] `max_num_segments`:: + diff --git a/docs/reference/indices/get-alias.asciidoc b/docs/reference/indices/get-alias.asciidoc index 7c2f414ce5f4..743aaf7aee17 100644 --- a/docs/reference/indices/get-alias.asciidoc +++ b/docs/reference/indices/get-alias.asciidoc @@ -47,11 +47,11 @@ omit this parameter or use `*` or `_all`. [[get-alias-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `all`. @@ -59,4 +59,4 @@ Defaults to `all`. (Optional, Boolean) If `false`, requests that include a missing data stream or index in the `` return an error. Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=local] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local] diff --git a/docs/reference/indices/get-data-stream.asciidoc b/docs/reference/indices/get-data-stream.asciidoc index d35e7a3d5e2e..240a33164b37 100644 --- a/docs/reference/indices/get-data-stream.asciidoc +++ b/docs/reference/indices/get-data-stream.asciidoc @@ -97,7 +97,7 @@ returned. [[get-data-stream-api-query-parms]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=ds-expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=ds-expand-wildcards] + Defaults to `open`. diff --git a/docs/reference/indices/get-field-mapping.asciidoc b/docs/reference/indices/get-field-mapping.asciidoc index f72210b58e8f..ac5895872fbb 100644 --- a/docs/reference/indices/get-field-mapping.asciidoc +++ b/docs/reference/indices/get-field-mapping.asciidoc @@ -47,13 +47,13 @@ limit returned information. [[get-field-mapping-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] `include_defaults`:: (Optional, Boolean) If `true`, the response includes default mapping values. diff --git a/docs/reference/indices/get-index-template-v1.asciidoc b/docs/reference/indices/get-index-template-v1.asciidoc index aa29786033d1..602ca2fe454a 100644 --- a/docs/reference/indices/get-index-template-v1.asciidoc +++ b/docs/reference/indices/get-index-template-v1.asciidoc @@ -52,7 +52,7 @@ privilege>> to use this API. [[get-template-v1-api-path-params]] ==== {api-path-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-template] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-template] + To return all index templates, omit this parameter or use a value of `_all` or `*`. @@ -61,11 +61,11 @@ or use a value of `_all` or `*`. [[get-template-v1-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=flat-settings] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=flat-settings] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=local] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] [[get-template-v1-api-example]] diff --git a/docs/reference/indices/get-index.asciidoc b/docs/reference/indices/get-index.asciidoc index 3e08cfc02c28..2551d25801d7 100644 --- a/docs/reference/indices/get-index.asciidoc +++ b/docs/reference/indices/get-index.asciidoc @@ -36,11 +36,11 @@ and indices, omit this parameter or use `*` or `_all`. [[get-index-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `open`. @@ -50,15 +50,15 @@ Return information about specific index features. Supports comma- separated values. Valid values are `aliases`, `mappings`, and `settings`. Defaults to `aliases,mappings,settings`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=flat-settings] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=flat-settings] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=include-defaults] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=include-defaults] `ignore_unavailable`:: (Optional, Boolean) If `false`, requests that target a missing index return an error. Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=local] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] diff --git a/docs/reference/indices/get-mapping.asciidoc b/docs/reference/indices/get-mapping.asciidoc index 90368126b57b..16dc8c66d071 100644 --- a/docs/reference/indices/get-mapping.asciidoc +++ b/docs/reference/indices/get-mapping.asciidoc @@ -38,19 +38,19 @@ and indices, omit this parameter or use `*` or `_all`. [[get-mapping-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `open`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=local] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] [[get-mapping-api-example]] diff --git a/docs/reference/indices/get-settings.asciidoc b/docs/reference/indices/get-settings.asciidoc index f7ba885b55cb..b6cb6d292638 100644 --- a/docs/reference/indices/get-settings.asciidoc +++ b/docs/reference/indices/get-settings.asciidoc @@ -44,23 +44,23 @@ used to limit the request. [[get-index-settings-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `open`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=flat-settings] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=flat-settings] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=include-defaults] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=include-defaults] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=local] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] [[get-index-settings-api-example]] diff --git a/docs/reference/indices/index-template-exists-v1.asciidoc b/docs/reference/indices/index-template-exists-v1.asciidoc index 888a967056ee..2358f0b1a376 100644 --- a/docs/reference/indices/index-template-exists-v1.asciidoc +++ b/docs/reference/indices/index-template-exists-v1.asciidoc @@ -43,17 +43,17 @@ and <> that can be applied automatically to new indices. [[template-exists-api-path-params]] ==== {api-path-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-template] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-template] [[template-exists-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=flat-settings] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=flat-settings] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=local] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] [[template-exists-api-response-codes]] diff --git a/docs/reference/indices/indices-exists.asciidoc b/docs/reference/indices/indices-exists.asciidoc index 934b224b8e79..d699d36add03 100644 --- a/docs/reference/indices/indices-exists.asciidoc +++ b/docs/reference/indices/indices-exists.asciidoc @@ -35,21 +35,21 @@ Supports wildcards (`*`). [[indices-exists-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `open`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=flat-settings] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=flat-settings] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=include-defaults] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=include-defaults] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=local] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local] [[indices-exists-api-response-codes]] ==== {api-response-codes-title} diff --git a/docs/reference/indices/open-close.asciidoc b/docs/reference/indices/open-close.asciidoc index bac5c07a0195..a077c4d19fd5 100644 --- a/docs/reference/indices/open-close.asciidoc +++ b/docs/reference/indices/open-close.asciidoc @@ -96,19 +96,19 @@ or using the <> API. [[open-index-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `closed`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [[open-index-api-example]] diff --git a/docs/reference/indices/put-component-template.asciidoc b/docs/reference/indices/put-component-template.asciidoc index faf7e67039de..0a0e36b63e6c 100644 --- a/docs/reference/indices/put-component-template.asciidoc +++ b/docs/reference/indices/put-component-template.asciidoc @@ -123,7 +123,7 @@ The exception of that rule are the `*@custom` component templates that let you s If `true`, this request cannot replace or update existing component templates. Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] [role="child_attributes"] [[put-component-template-api-request-body]] @@ -140,13 +140,13 @@ This is the template to be applied, may optionally include a `mappings`, `aliases`:: (Optional, object of objects) Aliases to add. + -include::{es-repo-dir}/indices/put-index-template.asciidoc[tag=template-ds-alias] +include::{es-ref-dir}/indices/put-index-template.asciidoc[tag=template-ds-alias] + -include::{es-repo-dir}/indices/create-index.asciidoc[tag=aliases-props] +include::{es-ref-dir}/indices/create-index.asciidoc[tag=aliases-props] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=mappings] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=mappings] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=settings] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=settings] ==== `version`:: diff --git a/docs/reference/indices/put-index-template-v1.asciidoc b/docs/reference/indices/put-index-template-v1.asciidoc index 5b7713656c4d..86a8a54edd97 100644 --- a/docs/reference/indices/put-index-template-v1.asciidoc +++ b/docs/reference/indices/put-index-template-v1.asciidoc @@ -112,7 +112,7 @@ Templates with lower `order` values are merged first. Templates with higher `order` values are merged later, overriding templates with lower values. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] [role="child_attributes"] [[put-index-template-v1-api-request-body]] @@ -126,11 +126,11 @@ used to match the names of indices during creation. `aliases`:: (Optional, object of objects) Aliases for the index. + -include::{es-repo-dir}/indices/create-index.asciidoc[tag=aliases-props] +include::{es-ref-dir}/indices/create-index.asciidoc[tag=aliases-props] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=mappings] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=mappings] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=settings] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=settings] `version`:: (Optional, integer) diff --git a/docs/reference/indices/put-index-template.asciidoc b/docs/reference/indices/put-index-template.asciidoc index b9460bda86a0..bcc7fa9caa81 100644 --- a/docs/reference/indices/put-index-template.asciidoc +++ b/docs/reference/indices/put-index-template.asciidoc @@ -156,7 +156,7 @@ aliases. Otherwise, these are index aliases. Data stream aliases ignore the `index_routing`, `routing`, and `search_routing` options. // end::template-ds-alias[] + -include::{es-repo-dir}/indices/create-index.asciidoc[tag=aliases-props] +include::{es-ref-dir}/indices/create-index.asciidoc[tag=aliases-props] include::{docdir}/rest-api/common-parms.asciidoc[tag=mappings] diff --git a/docs/reference/indices/put-mapping.asciidoc b/docs/reference/indices/put-mapping.asciidoc index bd2382c38910..dc6dbff1df42 100644 --- a/docs/reference/indices/put-mapping.asciidoc +++ b/docs/reference/indices/put-mapping.asciidoc @@ -50,17 +50,17 @@ and indices, omit this parameter or use `*` or `_all`. [[put-mapping-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `open`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] `write_index_only`:: (Optional, Boolean) diff --git a/docs/reference/indices/recovery.asciidoc b/docs/reference/indices/recovery.asciidoc index 81b3aa13580c..b4e4bd33f819 100644 --- a/docs/reference/indices/recovery.asciidoc +++ b/docs/reference/indices/recovery.asciidoc @@ -70,11 +70,11 @@ and indices, omit this parameter or use `*` or `_all`. [[index-recovery-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=active-only] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=active-only] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=detailed] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=detailed] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-query-parm] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-query-parm] [[index-recovery-api-response-body]] diff --git a/docs/reference/indices/refresh.asciidoc b/docs/reference/indices/refresh.asciidoc index f0f9d06d6fc4..bd8e821ff56b 100644 --- a/docs/reference/indices/refresh.asciidoc +++ b/docs/reference/indices/refresh.asciidoc @@ -81,15 +81,15 @@ omit this parameter or use `*` or `_all`. [[refresh-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `open`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] [[refresh-api-example]] diff --git a/docs/reference/indices/resolve-cluster.asciidoc b/docs/reference/indices/resolve-cluster.asciidoc index 8fa53bfc2705..48e6bfac4af1 100644 --- a/docs/reference/indices/resolve-cluster.asciidoc +++ b/docs/reference/indices/resolve-cluster.asciidoc @@ -91,15 +91,15 @@ Resources on <> can be specified using the [[resolve-cluster-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `open`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] + Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `true`. diff --git a/docs/reference/indices/resolve.asciidoc b/docs/reference/indices/resolve.asciidoc index c919bba5c765..856546b037fe 100644 --- a/docs/reference/indices/resolve.asciidoc +++ b/docs/reference/indices/resolve.asciidoc @@ -76,15 +76,15 @@ Resources on <> can be specified using the [[resolve-index-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `open`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] + Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `true`. diff --git a/docs/reference/indices/rollover-index.asciidoc b/docs/reference/indices/rollover-index.asciidoc index aa6f978b237d..2a47d28e5358 100644 --- a/docs/reference/indices/rollover-index.asciidoc +++ b/docs/reference/indices/rollover-index.asciidoc @@ -46,7 +46,7 @@ rollover also increments the data stream's generation. See [TIP] ==== -include::{es-repo-dir}/data-streams/set-up-a-data-stream.asciidoc[tag=time-series-alias-tip] +include::{es-ref-dir}/data-streams/set-up-a-data-stream.asciidoc[tag=time-series-alias-tip] See <>. ==== @@ -104,7 +104,7 @@ streams do not support this parameter. If the name of the alias's current write index does not end with `-` and a number, such as `my-index-000001` or `my-index-3`, this parameter is required. + -include::{es-repo-dir}/indices/create-index.asciidoc[tag=index-name-reqs] +include::{es-ref-dir}/indices/create-index.asciidoc[tag=index-name-reqs] [[rollover-index-api-query-params]] ==== {api-query-parms-title} @@ -119,9 +119,9 @@ If `true`, checks whether the current index satisfies the specified If `true`, signals that the data stream will be rolled over when the next indexing operation occurs. Applies only to data streams. Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [role="child_attributes"] [[rollover-index-api-request-body]] @@ -131,7 +131,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] (Optional, object of objects) Aliases for the target index. Data streams do not support this parameter. + -include::{es-repo-dir}/indices/create-index.asciidoc[tag=aliases-props] +include::{es-ref-dir}/indices/create-index.asciidoc[tag=aliases-props] [[rollover-conditions]] `conditions`:: @@ -152,14 +152,14 @@ instead. .Properties of `conditions` [%collapsible%open] ==== -include::{es-repo-dir}/ilm/actions/ilm-rollover.asciidoc[tag=rollover-conditions] +include::{es-ref-dir}/ilm/actions/ilm-rollover.asciidoc[tag=rollover-conditions] ==== -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=mappings] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=mappings] + Data streams do not support this parameter. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=settings] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=settings] + Data streams do not support this parameter. diff --git a/docs/reference/indices/segments.asciidoc b/docs/reference/indices/segments.asciidoc index db13243c62d2..8621bc7ae1de 100644 --- a/docs/reference/indices/segments.asciidoc +++ b/docs/reference/indices/segments.asciidoc @@ -40,50 +40,50 @@ and indices, omit this parameter or use `*` or `_all`. [[index-segments-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `open`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] [[index-segments-api-response-body]] ==== {api-response-body-title} ``:: (String) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=segment] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=segment] `generation`:: (Integer) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=generation] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=generation] `num_docs`:: (Integer) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=docs-count] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=docs-count] `deleted_docs`:: (Integer) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=docs-deleted] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=docs-deleted] `size_in_bytes`:: (Integer) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=segment-size] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=segment-size] `committed`:: (Boolean) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=committed] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=committed] `search`:: (Boolean) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=segment-search] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=segment-search] `version`:: (String) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=segment-version] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=segment-version] `compound`:: (Boolean) diff --git a/docs/reference/indices/shard-stores.asciidoc b/docs/reference/indices/shard-stores.asciidoc index 562c14694971..1b001a3175b8 100644 --- a/docs/reference/indices/shard-stores.asciidoc +++ b/docs/reference/indices/shard-stores.asciidoc @@ -58,15 +58,15 @@ and indices, omit this parameter or use `*` or `_all`. [[index-shard-stores-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `open`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] `status`:: + diff --git a/docs/reference/indices/shrink-index.asciidoc b/docs/reference/indices/shrink-index.asciidoc index 5d5e6c24d9e8..244733282d46 100644 --- a/docs/reference/indices/shrink-index.asciidoc +++ b/docs/reference/indices/shrink-index.asciidoc @@ -219,14 +219,14 @@ on index creation applies to the shrink index action as well. (Required, string) Name of the source index to shrink. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=target-index] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=target-index] [[shrink-index-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [role="child_attributes"] [[shrink-index-api-request-body]] @@ -235,9 +235,9 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] `aliases`:: (Optional, object of objects) Aliases for the resulting index. + -include::{es-repo-dir}/indices/create-index.asciidoc[tag=aliases-props] +include::{es-ref-dir}/indices/create-index.asciidoc[tag=aliases-props] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=target-index-settings] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=target-index-settings] `max_primary_shard_size`:: (Optional, <>) diff --git a/docs/reference/indices/simulate-template.asciidoc b/docs/reference/indices/simulate-template.asciidoc index 404aa70d72e7..c7397ace9788 100644 --- a/docs/reference/indices/simulate-template.asciidoc +++ b/docs/reference/indices/simulate-template.asciidoc @@ -132,7 +132,7 @@ The settings, mappings, and aliases that would be applied to matching indices. (Optional, object of objects) Aliases for the index. If the index template includes `data_stream`, this parameter is not supported. + -include::{es-repo-dir}/indices/create-index.asciidoc[tag=aliases-props] +include::{es-ref-dir}/indices/create-index.asciidoc[tag=aliases-props] include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=mappings] diff --git a/docs/reference/indices/split-index.asciidoc b/docs/reference/indices/split-index.asciidoc index 26ae0f19b177..0c93b572639d 100644 --- a/docs/reference/indices/split-index.asciidoc +++ b/docs/reference/indices/split-index.asciidoc @@ -255,15 +255,15 @@ on index creation applies to the split index action as well. (Required, string) Name of the source index to split. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=target-index] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=target-index] [[split-index-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=wait_for_active_shards] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [role="child_attributes"] [[split-index-api-request-body]] @@ -272,6 +272,6 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] `aliases`:: (Optional, object of objects) Aliases for the resulting index. + -include::{es-repo-dir}/indices/create-index.asciidoc[tag=aliases-props] +include::{es-ref-dir}/indices/create-index.asciidoc[tag=aliases-props] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=target-index-settings] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=target-index-settings] diff --git a/docs/reference/indices/stats.asciidoc b/docs/reference/indices/stats.asciidoc index 41b213b7a5b2..088d65c37ec6 100644 --- a/docs/reference/indices/stats.asciidoc +++ b/docs/reference/indices/stats.asciidoc @@ -64,34 +64,34 @@ to which the shard contributed. used to limit the request. Supports wildcards (`*`). To target all data streams and indices, omit this parameter or use `*` or `_all`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-metric] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-metric] [[index-stats-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `open`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=fields] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=fields] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=completion-fields] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=completion-fields] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=fielddata-fields] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=fielddata-fields] `forbid_closed_indices`:: (Optional, Boolean) If `true`, statistics are *not* collected from closed indices. Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=groups] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=groups] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=level] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=level] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=include-segment-file-sizes] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=include-segment-file-sizes] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=include-unloaded-segments] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=include-unloaded-segments] [[index-stats-api-example]] diff --git a/docs/reference/indices/update-settings.asciidoc b/docs/reference/indices/update-settings.asciidoc index 1ac9ecbb6a6a..3b29946d5ed7 100644 --- a/docs/reference/indices/update-settings.asciidoc +++ b/docs/reference/indices/update-settings.asciidoc @@ -44,17 +44,17 @@ and indices, omit this parameter or use `*` or `_all`. [[update-index-settings-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `open`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=flat-settings] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=flat-settings] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] `preserve_existing`:: (Optional, Boolean) If `true`, existing index settings remain unchanged. @@ -70,7 +70,7 @@ NOTE: Changing index settings on an automatically closed index using the `reopen parameter will result in the index becoming unavailable momentarily while the index is in the process of reopening. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [[update-index-settings-api-request-body]] diff --git a/docs/reference/ingest/apis/delete-pipeline.asciidoc b/docs/reference/ingest/apis/delete-pipeline.asciidoc index 368f9f2b0262..6f50251dbf1c 100644 --- a/docs/reference/ingest/apis/delete-pipeline.asciidoc +++ b/docs/reference/ingest/apis/delete-pipeline.asciidoc @@ -59,7 +59,7 @@ use a value of `*`. [[delete-pipeline-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [[delete-pipeline-api-api-example]] diff --git a/docs/reference/ingest/apis/get-pipeline.asciidoc b/docs/reference/ingest/apis/get-pipeline.asciidoc index 9208f0507876..71a261d97bde 100644 --- a/docs/reference/ingest/apis/get-pipeline.asciidoc +++ b/docs/reference/ingest/apis/get-pipeline.asciidoc @@ -62,7 +62,7 @@ To get all ingest pipelines, omit this parameter or use `*`. [[get-pipeline-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] [[get-pipeline-api-api-example]] diff --git a/docs/reference/ingest/apis/put-pipeline.asciidoc b/docs/reference/ingest/apis/put-pipeline.asciidoc index ab1139b99995..5b532dedf8e8 100644 --- a/docs/reference/ingest/apis/put-pipeline.asciidoc +++ b/docs/reference/ingest/apis/put-pipeline.asciidoc @@ -59,7 +59,7 @@ See also <>. version. If specified and the update is successful, the pipeline's version is incremented. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [[put-pipeline-api-request-body]] ==== {api-request-body-title} diff --git a/docs/reference/ingest/enrich.asciidoc b/docs/reference/ingest/enrich.asciidoc index 4688b21d37e7..6642cdc2a74c 100644 --- a/docs/reference/ingest/enrich.asciidoc +++ b/docs/reference/ingest/enrich.asciidoc @@ -112,7 +112,7 @@ that doesn't change frequently. [[enrich-prereqs]] ==== Prerequisites -include::{es-repo-dir}/ingest/apis/enrich/put-enrich-policy.asciidoc[tag=enrich-policy-api-prereqs] +include::{es-ref-dir}/ingest/apis/enrich/put-enrich-policy.asciidoc[tag=enrich-policy-api-prereqs] [[create-enrich-source-index]] ==== Add enrich data @@ -203,7 +203,7 @@ documents first and verifying enrich data was added correctly using the [[update-enrich-data]] ==== Update an enrich index -include::{es-repo-dir}/ingest/apis/enrich/execute-enrich-policy.asciidoc[tag=update-enrich-index] +include::{es-ref-dir}/ingest/apis/enrich/execute-enrich-policy.asciidoc[tag=update-enrich-index] If wanted, you can <> or <> any already ingested documents diff --git a/docs/reference/ingest/processors/inference.asciidoc b/docs/reference/ingest/processors/inference.asciidoc index 0995e3f64381..88d97d9422d5 100644 --- a/docs/reference/ingest/processors/inference.asciidoc +++ b/docs/reference/ingest/processors/inference.asciidoc @@ -113,23 +113,23 @@ Classification configuration for inference. `num_top_classes`:: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-num-top-classes] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-num-top-classes] `num_top_feature_importance_values`:: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-num-top-feature-importance-values] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-num-top-feature-importance-values] `results_field`:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field-processor] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field-processor] `top_classes_results_field`:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-top-classes-results-field] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-top-classes-results-field] `prediction_field_type`:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-prediction-field-type] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-prediction-field-type] [discrete] [[inference-processor-fill-mask-opt]] @@ -137,22 +137,22 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification `num_top_classes`:: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-num-top-classes] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-num-top-classes] `results_field`:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field-processor] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field-processor] `tokenization`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] + .Properties of tokenization [%collapsible%open] ===== `bert`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] + .Properties of bert [%collapsible%open] @@ -160,12 +160,12 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizati `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] + .Properties of roberta [%collapsible%open] @@ -173,12 +173,12 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizati `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `mpnet`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] + .Properties of mpnet [%collapsible%open] @@ -186,7 +186,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizati `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= ===== @@ -196,18 +196,18 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizati `results_field`:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field-processor] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field-processor] `tokenization`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] + .Properties of tokenization [%collapsible%open] ===== `bert`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] + .Properties of bert [%collapsible%open] @@ -215,12 +215,12 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizati `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] + .Properties of roberta [%collapsible%open] @@ -228,12 +228,12 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizati `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `mpnet`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] + .Properties of mpnet [%collapsible%open] @@ -241,7 +241,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizati `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= ===== @@ -253,11 +253,11 @@ Regression configuration for inference. `results_field`:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field-processor] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field-processor] `num_top_feature_importance_values`:: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-regression-num-top-feature-importance-values] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-regression-num-top-feature-importance-values] [discrete] [[inference-processor-text-classification-opt]] @@ -268,22 +268,22 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-regression-num `num_top_classes`:: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-num-top-classes] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-num-top-classes] `results_field`:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field-processor] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field-processor] `tokenization`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] + .Properties of tokenization [%collapsible%open] ===== `bert`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] + .Properties of bert [%collapsible%open] @@ -291,16 +291,16 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizati `span`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] + .Properties of roberta [%collapsible%open] @@ -308,16 +308,16 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizati `span`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `mpnet`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] + .Properties of mpnet [%collapsible%open] @@ -325,7 +325,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizati `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= ===== @@ -335,18 +335,18 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizati `results_field`:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field-processor] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field-processor] `tokenization`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] + .Properties of tokenization [%collapsible%open] ===== `bert`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] + .Properties of bert [%collapsible%open] @@ -354,12 +354,12 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizati `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] + .Properties of roberta [%collapsible%open] @@ -367,12 +367,12 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizati `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `mpnet`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] + .Properties of mpnet [%collapsible%open] @@ -380,7 +380,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizati `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= ===== @@ -391,19 +391,19 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizati `results_field`:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field-processor] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field-processor] `tokenization`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] + .Properties of tokenization [%collapsible%open] ===== `bert`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] + .Properties of bert [%collapsible%open] @@ -411,16 +411,16 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizati `span`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] + .Properties of roberta [%collapsible%open] @@ -428,16 +428,16 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizati `span`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `mpnet`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] + .Properties of mpnet [%collapsible%open] @@ -445,7 +445,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizati `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= ===== @@ -456,26 +456,26 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizati `labels`:: (Optional, array) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-zero-shot-classification-labels] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-zero-shot-classification-labels] `multi_label`:: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-zero-shot-classification-multi-label] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-zero-shot-classification-multi-label] `results_field`:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field-processor] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field-processor] `tokenization`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] + .Properties of tokenization [%collapsible%open] ===== `bert`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] + .Properties of bert [%collapsible%open] @@ -483,12 +483,12 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizati `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] + .Properties of roberta [%collapsible%open] @@ -496,12 +496,12 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizati `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `mpnet`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] + .Properties of mpnet [%collapsible%open] @@ -509,7 +509,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenizati `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= ===== diff --git a/docs/reference/mapping/explicit-mapping.asciidoc b/docs/reference/mapping/explicit-mapping.asciidoc index ba2d9ec16676..3b0de5fad051 100644 --- a/docs/reference/mapping/explicit-mapping.asciidoc +++ b/docs/reference/mapping/explicit-mapping.asciidoc @@ -63,9 +63,9 @@ PUT /my-index-000001/_mapping [[update-mapping]] === Update the mapping of a field -include::{es-repo-dir}/indices/put-mapping.asciidoc[tag=change-field-mapping] +include::{es-ref-dir}/indices/put-mapping.asciidoc[tag=change-field-mapping] -include::{es-repo-dir}/indices/put-mapping.asciidoc[tag=rename-field] +include::{es-ref-dir}/indices/put-mapping.asciidoc[tag=rename-field] [discrete] [[view-mapping]] diff --git a/docs/reference/mapping/mapping-settings-limit.asciidoc b/docs/reference/mapping/mapping-settings-limit.asciidoc index 6e05e6ea6085..197642216499 100644 --- a/docs/reference/mapping/mapping-settings-limit.asciidoc +++ b/docs/reference/mapping/mapping-settings-limit.asciidoc @@ -58,4 +58,4 @@ or setting the index setting `index.mapping.total_fields.ignore_dynamic_beyond_l unless a user starts to add a huge number of fields with really long names. Default is `Long.MAX_VALUE` (no limit). -include::{es-repo-dir}/data-streams/tsds-index-settings.asciidoc[tag=dimensions-limit] +include::{es-ref-dir}/data-streams/tsds-index-settings.asciidoc[tag=dimensions-limit] diff --git a/docs/reference/mapping/types/aggregate-metric-double.asciidoc b/docs/reference/mapping/types/aggregate-metric-double.asciidoc index cf9377dec005..e702d34f07d4 100644 --- a/docs/reference/mapping/types/aggregate-metric-double.asciidoc +++ b/docs/reference/mapping/types/aggregate-metric-double.asciidoc @@ -65,9 +65,9 @@ include::numeric.asciidoc[tag=time_series_metric] [%collapsible%open] ==== -include::{es-repo-dir}/data-streams/tsds.asciidoc[tag=time-series-metric-gauge] +include::{es-ref-dir}/data-streams/tsds.asciidoc[tag=time-series-metric-gauge] -include::{es-repo-dir}/data-streams/tsds.asciidoc[tag=time-series-metric-null] +include::{es-ref-dir}/data-streams/tsds.asciidoc[tag=time-series-metric-null] ==== [[aggregate-metric-double-uses]] diff --git a/docs/reference/mapping/types/dense-vector.asciidoc b/docs/reference/mapping/types/dense-vector.asciidoc index 14fe9d496397..6294423985ec 100644 --- a/docs/reference/mapping/types/dense-vector.asciidoc +++ b/docs/reference/mapping/types/dense-vector.asciidoc @@ -51,7 +51,7 @@ It is not possible to store multiple values in one `dense_vector` field. [[index-vectors-knn-search]] ==== Index vectors for kNN search -include::{es-repo-dir}/search/search-your-data/knn-search.asciidoc[tag=knn-def] +include::{es-ref-dir}/search/search-your-data/knn-search.asciidoc[tag=knn-def] Dense vector fields can be used to rank documents in <>. This lets you perform diff --git a/docs/reference/mapping/types/nested.asciidoc b/docs/reference/mapping/types/nested.asciidoc index 8611205cb749..5d6ede6acd5a 100644 --- a/docs/reference/mapping/types/nested.asciidoc +++ b/docs/reference/mapping/types/nested.asciidoc @@ -220,11 +220,11 @@ then 101 Lucene documents would be created: one for the parent document, and one nested object. Because of the expense associated with `nested` mappings, Elasticsearch puts settings in place to guard against performance problems: -include::{es-repo-dir}/mapping/mapping-settings-limit.asciidoc[tag=nested-fields-limit] +include::{es-ref-dir}/mapping/mapping-settings-limit.asciidoc[tag=nested-fields-limit] In the previous example, the `user` mapping would count as only 1 towards this limit. -include::{es-repo-dir}/mapping/mapping-settings-limit.asciidoc[tag=nested-objects-limit] +include::{es-ref-dir}/mapping/mapping-settings-limit.asciidoc[tag=nested-objects-limit] To illustrate how this setting works, consider adding another `nested` type called `comments` to the previous example mapping. For each document, the combined number of `user` and `comment` diff --git a/docs/reference/mapping/types/numeric.asciidoc b/docs/reference/mapping/types/numeric.asciidoc index a78611c4d8d3..32f4964e8ca4 100644 --- a/docs/reference/mapping/types/numeric.asciidoc +++ b/docs/reference/mapping/types/numeric.asciidoc @@ -200,11 +200,11 @@ metric type. You can't update this parameter for existing fields. .Valid `time_series_metric` values for numeric fields [%collapsible%open] ==== -include::{es-repo-dir}/data-streams/tsds.asciidoc[tag=time-series-metric-counter] +include::{es-ref-dir}/data-streams/tsds.asciidoc[tag=time-series-metric-counter] -include::{es-repo-dir}/data-streams/tsds.asciidoc[tag=time-series-metric-gauge] +include::{es-ref-dir}/data-streams/tsds.asciidoc[tag=time-series-metric-gauge] -include::{es-repo-dir}/data-streams/tsds.asciidoc[tag=time-series-metric-null] +include::{es-ref-dir}/data-streams/tsds.asciidoc[tag=time-series-metric-null] ==== + For a numeric time series metric, the `doc_values` parameter must be `true`. A diff --git a/docs/reference/migration/apis/deprecation.asciidoc b/docs/reference/migration/apis/deprecation.asciidoc index fd82bb3e0e6d..67b4c113af2b 100644 --- a/docs/reference/migration/apis/deprecation.asciidoc +++ b/docs/reference/migration/apis/deprecation.asciidoc @@ -5,7 +5,7 @@ Deprecation info ++++ -include::{es-repo-dir}/migration/apis/shared-migration-apis-tip.asciidoc[] +include::{es-ref-dir}/migration/apis/shared-migration-apis-tip.asciidoc[] The deprecation API is to be used to retrieve information about different cluster, node, and index level settings that use deprecated features that will diff --git a/docs/reference/migration/apis/feature-migration.asciidoc b/docs/reference/migration/apis/feature-migration.asciidoc index 9a6306dc2f59..e38639ac4453 100644 --- a/docs/reference/migration/apis/feature-migration.asciidoc +++ b/docs/reference/migration/apis/feature-migration.asciidoc @@ -5,7 +5,7 @@ Feature migration ++++ -include::{es-repo-dir}/migration/apis/shared-migration-apis-tip.asciidoc[] +include::{es-ref-dir}/migration/apis/shared-migration-apis-tip.asciidoc[] Version upgrades sometimes require changes to how features store configuration information and data in system indices. The feature migration APIs enable you to diff --git a/docs/reference/migration/migrate_8_0/rest-api-changes.asciidoc b/docs/reference/migration/migrate_8_0/rest-api-changes.asciidoc index 87e704110971..99c09b9b0538 100644 --- a/docs/reference/migration/migrate_8_0/rest-api-changes.asciidoc +++ b/docs/reference/migration/migrate_8_0/rest-api-changes.asciidoc @@ -190,7 +190,7 @@ version of a major version to a remote cluster running any minor version in the following major version. For example, a local 7.17 cluster can search any remote 8.x cluster. -include::{es-repo-dir}/search/search-your-data/ccs-version-compat-matrix.asciidoc[] +include::{es-ref-dir}/search/search-your-data/ccs-version-compat-matrix.asciidoc[] IMPORTANT: For the {ref}/eql-search-api.html[EQL search API], the local and remote clusters must use the same {es} version if they have versions prior to 7.17.7 (included) or prior to 8.5.1 (included). diff --git a/docs/reference/ml/anomaly-detection/apis/close-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/close-job.asciidoc index 988dff2d9232..f0cb968e082c 100644 --- a/docs/reference/ml/anomaly-detection/apis/close-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/close-job.asciidoc @@ -59,7 +59,7 @@ results the job might have recently produced or might produce in the future. ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection-wildcard] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection-wildcard] + You can close all jobs by using `_all` or by specifying `*` as the job identifier. @@ -69,7 +69,7 @@ identifier. `allow_no_match`:: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-jobs] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-jobs] `force`:: (Optional, Boolean) Use to close a failed job, or to forcefully close a job diff --git a/docs/reference/ml/anomaly-detection/apis/delete-calendar-event.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-calendar-event.asciidoc index 4e2501aa5a17..b80a248038ae 100644 --- a/docs/reference/ml/anomaly-detection/apis/delete-calendar-event.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/delete-calendar-event.asciidoc @@ -30,7 +30,7 @@ events and delete the calendar, see the ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=calendar-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=calendar-id] ``:: (Required, string) Identifier for the scheduled event. You can obtain this diff --git a/docs/reference/ml/anomaly-detection/apis/delete-calendar-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-calendar-job.asciidoc index 8b01fa1066f4..6720e236fd63 100644 --- a/docs/reference/ml/anomaly-detection/apis/delete-calendar-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/delete-calendar-job.asciidoc @@ -23,11 +23,11 @@ Requires the `manage_ml` cluster privilege. This privilege is included in the ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=calendar-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=calendar-id] ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection-list] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection-list] [[ml-delete-calendar-job-example]] == {api-examples-title} diff --git a/docs/reference/ml/anomaly-detection/apis/delete-calendar.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-calendar.asciidoc index 5f710ca50c55..6684366c6f33 100644 --- a/docs/reference/ml/anomaly-detection/apis/delete-calendar.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/delete-calendar.asciidoc @@ -29,7 +29,7 @@ calendar. ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=calendar-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=calendar-id] [[ml-delete-calendar-example]] == {api-examples-title} diff --git a/docs/reference/ml/anomaly-detection/apis/delete-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-datafeed.asciidoc index bf4b69b5d8bc..64a1e4c336fe 100644 --- a/docs/reference/ml/anomaly-detection/apis/delete-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/delete-datafeed.asciidoc @@ -27,7 +27,7 @@ can delete it. ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=datafeed-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=datafeed-id] [[ml-delete-datafeed-query-parms]] == {api-query-parms-title} diff --git a/docs/reference/ml/anomaly-detection/apis/delete-filter.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-filter.asciidoc index e59e04fe60c1..4b41347543e8 100644 --- a/docs/reference/ml/anomaly-detection/apis/delete-filter.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/delete-filter.asciidoc @@ -31,7 +31,7 @@ filter. For more information, see ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=filter-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=filter-id] [[ml-delete-filter-example]] == {api-examples-title} diff --git a/docs/reference/ml/anomaly-detection/apis/delete-forecast.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-forecast.asciidoc index b2866ee834a2..74e6ce27084a 100644 --- a/docs/reference/ml/anomaly-detection/apis/delete-forecast.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/delete-forecast.asciidoc @@ -45,7 +45,7 @@ forecasts from the job. ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] [[ml-delete-forecast-query-parms]] diff --git a/docs/reference/ml/anomaly-detection/apis/delete-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-job.asciidoc index 7520a45f9211..1bbe07fd44f4 100644 --- a/docs/reference/ml/anomaly-detection/apis/delete-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/delete-job.asciidoc @@ -43,7 +43,7 @@ parameters as the delete job request. ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] [[ml-delete-job-query-parms]] == {api-query-parms-title} diff --git a/docs/reference/ml/anomaly-detection/apis/delete-snapshot.asciidoc b/docs/reference/ml/anomaly-detection/apis/delete-snapshot.asciidoc index 6f39cdbb372e..ad10de7a2ba0 100644 --- a/docs/reference/ml/anomaly-detection/apis/delete-snapshot.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/delete-snapshot.asciidoc @@ -30,11 +30,11 @@ the `model_snapshot_id` in the results from the get jobs API. ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=snapshot-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=snapshot-id] [[ml-delete-snapshot-example]] == {api-examples-title} diff --git a/docs/reference/ml/anomaly-detection/apis/flush-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/flush-job.asciidoc index ea449c82efec..68ff601749b4 100644 --- a/docs/reference/ml/anomaly-detection/apis/flush-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/flush-job.asciidoc @@ -36,7 +36,7 @@ opened again before analyzing further data. ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] [[ml-flush-job-query-parms]] == {api-query-parms-title} diff --git a/docs/reference/ml/anomaly-detection/apis/forecast.asciidoc b/docs/reference/ml/anomaly-detection/apis/forecast.asciidoc index 80ee7c96b22a..3e6067ab0585 100644 --- a/docs/reference/ml/anomaly-detection/apis/forecast.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/forecast.asciidoc @@ -45,7 +45,7 @@ error occurs if you try to create a forecast for a job that has an ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] [[ml-forecast-query-parms]] == {api-query-parms-title} diff --git a/docs/reference/ml/anomaly-detection/apis/get-bucket.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-bucket.asciidoc index a234901be1c4..bca839d1db31 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-bucket.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-bucket.asciidoc @@ -32,7 +32,7 @@ bucket. ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] ``:: (Optional, string) The timestamp of a single bucket result. If you do not @@ -57,7 +57,7 @@ specific timestamps. `exclude_interim`:: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=exclude-interim-results] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=exclude-interim-results] `expand`:: (Optional, Boolean) If true, the output includes anomaly records. Defaults to `false`. @@ -137,11 +137,11 @@ initial value that was calculated at the time the bucket was processed. `is_interim`::: (Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=is-interim] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=is-interim] `job_id`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] `probability`::: (number) The probability that the bucket has this behavior, in the range 0 to 1. @@ -161,7 +161,7 @@ this. `bucket_span`:: (number) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=bucket-span-results] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=bucket-span-results] `event_count`:: (number) The number of input data records processed in this bucket. @@ -172,11 +172,11 @@ the initial value that was calculated at the time the bucket was processed. `is_interim`:: (Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=is-interim] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=is-interim] `job_id`:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] `processing_time_ms`:: (number) The amount of time, in milliseconds, that it took to analyze the diff --git a/docs/reference/ml/anomaly-detection/apis/get-calendar-event.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-calendar-event.asciidoc index 803871ddea34..fc06e286bf46 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-calendar-event.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-calendar-event.asciidoc @@ -31,7 +31,7 @@ For more information, see ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=calendar-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=calendar-id] + You can get scheduled event information for multiple calendars in a single API request by using a comma-separated list of ids or a wildcard expression. @@ -89,7 +89,7 @@ following properties: `calendar_id`:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=calendar-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=calendar-id] `description`:: (string) A description of the scheduled event. diff --git a/docs/reference/ml/anomaly-detection/apis/get-calendar.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-calendar.asciidoc index ffa19b3b1093..b2c46bbe16c0 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-calendar.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-calendar.asciidoc @@ -31,7 +31,7 @@ For more information, see ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=calendar-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=calendar-id] + You can get information for multiple calendars in a single API request by using a comma-separated list of ids or a wildcard expression. You can get information @@ -76,7 +76,7 @@ properties: `calendar_id`:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=calendar-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=calendar-id] `job_ids`:: (array) An array of {anomaly-job} identifiers. For example: diff --git a/docs/reference/ml/anomaly-detection/apis/get-category.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-category.asciidoc index 034a598b7337..33de5e0f71a0 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-category.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-category.asciidoc @@ -39,7 +39,7 @@ examine the description and examples of that category. For more information, see ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] ``:: (Optional, long) Identifier for the category, which is unique in the job. If you @@ -104,7 +104,7 @@ manual tweaking. `job_id`:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] `max_matching_length`:: (unsigned integer) The maximum length of the fields that matched the category. diff --git a/docs/reference/ml/anomaly-detection/apis/get-datafeed-stats.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-datafeed-stats.asciidoc index bb4ead5a6fd4..a224f3880a1a 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-datafeed-stats.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-datafeed-stats.asciidoc @@ -39,7 +39,7 @@ IMPORTANT: This API returns a maximum of 10,000 {dfeeds}. ``:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=datafeed-id-wildcard] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=datafeed-id-wildcard] + You can get statistics for multiple {dfeeds} in a single API request by using a comma-separated list of {dfeeds} or a wildcard expression. You can get @@ -51,7 +51,7 @@ identifier, or by omitting the identifier. `allow_no_match`:: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-datafeeds] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-datafeeds] [role="child_attributes"] [[ml-get-datafeed-stats-results]] @@ -62,30 +62,30 @@ informational; you cannot update their values. `assignment_explanation`:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=assignment-explanation-datafeeds] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=assignment-explanation-datafeeds] `datafeed_id`:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=datafeed-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=datafeed-id] `node`:: (object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-datafeeds] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-datafeeds] + -- [%collapsible%open] ==== `attributes`::: (object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-attributes] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-attributes] `ephemeral_id`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-ephemeral-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-ephemeral-id] `id`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-id] `name`::: (string) @@ -93,7 +93,7 @@ The node name. For example, `0-o0tOo`. `transport_address`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-transport-address] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-transport-address] ==== -- @@ -130,7 +130,7 @@ The start time as an epoch in milliseconds. `state`:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=state-datafeed] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=state-datafeed] `timing_stats`:: (object) An object that provides statistical information about timing aspect of @@ -141,24 +141,24 @@ this {dfeed}. ==== `average_search_time_per_bucket_ms`::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=search-bucket-avg] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=search-bucket-avg] `bucket_count`::: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=bucket-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=bucket-count] `exponential_average_search_time_per_hour_ms`::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=search-exp-avg-hour] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=search-exp-avg-hour] `job_id`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] `search_count`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=search-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=search-count] `total_search_time_ms`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=search-time] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=search-time] ==== -- diff --git a/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc index f3dd3b5f4da7..a986e2220f92 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-datafeed.asciidoc @@ -36,7 +36,7 @@ IMPORTANT: This API returns a maximum of 10,000 {dfeeds}. ``:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=datafeed-id-wildcard] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=datafeed-id-wildcard] + You can get information for multiple {dfeeds} in a single API request by using a comma-separated list of {dfeeds} or a wildcard expression. You can get @@ -48,11 +48,11 @@ information for all {dfeeds} by using `_all`, by specifying `*` as the `allow_no_match`:: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-datafeeds] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-datafeeds] `exclude_generated`:: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=exclude-generated] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=exclude-generated] [[ml-get-datafeed-results]] == {api-response-body-title} diff --git a/docs/reference/ml/anomaly-detection/apis/get-filter.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-filter.asciidoc index 29e11172b9e0..f73dcd236f1a 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-filter.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-filter.asciidoc @@ -31,7 +31,7 @@ You can get a single filter or all filters. For more information, see ``:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=filter-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=filter-id] [[ml-get-filter-query-parms]] == {api-query-parms-title} @@ -54,7 +54,7 @@ properties: `filter_id`:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=filter-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=filter-id] `items`:: (array of strings) An array of strings which is the filter item list. diff --git a/docs/reference/ml/anomaly-detection/apis/get-influencer.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-influencer.asciidoc index 76ead2921df6..31489e361a84 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-influencer.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-influencer.asciidoc @@ -30,7 +30,7 @@ the anomalies. Influencer results are available only if an ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] [[ml-get-influencer-query-parms]] @@ -38,7 +38,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] `desc`:: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=desc-results] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=desc-results] `end`:: (Optional, string) Returns influencers with timestamps earlier than this time. @@ -47,7 +47,7 @@ specific timestamps. `exclude_interim`:: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=exclude-interim-results] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=exclude-interim-results] `from`:: (Optional, integer) @@ -99,7 +99,7 @@ properties: `bucket_span`:: (number) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=bucket-span-results] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=bucket-span-results] `influencer_score`:: (number) A normalized score between 0-100, which is based on the probability of @@ -121,11 +121,11 @@ calculated at the time the bucket was processed. `is_interim`:: (Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=is-interim] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=is-interim] `job_id`:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] `probability`:: (number) The probability that the influencer has this behavior, in the range 0 @@ -138,7 +138,7 @@ human-readable and friendly interpretation of this. `timestamp`:: (date) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=timestamp-results] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=timestamp-results] NOTE: Additional influencer properties are added, depending on the fields being analyzed. For example, if it's analyzing `user_name` as an influencer, then a diff --git a/docs/reference/ml/anomaly-detection/apis/get-job-model-snapshot-upgrade-stats.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-job-model-snapshot-upgrade-stats.asciidoc index e1354648e385..0939282a7591 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-job-model-snapshot-upgrade-stats.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-job-model-snapshot-upgrade-stats.asciidoc @@ -36,7 +36,7 @@ returned. ``:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection-wildcard] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection-wildcard] ``:: (string) @@ -51,7 +51,7 @@ use wildcard expressions or `_all`. `allow_no_match`:: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-jobs] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-jobs] [role="child_attributes"] [[ml-get-job-model-snapshot-upgrade-stats-results]] @@ -62,11 +62,11 @@ All of these properties are informational; you cannot update their values. `assignment_explanation`:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=assignment-explanation-datafeeds] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=assignment-explanation-datafeeds] `job_id`:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] `node`:: (object) @@ -78,15 +78,15 @@ available only for upgrade tasks that are assigned to a node. ==== `attributes`::: (object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-attributes] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-attributes] `ephemeral_id`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-ephemeral-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-ephemeral-id] `id`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-id] `name`::: (string) @@ -94,13 +94,13 @@ The node name. For example, `0-o0tOo`. `transport_address`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-transport-address] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-transport-address] ==== -- `snapshot_id`:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-snapshot-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-snapshot-id] `state`:: (string) diff --git a/docs/reference/ml/anomaly-detection/apis/get-job-stats.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-job-stats.asciidoc index 4c1fbfe2da3d..a5bd188397bb 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-job-stats.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-job-stats.asciidoc @@ -46,7 +46,7 @@ omitting the identifier. `allow_no_match`:: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-jobs] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-jobs] [role="child_attributes"] [[ml-get-job-stats-results]] @@ -57,7 +57,7 @@ job: `assignment_explanation`:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=assignment-explanation-anomaly-jobs] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=assignment-explanation-anomaly-jobs] //Begin data_counts [[datacounts]]`data_counts`:: @@ -71,75 +71,75 @@ counts are not reset. ==== `bucket_count`::: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=bucket-count-anomaly-jobs] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=bucket-count-anomaly-jobs] `earliest_record_timestamp`::: (date) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=earliest-record-timestamp] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=earliest-record-timestamp] `empty_bucket_count`::: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=empty-bucket-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=empty-bucket-count] `input_bytes`::: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=input-bytes] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=input-bytes] `input_field_count`::: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=input-field-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=input-field-count] `input_record_count`::: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=input-record-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=input-record-count] `invalid_date_count`::: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=invalid-date-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=invalid-date-count] `job_id`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] `last_data_time`::: (date) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=last-data-time] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=last-data-time] `latest_empty_bucket_timestamp`::: (date) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=latest-empty-bucket-timestamp] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=latest-empty-bucket-timestamp] `latest_record_timestamp`::: (date) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=latest-record-timestamp] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=latest-record-timestamp] `latest_sparse_bucket_timestamp`::: (date) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=latest-sparse-record-timestamp] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=latest-sparse-record-timestamp] `log_time`::: (date) The timestamp of the `data_counts` according to server time. `missing_field_count`::: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=missing-field-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=missing-field-count] + The value of `processed_record_count` includes this count. `out_of_order_timestamp_count`::: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=out-of-order-timestamp-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=out-of-order-timestamp-count] `processed_field_count`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=processed-field-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=processed-field-count] `processed_record_count`::: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=processed-record-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=processed-record-count] `sparse_bucket_count`::: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=sparse-bucket-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=sparse-bucket-count] ==== //End data_counts @@ -183,13 +183,13 @@ forecasts related to this job. If there are no forecasts, this property is omitt `total`::: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=forecast-total] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=forecast-total] ==== //End forecasts_stats `job_id`:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] //Begin model_size_stats [[modelsizestats]]`model_size_stats`:: @@ -201,85 +201,85 @@ model. ==== `assignment_memory_basis`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=assignment-memory-basis] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=assignment-memory-basis] `bucket_allocation_failures_count`::: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=bucket-allocation-failures-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=bucket-allocation-failures-count] `categorized_doc_count`::: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=categorized-doc-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=categorized-doc-count] `categorization_status`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=categorization-status] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=categorization-status] `dead_category_count`::: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dead-category-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dead-category-count] `failed_category_count`::: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=failed-category-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=failed-category-count] `frequent_category_count`::: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=frequent-category-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=frequent-category-count] `job_id`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] `log_time`::: (date) The timestamp of the `model_size_stats` according to server time. `memory_status`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-memory-status] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-memory-status] `model_bytes`::: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-bytes] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-bytes] `model_bytes_exceeded`::: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-bytes-exceeded] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-bytes-exceeded] `model_bytes_memory_limit`::: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-memory-limit-anomaly-jobs] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-memory-limit-anomaly-jobs] `peak_model_bytes`::: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=peak-model-bytes] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=peak-model-bytes] `rare_category_count`::: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=rare-category-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=rare-category-count] `result_type`::: (string) For internal use. The type of result. `total_by_field_count`::: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=total-by-field-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=total-by-field-count] `total_category_count`::: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=total-category-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=total-category-count] `total_over_field_count`::: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=total-over-field-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=total-over-field-count] `total_partition_field_count`::: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=total-partition-field-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=total-partition-field-count] `timestamp`::: (date) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-timestamp] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-timestamp] ==== //End model_size_stats @@ -293,32 +293,32 @@ available only for open jobs. ==== `attributes`::: (object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-attributes] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-attributes] `ephemeral_id`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-ephemeral-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-ephemeral-id] `id`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-id] `name`::: (string) The node name. `transport_address`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-transport-address] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-transport-address] ==== //End node `open_time`:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=open-time] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=open-time] `state`:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=state-anomaly-job] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=state-anomaly-job] //Begin timing_stats [[timingstats]]`timing_stats`:: @@ -333,31 +333,31 @@ this job. `bucket_count`::: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=bucket-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=bucket-count] `exponential_average_bucket_processing_time_ms`::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=bucket-time-exponential-average] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=bucket-time-exponential-average] `exponential_average_bucket_processing_time_per_hour_ms`::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=bucket-time-exponential-average-hour] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=bucket-time-exponential-average-hour] `job_id`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] `maximum_bucket_processing_time_ms`::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=bucket-time-maximum] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=bucket-time-maximum] `minimum_bucket_processing_time_ms`::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=bucket-time-minimum] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=bucket-time-minimum] `total_bucket_processing_time_ms`::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=bucket-time-total] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=bucket-time-total] ==== //End timing_stats diff --git a/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc index ac8746d761e0..4ee6c429ce73 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-job.asciidoc @@ -46,11 +46,11 @@ omitting the identifier. `allow_no_match`:: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-jobs] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-jobs] `exclude_generated`:: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=exclude-generated] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=exclude-generated] [role="child_attributes"] [[ml-get-job-results]] @@ -121,83 +121,83 @@ the account name is listed in the response. `datafeed_id`::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=datafeed-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=datafeed-id] `aggregations`::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=aggregations] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=aggregations] `chunking_config`::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=chunking-config] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=chunking-config] + .Properties of `chunking_config` [%collapsible%open] ===== `mode`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=mode] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=mode] `time_span`::: (<>) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=time-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=time-span] ===== `delayed_data_check_config`::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=delayed-data-check-config] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=delayed-data-check-config] + .Properties of `delayed_data_check_config` [%collapsible%open] ===== `check_window`:: (<>) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=delayed-data-check-config-check-window] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=delayed-data-check-config-check-window] `enabled`:: (Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=delayed-data-check-config-enabled] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=delayed-data-check-config-enabled] ===== `frequency`::: (Optional, <>) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=frequency] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=frequency] `indices`::: (Required, array) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=indices] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=indices] `indices_options`::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=indices-options] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=indices-options] `job_id`::: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] `max_empty_searches`::: (Optional,integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=max-empty-searches] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=max-empty-searches] `query`::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=query] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=query] `query_delay`::: (Optional, <>) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=query-delay] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=query-delay] `runtime_mappings`::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=runtime-mappings] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=runtime-mappings] `script_fields`::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=script-fields] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=script-fields] `scroll_size`::: (Optional, unsigned integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=scroll-size] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=scroll-size] ==== `finished_time`:: @@ -218,7 +218,7 @@ independently. The `job_version` value represents the new version number. `model_snapshot_id`:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-snapshot-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-snapshot-id] [[ml-get-job-response-codes]] == {api-response-codes-title} diff --git a/docs/reference/ml/anomaly-detection/apis/get-overall-buckets.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-overall-buckets.asciidoc index cf8546d78fe6..b581b5c3a2eb 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-overall-buckets.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-overall-buckets.asciidoc @@ -49,7 +49,7 @@ a span equal to the jobs' largest bucket span. ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection-wildcard-list] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection-wildcard-list] + You can summarize the bucket results for all {anomaly-jobs} by using `_all` or by specifying `*` as the job identifier. @@ -59,7 +59,7 @@ by specifying `*` as the job identifier. `allow_no_match`:: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-jobs] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-jobs] `bucket_span`:: (Optional, string) The span of the overall buckets. Must be greater or equal to @@ -108,7 +108,7 @@ of the job with the longest one. `is_interim`:: (Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=is-interim] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=is-interim] `jobs`:: (array) An array of objects that contain the `max_anomaly_score` per `job_id`. @@ -121,7 +121,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=is-interim] `timestamp`:: (date) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=timestamp-results] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=timestamp-results] [[ml-get-overall-buckets-example]] == {api-examples-title} diff --git a/docs/reference/ml/anomaly-detection/apis/get-record.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-record.asciidoc index 686632366315..e74ab3ecb4b1 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-record.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-record.asciidoc @@ -39,14 +39,14 @@ of detectors. ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] [[ml-get-record-query-parms]] == {api-query-parms-title} `desc`:: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=desc-results] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=desc-results] `end`:: (Optional, string) Returns records with timestamps earlier than this time. @@ -55,7 +55,7 @@ specific timestamps. `exclude_interim`:: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=exclude-interim-results] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=exclude-interim-results] `from`:: (Optional, integer) Skips the specified number of records. Defaults to `0`. @@ -158,11 +158,11 @@ current bucket. `bucket_span`:: (number) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=bucket-span-results] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=bucket-span-results] `by_field_name`:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=by-field-name] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=by-field-name] `by_field_value`:: (string) The value of `by_field_name`. @@ -220,11 +220,11 @@ at the time the bucket was processed. `is_interim`:: (Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=is-interim] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=is-interim] `job_id`:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] `multi_bucket_impact`:: (number) An indication of how strongly an anomaly is multi bucket or single @@ -234,14 +234,14 @@ bucket. `over_field_name`:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=over-field-name] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=over-field-name] `over_field_value`:: (string) The value of `over_field_name`. `partition_field_name`:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=partition-field-name] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=partition-field-name] `partition_field_value`:: (string) The value of `partition_field_name`. @@ -262,7 +262,7 @@ be updated by a re-normalization process as new data is analyzed. `timestamp`:: (date) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=timestamp-results] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=timestamp-results] `typical`:: (array) The typical value for the bucket, according to analytical modeling. diff --git a/docs/reference/ml/anomaly-detection/apis/get-snapshot.asciidoc b/docs/reference/ml/anomaly-detection/apis/get-snapshot.asciidoc index 0132f57d412e..d94bd4060854 100644 --- a/docs/reference/ml/anomaly-detection/apis/get-snapshot.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/get-snapshot.asciidoc @@ -26,11 +26,11 @@ Requires the `monitor_ml` cluster privilege. This privilege is included in the ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] ``:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=snapshot-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=snapshot-id] + -- You can get information for multiple snapshots by using a comma-separated list @@ -122,7 +122,7 @@ independently. The `min_version` value represents the new version number. ==== `assignment_memory_basis`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=assignment-memory-basis] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=assignment-memory-basis] `bucket_allocation_failures_count`::: (long) The number of buckets for which entities were not processed due to memory @@ -155,7 +155,7 @@ side effect of the way categorization has no prior training.) `failed_category_count`::: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=failed-category-count] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=failed-category-count] `frequent_category_count`::: (long) The number of categories that match more than 1% of categorized @@ -163,7 +163,7 @@ documents. `job_id`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] `log_time`::: (date) The timestamp that the `model_size_stats` were recorded, according to @@ -223,7 +223,7 @@ separately for each detector and partition. `retain`:: (Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=retain] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=retain] `snapshot_id`:: (string) A numerical character string that uniquely identifies the model diff --git a/docs/reference/ml/anomaly-detection/apis/open-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/open-job.asciidoc index 92d8908baab4..385f672f467f 100644 --- a/docs/reference/ml/anomaly-detection/apis/open-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/open-job.asciidoc @@ -36,7 +36,7 @@ data is received. ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] [[ml-open-job-query-parms]] == {api-query-parms-title} diff --git a/docs/reference/ml/anomaly-detection/apis/post-data.asciidoc b/docs/reference/ml/anomaly-detection/apis/post-data.asciidoc index e5b8a30055f9..931efcf8c2a5 100644 --- a/docs/reference/ml/anomaly-detection/apis/post-data.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/post-data.asciidoc @@ -52,7 +52,7 @@ or a comma-separated list. ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] [[ml-post-data-query-parms]] == {api-query-parms-title} diff --git a/docs/reference/ml/anomaly-detection/apis/preview-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/preview-datafeed.asciidoc index c838af78871d..243cd2a5f32a 100644 --- a/docs/reference/ml/anomaly-detection/apis/preview-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/preview-datafeed.asciidoc @@ -52,7 +52,7 @@ supply the credentials. ``:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=datafeed-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=datafeed-id] + NOTE: If you provide the `` as a path parameter, you cannot provide {dfeed} or {anomaly-job} configuration details in the request body. diff --git a/docs/reference/ml/anomaly-detection/apis/put-calendar-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/put-calendar-job.asciidoc index 2483f24f6afd..0c19a08cbd74 100644 --- a/docs/reference/ml/anomaly-detection/apis/put-calendar-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/put-calendar-job.asciidoc @@ -23,11 +23,11 @@ Requires the `manage_ml` cluster privilege. This privilege is included in the ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=calendar-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=calendar-id] ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection-list] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection-list] [[ml-put-calendar-job-example]] == {api-examples-title} diff --git a/docs/reference/ml/anomaly-detection/apis/put-calendar.asciidoc b/docs/reference/ml/anomaly-detection/apis/put-calendar.asciidoc index 8ddaea995c71..fd2b58a31737 100644 --- a/docs/reference/ml/anomaly-detection/apis/put-calendar.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/put-calendar.asciidoc @@ -29,7 +29,7 @@ For more information, see ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=calendar-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=calendar-id] [[ml-put-calendar-request-body]] == {api-request-body-title} diff --git a/docs/reference/ml/anomaly-detection/apis/put-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/put-datafeed.asciidoc index 478a70e23b93..47e3059666d7 100644 --- a/docs/reference/ml/anomaly-detection/apis/put-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/put-datafeed.asciidoc @@ -52,7 +52,7 @@ credentials are used instead. ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=datafeed-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=datafeed-id] [[ml-put-datafeed-query-params]] == {api-query-parms-title} @@ -62,11 +62,11 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=datafeed-id] concrete indices are ignored. This includes the `_all` string or when no indices are specified. Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `open`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=ignore_throttled] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=ignore_throttled] `ignore_unavailable`:: (Optional, Boolean) If `true`, unavailable indices (missing or closed) are @@ -79,79 +79,79 @@ ignored. Defaults to `false`. `aggregations`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=aggregations] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=aggregations] `chunking_config`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=chunking-config] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=chunking-config] + .Properties of `chunking_config` [%collapsible%open] ==== `mode`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=mode] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=mode] `time_span`::: (<>) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=time-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=time-span] ==== `delayed_data_check_config`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=delayed-data-check-config] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=delayed-data-check-config] + .Properties of `delayed_data_check_config` [%collapsible%open] ==== `check_window`:: (<>) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=delayed-data-check-config-check-window] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=delayed-data-check-config-check-window] `enabled`:: (Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=delayed-data-check-config-enabled] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=delayed-data-check-config-enabled] ==== `frequency`:: (Optional, <>) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=frequency] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=frequency] `indices`:: (Required, array) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=indices] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=indices] `indices_options`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=indices-options] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=indices-options] `job_id`:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] `max_empty_searches`:: (Optional,integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=max-empty-searches] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=max-empty-searches] `query`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=query] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=query] `query_delay`:: (Optional, <>) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=query-delay] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=query-delay] `runtime_mappings`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=runtime-mappings] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=runtime-mappings] `script_fields`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=script-fields] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=script-fields] `scroll_size`:: (Optional, unsigned integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=scroll-size] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=scroll-size] [[ml-put-datafeed-example]] == {api-examples-title} diff --git a/docs/reference/ml/anomaly-detection/apis/put-filter.asciidoc b/docs/reference/ml/anomaly-detection/apis/put-filter.asciidoc index 4e0d2ed1fabc..b50ba8cb1e23 100644 --- a/docs/reference/ml/anomaly-detection/apis/put-filter.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/put-filter.asciidoc @@ -31,7 +31,7 @@ configuration objects. For more information, see ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=filter-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=filter-id] [[ml-put-filter-request-body]] == {api-request-body-title} diff --git a/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc index e4e10e2ae2fc..012904a9affa 100644 --- a/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/put-job.asciidoc @@ -42,7 +42,7 @@ credentials are used instead. ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection-define] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection-define] [role="child_attributes"] [[ml-put-job-request-body]] @@ -50,31 +50,31 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection-define `allow_lazy_open`:: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-lazy-open] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=allow-lazy-open] //Begin analysis_config [[put-analysisconfig]]`analysis_config`:: (Required, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=analysis-config] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=analysis-config] + .Properties of `analysis_config` [%collapsible%open] ==== `bucket_span`::: (<>) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=bucket-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=bucket-span] `categorization_analyzer`::: (object or string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=categorization-analyzer] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=categorization-analyzer] `categorization_field_name`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=categorization-field-name] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=categorization-field-name] `categorization_filters`::: (array of strings) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=categorization-filters] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=categorization-filters] //Begin analysis_config.detectors `detectors`::: @@ -90,12 +90,12 @@ no analysis can occur and an error is returned. ===== `by_field_name`:::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=by-field-name] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=by-field-name] //Begin analysis_config.detectors.custom_rules [[put-customrules]]`custom_rules`:::: (array) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules] + .Properties of `custom_rules` [%collapsible%open] @@ -103,45 +103,45 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules] `actions`::: (array) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules-actions] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-actions] //Begin analysis_config.detectors.custom_rules.conditions `conditions`::: (array) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules-conditions] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-conditions] + .Properties of `conditions` [%collapsible%open] ======= `applies_to`:::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules-conditions-applies-to] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-conditions-applies-to] `operator`:::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules-conditions-operator] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-conditions-operator] `value`:::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules-conditions-value] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-conditions-value] ======= //End analysis_config.detectors.custom_rules.conditions //Begin analysis_config.detectors.custom_rules.scope `scope`::: (object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules-scope] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-scope] + .Properties of `scope` [%collapsible%open] ======= `filter_id`:::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules-scope-filter-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-scope-filter-id] `filter_type`:::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules-scope-filter-type] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-scope-filter-type] ======= //End analysis_config.detectors.custom_rules.scope ====== @@ -149,114 +149,114 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules-scope-filter-type] `detector_description`:::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=detector-description] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=detector-description] `detector_index`:::: (integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=detector-index] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=detector-index] + If you specify a value for this property, it is ignored. `exclude_frequent`:::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=exclude-frequent] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=exclude-frequent] `field_name`:::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=detector-field-name] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=detector-field-name] `function`:::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=function] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=function] `over_field_name`:::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=over-field-name] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=over-field-name] `partition_field_name`:::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=partition-field-name] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=partition-field-name] `use_null`:::: (Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=use-null] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=use-null] ===== //End analysis_config.detectors `influencers`::: (array of strings) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=influencers] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=influencers] `latency`::: (time units) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=latency] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=latency] `model_prune_window`::: (Optional, <>) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-prune-window] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-prune-window] `multivariate_by_fields`::: (Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=multivariate-by-fields] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=multivariate-by-fields] //Begin analysis_config.per_partition_categorization `per_partition_categorization`::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=per-partition-categorization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=per-partition-categorization] + .Properties of `per_partition_categorization` [%collapsible%open] ===== `enabled`:::: (Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=per-partition-categorization-enabled] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=per-partition-categorization-enabled] `stop_on_warn`:::: (Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=per-partition-categorization-stop-on-warn] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=per-partition-categorization-stop-on-warn] ===== //End analysis_config.per_partition_categorization `summary_count_field_name`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=summary-count-field-name] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=summary-count-field-name] ==== //End analysis_config //Begin analysis_limits [[put-analysislimits]]`analysis_limits`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=analysis-limits] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=analysis-limits] + .Properties of `analysis_limits` [%collapsible%open] ==== `categorization_examples_limit`::: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=categorization-examples-limit] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=categorization-examples-limit] `model_memory_limit`::: (long or string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-memory-limit-ad] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-memory-limit-ad] ==== //End analysis_limits `background_persist_interval`:: (Optional, <>) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=background-persist-interval] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=background-persist-interval] [[put-customsettings]]`custom_settings`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-settings] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-settings] [[put-dailymodelsnapshotretentionafterdays]]`daily_model_snapshot_retention_after_days`:: (Optional, long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=daily-model-snapshot-retention-after-days] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=daily-model-snapshot-retention-after-days] //Begin data_description [[put-datadescription]]`data_description`:: (Required, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=data-description] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=data-description] //End data_description [[put-datafeedconfig]]`datafeed_config`:: @@ -269,81 +269,81 @@ from {es} for analysis by the job. You can associate only one {dfeed} with each ==== `aggregations`::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=aggregations] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=aggregations] `chunking_config`::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=chunking-config] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=chunking-config] + .Properties of `chunking_config` [%collapsible%open] ===== `mode`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=mode] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=mode] `time_span`::: (<>) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=time-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=time-span] ===== `datafeed_id`::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=datafeed-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=datafeed-id] + Defaults to the same ID as the {anomaly-job}. `delayed_data_check_config`::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=delayed-data-check-config] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=delayed-data-check-config] + .Properties of `delayed_data_check_config` [%collapsible%open] ===== `check_window`:: (<>) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=delayed-data-check-config-check-window] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=delayed-data-check-config-check-window] `enabled`:: (Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=delayed-data-check-config-enabled] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=delayed-data-check-config-enabled] ===== `frequency`::: (Optional, <>) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=frequency] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=frequency] `indices`::: (Required, array) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=indices] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=indices] `indices_options`::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=indices-options] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=indices-options] `max_empty_searches`::: (Optional,integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=max-empty-searches] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=max-empty-searches] `query`::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=query] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=query] `query_delay`::: (Optional, <>) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=query-delay] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=query-delay] `runtime_mappings`::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=runtime-mappings] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=runtime-mappings] `script_fields`::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=script-fields] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=script-fields] `scroll_size`::: (Optional, unsigned integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=scroll-size] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=scroll-size] ==== `description`:: @@ -351,45 +351,45 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=scroll-size] `groups`:: (Optional, array of strings) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=groups] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=groups] //Begin model_plot_config `model_plot_config`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-plot-config] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-plot-config] + .Properties of `model_plot_config` [%collapsible%open] ==== `annotations_enabled`::: (Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-plot-config-annotations-enabled] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-plot-config-annotations-enabled] `enabled`::: (Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-plot-config-enabled] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-plot-config-enabled] `terms`::: experimental[] (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-plot-config-terms] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-plot-config-terms] ==== //End model_plot_config [[put-modelsnapshotretentiondays]]`model_snapshot_retention_days`:: (Optional, long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-snapshot-retention-days] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-snapshot-retention-days] [[put-renormalizationwindowdays]]`renormalization_window_days`:: (Optional, long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=renormalization-window-days] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=renormalization-window-days] [[put-resultsindexname]]`results_index_name`:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=results-index-name] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=results-index-name] [[put-resultsretentiondays]]`results_retention_days`:: (Optional, long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=results-retention-days] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=results-retention-days] [[ml-put-job-example]] == {api-examples-title} diff --git a/docs/reference/ml/anomaly-detection/apis/reset-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/reset-job.asciidoc index 4076e4a8038e..9009d634a2e9 100644 --- a/docs/reference/ml/anomaly-detection/apis/reset-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/reset-job.asciidoc @@ -35,7 +35,7 @@ separated list. ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] [[ml-reset-job-query-parms]] == {api-query-parms-title} diff --git a/docs/reference/ml/anomaly-detection/apis/revert-snapshot.asciidoc b/docs/reference/ml/anomaly-detection/apis/revert-snapshot.asciidoc index 4b3a61133dfb..c8d7a27ee204 100644 --- a/docs/reference/ml/anomaly-detection/apis/revert-snapshot.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/revert-snapshot.asciidoc @@ -39,11 +39,11 @@ NOTE: Reverting to a snapshot does not change the `data_counts` values of the ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=snapshot-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=snapshot-id] + -- You can specify `empty` as the . Reverting to the `empty` snapshot diff --git a/docs/reference/ml/anomaly-detection/apis/start-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/start-datafeed.asciidoc index 6d958383f337..b54c80133d7d 100644 --- a/docs/reference/ml/anomaly-detection/apis/start-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/start-datafeed.asciidoc @@ -43,7 +43,7 @@ you created or updated the {dfeed}, those credentials are used instead. ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=datafeed-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=datafeed-id] [[ml-start-datafeed-query-parms]] == {api-query-parms-title} diff --git a/docs/reference/ml/anomaly-detection/apis/stop-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/stop-datafeed.asciidoc index eb50e0a154a5..bc15a1de8a05 100644 --- a/docs/reference/ml/anomaly-detection/apis/stop-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/stop-datafeed.asciidoc @@ -45,7 +45,7 @@ identifier. `allow_no_match`:: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-datafeeds] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-datafeeds] `force`:: (Optional, Boolean) If true, the {dfeed} is stopped forcefully. diff --git a/docs/reference/ml/anomaly-detection/apis/update-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/update-datafeed.asciidoc index 5e6121cd01ac..b3920d9d4f80 100644 --- a/docs/reference/ml/anomaly-detection/apis/update-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/update-datafeed.asciidoc @@ -40,7 +40,7 @@ credentials are used instead. ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=datafeed-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=datafeed-id] [[ml-update-datafeed-query-params]] == {api-query-parms-title} @@ -50,11 +50,11 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=datafeed-id] concrete indices are ignored. This includes the `_all` string or when no indices are specified. Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `open`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=ignore_throttled] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=ignore_throttled] `ignore_unavailable`:: (Optional, Boolean) If `true`, unavailable indices (missing or closed) are @@ -68,55 +68,55 @@ The following properties can be updated after the {dfeed} is created: `aggregations`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=aggregations] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=aggregations] `chunking_config`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=chunking-config] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=chunking-config] + .Properties of `chunking_config` [%collapsible%open] ==== `mode`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=mode] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=mode] `time_span`::: (<>) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=time-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=time-span] ==== `delayed_data_check_config`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=delayed-data-check-config] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=delayed-data-check-config] + .Properties of `delayed_data_check_config` [%collapsible%open] ==== `check_window`:: (<>) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=delayed-data-check-config-check-window] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=delayed-data-check-config-check-window] `enabled`:: (Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=delayed-data-check-config-enabled] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=delayed-data-check-config-enabled] ==== `frequency`:: (Optional, <>) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=frequency] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=frequency] `indices`:: (Optional, array) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=indices] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=indices] `indices_options`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=indices-options] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=indices-options] `max_empty_searches`:: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=max-empty-searches] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=max-empty-searches] + -- The special value `-1` unsets this setting. @@ -124,7 +124,7 @@ The special value `-1` unsets this setting. `query`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=query] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=query] + -- WARNING: If you change the query, the analyzed data is also changed. Therefore, @@ -138,19 +138,19 @@ the results of the other job. `query_delay`:: (Optional, <>) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=query-delay] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=query-delay] `runtime_mappings`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=runtime-mappings] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=runtime-mappings] `script_fields`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=script-fields] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=script-fields] `scroll_size`:: (Optional, unsigned integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=scroll-size] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=scroll-size] [[ml-update-datafeed-example]] diff --git a/docs/reference/ml/anomaly-detection/apis/update-filter.asciidoc b/docs/reference/ml/anomaly-detection/apis/update-filter.asciidoc index 46e853e08265..a4221c37a438 100644 --- a/docs/reference/ml/anomaly-detection/apis/update-filter.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/update-filter.asciidoc @@ -23,7 +23,7 @@ Requires the `manage_ml` cluster privilege. This privilege is included in the ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=filter-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=filter-id] [[ml-update-filter-request-body]] == {api-request-body-title} diff --git a/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc b/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc index fd645fe0d9bf..6953235c854c 100644 --- a/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/update-job.asciidoc @@ -23,7 +23,7 @@ Requires the `manage_ml` cluster privilege. This privilege is included in the ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] [role="child_attributes"] [[ml-update-job-request-body]] @@ -33,7 +33,7 @@ The following properties can be updated after the job is created: `allow_lazy_open`:: (Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-lazy-open] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=allow-lazy-open] + -- NOTE: If the job is open when you make the update, you must stop the {dfeed}, @@ -43,7 +43,7 @@ close the job, then reopen the job and restart the {dfeed} for the changes to ta //Begin analysis_limits [[update-analysislimits]]`analysis_limits`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=analysis-limits] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=analysis-limits] + You can update the `analysis_limits` only while the job is closed. + @@ -52,7 +52,7 @@ You can update the `analysis_limits` only while the job is closed. ==== `model_memory_limit`::: (long or string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-memory-limit-ad] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-memory-limit-ad] + -- [NOTE] @@ -71,7 +71,7 @@ to re-run the job with an increased `model_memory_limit`. `background_persist_interval`:: (<>) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=background-persist-interval] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=background-persist-interval] + -- NOTE: If the job is open when you make the update, you must stop the {dfeed}, @@ -81,11 +81,11 @@ close the job, then reopen the job and restart the {dfeed} for the changes to ta [[update-customsettings]]`custom_settings`:: (object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-settings] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-settings] `daily_model_snapshot_retention_after_days`:: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=daily-model-snapshot-retention-after-days] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=daily-model-snapshot-retention-after-days] `description`:: (string) A description of the job. @@ -101,7 +101,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=daily-model-snapshot-retention- //Begin detectors.custom_rules `custom_rules`::: (array) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules] + .Properties of `custom_rules` [%collapsible%open] @@ -109,12 +109,12 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules] `actions`::: (array) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules-actions] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-actions] // Begin detectors.custom_rules.conditions `conditions`::: (array) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules-conditions] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-conditions] + .Properties of `conditions` [%collapsible%open] @@ -122,33 +122,33 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules-conditions] `applies_to`:::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules-conditions-applies-to] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-conditions-applies-to] `operator`:::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules-conditions-operator] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-conditions-operator] `value`:::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules-conditions-value] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-conditions-value] ====== //End detectors.custom_rules.conditions //Begin detectors.custom_rules.scope `scope`::: (object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules-scope] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-scope] + .Properties of `scope` [%collapsible%open] ====== `filter_id`:::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules-scope-filter-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-scope-filter-id] `filter_type`:::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules-scope-filter-type] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-scope-filter-type] ====== //End detectors.custom_rules.scope ===== @@ -156,11 +156,11 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules-scope-filter-type] `description`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=detector-description] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=detector-description] `detector_index`::: (integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=detector-index] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=detector-index] + -- If you want to update a specific detector, you must use this identifier. You @@ -171,59 +171,59 @@ cannot, however, change the `detector_index` value for a detector. `groups`:: (array of strings) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=groups] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=groups] //Begin model_plot_config `model_plot_config`:: (object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-plot-config] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-plot-config] + .Properties of `model_plot_config` [%collapsible%open] ==== `annotations_enabled`::: (Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-plot-config-annotations-enabled] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-plot-config-annotations-enabled] `enabled`::: (Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-plot-config-enabled] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-plot-config-enabled] `terms`::: experimental[] (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-plot-config-terms] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-plot-config-terms] ==== //End model_plot_config `model_prune_window`:: (<>) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-prune-window] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-prune-window] `model_snapshot_retention_days`:: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-snapshot-retention-days] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-snapshot-retention-days] //Begin per_partition_categorization `per_partition_categorization`::: (object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=per-partition-categorization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=per-partition-categorization] + .Properties of `per_partition_categorization` [%collapsible%open] ==== `enabled`::: (Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=per-partition-categorization-enabled] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=per-partition-categorization-enabled] `stop_on_warn`::: (Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=per-partition-categorization-stop-on-warn] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=per-partition-categorization-stop-on-warn] ==== //End per_partition_categorization `renormalization_window_days`:: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=renormalization-window-days] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=renormalization-window-days] + -- NOTE: If the job is open when you make the update, you must stop the {dfeed}, @@ -233,7 +233,7 @@ close the job, then reopen the job and restart the {dfeed} for the changes to ta `results_retention_days`:: (long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=results-retention-days] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=results-retention-days] [[ml-update-job-example]] diff --git a/docs/reference/ml/anomaly-detection/apis/update-snapshot.asciidoc b/docs/reference/ml/anomaly-detection/apis/update-snapshot.asciidoc index 2978d64f0122..f8c038486002 100644 --- a/docs/reference/ml/anomaly-detection/apis/update-snapshot.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/update-snapshot.asciidoc @@ -23,11 +23,11 @@ Requires the `manage_ml` cluster privilege. This privilege is included in the ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=snapshot-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=snapshot-id] [[ml-update-snapshot-request-body]] == {api-request-body-title} @@ -39,7 +39,7 @@ The following properties can be updated after the model snapshot is created: `retain`:: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=retain] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=retain] [[ml-update-snapshot-example]] diff --git a/docs/reference/ml/anomaly-detection/apis/upgrade-job-model-snapshot.asciidoc b/docs/reference/ml/anomaly-detection/apis/upgrade-job-model-snapshot.asciidoc index 3f09b292dc2d..3a74e3b2296d 100644 --- a/docs/reference/ml/anomaly-detection/apis/upgrade-job-model-snapshot.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/upgrade-job-model-snapshot.asciidoc @@ -42,11 +42,11 @@ snapshot cannot be the current snapshot of the {anomaly-job}. ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-anomaly-detection] ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=snapshot-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=snapshot-id] [[ml-upgrade-job-model-snapshot-query-parms]] diff --git a/docs/reference/ml/anomaly-detection/apis/validate-detector.asciidoc b/docs/reference/ml/anomaly-detection/apis/validate-detector.asciidoc index 5c312264cf71..c71673be7dc0 100644 --- a/docs/reference/ml/anomaly-detection/apis/validate-detector.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/validate-detector.asciidoc @@ -29,78 +29,78 @@ before you create an {anomaly-job}. `by_field_name`:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=by-field-name] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=by-field-name] `custom_rules`:: + -- (array) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules] `analysis_config`.`detectors`.`custom_rules`.`actions`::: (array) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules-actions] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-actions] `analysis_config`.`detectors`.`custom_rules`.`scope`::: (object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules-scope] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-scope] `analysis_config`.`detectors`.`custom_rules`.`scope`.`filter_id`:::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules-scope-filter-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-scope-filter-id] `analysis_config`.`detectors`.`custom_rules`.`scope`.`filter_type`:::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules-scope-filter-type] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-scope-filter-type] `analysis_config`.`detectors`.`custom_rules`.`conditions`::: (array) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules-conditions] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-conditions] `analysis_config`.`detectors`.`custom_rules`.`conditions`.`applies_to`:::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules-conditions-applies-to] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-conditions-applies-to] `analysis_config`.`detectors`.`custom_rules`.`conditions`.`operator`:::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules-conditions-operator] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-conditions-operator] `analysis_config`.`detectors`.`custom_rules`.`conditions`.`value`:::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-rules-conditions-value] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-rules-conditions-value] -- `detector_description`:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=detector-description] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=detector-description] `detector_index`:: (integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=detector-index] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=detector-index] `exclude_frequent`:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=exclude-frequent] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=exclude-frequent] `field_name`:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=detector-field-name] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=detector-field-name] `function`:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=function] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=function] `over_field_name`:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=over-field-name] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=over-field-name] `partition_field_name`:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=partition-field-name] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=partition-field-name] `use_null`:: (Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=use-null] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=use-null] [[ml-valid-detector-example]] == {api-examples-title} diff --git a/docs/reference/ml/common/apis/get-ml-memory.asciidoc b/docs/reference/ml/common/apis/get-ml-memory.asciidoc index 2dd79a969aca..81e0f59a97e5 100644 --- a/docs/reference/ml/common/apis/get-ml-memory.asciidoc +++ b/docs/reference/ml/common/apis/get-ml-memory.asciidoc @@ -42,7 +42,7 @@ node, both within the JVM heap, and natively, outside of the JVM. Specify this query parameter to include the fields with units in the response. Otherwise only the `_in_bytes` sizes are returned in the response. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [role="child_attributes"] [[get-ml-memory-response-body]] @@ -89,11 +89,11 @@ Contains statistics for the node. ===== `attributes`:: (object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-attributes] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-attributes] `ephemeral_id`:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-ephemeral-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-ephemeral-id] `jvm`:: (object) @@ -216,7 +216,7 @@ Roles assigned to the node. See <>. `transport_address`:: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-transport-address] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-transport-address] ===== ==== diff --git a/docs/reference/ml/df-analytics/apis/delete-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/delete-dfanalytics.asciidoc index 8631e1e7ae16..b505da570244 100644 --- a/docs/reference/ml/df-analytics/apis/delete-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/delete-dfanalytics.asciidoc @@ -27,7 +27,7 @@ Requires the `manage_ml` cluster privilege. This privilege is included in the ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-data-frame-analytics] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-data-frame-analytics] [[ml-delete-dfanalytics-query-params]] == {api-query-parms-title} diff --git a/docs/reference/ml/df-analytics/apis/explain-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/explain-dfanalytics.asciidoc index ed0a98474b5c..0ee7ec563458 100644 --- a/docs/reference/ml/df-analytics/apis/explain-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/explain-dfanalytics.asciidoc @@ -52,7 +52,7 @@ they are not included in the explanation. ``:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-data-frame-analytics] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-data-frame-analytics] [[ml-explain-dfanalytics-request-body]] == {api-request-body-title} diff --git a/docs/reference/ml/df-analytics/apis/get-dfanalytics-stats.asciidoc b/docs/reference/ml/df-analytics/apis/get-dfanalytics-stats.asciidoc index b18899cc8655..b193379084db 100644 --- a/docs/reference/ml/df-analytics/apis/get-dfanalytics-stats.asciidoc +++ b/docs/reference/ml/df-analytics/apis/get-dfanalytics-stats.asciidoc @@ -35,7 +35,7 @@ Requires the `monitor_ml` cluster privilege. This privilege is included in the ``:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-data-frame-analytics-default] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-data-frame-analytics-default] [[ml-get-dfanalytics-stats-query-params]] @@ -43,19 +43,19 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-data-frame-analytics-def `allow_no_match`:: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-dfa-jobs] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-dfa-jobs] `from`:: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=from] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=from] `size`:: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=size] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=size] `verbose`:: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=verbose] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=verbose] [role="child_attributes"] [[ml-get-dfanalytics-stats-response-body]] @@ -95,106 +95,106 @@ An object containing the parameters of the {classanalysis} job. ======= `alpha`:::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-alpha] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-alpha] `class_assignment_objective`:::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=class-assignment-objective] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=class-assignment-objective] `downsample_factor`:::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-downsample-factor] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-downsample-factor] `eta`:::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=eta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=eta] `eta_growth_rate_per_tree`:::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-eta-growth] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-eta-growth] `feature_bag_fraction`:::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=feature-bag-fraction] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=feature-bag-fraction] `gamma`:::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=gamma] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=gamma] `lambda`:::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=lambda] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=lambda] `max_attempts_to_add_tree`:::: (integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-max-attempts] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-max-attempts] `max_optimization_rounds_per_hyperparameter`:::: (integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-max-optimization-rounds] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-max-optimization-rounds] `max_trees`:::: (integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=max-trees] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=max-trees] `num_folds`:::: (integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-num-folds] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-num-folds] `num_splits_per_feature`:::: (integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-num-splits] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-num-splits] `soft_tree_depth_limit`:::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-soft-limit] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-soft-limit] `soft_tree_depth_tolerance`:::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-soft-tolerance] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-soft-tolerance] ======= //End class_hyperparameters `iteration`:::: (integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-iteration] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-iteration] `timestamp`:::: (date) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-timestamp] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-timestamp] //Begin class_timing_stats `timing_stats`:::: (object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-timing-stats] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-timing-stats] + .Properties of `timing_stats` [%collapsible%open] ======= `elapsed_time`:::: (integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-timing-stats-elapsed] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-timing-stats-elapsed] `iteration_time`:::: (integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-timing-stats-iteration] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-timing-stats-iteration] ======= //End class_timing_stats //Begin class_validation_loss `validation_loss`:::: (object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-validation-loss] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-validation-loss] + .Properties of `validation_loss` [%collapsible%open] ======= `fold_values`:::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-validation-loss-fold] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-validation-loss-fold] `loss_type`:::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-validation-loss-type] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-validation-loss-type] ======= //End class_validation_loss ====== @@ -219,45 +219,45 @@ heuristics. ======= `compute_feature_influence`:::: (Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=compute-feature-influence] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=compute-feature-influence] `feature_influence_threshold`:::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=feature-influence-threshold] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=feature-influence-threshold] `method`:::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=method] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=method] `n_neighbors`:::: (integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=n-neighbors] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=n-neighbors] `outlier_fraction`:::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=outlier-fraction] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=outlier-fraction] `standardization_enabled`:::: (Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=standardization-enabled] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=standardization-enabled] ======= //End parameters `timestamp`:::: (date) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-timestamp] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-timestamp] //Begin od_timing_stats `timing_stats`:::: (object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-timing-stats] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-timing-stats] + .Property of `timing_stats` [%collapsible%open] ======= `elapsed_time`:::: (integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-timing-stats-elapsed] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-timing-stats-elapsed] ======= //End od_timing_stats ====== @@ -281,103 +281,103 @@ An object containing the parameters of the {reganalysis} job. ======= `alpha`:::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-alpha] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-alpha] `downsample_factor`:::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-downsample-factor] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-downsample-factor] `eta`:::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=eta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=eta] `eta_growth_rate_per_tree`:::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-eta-growth] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-eta-growth] `feature_bag_fraction`:::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=feature-bag-fraction] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=feature-bag-fraction] `gamma`:::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=gamma] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=gamma] `lambda`:::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=lambda] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=lambda] `max_attempts_to_add_tree`:::: (integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-max-attempts] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-max-attempts] `max_optimization_rounds_per_hyperparameter`:::: (integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-max-optimization-rounds] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-max-optimization-rounds] `max_trees`:::: (integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=max-trees] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=max-trees] `num_folds`:::: (integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-num-folds] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-num-folds] `num_splits_per_feature`:::: (integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-num-splits] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-num-splits] `soft_tree_depth_limit`:::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-soft-limit] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-soft-limit] `soft_tree_depth_tolerance`:::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-soft-tolerance] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-soft-tolerance] ======= //End reg_hyperparameters `iteration`:::: (integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-iteration] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-iteration] `timestamp`:::: (date) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-timestamp] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-timestamp] //Begin reg_timing_stats `timing_stats`:::: (object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-timing-stats] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-timing-stats] + .Propertis of `timing_stats` [%collapsible%open] ======= `elapsed_time`:::: (integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-timing-stats-elapsed] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-timing-stats-elapsed] `iteration_time`:::: (integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-timing-stats-iteration] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-timing-stats-iteration] ======= //End reg_timing_stats //Begin reg_validation_loss `validation_loss`:::: (object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-validation-loss] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-validation-loss] + .Properties of `validation_loss` [%collapsible%open] ======= `fold_values`:::: (array of strings) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-validation-loss-fold] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-validation-loss-fold] `loss_type`:::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-validation-loss-type] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-validation-loss-type] ======= //End reg_validation_loss ====== @@ -463,15 +463,15 @@ available only for running jobs. ===== `attributes`:::: (object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-attributes] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-attributes] `ephemeral_id`:::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-ephemeral-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-ephemeral-id] `id`:::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-id] `name`:::: (string) @@ -479,7 +479,7 @@ The node name. `transport_address`:::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-transport-address] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-transport-address] ===== `progress`::: diff --git a/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc index 7154dd607e77..c2a4caa981da 100644 --- a/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/get-dfanalytics.asciidoc @@ -39,7 +39,7 @@ by using a comma-separated list of {dfanalytics-jobs} or a wildcard expression. ``:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-data-frame-analytics-default] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-data-frame-analytics-default] + -- You can get information for all {dfanalytics-jobs} by using _all, by specifying @@ -53,19 +53,19 @@ You can get information for all {dfanalytics-jobs} by using _all, by specifying `allow_no_match`:: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-dfa-jobs] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-dfa-jobs] `exclude_generated`:: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=exclude-generated] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=exclude-generated] `from`:: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=from] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=from] `size`:: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=size] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=size] [role="child_attributes"] [[ml-get-dfanalytics-results]] diff --git a/docs/reference/ml/df-analytics/apis/preview-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/preview-dfanalytics.asciidoc index 5e67a0cf7832..2c61c3263992 100644 --- a/docs/reference/ml/df-analytics/apis/preview-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/preview-dfanalytics.asciidoc @@ -41,7 +41,7 @@ that either exists already or one that has not been created yet. ``:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-data-frame-analytics] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-data-frame-analytics] [[ml-preview-dfanalytics-request-body]] == {api-request-body-title} diff --git a/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc index f11166f9c1a6..54cbe78b3445 100644 --- a/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc @@ -51,7 +51,7 @@ determines a value for each of the undefined parameters. ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-data-frame-analytics-define] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-data-frame-analytics-define] [role="child_attributes"] [[ml-put-dfanalytics-request-body]] @@ -93,16 +93,16 @@ understand the function of these parameters. ===== `alpha`:::: (Optional, double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-alpha] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-alpha] `class_assignment_objective`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=class-assignment-objective] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=class-assignment-objective] `dependent_variable`:::: (Required, string) + -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dependent-variable] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dependent-variable] + The data type of the field must be numeric (`integer`, `short`, `long`, `byte`), categorical (`ip` or `keyword`), or boolean. There must be no more than 100 @@ -110,148 +110,148 @@ different values in this field. `downsample_factor`:::: (Optional, double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-downsample-factor] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-downsample-factor] `early_stopping_enabled`:::: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-early-stopping-enabled] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-early-stopping-enabled] `eta`:::: (Optional, double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=eta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=eta] `eta_growth_rate_per_tree`:::: (Optional, double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-eta-growth] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-eta-growth] `feature_bag_fraction`:::: (Optional, double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=feature-bag-fraction] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=feature-bag-fraction] `feature_processors`:::: (Optional, list) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors] + .Properties of `feature_processors` [%collapsible%open] ====== `frequency_encoding`:::: (object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-frequency] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-frequency] + .Properties of `frequency_encoding` [%collapsible%open] ======= `feature_name`:::: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-feat-name] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-feat-name] `field`:::: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-field] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-field] `frequency_map`:::: (Required, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-frequency-map] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-frequency-map] ======= `multi_encoding`:::: (object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-multi] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-multi] + .Properties of `multi_encoding` [%collapsible%open] ======= `processors`:::: (Required, array) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-multi-proc] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-multi-proc] ======= `n_gram_encoding`:::: (object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-ngram] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-ngram] + .Properties of `n_gram_encoding` [%collapsible%open] ======= `feature_prefix`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-ngram-feat-pref] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-ngram-feat-pref] `field`:::: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-ngram-field] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-ngram-field] `length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-ngram-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-ngram-length] `n_grams`:::: (Required, array) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-ngram-ngrams] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-ngram-ngrams] `start`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-ngram-start] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-ngram-start] ======= `one_hot_encoding`:::: (object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-one-hot] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-one-hot] + .Properties of `one_hot_encoding` [%collapsible%open] ======= `field`:::: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-field] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-field] `hot_map`:::: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-one-hot-map] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-one-hot-map] ======= `target_mean_encoding`:::: (object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-target-mean] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-target-mean] + .Properties of `target_mean_encoding` [%collapsible%open] ======= `default_value`:::: (Required, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-target-mean-default] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-target-mean-default] `feature_name`:::: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-feat-name] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-feat-name] `field`:::: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-field] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-field] `target_map`:::: (Required, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-target-mean-map] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors-target-mean-map] ======= ====== `gamma`:::: (Optional, double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=gamma] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=gamma] `lambda`:::: (Optional, double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=lambda] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=lambda] `max_optimization_rounds_per_hyperparameter`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-max-optimization-rounds] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-max-optimization-rounds] `max_trees`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=max-trees] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=max-trees] `num_top_classes`:::: (Optional, integer) @@ -276,23 +276,23 @@ By default, it is zero and no {feat-imp} calculation occurs. `prediction_field_name`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=prediction-field-name] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=prediction-field-name] `randomize_seed`:::: (Optional, long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=randomize-seed] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=randomize-seed] `soft_tree_depth_limit`:::: (Optional, double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-soft-limit] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-soft-limit] `soft_tree_depth_tolerance`:::: (Optional, double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-soft-tolerance] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-soft-tolerance] `training_percent`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=training-percent] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=training-percent] //End classification ===== //Begin outlier_detection @@ -306,27 +306,27 @@ The configuration information necessary to perform ===== `compute_feature_influence`:::: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=compute-feature-influence] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=compute-feature-influence] `feature_influence_threshold`:::: (Optional, double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=feature-influence-threshold] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=feature-influence-threshold] `method`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=method] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=method] `n_neighbors`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=n-neighbors] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=n-neighbors] `outlier_fraction`:::: (Optional, double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=outlier-fraction] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=outlier-fraction] `standardization_enabled`:::: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=standardization-enabled] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=standardization-enabled] //End outlier_detection ===== //Begin regression @@ -345,46 +345,46 @@ understand the function of these parameters. ===== `alpha`:::: (Optional, double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-alpha] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-alpha] `dependent_variable`:::: (Required, string) + -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dependent-variable] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dependent-variable] + The data type of the field must be numeric. `downsample_factor`:::: (Optional, double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-downsample-factor] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-downsample-factor] `early_stopping_enabled`:::: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-early-stopping-enabled] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-early-stopping-enabled] `eta`:::: (Optional, double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=eta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=eta] `eta_growth_rate_per_tree`:::: (Optional, double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-eta-growth] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-eta-growth] `feature_bag_fraction`:::: (Optional, double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=feature-bag-fraction] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=feature-bag-fraction] `feature_processors`:::: (Optional, list) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-feature-processors] `gamma`:::: (Optional, double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=gamma] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=gamma] `lambda`:::: (Optional, double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=lambda] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=lambda] `loss_function`:::: (Optional, string) @@ -401,11 +401,11 @@ A positive number that is used as a parameter to the `loss_function`. `max_optimization_rounds_per_hyperparameter`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-max-optimization-rounds] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-max-optimization-rounds] `max_trees`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=max-trees] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=max-trees] `num_top_feature_importance_values`:::: (Optional, integer) @@ -415,23 +415,23 @@ By default, it is zero and no {feat-imp} calculation occurs. `prediction_field_name`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=prediction-field-name] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=prediction-field-name] `randomize_seed`:::: (Optional, long) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=randomize-seed] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=randomize-seed] `soft_tree_depth_limit`:::: (Optional, double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-soft-limit] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-soft-limit] `soft_tree_depth_tolerance`:::: (Optional, double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dfas-soft-tolerance] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dfas-soft-tolerance] `training_percent`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=training-percent] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=training-percent] ===== //End regression ==== @@ -493,11 +493,11 @@ analysis. `description`:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=description-dfa] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=description-dfa] `dest`:: (Required, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=dest] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=dest] `max_num_threads`:: (Optional, integer) @@ -509,11 +509,11 @@ functionality other than the analysis itself. `_meta`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=meta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=meta] `model_memory_limit`:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-memory-limit-dfa] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-memory-limit-dfa] `source`:: (object) diff --git a/docs/reference/ml/df-analytics/apis/start-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/start-dfanalytics.asciidoc index 8ed1fab20206..70e996ef8dd0 100644 --- a/docs/reference/ml/df-analytics/apis/start-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/start-dfanalytics.asciidoc @@ -53,14 +53,14 @@ when you created the job, those credentials are used. ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-data-frame-analytics-define] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-data-frame-analytics-define] [[ml-start-dfanalytics-query-params]] == {api-query-parms-title} `timeout`:: (Optional, <>) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=timeout-start] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=timeout-start] [[ml-start-dfanalytics-response-body]] == {api-response-body-title} diff --git a/docs/reference/ml/df-analytics/apis/stop-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/stop-dfanalytics.asciidoc index 3ac7be860fd1..2fa3bc4413d7 100644 --- a/docs/reference/ml/df-analytics/apis/stop-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/stop-dfanalytics.asciidoc @@ -42,14 +42,14 @@ stop all {dfanalytics-job} by using _all or by specifying * as the ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-data-frame-analytics-define] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-data-frame-analytics-define] [[ml-stop-dfanalytics-query-params]] == {api-query-parms-title} `allow_no_match`:: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-dfa-jobs] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-dfa-jobs] `force`:: @@ -57,7 +57,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-dfa-jobs] `timeout`:: (Optional, <>) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=timeout-stop] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=timeout-stop] [[ml-stop-dfanalytics-example]] diff --git a/docs/reference/ml/df-analytics/apis/update-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/update-dfanalytics.asciidoc index f78b0516a621..49cca176be69 100644 --- a/docs/reference/ml/df-analytics/apis/update-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/update-dfanalytics.asciidoc @@ -44,7 +44,7 @@ indices and stores the outcome in a destination index. ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=job-id-data-frame-analytics-define] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=job-id-data-frame-analytics-define] [role="child_attributes"] [[ml-update-dfanalytics-request-body]] @@ -62,7 +62,7 @@ the `starting` state until sufficient {ml} node capacity is available. `description`:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=description-dfa] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=description-dfa] `max_num_threads`:: (Optional, integer) @@ -74,11 +74,11 @@ functionality other than the analysis itself. `_meta`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=meta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=meta] `model_memory_limit`:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-memory-limit-dfa] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-memory-limit-dfa] [[ml-update-dfanalytics-example]] == {api-examples-title} diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc index cc6d9037bd59..6bbc98db1c2e 100644 --- a/docs/reference/ml/ml-shared.asciidoc +++ b/docs/reference/ml/ml-shared.asciidoc @@ -234,15 +234,15 @@ is an object it has the following properties: ===== `char_filter`:::: (array of strings or objects) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=char-filter] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=char-filter] `tokenizer`:::: (string or object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=tokenizer] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=tokenizer] `filter`:::: (array of strings or objects) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=filter] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=filter] ===== end::categorization-analyzer[] @@ -448,7 +448,7 @@ it is not stored in {es}. Only the results for {anomaly-detect} are retained. `time_format`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=time-format] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=time-format] ==== end::data-description[] diff --git a/docs/reference/ml/trained-models/apis/clear-trained-model-deployment-cache.asciidoc b/docs/reference/ml/trained-models/apis/clear-trained-model-deployment-cache.asciidoc index 642afc520e6a..f24379705fc7 100644 --- a/docs/reference/ml/trained-models/apis/clear-trained-model-deployment-cache.asciidoc +++ b/docs/reference/ml/trained-models/apis/clear-trained-model-deployment-cache.asciidoc @@ -31,7 +31,7 @@ node. Calling this API clears the caches without restarting the deployment. `deployment_id`:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=deployment-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=deployment-id] [[clear-trained-model-deployment-cache-example]] == {api-examples-title} diff --git a/docs/reference/ml/trained-models/apis/delete-trained-models.asciidoc b/docs/reference/ml/trained-models/apis/delete-trained-models.asciidoc index bba39d876ebd..1b54343d1f1c 100644 --- a/docs/reference/ml/trained-models/apis/delete-trained-models.asciidoc +++ b/docs/reference/ml/trained-models/apis/delete-trained-models.asciidoc @@ -27,7 +27,7 @@ Requires the `manage_ml` cluster privilege. This privilege is included in the ``:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-id] [[ml-delete-trained-models-query-parms]] == {api-query-parms-title} diff --git a/docs/reference/ml/trained-models/apis/get-trained-models-stats.asciidoc b/docs/reference/ml/trained-models/apis/get-trained-models-stats.asciidoc index aec997855902..beff87e6ec6e 100644 --- a/docs/reference/ml/trained-models/apis/get-trained-models-stats.asciidoc +++ b/docs/reference/ml/trained-models/apis/get-trained-models-stats.asciidoc @@ -53,15 +53,15 @@ model. `allow_no_match`:: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-models] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-models] `from`:: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=from-models] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=from-models] `size`:: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=size-models] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=size-models] [role="child_attributes"] [[ml-get-trained-models-stats-results]] @@ -120,7 +120,7 @@ The desired number of nodes for model allocation. ====== `deployment_id`::: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=deployment-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=deployment-id] `error_count`::: (integer) @@ -132,7 +132,7 @@ The sum of `inference_count` for all nodes in the deployment. `model_id`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-id] `nodes`::: (array of objects) @@ -190,22 +190,22 @@ Information pertaining to the node. ======== `attributes`::: (object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-attributes] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-attributes] `ephemeral_id`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-ephemeral-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-ephemeral-id] `id`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-id] `name`::: (string) The node name. `transport_address`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=node-transport-address] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=node-transport-address] ======== `number_of_allocations`::: @@ -365,7 +365,7 @@ section in <>. `model_id`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-id] `model_size_stats`::: (object) diff --git a/docs/reference/ml/trained-models/apis/get-trained-models.asciidoc b/docs/reference/ml/trained-models/apis/get-trained-models.asciidoc index 648595a83e8f..f203578c6f1c 100644 --- a/docs/reference/ml/trained-models/apis/get-trained-models.asciidoc +++ b/docs/reference/ml/trained-models/apis/get-trained-models.asciidoc @@ -38,7 +38,7 @@ Requires the `monitor_ml` cluster privilege. This privilege is included in the ``:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-id-or-alias] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-id-or-alias] + You can get information for multiple trained models in a single API request by using a comma-separated list of model IDs or a wildcard expression. @@ -48,7 +48,7 @@ using a comma-separated list of model IDs or a wildcard expression. `allow_no_match`:: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-models] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-models] `decompress_definition`:: (Optional, Boolean) @@ -57,11 +57,11 @@ Specifies whether the included model definition should be returned as a JSON map `exclude_generated`:: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=exclude-generated] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=exclude-generated] `from`:: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=from-models] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=from-models] `include`:: (Optional, string) @@ -83,11 +83,11 @@ in the response body. `size`:: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=size-models] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=size-models] `tags`:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=tags] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=tags] [role="child_attributes"] [[ml-get-trained-models-results]] @@ -148,157 +148,157 @@ Classification configuration for inference. ====== `num_top_classes`::: (integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-num-top-classes] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-num-top-classes] `num_top_feature_importance_values`::: (integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-num-top-feature-importance-values] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-num-top-feature-importance-values] `prediction_field_type`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-prediction-field-type] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-prediction-field-type] `results_field`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] `top_classes_results_field`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-top-classes-results-field] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-top-classes-results-field] ====== `fill_mask`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-fill-mask] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-fill-mask] + .Properties of fill_mask inference [%collapsible%open] ====== `mask_token`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-mask-token] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-mask-token] `tokenization`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] + .Properties of tokenization [%collapsible%open] ======= `bert`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] + .Properties of bert [%collapsible%open] ======== `do_lower_case`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-with-special-tokens] ======== `roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] + .Properties of roberta [%collapsible%open] ======== `add_prefix_space`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-add-prefix-space] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-add-prefix-space] `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] ======== `mpnet`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] + .Properties of mpnet [%collapsible%open] ======== `do_lower_case`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet-with-special-tokens] ======== `xlm_roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] + .Properties of xlm_roberta [%collapsible%open] ======== `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] ======== `bert_ja`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] + .Properties of bert_ja [%collapsible%open] ======== `do_lower_case`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja-with-special-tokens] ======== ======= `vocabulary`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-vocabulary] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-vocabulary] + .Properties of vocabulary [%collapsible%open] @@ -311,7 +311,7 @@ The index where the vocabulary is stored. `ner`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-ner] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-ner] + .Properties of ner inference [%collapsible%open] @@ -325,126 +325,126 @@ and miscellaneous. For example: `tokenization`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] + .Properties of tokenization [%collapsible%open] ======= `bert`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] + .Properties of bert [%collapsible%open] ======== `do_lower_case`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-with-special-tokens] ======== `roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] + .Properties of roberta [%collapsible%open] ======== `add_prefix_space`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-add-prefix-space] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-add-prefix-space] `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] ======== `mpnet`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] + .Properties of mpnet [%collapsible%open] ======== `do_lower_case`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet-with-special-tokens] ======== `xlm_roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] + .Properties of xlm_roberta [%collapsible%open] ======== `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] ======== `bert_ja`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] + .Properties of bert_ja [%collapsible%open] ======== `do_lower_case`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja-with-special-tokens] ======== ======= `vocabulary`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-vocabulary] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-vocabulary] + .Properties of vocabulary [%collapsible%open] @@ -457,133 +457,133 @@ The index where the vocabulary is stored `pass_through`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-pass-through] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-pass-through] + .Properties of pass_through inference [%collapsible%open] ====== `tokenization`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] + .Properties of tokenization [%collapsible%open] ======= `bert`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] + .Properties of bert [%collapsible%open] ======== `do_lower_case`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-with-special-tokens] ======== `roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] + .Properties of roberta [%collapsible%open] ======== `add_prefix_space`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-add-prefix-space] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-add-prefix-space] `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] ======== `mpnet`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] + .Properties of mpnet [%collapsible%open] ======== `do_lower_case`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet-with-special-tokens] ======== `xlm_roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] + .Properties of xlm_roberta [%collapsible%open] ======== `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] ======== `bert_ja`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] + .Properties of bert_ja [%collapsible%open] ======== `do_lower_case`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja-with-special-tokens] ======== ======= `vocabulary`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-vocabulary] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-vocabulary] + .Properties of vocabulary [%collapsible%open] @@ -603,15 +603,15 @@ Regression configuration for inference. ====== `num_top_feature_importance_values`::: (integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-regression-num-top-feature-importance-values] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-regression-num-top-feature-importance-values] `results_field`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] ====== `text_classification`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-classification] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-classification] + .Properties of text_classification inference [%collapsible%open] @@ -626,147 +626,147 @@ Specifies the number of top class predictions to return. Defaults to all classes `tokenization`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] + .Properties of tokenization [%collapsible%open] ======= `bert`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] + .Properties of bert [%collapsible%open] ======== `do_lower_case`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `span`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-with-special-tokens] ======== `roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] + .Properties of roberta [%collapsible%open] ======== `add_prefix_space`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-add-prefix-space] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-add-prefix-space] `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `span`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] ======== `mpnet`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] + .Properties of mpnet [%collapsible%open] ======== `do_lower_case`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `span`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet-with-special-tokens] ======== `xlm_roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] + .Properties of xlm_roberta [%collapsible%open] ======== `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `span`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] ======== `bert_ja`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] + .Properties of bert_ja [%collapsible%open] ======== `do_lower_case`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `span`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja-with-special-tokens] ======== ======= `vocabulary`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-vocabulary] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-vocabulary] + .Properties of vocabulary [%collapsible%open] @@ -778,141 +778,141 @@ The index where the vocabulary is stored. ====== `text_embedding`:::: (Object, optional) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-embedding] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-embedding] + .Properties of text_embedding inference [%collapsible%open] ====== `embedding_size`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-embedding-size] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-embedding-size] `results_field`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] `tokenization`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] + .Properties of tokenization [%collapsible%open] ======= `bert`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] + .Properties of bert [%collapsible%open] ======== `do_lower_case`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-with-special-tokens] ======== `roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] + .Properties of roberta [%collapsible%open] ======== `add_prefix_space`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-add-prefix-space] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-add-prefix-space] `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] ======== `mpnet`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] + .Properties of mpnet [%collapsible%open] ======== `do_lower_case`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet-with-special-tokens] ======== `xlm_roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] + .Properties of xlm_roberta [%collapsible%open] ======== `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] ======== `bert_ja`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] + .Properties of bert_ja [%collapsible%open] ======== `do_lower_case`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja-with-special-tokens] ======== ======= `vocabulary`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-vocabulary] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-vocabulary] + .Properties of vocabulary [%collapsible%open] @@ -924,157 +924,157 @@ The index where the vocabulary is stored. ====== `text_similarity`:::: (Object, optional) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-similarity] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-similarity] + .Properties of text_similarity inference [%collapsible%open] ====== `span_score_combination_function`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-similarity-span-score-func] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-similarity-span-score-func] `tokenization`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] + .Properties of tokenization [%collapsible%open] ======= `bert`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] + .Properties of bert [%collapsible%open] ======== `do_lower_case`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `span`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-with-special-tokens] ======== `roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] + .Properties of roberta [%collapsible%open] ======== `add_prefix_space`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-add-prefix-space] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-add-prefix-space] `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `span`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] ======== `mpnet`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] + .Properties of mpnet [%collapsible%open] ======== `do_lower_case`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `span`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet-with-special-tokens] ======== `xlm_roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] + .Properties of xlm_roberta [%collapsible%open] ======== `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `span`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] ======== `bert_ja`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] + .Properties of bert_ja [%collapsible%open] ======== `do_lower_case`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `span`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja-with-special-tokens] ======== ======= `vocabulary`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-vocabulary] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-vocabulary] + .Properties of vocabulary [%collapsible%open] @@ -1086,149 +1086,149 @@ The index where the vocabulary is stored. ====== `zero_shot_classification`:::: (Object, optional) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-zero-shot-classification] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-zero-shot-classification] + .Properties of zero_shot_classification inference [%collapsible%open] ====== `classification_labels`:::: (Required, array) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-zero-shot-classification-classification-labels] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-zero-shot-classification-classification-labels] `hypothesis_template`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-zero-shot-classification-hypothesis-template] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-zero-shot-classification-hypothesis-template] `labels`:::: (Optional, array) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-zero-shot-classification-labels] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-zero-shot-classification-labels] `multi_label`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-zero-shot-classification-multi-label] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-zero-shot-classification-multi-label] `tokenization`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] + .Properties of tokenization [%collapsible%open] ======= `bert`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] + .Properties of bert [%collapsible%open] ======== `do_lower_case`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-with-special-tokens] ======== `roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] + .Properties of roberta [%collapsible%open] ======== `add_prefix_space`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-add-prefix-space] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-add-prefix-space] `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] ======== `mpnet`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] + .Properties of mpnet [%collapsible%open] ======== `do_lower_case`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet-with-special-tokens] ======== `xlm_roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] + .Properties of xlm_roberta [%collapsible%open] ======== `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] ======== `bert_ja`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] + .Properties of bert_ja [%collapsible%open] ======== `do_lower_case`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] `max_sequence_length`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja-with-special-tokens] ======== ======= `vocabulary`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-vocabulary] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-vocabulary] + .Properties of vocabulary [%collapsible%open] @@ -1307,7 +1307,7 @@ hyperparameter optimization. `max_trees`:::: (integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=max-trees-trained-models] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=max-trees-trained-models] `name`:::: (string) @@ -1344,7 +1344,7 @@ request parameter. `feature_name`::: (string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-metadata-feature-importance-feature-name] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-metadata-feature-importance-feature-name] `importance`::: (object) @@ -1355,15 +1355,15 @@ A collection of {feat-imp} statistics related to the training data set for this ======= `mean_magnitude`::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-metadata-feature-importance-magnitude] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-metadata-feature-importance-magnitude] `max`::: (integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-metadata-feature-importance-max] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-metadata-feature-importance-max] `min`::: (integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-metadata-feature-importance-min] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-metadata-feature-importance-min] ======= @@ -1390,15 +1390,15 @@ A collection of {feat-imp} statistics related to the training data set for this ======== `mean_magnitude`::: (double) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-metadata-feature-importance-magnitude] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-metadata-feature-importance-magnitude] `max`::: (int) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-metadata-feature-importance-max] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-metadata-feature-importance-max] `min`::: (int) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-metadata-feature-importance-min] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-metadata-feature-importance-min] ======== diff --git a/docs/reference/ml/trained-models/apis/infer-trained-model-deployment.asciidoc b/docs/reference/ml/trained-models/apis/infer-trained-model-deployment.asciidoc index d92d74d894a3..83bc56d18df6 100644 --- a/docs/reference/ml/trained-models/apis/infer-trained-model-deployment.asciidoc +++ b/docs/reference/ml/trained-models/apis/infer-trained-model-deployment.asciidoc @@ -31,7 +31,7 @@ deprecated::[8.3.0,Replaced by <>.] ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-id-or-alias] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-id-or-alias] [[infer-trained-model-deployment-query-params]] == {api-query-parms-title} diff --git a/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc b/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc index d7201fcf42c0..9aac913e7559 100644 --- a/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc +++ b/docs/reference/ml/trained-models/apis/infer-trained-model.asciidoc @@ -34,14 +34,14 @@ directly from the {infer} cache. ``:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-id-or-alias] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-id-or-alias] If you specify the `model_id` in the API call, and the model has multiple deployments, a random deployment will be used. If the `model_id` matches the ID of one of the deployments, that deployment will be used. ``:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=deployment-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=deployment-id] [[infer-trained-model-query-params]] == {api-query-parms-title} @@ -84,28 +84,28 @@ Classification configuration for inference. ===== `num_top_classes`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-num-top-classes] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-num-top-classes] `num_top_feature_importance_values`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-num-top-feature-importance-values] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-num-top-feature-importance-values] `prediction_field_type`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-prediction-field-type] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-prediction-field-type] `results_field`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] `top_classes_results_field`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-top-classes-results-field] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-top-classes-results-field] ===== `fill_mask`::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-fill-mask] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-fill-mask] + .Properties of fill_mask inference [%collapsible%open] @@ -117,228 +117,228 @@ to `0`. `results_field`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] `tokenization`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] + .Properties of tokenization [%collapsible%open] ====== `bert`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] + .Properties of bert [%collapsible%open] ======= `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] + .Properties of roberta [%collapsible%open] ======= `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `mpnet`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] + .Properties of mpnet [%collapsible%open] ======= `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `xlm_roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] + .Properties of xlm_roberta [%collapsible%open] ======= `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `bert_ja`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] + .Properties of bert_ja [%collapsible%open] ======= `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= ====== ===== `ner`::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-ner] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-ner] + .Properties of ner inference [%collapsible%open] ===== `results_field`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] `tokenization`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] + .Properties of tokenization [%collapsible%open] ====== `bert`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] + .Properties of bert [%collapsible%open] ======= `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] + .Properties of roberta [%collapsible%open] ======= `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `mpnet`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] + .Properties of mpnet [%collapsible%open] ======= `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `xlm_roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] + .Properties of xlm_roberta [%collapsible%open] ======= `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `bert_ja`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] + .Properties of bert_ja [%collapsible%open] ======= `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= ====== ===== `pass_through`::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-pass-through] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-pass-through] + .Properties of pass_through inference [%collapsible%open] ===== `results_field`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] `tokenization`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] + .Properties of tokenization [%collapsible%open] ====== `bert`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] + .Properties of bert [%collapsible%open] ======= `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] + .Properties of roberta [%collapsible%open] ======= `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `mpnet`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] + .Properties of mpnet [%collapsible%open] ======= `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `xlm_roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] + .Properties of xlm_roberta [%collapsible%open] ======= `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `bert_ja`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] + .Properties of bert_ja [%collapsible%open] ======= `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= ====== ===== `question_answering`::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-question-answering] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-question-answering] + .Properties of question_answering inference [%collapsible%open] @@ -358,11 +358,11 @@ The question to use when extracting an answer `results_field`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] `tokenization`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] + Recommended to set `max_sequence_length` to `386` with `128` of `span` and set `truncate` to `none`. @@ -372,78 +372,78 @@ Recommended to set `max_sequence_length` to `386` with `128` of `span` and set ====== `bert`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] + .Properties of bert [%collapsible%open] ======= `span`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] + .Properties of roberta [%collapsible%open] ======= `span`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `mpnet`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] + .Properties of mpnet [%collapsible%open] ======= `span`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `xlm_roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] + .Properties of xlm_roberta [%collapsible%open] ======= `span`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `bert_ja`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] + .Properties of bert_ja [%collapsible%open] ======= `span`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= ====== ===== @@ -457,16 +457,16 @@ Regression configuration for inference. ===== `num_top_feature_importance_values`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-regression-num-top-feature-importance-values] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-regression-num-top-feature-importance-values] `results_field`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] ===== `text_classification`::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-classification] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-classification] + .Properties of text_classification inference [%collapsible%open] @@ -481,346 +481,346 @@ Specifies the number of top class predictions to return. Defaults to all classes `results_field`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] `tokenization`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] + .Properties of tokenization [%collapsible%open] ====== `bert`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] + .Properties of bert [%collapsible%open] ======= `span`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] + .Properties of roberta [%collapsible%open] ======= `span`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `mpnet`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] + .Properties of mpnet [%collapsible%open] ======= `span`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `xlm_roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] + .Properties of xlm_roberta [%collapsible%open] ======= `span`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `bert_ja`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] + .Properties of bert_ja [%collapsible%open] ======= `span`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= ====== ===== `text_embedding`::: (Object, optional) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-embedding] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-embedding] + .Properties of text_embedding inference [%collapsible%open] ===== `results_field`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] `tokenization`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] + .Properties of tokenization [%collapsible%open] ====== `bert`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] + .Properties of bert [%collapsible%open] ======= `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] + .Properties of roberta [%collapsible%open] ======= `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `mpnet`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] + .Properties of mpnet [%collapsible%open] ======= `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `xlm_roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] + .Properties of xlm_roberta [%collapsible%open] ======= `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `bert_ja`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] + .Properties of bert_ja [%collapsible%open] ======= `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= ====== ===== `text_similarity`::: (Object, optional) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-similarity] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-similarity] + .Properties of text_similarity inference [%collapsible%open] ===== `span_score_combination_function`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-similarity-span-score-func] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-similarity-span-score-func] `text`:::: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-similarity-text] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-similarity-text] `tokenization`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] + .Properties of tokenization [%collapsible%open] ====== `bert`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] + .Properties of bert [%collapsible%open] ======= `span`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-with-special-tokens] ======= `roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] + .Properties of roberta [%collapsible%open] ======= `span`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `mpnet`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] + .Properties of mpnet [%collapsible%open] ======= `span`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `xlm_roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] + .Properties of xlm_roberta [%collapsible%open] ======= `span`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `bert_ja`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] + .Properties of bert_ja [%collapsible%open] ======= `span`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `with_special_tokens`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja-with-special-tokens] ======= ====== ===== `zero_shot_classification`::: (Object, optional) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-zero-shot-classification] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-zero-shot-classification] + .Properties of zero_shot_classification inference [%collapsible%open] ===== `labels`:::: (Optional, array) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-zero-shot-classification-labels] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-zero-shot-classification-labels] `multi_label`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-zero-shot-classification-multi-label] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-zero-shot-classification-multi-label] `results_field`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] `tokenization`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] + .Properties of tokenization [%collapsible%open] ====== `bert`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] + .Properties of bert [%collapsible%open] ======= `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] + .Properties of roberta [%collapsible%open] ======= `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `mpnet`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] + .Properties of mpnet [%collapsible%open] ======= `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `xlm_roberta`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] + .Properties of xlm_roberta [%collapsible%open] ======= `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= `bert_ja`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] + .Properties of bert_ja [%collapsible%open] ======= `truncate`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] ======= ====== ===== diff --git a/docs/reference/ml/trained-models/apis/put-trained-model-definition-part.asciidoc b/docs/reference/ml/trained-models/apis/put-trained-model-definition-part.asciidoc index 53cad5cffa37..d1da29abffcd 100644 --- a/docs/reference/ml/trained-models/apis/put-trained-model-definition-part.asciidoc +++ b/docs/reference/ml/trained-models/apis/put-trained-model-definition-part.asciidoc @@ -26,7 +26,7 @@ Requires the `manage_ml` cluster privilege. This privilege is included in the ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-id] ``:: (Required, number) diff --git a/docs/reference/ml/trained-models/apis/put-trained-model-vocabulary.asciidoc b/docs/reference/ml/trained-models/apis/put-trained-model-vocabulary.asciidoc index 8541583f368f..2fdf86259388 100644 --- a/docs/reference/ml/trained-models/apis/put-trained-model-vocabulary.asciidoc +++ b/docs/reference/ml/trained-models/apis/put-trained-model-vocabulary.asciidoc @@ -34,7 +34,7 @@ The vocabulary is stored in the index as described in ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-id] [[ml-put-trained-model-vocabulary-request-body]] == {api-request-body-title} diff --git a/docs/reference/ml/trained-models/apis/put-trained-models.asciidoc b/docs/reference/ml/trained-models/apis/put-trained-models.asciidoc index 433286a8e0c2..eef90630eb35 100644 --- a/docs/reference/ml/trained-models/apis/put-trained-models.asciidoc +++ b/docs/reference/ml/trained-models/apis/put-trained-models.asciidoc @@ -40,7 +40,7 @@ created by {dfanalytics}. ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-id] [[ml-put-trained-models-query-params]] == {api-query-parms-title} @@ -104,7 +104,7 @@ The field name to encode. Object that maps the field value to the frequency encoded value. `custom`:: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-preprocessor] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-preprocessor] ====== //End frequency encoding @@ -126,7 +126,7 @@ The field name to encode. String map of "field_value: one_hot_column_name". `custom`:: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-preprocessor] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-preprocessor] ====== //End one hot encoding @@ -156,7 +156,7 @@ The field name to encode. Object that maps the field value to the target mean value. `custom`:: -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=custom-preprocessor] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=custom-preprocessor] ====== //End target mean encoding @@ -409,28 +409,28 @@ Classification configuration for inference. ===== `num_top_classes`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-num-top-classes] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-num-top-classes] `num_top_feature_importance_values`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-num-top-feature-importance-values] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-num-top-feature-importance-values] `prediction_field_type`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-prediction-field-type] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-prediction-field-type] `results_field`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] `top_classes_results_field`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-top-classes-results-field] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-classification-top-classes-results-field] ===== `fill_mask`::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-fill-mask] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-fill-mask] + .Properties of fill_mask inference [%collapsible%open] @@ -441,11 +441,11 @@ Number of top predicted tokens to return for replacing the mask token. Defaults `results_field`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] `tokenization`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] + Refer to <> to review the properties of the `tokenization` object. @@ -453,7 +453,7 @@ Refer to <> to review the properties of the `ner`::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-ner] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-ner] + .Properties of ner inference [%collapsible%open] @@ -467,11 +467,11 @@ Example: ["O", "B-PER", "I-PER", "B-ORG", "I-ORG", "B-LOC", "I-LOC", "B-MISC", `results_field`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] `tokenization`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] + Refer to <> to review the properties of the `tokenization` object. @@ -479,18 +479,18 @@ properties of the `tokenization` object. `pass_through`::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-pass-through] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-pass-through] + .Properties of pass_through inference [%collapsible%open] ===== `results_field`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] `tokenization`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] + Refer to <> to review the properties of the `tokenization` object. @@ -498,7 +498,7 @@ Refer to <> to review the properties of the `question_answering`::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-question-answering] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-question-answering] + .Properties of question_answering inference [%collapsible%open] @@ -509,11 +509,11 @@ The maximum amount of words in the answer. Defaults to `15`. `results_field`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] `tokenization`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] + Recommended to set `max_sentence_length` to `386` with `128` of `span` and set `truncate` to `none`. @@ -531,16 +531,16 @@ Regression configuration for inference. ===== `num_top_feature_importance_values`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-regression-num-top-feature-importance-values] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-regression-num-top-feature-importance-values] `results_field`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] ===== `text_classification`::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-classification] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-classification] + .Properties of text_classification inference [%collapsible%open] @@ -555,11 +555,11 @@ Specifies the number of top class predictions to return. Defaults to all classes `results_field`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] `tokenization`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] + Refer to <> to review the properties of the `tokenization` object. @@ -567,22 +567,22 @@ Refer to <> to review the properties of the `text_embedding`::: (Object, optional) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-embedding] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-embedding] + .Properties of text_embedding inference [%collapsible%open] ===== `embedding_size`:::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-embedding-size] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-embedding-size] `results_field`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] `tokenization`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] + Refer to <> to review the properties of the `tokenization` object. @@ -590,18 +590,18 @@ Refer to <> to review the properties of the `text_similarity`:::: (Object, optional) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-similarity] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-similarity] + .Properties of text_similarity inference [%collapsible%open] ===== `span_score_combination_function`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-similarity-span-score-func] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-similarity-span-score-func] `tokenization`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] + Refer to <> to review the properties of the `tokenization` object. @@ -609,34 +609,34 @@ Refer to <> to review the properties of the `zero_shot_classification`::: (Object, optional) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-zero-shot-classification] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-zero-shot-classification] + .Properties of zero_shot_classification inference [%collapsible%open] ===== `classification_labels`:::: (Required, array) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-zero-shot-classification-classification-labels] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-zero-shot-classification-classification-labels] `hypothesis_template`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-zero-shot-classification-hypothesis-template] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-zero-shot-classification-hypothesis-template] `labels`:::: (Optional, array) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-zero-shot-classification-labels] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-zero-shot-classification-labels] `multi_label`:::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-zero-shot-classification-multi-label] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-zero-shot-classification-multi-label] `results_field`:::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] `tokenization`:::: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] + Refer to <> to review the properties of the `tokenization` object. @@ -748,134 +748,134 @@ The `tokenization` object has the following properties. `bert`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert] + .Properties of bert [%collapsible%open] ==== `do_lower_case`::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] `max_sequence_length`::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `span`::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-with-special-tokens] ==== `roberta`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta] + .Properties of roberta [%collapsible%open] ==== `add_prefix_space`::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-add-prefix-space] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-add-prefix-space] `max_sequence_length`::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `span`::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] ==== `mpnet`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet] + .Properties of mpnet [%collapsible%open] ==== `do_lower_case`::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] `max_sequence_length`::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `span`::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-mpnet-with-special-tokens] ==== `xlm_roberta`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-xlm-roberta] + .Properties of xlm_roberta [%collapsible%open] ==== `max_sequence_length`::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `span`::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-roberta-with-special-tokens] ==== `bert_ja`:: (Optional, object) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja] + .Properties of bert_ja [%collapsible%open] ==== `do_lower_case`::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-do-lower-case] `max_sequence_length`::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-max-sequence-length] `span`::: (Optional, integer) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-span] `truncate`::: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-truncate] `with_special_tokens`::: (Optional, boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja-with-special-tokens] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization-bert-ja-with-special-tokens] ==== diff --git a/docs/reference/ml/trained-models/apis/start-trained-model-deployment.asciidoc b/docs/reference/ml/trained-models/apis/start-trained-model-deployment.asciidoc index 5771579cfaf3..50754ac55443 100644 --- a/docs/reference/ml/trained-models/apis/start-trained-model-deployment.asciidoc +++ b/docs/reference/ml/trained-models/apis/start-trained-model-deployment.asciidoc @@ -53,7 +53,7 @@ the node's allocated processors. ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-id] [[start-trained-model-deployment-query-params]] == {api-query-parms-title} @@ -67,7 +67,7 @@ cache, `0b` can be provided. `deployment_id`:: (Optional, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=deployment-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=deployment-id] Defaults to `model_id`. `number_of_allocations`:: diff --git a/docs/reference/ml/trained-models/apis/stop-trained-model-deployment.asciidoc b/docs/reference/ml/trained-models/apis/stop-trained-model-deployment.asciidoc index 335d76ffcd56..622b440622cd 100644 --- a/docs/reference/ml/trained-models/apis/stop-trained-model-deployment.asciidoc +++ b/docs/reference/ml/trained-models/apis/stop-trained-model-deployment.asciidoc @@ -29,7 +29,7 @@ Deployment is required only for trained models that have a PyTorch `model_type`. ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=deployment-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=deployment-id] [[stop-trained-model-deployment-query-params]] @@ -37,7 +37,7 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=deployment-id] `allow_no_match`:: (Optional, Boolean) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-deployments] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=allow-no-match-deployments] `force`:: (Optional, Boolean) If true, the deployment is stopped even if it or one of its diff --git a/docs/reference/ml/trained-models/apis/update-trained-model-deployment.asciidoc b/docs/reference/ml/trained-models/apis/update-trained-model-deployment.asciidoc index 1b5f94cceeaf..93547c1d3e9b 100644 --- a/docs/reference/ml/trained-models/apis/update-trained-model-deployment.asciidoc +++ b/docs/reference/ml/trained-models/apis/update-trained-model-deployment.asciidoc @@ -34,7 +34,7 @@ You can either increase or decrease the number of allocations of such a deployme ``:: (Required, string) -include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=deployment-id] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=deployment-id] [[update-trained-model-deployment-request-body]] == {api-request-body-title} diff --git a/docs/reference/modules/cluster/misc.asciidoc b/docs/reference/modules/cluster/misc.asciidoc index 7eb1cf357498..3da5df4f1641 100644 --- a/docs/reference/modules/cluster/misc.asciidoc +++ b/docs/reference/modules/cluster/misc.asciidoc @@ -163,7 +163,7 @@ increase it if you expect nodes to be absent from the cluster and miss more than 500 deletes. We think that is rare, thus the default. Tombstones don't take up much space, but we also think that a number like 50,000 is probably too big. -include::{es-repo-dir}/indices/dangling-indices-list.asciidoc[tag=dangling-index-description] +include::{es-ref-dir}/indices/dangling-indices-list.asciidoc[tag=dangling-index-description] You can use the <> to manage this situation. diff --git a/docs/reference/modules/cluster/remote-clusters-api-key.asciidoc b/docs/reference/modules/cluster/remote-clusters-api-key.asciidoc index 0cac52deaae4..506e834e0b1c 100644 --- a/docs/reference/modules/cluster/remote-clusters-api-key.asciidoc +++ b/docs/reference/modules/cluster/remote-clusters-api-key.asciidoc @@ -198,4 +198,4 @@ Configuring the `remote_cluster_client` settings in `elasticsearch.yml` still re include::remote-clusters-connect.asciidoc[] :!trust-mechanism: -include::{es-repo-dir}/security/authentication/remote-clusters-privileges-api-key.asciidoc[leveloffset=+1] +include::{es-ref-dir}/security/authentication/remote-clusters-privileges-api-key.asciidoc[leveloffset=+1] diff --git a/docs/reference/modules/cluster/remote-clusters-cert.asciidoc b/docs/reference/modules/cluster/remote-clusters-cert.asciidoc index 11d71955cfe6..6602c807f5b6 100644 --- a/docs/reference/modules/cluster/remote-clusters-cert.asciidoc +++ b/docs/reference/modules/cluster/remote-clusters-cert.asciidoc @@ -80,4 +80,4 @@ generate certificates for all nodes simplifies this task. include::remote-clusters-connect.asciidoc[] :!trust-mechanism: -include::{es-repo-dir}/security/authentication/remote-clusters-privileges-cert.asciidoc[leveloffset=+1] \ No newline at end of file +include::{es-ref-dir}/security/authentication/remote-clusters-privileges-cert.asciidoc[leveloffset=+1] \ No newline at end of file diff --git a/docs/reference/modules/indices/index_management.asciidoc b/docs/reference/modules/indices/index_management.asciidoc index cdb8af570c6d..5f7274b2271d 100644 --- a/docs/reference/modules/indices/index_management.asciidoc +++ b/docs/reference/modules/indices/index_management.asciidoc @@ -46,9 +46,9 @@ to `true`. This setting affects the following built-in index templates: -include::{es-repo-dir}/indices/index-templates.asciidoc[tag=built-in-index-template-patterns] +include::{es-ref-dir}/indices/index-templates.asciidoc[tag=built-in-index-template-patterns] This setting also affects the following built-in component templates: -include::{es-repo-dir}/indices/put-component-template.asciidoc[tag=built-in-component-templates] +include::{es-ref-dir}/indices/put-component-template.asciidoc[tag=built-in-component-templates] -- diff --git a/docs/reference/modules/node.asciidoc b/docs/reference/modules/node.asciidoc index 8a42d11f6367..81df2cf4a2a6 100644 --- a/docs/reference/modules/node.asciidoc +++ b/docs/reference/modules/node.asciidoc @@ -253,7 +253,7 @@ node.roles: [ data ] ===== Content data node Content data nodes are part of the content tier. -include::{es-repo-dir}/datatiers.asciidoc[tag=content-tier] +include::{es-ref-dir}/datatiers.asciidoc[tag=content-tier] To create a dedicated content node, set: [source,yaml] @@ -265,7 +265,7 @@ node.roles: [ data_content ] ===== Hot data node Hot data nodes are part of the hot tier. -include::{es-repo-dir}/datatiers.asciidoc[tag=hot-tier] +include::{es-ref-dir}/datatiers.asciidoc[tag=hot-tier] To create a dedicated hot node, set: [source,yaml] @@ -277,7 +277,7 @@ node.roles: [ data_hot ] ===== Warm data node Warm data nodes are part of the warm tier. -include::{es-repo-dir}/datatiers.asciidoc[tag=warm-tier] +include::{es-ref-dir}/datatiers.asciidoc[tag=warm-tier] To create a dedicated warm node, set: [source,yaml] @@ -289,7 +289,7 @@ node.roles: [ data_warm ] ===== Cold data node Cold data nodes are part of the cold tier. -include::{es-repo-dir}/datatiers.asciidoc[tag=cold-tier] +include::{es-ref-dir}/datatiers.asciidoc[tag=cold-tier] To create a dedicated cold node, set: [source,yaml] @@ -301,7 +301,7 @@ node.roles: [ data_cold ] ===== Frozen data node Frozen data nodes are part of the frozen tier. -include::{es-repo-dir}/datatiers.asciidoc[tag=frozen-tier] +include::{es-ref-dir}/datatiers.asciidoc[tag=frozen-tier] To create a dedicated frozen node, set: [source,yaml] diff --git a/docs/reference/query-dsl/knn-query.asciidoc b/docs/reference/query-dsl/knn-query.asciidoc index 56f90c7d5da0..b7ded6929ed2 100644 --- a/docs/reference/query-dsl/knn-query.asciidoc +++ b/docs/reference/query-dsl/knn-query.asciidoc @@ -96,7 +96,7 @@ Either this or `query_vector_builder` must be provided. + -- (Optional, object) Query vector builder. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-query-vector-builder] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=knn-query-vector-builder] -- diff --git a/docs/reference/query-dsl/script-score-query.asciidoc b/docs/reference/query-dsl/script-score-query.asciidoc index 08522708554f..9291b8c15f0d 100644 --- a/docs/reference/query-dsl/script-score-query.asciidoc +++ b/docs/reference/query-dsl/script-score-query.asciidoc @@ -335,7 +335,7 @@ through a script: The `script_score` query has equivalent <> that can be used in scripts. -include::{es-repo-dir}/vectors/vector-functions.asciidoc[] +include::{es-ref-dir}/vectors/vector-functions.asciidoc[] [[score-explanation]] ===== Explain request diff --git a/docs/reference/repositories-metering-api/apis/clear-repositories-metering-archive.asciidoc b/docs/reference/repositories-metering-api/apis/clear-repositories-metering-archive.asciidoc index 991b015f6356..addcd65f0e84 100644 --- a/docs/reference/repositories-metering-api/apis/clear-repositories-metering-archive.asciidoc +++ b/docs/reference/repositories-metering-api/apis/clear-repositories-metering-archive.asciidoc @@ -26,7 +26,7 @@ You can use this API to clear the archived repositories metering information in [[clear-repositories-metering-archive-api-path-params]] ==== {api-path-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=node-id] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=node-id] ``:: (long) Specifies the maximum <> to be cleared from the archive. @@ -37,4 +37,4 @@ All the nodes selective options are explained <>. ==== {api-response-body-title} Returns the deleted archived repositories metering information. -include::{es-repo-dir}/repositories-metering-api/apis/repositories-meterings-body.asciidoc[tag=repositories-metering-body] +include::{es-ref-dir}/repositories-metering-api/apis/repositories-meterings-body.asciidoc[tag=repositories-metering-body] diff --git a/docs/reference/repositories-metering-api/apis/get-repositories-metering.asciidoc b/docs/reference/repositories-metering-api/apis/get-repositories-metering.asciidoc index 4e7eb876ad72..314f85a7dba5 100644 --- a/docs/reference/repositories-metering-api/apis/get-repositories-metering.asciidoc +++ b/docs/reference/repositories-metering-api/apis/get-repositories-metering.asciidoc @@ -30,11 +30,11 @@ exposed by this API is volatile, meaning that it won't be present after node res [[get-repositories-metering-api-path-params]] ==== {api-path-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=node-id] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=node-id] All the nodes selective options are explained <>. [role="child_attributes"] [[get-repositories-metering-api-response-body]] ==== {api-response-body-title} -include::{es-repo-dir}/repositories-metering-api/apis/repositories-meterings-body.asciidoc[tag=repositories-metering-body] +include::{es-ref-dir}/repositories-metering-api/apis/repositories-meterings-body.asciidoc[tag=repositories-metering-body] diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc index 39d9e90079bf..e63f66217d8d 100644 --- a/docs/reference/rest-api/common-parms.asciidoc +++ b/docs/reference/rest-api/common-parms.asciidoc @@ -1180,7 +1180,7 @@ tag::target-index[] (Required, string) Name of the target index to create. -include::{es-repo-dir}/indices/create-index.asciidoc[tag=index-name-reqs] +include::{es-ref-dir}/indices/create-index.asciidoc[tag=index-name-reqs] -- end::target-index[] diff --git a/docs/reference/rest-api/defs.asciidoc b/docs/reference/rest-api/defs.asciidoc index a80d4a42eb30..52b05daaefd5 100644 --- a/docs/reference/rest-api/defs.asciidoc +++ b/docs/reference/rest-api/defs.asciidoc @@ -8,4 +8,4 @@ to {security-features}. * <> -include::{es-repo-dir}/rest-api/security/role-mapping-resources.asciidoc[] +include::{es-ref-dir}/rest-api/security/role-mapping-resources.asciidoc[] diff --git a/docs/reference/rest-api/index.asciidoc b/docs/reference/rest-api/index.asciidoc index fa0d3babb3a0..50c9f96ad81b 100644 --- a/docs/reference/rest-api/index.asciidoc +++ b/docs/reference/rest-api/index.asciidoc @@ -60,52 +60,52 @@ not be included yet. * <> -- -include::{es-repo-dir}/api-conventions.asciidoc[] -include::{es-repo-dir}/rest-api/common-options.asciidoc[] -include::{es-repo-dir}/rest-api/rest-api-compatibility.asciidoc[] -include::{es-repo-dir}/autoscaling/apis/autoscaling-apis.asciidoc[] -include::{es-repo-dir}/behavioral-analytics/apis/index.asciidoc[] -include::{es-repo-dir}/cat.asciidoc[] -include::{es-repo-dir}/cluster.asciidoc[] -include::{es-repo-dir}/ccr/apis/ccr-apis.asciidoc[] -include::{es-repo-dir}/connector/apis/connector-apis.asciidoc[] -include::{es-repo-dir}/data-streams/data-stream-apis.asciidoc[] -include::{es-repo-dir}/docs.asciidoc[] -include::{es-repo-dir}/ingest/apis/enrich/index.asciidoc[] -include::{es-repo-dir}/eql/eql-apis.asciidoc[] -include::{es-repo-dir}/esql/esql-apis.asciidoc[] -include::{es-repo-dir}/features/apis/features-apis.asciidoc[] -include::{es-repo-dir}/fleet/index.asciidoc[] -include::{es-repo-dir}/graph/explore.asciidoc[] -include::{es-repo-dir}/indices.asciidoc[] -include::{es-repo-dir}/ilm/apis/ilm-api.asciidoc[] -include::{es-repo-dir}/inference/inference-apis.asciidoc[] +include::{es-ref-dir}/api-conventions.asciidoc[] +include::{es-ref-dir}/rest-api/common-options.asciidoc[] +include::{es-ref-dir}/rest-api/rest-api-compatibility.asciidoc[] +include::{es-ref-dir}/autoscaling/apis/autoscaling-apis.asciidoc[] +include::{es-ref-dir}/behavioral-analytics/apis/index.asciidoc[] +include::{es-ref-dir}/cat.asciidoc[] +include::{es-ref-dir}/cluster.asciidoc[] +include::{es-ref-dir}/ccr/apis/ccr-apis.asciidoc[] +include::{es-ref-dir}/connector/apis/connector-apis.asciidoc[] +include::{es-ref-dir}/data-streams/data-stream-apis.asciidoc[] +include::{es-ref-dir}/docs.asciidoc[] +include::{es-ref-dir}/ingest/apis/enrich/index.asciidoc[] +include::{es-ref-dir}/eql/eql-apis.asciidoc[] +include::{es-ref-dir}/esql/esql-apis.asciidoc[] +include::{es-ref-dir}/features/apis/features-apis.asciidoc[] +include::{es-ref-dir}/fleet/index.asciidoc[] +include::{es-ref-dir}/graph/explore.asciidoc[] +include::{es-ref-dir}/indices.asciidoc[] +include::{es-ref-dir}/ilm/apis/ilm-api.asciidoc[] +include::{es-ref-dir}/inference/inference-apis.asciidoc[] include::info.asciidoc[] -include::{es-repo-dir}/ingest/apis/index.asciidoc[] -include::{es-repo-dir}/licensing/index.asciidoc[] -include::{es-repo-dir}/rest-api/logstash/index.asciidoc[] -include::{es-repo-dir}/ml/common/apis/index.asciidoc[] -include::{es-repo-dir}/ml/anomaly-detection/apis/index.asciidoc[] -include::{es-repo-dir}/ml/df-analytics/apis/index.asciidoc[] -include::{es-repo-dir}/ml/trained-models/apis/index.asciidoc[] -include::{es-repo-dir}/migration/migration.asciidoc[] -include::{es-repo-dir}/shutdown/apis/shutdown-api.asciidoc[] -include::{es-repo-dir}/query-rules/apis/index.asciidoc[] -include::{es-repo-dir}/indices/apis/reload-analyzers.asciidoc[] -include::{es-repo-dir}/repositories-metering-api/repositories-metering-apis.asciidoc[] -include::{es-repo-dir}/rollup/rollup-apis.asciidoc[] -include::{es-repo-dir}/rest-api/root.asciidoc[] -include::{es-repo-dir}/scripting/apis/script-apis.asciidoc[] -include::{es-repo-dir}/search.asciidoc[] -include::{es-repo-dir}/search-application/apis/index.asciidoc[] -include::{es-repo-dir}/searchable-snapshots/apis/searchable-snapshots-apis.asciidoc[] -include::{es-repo-dir}/rest-api/security.asciidoc[] -include::{es-repo-dir}/snapshot-restore/apis/snapshot-restore-apis.asciidoc[] -include::{es-repo-dir}/slm/apis/slm-api.asciidoc[] -include::{es-repo-dir}/sql/apis/sql-apis.asciidoc[] -include::{es-repo-dir}/synonyms/apis/synonyms-apis.asciidoc[] -include::{es-repo-dir}/text-structure/apis/index.asciidoc[] -include::{es-repo-dir}/transform/apis/index.asciidoc[] +include::{es-ref-dir}/ingest/apis/index.asciidoc[] +include::{es-ref-dir}/licensing/index.asciidoc[] +include::{es-ref-dir}/rest-api/logstash/index.asciidoc[] +include::{es-ref-dir}/ml/common/apis/index.asciidoc[] +include::{es-ref-dir}/ml/anomaly-detection/apis/index.asciidoc[] +include::{es-ref-dir}/ml/df-analytics/apis/index.asciidoc[] +include::{es-ref-dir}/ml/trained-models/apis/index.asciidoc[] +include::{es-ref-dir}/migration/migration.asciidoc[] +include::{es-ref-dir}/shutdown/apis/shutdown-api.asciidoc[] +include::{es-ref-dir}/query-rules/apis/index.asciidoc[] +include::{es-ref-dir}/indices/apis/reload-analyzers.asciidoc[] +include::{es-ref-dir}/repositories-metering-api/repositories-metering-apis.asciidoc[] +include::{es-ref-dir}/rollup/rollup-apis.asciidoc[] +include::{es-ref-dir}/rest-api/root.asciidoc[] +include::{es-ref-dir}/scripting/apis/script-apis.asciidoc[] +include::{es-ref-dir}/search.asciidoc[] +include::{es-ref-dir}/search-application/apis/index.asciidoc[] +include::{es-ref-dir}/searchable-snapshots/apis/searchable-snapshots-apis.asciidoc[] +include::{es-ref-dir}/rest-api/security.asciidoc[] +include::{es-ref-dir}/snapshot-restore/apis/snapshot-restore-apis.asciidoc[] +include::{es-ref-dir}/slm/apis/slm-api.asciidoc[] +include::{es-ref-dir}/sql/apis/sql-apis.asciidoc[] +include::{es-ref-dir}/synonyms/apis/synonyms-apis.asciidoc[] +include::{es-ref-dir}/text-structure/apis/index.asciidoc[] +include::{es-ref-dir}/transform/apis/index.asciidoc[] include::usage.asciidoc[] -include::{es-repo-dir}/rest-api/watcher.asciidoc[] +include::{es-ref-dir}/rest-api/watcher.asciidoc[] include::defs.asciidoc[] diff --git a/docs/reference/rest-api/security/disable-user-profile.asciidoc b/docs/reference/rest-api/security/disable-user-profile.asciidoc index a25b4a311aa8..35658f071679 100644 --- a/docs/reference/rest-api/security/disable-user-profile.asciidoc +++ b/docs/reference/rest-api/security/disable-user-profile.asciidoc @@ -45,7 +45,7 @@ To re-enable a disabled user profile, use the [[security-api-disable-user-profile-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=refresh] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=refresh] diff --git a/docs/reference/rest-api/security/enable-user-profile.asciidoc b/docs/reference/rest-api/security/enable-user-profile.asciidoc index c61e847cfd1f..e27673b07f59 100644 --- a/docs/reference/rest-api/security/enable-user-profile.asciidoc +++ b/docs/reference/rest-api/security/enable-user-profile.asciidoc @@ -42,7 +42,7 @@ enable user profile API to make the profile visible in these searches again. [[security-api-enable-user-profile-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=refresh] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=refresh] [[security-api-enable-user-profile-example]] ==== {api-examples-title} diff --git a/docs/reference/rest-api/security/query-api-key.asciidoc b/docs/reference/rest-api/security/query-api-key.asciidoc index ad4184ec34a2..5760968cc7e9 100644 --- a/docs/reference/rest-api/security/query-api-key.asciidoc +++ b/docs/reference/rest-api/security/query-api-key.asciidoc @@ -242,7 +242,7 @@ This supports only a subset of aggregation types, namely: <>, and <>. Additionally, aggregations only run over the same subset of fields that `query` works with. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=from] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=from] + By default, you cannot page through more than 10,000 hits using the `from` and `size` parameters. To page through more hits, use the diff --git a/docs/reference/rest-api/security/query-user.asciidoc b/docs/reference/rest-api/security/query-user.asciidoc index 92c293d1fe59..d0c7b44faf3b 100644 --- a/docs/reference/rest-api/security/query-user.asciidoc +++ b/docs/reference/rest-api/security/query-user.asciidoc @@ -70,7 +70,7 @@ Specifies whether the user is enabled. (Optional, boolean) Determines whether to retrieve the <> `uid`, if exists, for the users. Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=from] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=from] + By default, you cannot page through more than 10,000 hits using the `from` and `size` parameters. To page through more hits, use the diff --git a/docs/reference/rest-api/security/update-user-profile-data.asciidoc b/docs/reference/rest-api/security/update-user-profile-data.asciidoc index d4c89558c3fc..01fa5e11d10e 100644 --- a/docs/reference/rest-api/security/update-user-profile-data.asciidoc +++ b/docs/reference/rest-api/security/update-user-profile-data.asciidoc @@ -48,11 +48,11 @@ the allowed namespaces. [[security-api-update-user-profile-data-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=if_seq_no] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=if_seq_no] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=if_primary_term] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=if_primary_term] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=refresh] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=refresh] `uid`:: (Required, string) A unique identifier for the user profile. diff --git a/docs/reference/rest-api/usage.asciidoc b/docs/reference/rest-api/usage.asciidoc index bbbd73a0523f..6bdfaab17a4d 100644 --- a/docs/reference/rest-api/usage.asciidoc +++ b/docs/reference/rest-api/usage.asciidoc @@ -28,7 +28,7 @@ available under the current license and some usage statistics. [[usage-api-query-parms]] === {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] [discrete] [[usage-api-example]] diff --git a/docs/reference/rest-api/watcher/start.asciidoc b/docs/reference/rest-api/watcher/start.asciidoc index 10811ac0b861..565ef60160a9 100644 --- a/docs/reference/rest-api/watcher/start.asciidoc +++ b/docs/reference/rest-api/watcher/start.asciidoc @@ -27,7 +27,7 @@ information, see <>. [[watcher-api-start-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] //[[watcher-api-start-request-body]] //==== {api-request-body-title} diff --git a/docs/reference/rest-api/watcher/stop.asciidoc b/docs/reference/rest-api/watcher/stop.asciidoc index c06090a3cd99..50acd6e9eb2d 100644 --- a/docs/reference/rest-api/watcher/stop.asciidoc +++ b/docs/reference/rest-api/watcher/stop.asciidoc @@ -27,7 +27,7 @@ information, see <>. [[watcher-api-stop-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] //[[watcher-api-stop-request-body]] //==== {api-request-body-title} diff --git a/docs/reference/scripting/apis/create-stored-script-api.asciidoc b/docs/reference/scripting/apis/create-stored-script-api.asciidoc index 6108831a836b..dab1314e65dc 100644 --- a/docs/reference/scripting/apis/create-stored-script-api.asciidoc +++ b/docs/reference/scripting/apis/create-stored-script-api.asciidoc @@ -67,7 +67,7 @@ the API immediately compiles the script or template in this context. If you specify both this and the `` request path parameter, the API uses the request path parameter. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [role="child_attributes"] [[create-stored-script-api-request-body]] diff --git a/docs/reference/scripting/apis/delete-stored-script-api.asciidoc b/docs/reference/scripting/apis/delete-stored-script-api.asciidoc index 038d1916f76f..e233922c9a7d 100644 --- a/docs/reference/scripting/apis/delete-stored-script-api.asciidoc +++ b/docs/reference/scripting/apis/delete-stored-script-api.asciidoc @@ -47,4 +47,4 @@ Identifier for the stored script or search template. [[delete-stored-script-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] diff --git a/docs/reference/scripting/apis/get-stored-script-api.asciidoc b/docs/reference/scripting/apis/get-stored-script-api.asciidoc index 6b6f6648a7ed..fffeb24e0331 100644 --- a/docs/reference/scripting/apis/get-stored-script-api.asciidoc +++ b/docs/reference/scripting/apis/get-stored-script-api.asciidoc @@ -55,4 +55,4 @@ Identifier for the stored script or search template. [[get-stored-script-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] diff --git a/docs/reference/scripting/engine.asciidoc b/docs/reference/scripting/engine.asciidoc index 54d85e6e8236..ef062e8afab1 100644 --- a/docs/reference/scripting/engine.asciidoc +++ b/docs/reference/scripting/engine.asciidoc @@ -17,7 +17,7 @@ the document frequency of a provided term. ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- -include-tagged::{plugins-examples-dir}/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java[expert_engine] +include-tagged::{elasticsearch-root}/plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java[expert_engine] -------------------------------------------------- You can execute the script by specifying its `lang` as `expert_scripts`, and the name diff --git a/docs/reference/search/count.asciidoc b/docs/reference/search/count.asciidoc index 152571a25ec1..399545adf8d1 100644 --- a/docs/reference/search/count.asciidoc +++ b/docs/reference/search/count.asciidoc @@ -56,46 +56,46 @@ this parameter or use `*` or `_all`. [[search-count-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=analyzer] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=analyzer] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=analyze_wildcard] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=analyze_wildcard] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=default_operator] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=default_operator] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=df] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=df] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `open`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=ignore_throttled] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=ignore_throttled] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=lenient] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=lenient] `min_score`:: (Optional, float) Sets the minimum `_score` value that documents must have to be included in the result. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=preference] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=preference] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=search-q] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=search-q] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=routing] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=routing] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=terminate_after] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=terminate_after] [[search-count-request-body]] ==== {api-request-body-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=query] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=query] [[search-count-api-example]] diff --git a/docs/reference/search/explain.asciidoc b/docs/reference/search/explain.asciidoc index c9e297c93413..77e2d5bd63ef 100644 --- a/docs/reference/search/explain.asciidoc +++ b/docs/reference/search/explain.asciidoc @@ -59,37 +59,37 @@ Only a single index name can be provided to this parameter. [[search-explain-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=analyzer] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=analyzer] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=analyze_wildcard] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=analyze_wildcard] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=default_operator] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=default_operator] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=df] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=df] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=lenient] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=lenient] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=preference] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=preference] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=search-q] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=search-q] `stored_fields`:: (Optional, string) A comma-separated list of stored fields to return in the response. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=routing] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=routing] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=source] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source_excludes] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=source_excludes] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source_includes] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=source_includes] [[search-explain-api-request-body]] ==== {api-request-body-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=query] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=query] [[search-explain-api-example]] diff --git a/docs/reference/search/field-caps.asciidoc b/docs/reference/search/field-caps.asciidoc index 5fe924d38e02..2ff2b8d18604 100644 --- a/docs/reference/search/field-caps.asciidoc +++ b/docs/reference/search/field-caps.asciidoc @@ -61,17 +61,17 @@ and indices, omit this parameter or use `*` or `_all`. Comma-separated list of fields to retrieve capabilities for. Wildcard (`*`) expressions are supported. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + -- Defaults to `open`. -- -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] `include_unmapped`:: (Optional, Boolean) If `true`, unmapped fields that are mapped in one index but not in another are included in the response. Fields that don't have any mapping are never included. diff --git a/docs/reference/search/knn-search.asciidoc b/docs/reference/search/knn-search.asciidoc index 7947c688a807..78e3e13b09fe 100644 --- a/docs/reference/search/knn-search.asciidoc +++ b/docs/reference/search/knn-search.asciidoc @@ -91,7 +91,7 @@ use `*` or `_all`. [[knn-search-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=routing] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=routing] [role="child_attributes"] [[knn-search-api-request-body]] @@ -99,36 +99,36 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=routing] `filter`:: (Optional, <>) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-filter] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=knn-filter] `knn`:: (Required, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=knn] + .Properties of `knn` object [%collapsible%open] ==== `field`:: (Required, string) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-field] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=knn-field] `k`:: (Optional, integer) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-k] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=knn-k] `num_candidates`:: (Optional, integer) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-num-candidates] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=knn-num-candidates] `query_vector`:: (Required, array of floats or string) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-query-vector] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=knn-query-vector] ==== -include::{es-repo-dir}/search/search.asciidoc[tag=docvalue-fields-def] -include::{es-repo-dir}/search/search.asciidoc[tag=fields-param-def] -include::{es-repo-dir}/search/search.asciidoc[tag=source-filtering-def] -include::{es-repo-dir}/search/search.asciidoc[tag=stored-fields-def] +include::{es-ref-dir}/search/search.asciidoc[tag=docvalue-fields-def] +include::{es-ref-dir}/search/search.asciidoc[tag=fields-param-def] +include::{es-ref-dir}/search/search.asciidoc[tag=source-filtering-def] +include::{es-ref-dir}/search/search.asciidoc[tag=stored-fields-def] [role="child_attributes"] [[knn-search-api-response-body]] diff --git a/docs/reference/search/multi-search-template-api.asciidoc b/docs/reference/search/multi-search-template-api.asciidoc index a680795737ff..c8eea52a6fd9 100644 --- a/docs/reference/search/multi-search-template-api.asciidoc +++ b/docs/reference/search/multi-search-template-api.asciidoc @@ -89,7 +89,7 @@ min(<>, 10)))+. (Optional, Boolean) If `true`, the response returns `hits.total` as an integer. If false, it returns `hits.total` as an object. Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=search_type] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=search_type] `typed_keys`:: (Optional, Boolean) If `true`, the response prefixes aggregation and suggester diff --git a/docs/reference/search/multi-search.asciidoc b/docs/reference/search/multi-search.asciidoc index 90056d503655..9cafa756f035 100644 --- a/docs/reference/search/multi-search.asciidoc +++ b/docs/reference/search/multi-search.asciidoc @@ -72,7 +72,7 @@ in a cluster, omit this parameter or use `_all` or `*`. [[search-multi-search-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] `ccs_minimize_roundtrips`:: (Optional, Boolean) @@ -80,13 +80,13 @@ If `true`, network roundtrips between the coordinating node and remote clusters are minimized for {ccs} requests. Defaults to `true`. See <>. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `open`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=ignore_throttled] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=ignore_throttled] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] `max_concurrent_searches`:: (Optional, integer) @@ -184,7 +184,7 @@ If `true`, the request does *not* return an error if a wildcard expression or This parameter also applies to <> that point to a missing or index. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `open`. diff --git a/docs/reference/search/rank-eval.asciidoc b/docs/reference/search/rank-eval.asciidoc index e008b9a8aef8..05862ebbbcca 100644 --- a/docs/reference/search/rank-eval.asciidoc +++ b/docs/reference/search/rank-eval.asciidoc @@ -78,17 +78,17 @@ and indices, omit this parameter or use `*` or `_all`. [[search-rank-eval-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + -- Defaults to `open`. -- -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] [[search-rank-eval-api-example]] diff --git a/docs/reference/search/retriever.asciidoc b/docs/reference/search/retriever.asciidoc index a872f1866f42..42d2129f0fde 100644 --- a/docs/reference/search/retriever.asciidoc +++ b/docs/reference/search/retriever.asciidoc @@ -46,7 +46,7 @@ where all documents must match this query but do not contribute to the score. + Defines a search after object parameter used for pagination. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=terminate_after] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=terminate_after] `sort`:: + @@ -97,12 +97,12 @@ A kNN retriever returns top documents from a <>) + -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-filter] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=knn-filter] `similarity`:: (Optional, float) + -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-similarity] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=knn-similarity] ===== Restrictions @@ -160,11 +160,11 @@ equally weighting two or more child retrievers. ===== Parameters -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=rrf-retrievers] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=rrf-retrievers] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=rrf-rank-constant] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=rrf-rank-constant] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=rrf-window-size] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=rrf-window-size] ===== Restrictions diff --git a/docs/reference/search/rrf.asciidoc b/docs/reference/search/rrf.asciidoc index 813d1f6557be..96477cdee45f 100644 --- a/docs/reference/search/rrf.asciidoc +++ b/docs/reference/search/rrf.asciidoc @@ -41,11 +41,11 @@ An RRF retriever is an optional object defined as part of a search request's <>. The RRF retriever object contains the following parameters: -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=rrf-retrievers] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=rrf-retrievers] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=rrf-rank-constant] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=rrf-rank-constant] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=rrf-window-size] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=rrf-window-size] An example request using RRF: diff --git a/docs/reference/search/search-shards.asciidoc b/docs/reference/search/search-shards.asciidoc index d0a518865baf..b9646f4d3730 100644 --- a/docs/reference/search/search-shards.asciidoc +++ b/docs/reference/search/search-shards.asciidoc @@ -45,23 +45,23 @@ this parameter or use `*` or `_all`. [[search-shards-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + -- Defaults to `open`. -- -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=local] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=preference] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=preference] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=routing] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=routing] [[search-shards-api-example]] diff --git a/docs/reference/search/search-template-api.asciidoc b/docs/reference/search/search-template-api.asciidoc index 539048a32474..038396e55860 100644 --- a/docs/reference/search/search-template-api.asciidoc +++ b/docs/reference/search/search-template-api.asciidoc @@ -78,7 +78,7 @@ this parameter or use `*`. [[search-template-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `true`. @@ -86,27 +86,27 @@ Defaults to `true`. (Optional, Boolean) If `true`, network round-trips are minimized for cross-cluster search requests. Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] `explain`:: (Optional, Boolean) If `true`, the response includes additional details about score computation as part of a hit. Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=ignore_throttled] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=ignore_throttled] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=preference] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=preference] `rest_total_hits_as_int`:: (Optional, Boolean) If `true`, the response returns `hits.total` as an integer. If false, it returns `hits.total` as an object. Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=routing] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=routing] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=scroll] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=scroll] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=search_type] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=search_type] `typed_keys`:: (Optional, Boolean) If `true`, the response prefixes aggregation and suggester diff --git a/docs/reference/search/search-vector-tile-api.asciidoc b/docs/reference/search/search-vector-tile-api.asciidoc index 97d2e76e7fcd..2cdc29918a69 100644 --- a/docs/reference/search/search-vector-tile-api.asciidoc +++ b/docs/reference/search/search-vector-tile-api.asciidoc @@ -429,7 +429,7 @@ include::search-vector-tile-api.asciidoc[tag=grid-type] (Optional, object) <> used to filter documents for the search. -include::{es-repo-dir}/search/search.asciidoc[tag=runtime-mappings-def] +include::{es-ref-dir}/search/search.asciidoc[tag=runtime-mappings-def] include::search-vector-tile-api.asciidoc[tag=size] diff --git a/docs/reference/search/search-your-data/knn-search.asciidoc b/docs/reference/search/search-your-data/knn-search.asciidoc index ffac84c11a77..db4b0febb07b 100644 --- a/docs/reference/search/search-your-data/knn-search.asciidoc +++ b/docs/reference/search/search-your-data/knn-search.asciidoc @@ -1041,7 +1041,7 @@ PUT image-index option is not supported. * {blank} -include::{es-repo-dir}/search/knn-search.asciidoc[tag=hnsw-algorithm] +include::{es-ref-dir}/search/knn-search.asciidoc[tag=hnsw-algorithm] NOTE: Approximate kNN search always uses the <> search type in order to gather diff --git a/docs/reference/search/search-your-data/search-across-clusters.asciidoc b/docs/reference/search/search-your-data/search-across-clusters.asciidoc index ee1d9fcae18e..2573722b6d2e 100644 --- a/docs/reference/search/search-your-data/search-across-clusters.asciidoc +++ b/docs/reference/search/search-your-data/search-across-clusters.asciidoc @@ -1366,7 +1366,7 @@ following major version. For example, a local 7.17 cluster can search any remote 8.x cluster. [[ccs-version-compatibility]] -include::{es-repo-dir}/search/search-your-data/ccs-version-compat-matrix.asciidoc[] +include::{es-ref-dir}/search/search-your-data/ccs-version-compat-matrix.asciidoc[] IMPORTANT: For the <>, the local and remote clusters must use the same {es} version if they have versions prior to 7.17.7 (included) or prior to 8.5.1 (included). diff --git a/docs/reference/search/search-your-data/search-with-synonyms.asciidoc b/docs/reference/search/search-your-data/search-with-synonyms.asciidoc index 16952f94890c..596af695b791 100644 --- a/docs/reference/search/search-your-data/search-with-synonyms.asciidoc +++ b/docs/reference/search/search-your-data/search-with-synonyms.asciidoc @@ -47,7 +47,7 @@ An example synonyms file: [source,synonyms] -------------------------------------------------- -include::{es-test-dir}/cluster/config/analysis/synonym.txt[] +include::{elasticsearch-root}/docs/src/test/cluster/config/analysis/synonym.txt[] -------------------------------------------------- To update an existing synonyms set, upload new files to your cluster. diff --git a/docs/reference/search/search-your-data/semantic-search-inference.asciidoc b/docs/reference/search/search-your-data/semantic-search-inference.asciidoc index 53abf0f0458a..e7e16d74764f 100644 --- a/docs/reference/search/search-your-data/semantic-search-inference.asciidoc +++ b/docs/reference/search/search-your-data/semantic-search-inference.asciidoc @@ -20,7 +20,7 @@ review the corresponding instructions. [[infer-service-requirements]] ==== Requirements -include::{es-repo-dir}/tab-widgets/inference-api/infer-api-requirements-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/inference-api/infer-api-requirements-widget.asciidoc[] [discrete] @@ -29,7 +29,7 @@ include::{es-repo-dir}/tab-widgets/inference-api/infer-api-requirements-widget.a Create an {infer} endpoint by using the <>: -include::{es-repo-dir}/tab-widgets/inference-api/infer-api-task-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/inference-api/infer-api-task-widget.asciidoc[] [discrete] @@ -41,7 +41,7 @@ that the model will create based on your input text - must be created. The destination index must have a field with the <> field type to index the output of the used model. -include::{es-repo-dir}/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc[] [discrete] @@ -52,7 +52,7 @@ Create an <> with an <> and use the model you created above to infer against the data that is being ingested in the pipeline. -include::{es-repo-dir}/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/inference-api/infer-api-ingest-pipeline-widget.asciidoc[] [discrete] @@ -82,7 +82,7 @@ you can see an index named `test-data` with 182469 documents. Create the embeddings from the text by reindexing the data through the {infer} pipeline that uses the chosen model as the inference model. -include::{es-repo-dir}/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/inference-api/infer-api-reindex-widget.asciidoc[] The call returns a task ID to monitor the progress: @@ -114,4 +114,4 @@ provide the query text and the model you have used to create the embeddings. NOTE: If you cancelled the reindexing process, you run the query only a part of the data which affects the quality of your results. -include::{es-repo-dir}/tab-widgets/inference-api/infer-api-search-widget.asciidoc[] \ No newline at end of file +include::{es-ref-dir}/tab-widgets/inference-api/infer-api-search-widget.asciidoc[] \ No newline at end of file diff --git a/docs/reference/search/search-your-data/semantic-search.asciidoc b/docs/reference/search/search-your-data/semantic-search.asciidoc index f4768e5c3a23..a4d892c98645 100644 --- a/docs/reference/search/search-your-data/semantic-search.asciidoc +++ b/docs/reference/search/search-your-data/semantic-search.asciidoc @@ -64,7 +64,7 @@ important each is. After you decide which model you want to use for implementing semantic search, you need to deploy the model in {es}. -include::{es-repo-dir}/tab-widgets/semantic-search/deploy-nlp-model-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/semantic-search/deploy-nlp-model-widget.asciidoc[] [discrete] [[semantic-search-field-mappings]] @@ -74,7 +74,7 @@ Before you start using the deployed model to generate embeddings based on your input text, you need to prepare your index mapping first. The mapping of the index depends on the type of model. -include::{es-repo-dir}/tab-widgets/semantic-search/field-mappings-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/semantic-search/field-mappings-widget.asciidoc[] [discrete] [[semantic-search-generate-embeddings]] @@ -89,7 +89,7 @@ infer against the data ingested through the pipeline. After you created the ingest pipeline with the inference processor, you can ingest your data through it to generate the model output. -include::{es-repo-dir}/tab-widgets/semantic-search/generate-embeddings-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/semantic-search/generate-embeddings-widget.asciidoc[] Now it is time to perform semantic search! @@ -100,7 +100,7 @@ Now it is time to perform semantic search! Depending on the type of model you have deployed, you can query rank features with a text expansion query, or dense vectors with a kNN search. -include::{es-repo-dir}/tab-widgets/semantic-search/search-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/semantic-search/search-widget.asciidoc[] [discrete] [[semantic-search-hybrid-search]] @@ -114,7 +114,7 @@ Combining semantic and lexical search into one hybrid search request using but hybrid search using reciprocal rank fusion {blog-ref}improving-information-retrieval-elastic-stack-hybrid[has been shown to perform better in general]. -include::{es-repo-dir}/tab-widgets/semantic-search/hybrid-search-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/semantic-search/hybrid-search-widget.asciidoc[] [discrete] [[semantic-search-read-more]] diff --git a/docs/reference/search/search.asciidoc b/docs/reference/search/search.asciidoc index 3be30c98261d..15985088a6ff 100644 --- a/docs/reference/search/search.asciidoc +++ b/docs/reference/search/search.asciidoc @@ -56,7 +56,7 @@ IMPORTANT: Several options for this API can be specified using a query parameter or a request body parameter. If both parameters are specified, only the query parameter is used. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `true`. @@ -70,9 +70,9 @@ no partial results. Defaults to `true`. To override the default for this field, set the `search.default_allow_partial_results` cluster setting to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=analyzer] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=analyzer] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=analyze_wildcard] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=analyze_wildcard] `batched_reduce_size`:: (Optional, integer) The number of shard results that should be reduced at once @@ -86,15 +86,15 @@ shards in the request can be large. Defaults to `512`. coordinating node and the remote clusters are minimized when executing {ccs} (CCS) requests. See <>. Defaults to `true`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=default_operator] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=default_operator] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=df] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=df] `docvalue_fields`:: (Optional, string) A comma-separated list of fields to return as the docvalue representation of a field for each hit. See <>. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] + Defaults to `open`. @@ -103,13 +103,13 @@ Defaults to `open`. computation as part of a hit. Defaults to `false`. [[search-from-param]] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=from] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=from] + By default, you cannot page through more than 10,000 hits using the `from` and `size` parameters. To page through more hits, use the <> parameter. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=ignore_throttled] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=ignore_throttled] `include_named_queries_score`:: (Optional, Boolean) If `true`, includes the score contribution from any named queries. @@ -118,9 +118,9 @@ response. Typically, this adds a small overhead to a request. However, using computationally expensive named queries on a large number of hits may add significant overhead. Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=lenient] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=lenient] `max_concurrent_shard_requests`:: (Optional, integer) Defines the number of concurrent shard requests per node @@ -182,7 +182,7 @@ end::search-preference[] [[search-api-query-params-q]] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=search-q] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=search-q] + You can use the `q` parameter to run a query parameter search. Query parameter searches do not support the full {es} <> but are handy for @@ -201,7 +201,7 @@ level settings. (Optional, Boolean) Indicates whether hits.total should be rendered as an integer or an object in the rest search response. Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=routing] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=routing] [[search-api-scroll-query-param]] `scroll`:: @@ -275,9 +275,9 @@ Comma-separated list of source fields to return. Wildcard (`*`) patterns are supported. ==== -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source_excludes] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=source_excludes] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source_includes] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=source_includes] `stats`:: (Optional, string) Specific `tag` of the request for logging and statistical @@ -322,7 +322,7 @@ returned. This parameter can only be used when the `suggest_field` query string parameter is specified. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=terminate_after] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=terminate_after] + Defaults to `0`, which does not terminate query execution early. @@ -466,7 +466,7 @@ search response. (Optional, Boolean) If `true`, returns detailed information about score computation as part of a hit. Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=from] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=from] + By default, you cannot page through more than 10,000 hits using the `from` and `size` parameters. To page through more hits, use the @@ -493,30 +493,30 @@ A boost value greater than `1.0` increases the score. A boost value between [[search-api-knn]] `knn`:: (Optional, object or array of objects) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=knn] + .Properties of `knn` object [%collapsible%open] ==== `field`:: (Required, string) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-field] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=knn-field] `filter`:: (Optional, <>) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-filter] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=knn-filter] `k`:: (Optional, integer) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-k] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=knn-k] `num_candidates`:: (Optional, integer) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-num-candidates] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=knn-num-candidates] `query_vector`:: (Optional, array of floats) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-query-vector] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=knn-query-vector] `query_vector_builder`:: (Optional, object) @@ -526,7 +526,7 @@ not both. Refer to <> to learn more. `similarity`:: (Optional, float) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=knn-similarity] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=knn-similarity] ==== @@ -714,7 +714,7 @@ aggregation for its associated searches. You can retrieve these stats using the <>. [[request-body-search-terminate-after]] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=terminate_after] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=terminate_after] + Defaults to `0`, which does not terminate query execution early. diff --git a/docs/reference/search/validate.asciidoc b/docs/reference/search/validate.asciidoc index 528e24151c47..ce682e485cd2 100644 --- a/docs/reference/search/validate.asciidoc +++ b/docs/reference/search/validate.asciidoc @@ -41,7 +41,7 @@ request body. search. Supports wildcards (`*`). To search all data streams or indices, omit this parameter or use `*` or `_all`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=query] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=query] [[search-validate-api-query-params]] @@ -51,33 +51,33 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=query] (Optional, Boolean) If `true`, the validation is executed on all shards instead of one random shard per index. Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-indices] + Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=analyzer] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=analyzer] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=analyze_wildcard] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=analyze_wildcard] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=default_operator] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=default_operator] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=df] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=df] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=expand-wildcards] `explain`:: (Optional, Boolean) If `true`, the response returns detailed information if an error has occurred. Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailable] -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=lenient] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=lenient] `rewrite`:: (Optional, Boolean) If `true`, returns a more detailed explanation showing the actual Lucene query that will be executed. Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=search-q] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=search-q] [[search-validate-api-example]] diff --git a/docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc b/docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc index 255b78993e1c..ba25cebcd1e1 100644 --- a/docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc +++ b/docs/reference/searchable-snapshots/apis/mount-snapshot.asciidoc @@ -40,7 +40,7 @@ to mount. [[searchable-snapshots-api-mount-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] `wait_for_completion`:: (Optional, Boolean) If `true`, the request blocks until the operation is complete. diff --git a/docs/reference/searchable-snapshots/apis/node-cache-stats.asciidoc b/docs/reference/searchable-snapshots/apis/node-cache-stats.asciidoc index ce97a912d875..50314b6d36f2 100644 --- a/docs/reference/searchable-snapshots/apis/node-cache-stats.asciidoc +++ b/docs/reference/searchable-snapshots/apis/node-cache-stats.asciidoc @@ -33,7 +33,7 @@ For more information, see <>. [[searchable-snapshots-api-cache-stats-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] [role="child_attributes"] [[searchable-snapshots-api-cache-stats-response-body]] diff --git a/docs/reference/settings/monitoring-settings.asciidoc b/docs/reference/settings/monitoring-settings.asciidoc index a10116005a6b..8fdad409ee98 100644 --- a/docs/reference/settings/monitoring-settings.asciidoc +++ b/docs/reference/settings/monitoring-settings.asciidoc @@ -286,18 +286,18 @@ You can configure the following TLS/SSL settings. +{ssl-prefix}.ssl.supported_protocols+:: (<>) deprecated:[7.16.0] -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-supported-protocols] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-supported-protocols] ifdef::verifies[] +{ssl-prefix}.ssl.verification_mode+:: (<>) deprecated:[7.16.0] Controls the verification of certificates. -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-verification-mode-values] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-verification-mode-values] endif::verifies[] +{ssl-prefix}.ssl.cipher_suites+:: (<>) deprecated:[7.16.0] -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-cipher-suites-values] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-cipher-suites-values] [#{ssl-context}-tls-ssl-key-trusted-certificate-settings] ===== {component} TLS/SSL key and trusted certificate settings @@ -315,23 +315,23 @@ When using PEM encoded files, use the following settings: +{ssl-prefix}.ssl.key+:: (<>) deprecated:[7.16.0] -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-key-pem] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-key-pem] +{ssl-prefix}.ssl.key_passphrase+:: (<>) deprecated:[7.16.0] -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-key-passphrase] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-key-passphrase] +{ssl-prefix}.ssl.secure_key_passphrase+:: (<>) deprecated:[7.16.0] -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-secure-key-passphrase] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-secure-key-passphrase] +{ssl-prefix}.ssl.certificate+:: (<>) deprecated:[7.16.0] -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-certificate] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-certificate] +{ssl-prefix}.ssl.certificate_authorities+:: (<>) deprecated:[7.16.0] -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-certificate-authorities] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-certificate-authorities] ===== Java keystore files @@ -340,35 +340,35 @@ and certificates that should be trusted, use the following settings: +{ssl-prefix}.ssl.keystore.path+:: (<>) deprecated:[7.16.0] -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-path] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-path] +{ssl-prefix}.ssl.keystore.password+:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-password] +{ssl-prefix}.ssl.keystore.secure_password+:: (<>) deprecated:[7.16.0] -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-password] +{ssl-prefix}.ssl.keystore.key_password+:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-key-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-key-password] +{ssl-prefix}.ssl.keystore.secure_key_password+:: (<>) deprecated:[7.16.0] -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-key-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-key-password] +{ssl-prefix}.ssl.truststore.path+:: (<>) deprecated:[7.16.0] -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-path] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-path] +{ssl-prefix}.ssl.truststore.password+:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-password] +{ssl-prefix}.ssl.truststore.secure_password+:: (<>) deprecated:[7.16.0] -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-secure-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-secure-password] [#{ssl-context}-pkcs12-files] ===== PKCS#12 files @@ -380,31 +380,31 @@ PKCS#12 files are configured in the same way as Java keystore files: +{ssl-prefix}.ssl.keystore.path+:: (<>) deprecated:[7.16.0] -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-path] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-path] +{ssl-prefix}.ssl.keystore.type+:: (<>) deprecated:[7.16.0] -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-type-pkcs12] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-type-pkcs12] +{ssl-prefix}.ssl.keystore.password+:: (<>) deprecated:[7.16.0] -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-password] +{ssl-prefix}.ssl.keystore.secure_password+:: (<>) deprecated:[7.16.0] -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-password] +{ssl-prefix}.ssl.keystore.key_password+:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-key-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-key-password] +{ssl-prefix}.ssl.keystore.secure_key_password+:: (<>) deprecated:[7.16.0] -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-key-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-key-password] +{ssl-prefix}.ssl.truststore.path+:: (<>) deprecated:[7.16.0] -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-path] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-path] +{ssl-prefix}.ssl.truststore.type+:: (<>) deprecated:[7.16.0] @@ -413,9 +413,9 @@ Set this to `PKCS12` to indicate that the truststore is a PKCS#12 file. +{ssl-prefix}.ssl.truststore.password+:: (<>) deprecated:[7.16.0] -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-password] +{ssl-prefix}.ssl.truststore.secure_password+:: (<>) deprecated:[7.16.0] -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-secure-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-secure-password] diff --git a/docs/reference/settings/security-settings.asciidoc b/docs/reference/settings/security-settings.asciidoc index ae8bf5d4e900..c620d97fda42 100644 --- a/docs/reference/settings/security-settings.asciidoc +++ b/docs/reference/settings/security-settings.asciidoc @@ -613,90 +613,90 @@ Defaults to `5s` (5 seconds ). `ssl.key`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-key-pem] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-key-pem] + If the LDAP server requires client authentication, it uses this file. You cannot use this setting and `ssl.keystore.path` at the same time. `ssl.key_passphrase`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-key-passphrase] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-key-passphrase] `ssl.secure_key_passphrase`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-secure-key-passphrase] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-secure-key-passphrase] `ssl.certificate`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-certificate] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-certificate] + This certificate is presented to clients when they connect. `ssl.certificate_authorities`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-certificate-authorities] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-certificate-authorities] + You cannot use this setting and `ssl.truststore.path` at the same time. `ssl.keystore.path`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-path] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-path] + You cannot use this setting and `ssl.key` at the same time. `ssl.keystore.type`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-type-pkcs12] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-type-pkcs12] `ssl.keystore.password`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-password] `ssl.keystore.secure_password`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-password] `ssl.keystore.key_password`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-key-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-key-password] `ssl.keystore.secure_key_password`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-key-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-key-password] `ssl.truststore.path`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-path] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-path] + You cannot use this setting and `ssl.certificate_authorities` at the same time. `ssl.truststore.password`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-password] `ssl.truststore.secure_password`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-secure-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-secure-password] `ssl.truststore.type`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-type] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-type] `ssl.verification_mode`:: (<>) Indicates the type of verification when using `ldaps` to protect against man in the middle attacks and certificate forgery. -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-verification-mode-values] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-verification-mode-values] `ssl.supported_protocols`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-supported-protocols] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-supported-protocols] `ssl.cipher_suites`:: (<>) Specifies the cipher suites that should be supported when communicating with the LDAP server. -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-cipher-suites-values] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-cipher-suites-values] //TBD: Can this be updated to using the Java 11 URL instead or does it have to stay java8? `cache.ttl`:: @@ -915,91 +915,91 @@ Defaults to `5s` (5 seconds ). `ssl.certificate`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-certificate] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-certificate] + This certificate is presented to clients when they connect. `ssl.certificate_authorities`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-certificate-authorities] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-certificate-authorities] + You cannot use this setting and `ssl.truststore.path` at the same time. `ssl.key`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-key-pem] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-key-pem] + If the Active Directory server requires client authentication, it uses this file. You cannot use this setting and `ssl.keystore.path` at the same time. `ssl.key_passphrase`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-key-passphrase] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-key-passphrase] `ssl.secure_key_passphrase`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-secure-key-passphrase] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-secure-key-passphrase] `ssl.keystore.key_password`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-key-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-key-password] `ssl.keystore.secure_key_password`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-key-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-key-password] `ssl.keystore.password`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-password] `ssl.secure_keystore.password`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-password] //TBD: Why/how is this different than `ssl.keystore.secure_password`? `ssl.keystore.path`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-path] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-path] + You cannot use this setting and `ssl.key` at the same time. `ssl.keystore.type`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-type-pkcs12] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-type-pkcs12] `ssl.truststore.password`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-password] `ssl.truststore.secure_password`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-secure-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-secure-password] `ssl.truststore.path`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-path] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-path] + You cannot use this setting and `ssl.certificate_authorities` at the same time. `ssl.truststore.type`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-type] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-type] `ssl.verification_mode`:: (<>) Indicates the type of verification when using `ldaps` to protect against man in the middle attacks and certificate forgery. -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-verification-mode-values] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-verification-mode-values] `ssl.supported_protocols`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-supported-protocols] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-supported-protocols] `ssl.cipher_suites`:: (<>) Specifies the cipher suites that should be supported when communicating with the Active Directory server. -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-cipher-suites-values] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-cipher-suites-values] //TBD: Can this be updated to using the Java 11 URL instead or does it have to stay java8? `cache.ttl`:: @@ -1060,12 +1060,12 @@ Algorithm for the truststore. Defaults to `SunX509`. `truststore.password`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-password] If `truststore.path` is set, this setting is required. `truststore.secure_password`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-secure-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-secure-password] `truststore.path`:: (<>) @@ -1522,91 +1522,91 @@ over https. // tag::saml-ssl-key-tag[] `ssl.key` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-key-pem] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-key-pem] // end::saml-ssl-key-tag[] // tag::saml-ssl-key-passphrase-tag[] `ssl.key_passphrase` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-key-passphrase] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-key-passphrase] // end::saml-ssl-key-passphrase-tag[] `ssl.secure_key_passphrase`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-secure-key-passphrase] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-secure-key-passphrase] + You cannot use this setting and `ssl.key_passphrase` at the same time. // tag::saml-ssl-certificate-tag[] `ssl.certificate` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-certificate] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-certificate] // end::saml-ssl-certificate-tag[] // tag::saml-ssl-certificate-authorities-tag[] `ssl.certificate_authorities` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-certificate-authorities] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-certificate-authorities] // end::saml-ssl-certificate-authorities-tag[] // tag::saml-ssl-keystore-path-tag[] `ssl.keystore.path` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-path] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-path] // end::saml-ssl-keystore-path-tag[] // tag::saml-ssl-keystore-type[] `ssl.keystore.type` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-type-pkcs12] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-type-pkcs12] // end::saml-ssl-keystore-type[] // tag::saml-ssl-keystore-password-tag[] `ssl.keystore.password` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-password] // end::saml-ssl-keystore-password-tag[] `ssl.keystore.secure_password`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-password] + You cannot use this setting and `ssl.keystore.password` at the same time. //TBD: Why is this different name than `ssl.secure_keystore.password` elsewhere in this file? `ssl.keystore.key_password`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-key-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-key-password] + You cannot use this setting and `ssl.keystore.secure_key_password` at the same time. `ssl.keystore.secure_key_password`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-key-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-key-password] You cannot use this setting and `ssl.keystore.key_password` at the same time. // tag::saml-ssl-truststore-path-tag[] `ssl.truststore.path` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-path] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-path] // end::saml-ssl-truststore-path-tag[] // tag::saml-ssl-truststore-type-tag[] `ssl.truststore.type` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-type] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-type] // end::saml-ssl-truststore-type-tag[] // tag::saml-ssl-truststore-password-tag[] `ssl.truststore.password` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-password] // end::saml-ssl-truststore-password-tag[] `ssl.truststore.secure_password`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-secure-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-secure-password] + This setting cannot be used with `ssl.truststore.password`. @@ -1614,19 +1614,19 @@ This setting cannot be used with `ssl.truststore.password`. `ssl.verification_mode` {ess-icon}:: (<>) Controls the verification of certificates. -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-verification-mode-values] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-verification-mode-values] // end::saml-ssl-verification-mode-tag[] // tag::saml-ssl-supported-prototols-tag[] `ssl.supported_protocols` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-supported-protocols] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-supported-protocols] // end::saml-ssl-supported-prototols-tag[] // tag::saml-ssl-cipher-suites-tag[] `ssl.cipher_suites` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-cipher-suites-values] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-cipher-suites-values] // end::saml-ssl-cipher-suites-tag[] [discrete] @@ -2008,91 +2008,91 @@ NOTE: These settings are _only_ used for the back-channel communication between // tag::oidc-ssl-key-tag[] `ssl.key` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-key-pem] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-key-pem] // end::oidc-ssl-key-tag[] // tag::oidc-ssl-key-passphrase-tag[] `ssl.key_passphrase` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-key-passphrase] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-key-passphrase] // end::oidc-ssl-key-passphrase-tag[] `ssl.secure_key_passphrase`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-secure-key-passphrase] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-secure-key-passphrase] + You cannot use this setting and `ssl.key_passphrase` at the same time. // tag::oidc-ssl-certificate-tag[] `ssl.certificate` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-certificate] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-certificate] // end::oidc-ssl-certificate-tag[] // tag::oidc-ssl-certificate-authorities-tag[] `ssl.certificate_authorities` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-certificate-authorities] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-certificate-authorities] // end::oidc-ssl-certificate-authorities-tag[] // tag::oidc-ssl-keystore-path-tag[] `ssl.keystore.path` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-path] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-path] // end::oidc-ssl-keystore-path-tag[] // tag::oidc-ssl-keystore-type-tag[] `ssl.keystore.type` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-type-pkcs12] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-type-pkcs12] // end::oidc-ssl-keystore-type-tag[] // tag::oidc-ssl-keystore-password-tag[] `ssl.keystore.password` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-password] // end::oidc-ssl-keystore-password-tag[] `ssl.keystore.secure_password`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-password] + You cannot use this setting and `ssl.keystore.password` at the same time. `ssl.keystore.key_password`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-key-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-key-password] + You cannot use this setting and `ssl.keystore.secure_key_password` at the same time. `ssl.keystore.secure_key_password`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-key-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-key-password] + You cannot use this setting and `ssl.keystore.key_password` at the same time. // tag::oidc-ssl-truststore-path-tag[] `ssl.truststore.path` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-path] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-path] // end::oidc-ssl-truststore-path-tag[] // tag::oidc-ssl-truststore-type-tag[] `ssl.truststore.type` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-type] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-type] // end::oidc-ssl-truststore-type-tag[] // tag::oidc-ssl-truststore-password-tag[] `ssl.truststore.password` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-password] // end::oidc-ssl-truststore-password-tag[] `ssl.truststore.secure_password`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-secure-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-secure-password] + You cannot use this setting and `ssl.truststore.password` at the same time. @@ -2100,19 +2100,19 @@ You cannot use this setting and `ssl.truststore.password` at the same time. `ssl.verification_mode` {ess-icon}:: (<>) Controls the verification of certificates. -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-verification-mode-values] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-verification-mode-values] // end::oidc-ssl-verification-mode-tag[] // tag::oidc-ssl-supported-protocols-tag[] `ssl.supported_protocols` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-supported-protocols] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-supported-protocols] // end::oidc-ssl-supported-protocols-tag[] // tag::oidc-ssl-cipher-suites-tag[] `ssl.cipher_suites` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-cipher-suites-values] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-cipher-suites-values] // end::oidc-ssl-cipher-suites-tag[] [[ref-jwt-settings]] @@ -2443,91 +2443,91 @@ NOTE: These settings are _only_ used for the back-channel communication between // tag::jwt-ssl-key-tag[] `ssl.key` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-key-pem] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-key-pem] // end::jwt-ssl-key-tag[] // tag::jwt-ssl-key-passphrase-tag[] `ssl.key_passphrase` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-key-passphrase] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-key-passphrase] // end::jwt-ssl-key-passphrase-tag[] `ssl.secure_key_passphrase`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-secure-key-passphrase] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-secure-key-passphrase] + You cannot use this setting and `ssl.key_passphrase` at the same time. // tag::jwt-ssl-certificate-tag[] `ssl.certificate` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-certificate] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-certificate] // end::jwt-ssl-certificate-tag[] // tag::jwt-ssl-certificate-authorities-tag[] `ssl.certificate_authorities` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-certificate-authorities] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-certificate-authorities] // end::jwt-ssl-certificate-authorities-tag[] // tag::jwt-ssl-keystore-path-tag[] `ssl.keystore.path` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-path] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-path] // end::jwt-ssl-keystore-path-tag[] // tag::jwt-ssl-keystore-type-tag[] `ssl.keystore.type` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-type-pkcs12] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-type-pkcs12] // end::jwt-ssl-keystore-type-tag[] // tag::jwt-ssl-keystore-password-tag[] `ssl.keystore.password` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-password] // end::jwt-ssl-keystore-password-tag[] `ssl.keystore.secure_password`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-password] + You cannot use this setting and `ssl.keystore.password` at the same time. `ssl.keystore.key_password`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-key-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-key-password] + You cannot use this setting and `ssl.keystore.secure_key_password` at the same time. `ssl.keystore.secure_key_password`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-key-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-key-password] + You cannot use this setting and `ssl.keystore.key_password` at the same time. // tag::jwt-ssl-truststore-path-tag[] `ssl.truststore.path` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-path] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-path] // end::jwt-ssl-truststore-path-tag[] // tag::jwt-ssl-truststore-type-tag[] `ssl.truststore.type` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-type] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-type] // end::jwt-ssl-truststore-type-tag[] // tag::jwt-ssl-truststore-password-tag[] `ssl.truststore.password` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-password] // end::jwt-ssl-truststore-password-tag[] `ssl.truststore.secure_password`:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-secure-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-secure-password] + You cannot use this setting and `ssl.truststore.password` at the same time. @@ -2535,19 +2535,19 @@ You cannot use this setting and `ssl.truststore.password` at the same time. `ssl.verification_mode` {ess-icon}:: (<>) Controls the verification of certificates. -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-verification-mode-values] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-verification-mode-values] // end::jwt-ssl-verification-mode-tag[] // tag::jwt-ssl-supported-protocols-tag[] `ssl.supported_protocols` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-supported-protocols] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-supported-protocols] // end::jwt-ssl-supported-protocols-tag[] // tag::jwt-ssl-cipher-suites-tag[] `ssl.cipher_suites` {ess-icon}:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-cipher-suites-values] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-cipher-suites-values] // end::jwt-ssl-cipher-suites-tag[] [discrete] diff --git a/docs/reference/settings/ssl-settings.asciidoc b/docs/reference/settings/ssl-settings.asciidoc index 2ab32c1a65c2..1c484a444743 100644 --- a/docs/reference/settings/ssl-settings.asciidoc +++ b/docs/reference/settings/ssl-settings.asciidoc @@ -15,7 +15,7 @@ endif::no-enabled-setting[] +{ssl-prefix}.ssl.supported_protocols+:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-supported-protocols] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-supported-protocols] ifdef::server[] +{ssl-prefix}.ssl.client_authentication+:: @@ -41,11 +41,11 @@ a TLS _server_ is discouraged. endif::verifies[] Defines how to verify the certificates presented by another party in the TLS connection: -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-verification-mode-values] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-verification-mode-values] +{ssl-prefix}.ssl.cipher_suites+:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-cipher-suites-values] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-cipher-suites-values] [#{ssl-context}-tls-ssl-key-trusted-certificate-settings] ===== {component} TLS/SSL key and trusted certificate settings @@ -66,25 +66,25 @@ When using PEM encoded files, use the following settings: +{ssl-prefix}.ssl.key+:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-key-pem] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-key-pem] ifndef::secure-pass[] +{ssl-prefix}.ssl.key_passphrase+:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-key-passphrase] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-key-passphrase] endif::secure-pass[] +{ssl-prefix}.ssl.secure_key_passphrase+:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-secure-key-passphrase] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-secure-key-passphrase] +{ssl-prefix}.ssl.certificate+:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-certificate] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-certificate] +{ssl-prefix}.ssl.certificate_authorities+:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-certificate-authorities] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-certificate-authorities] ===== Java keystore files @@ -93,41 +93,41 @@ and certificates that should be trusted, use the following settings: +{ssl-prefix}.ssl.keystore.path+:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-path] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-path] ifndef::secure-pass[] +{ssl-prefix}.ssl.keystore.password+:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-password] endif::secure-pass[] +{ssl-prefix}.ssl.keystore.secure_password+:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-password] ifndef::secure-pass[] +{ssl-prefix}.ssl.keystore.key_password+:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-key-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-key-password] endif::secure-pass[] +{ssl-prefix}.ssl.keystore.secure_key_password+:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-key-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-key-password] +{ssl-prefix}.ssl.truststore.path+:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-path] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-path] ifndef::secure-pass[] +{ssl-prefix}.ssl.truststore.password+:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-password] endif::secure-pass[] +{ssl-prefix}.ssl.truststore.secure_password+:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-secure-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-secure-password] [#{ssl-context}-pkcs12-files] ===== PKCS#12 files @@ -139,35 +139,35 @@ PKCS#12 files are configured in the same way as Java keystore files: +{ssl-prefix}.ssl.keystore.path+:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-path] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-path] +{ssl-prefix}.ssl.keystore.type+:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-type-pkcs12] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-type-pkcs12] ifndef::secure-pass[] +{ssl-prefix}.ssl.keystore.password+:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-password] endif::secure-pass[] +{ssl-prefix}.ssl.keystore.secure_password+:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-password] ifndef::secure-pass[] +{ssl-prefix}.ssl.keystore.key_password+:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-key-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-key-password] endif::secure-pass[] +{ssl-prefix}.ssl.keystore.secure_key_password+:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-key-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-key-password] +{ssl-prefix}.ssl.truststore.path+:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-path] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-path] +{ssl-prefix}.ssl.truststore.type+:: (<>) @@ -177,10 +177,10 @@ Set this to `PKCS12` to indicate that the truststore is a PKCS#12 file. ifndef::secure-pass[] +{ssl-prefix}.ssl.truststore.password+:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-password] endif::secure-pass[] +{ssl-prefix}.ssl.truststore.secure_password+:: (<>) -include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-secure-password] +include::{es-ref-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-secure-password] diff --git a/docs/reference/setup/add-nodes.asciidoc b/docs/reference/setup/add-nodes.asciidoc index e65dd0ba7af1..ba749782c092 100644 --- a/docs/reference/setup/add-nodes.asciidoc +++ b/docs/reference/setup/add-nodes.asciidoc @@ -37,7 +37,7 @@ To add a node to a cluster running on multiple machines, you must also set the rest of its cluster. ==== -include::{es-repo-dir}/security/enroll-nodes.asciidoc[] +include::{es-ref-dir}/security/enroll-nodes.asciidoc[] For more information about discovery and shard allocation, refer to <> and <>. diff --git a/docs/reference/setup/configuration.asciidoc b/docs/reference/setup/configuration.asciidoc index 1d111a07f9c2..eea0541b898c 100644 --- a/docs/reference/setup/configuration.asciidoc +++ b/docs/reference/setup/configuration.asciidoc @@ -159,7 +159,7 @@ cluster settings and node settings. The API doesn't require a restart and ensures a setting's value is the same on all nodes. ==== -include::{es-repo-dir}/cluster/update-settings.asciidoc[tag=transient-settings-warning] +include::{es-ref-dir}/cluster/update-settings.asciidoc[tag=transient-settings-warning] // end::cluster-setting-precedence[] -- diff --git a/docs/reference/setup/important-settings/path-settings.asciidoc b/docs/reference/setup/important-settings/path-settings.asciidoc index 3e87d504963a..a0a444ca5090 100644 --- a/docs/reference/setup/important-settings/path-settings.asciidoc +++ b/docs/reference/setup/important-settings/path-settings.asciidoc @@ -18,9 +18,9 @@ data and log to locations outside of `$ES_HOME` by default. Supported `path.data` and `path.logs` values vary by platform: -include::{es-repo-dir}/tab-widgets/customize-data-log-path-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/customize-data-log-path-widget.asciidoc[] -include::{es-repo-dir}/modules/node.asciidoc[tag=modules-node-data-path-warning-tag] +include::{es-ref-dir}/modules/node.asciidoc[tag=modules-node-data-path-warning-tag] [discrete] ==== Multiple data paths @@ -36,7 +36,7 @@ the node, even if the node’s other paths have available disk space. If you nee additional disk space, we recommend you add a new node rather than additional data paths. -include::{es-repo-dir}/tab-widgets/multi-data-path-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/multi-data-path-widget.asciidoc[] [discrete] [[mdp-migrate]] diff --git a/docs/reference/setup/install/deb.asciidoc b/docs/reference/setup/install/deb.asciidoc index edaed7c78548..c7e146a5442c 100644 --- a/docs/reference/setup/install/deb.asciidoc +++ b/docs/reference/setup/install/deb.asciidoc @@ -1,6 +1,8 @@ [[deb]] === Install Elasticsearch with Debian Package +:include-xpack: true + The Debian package for Elasticsearch can be <> or from our <>. It can be used to install Elasticsearch on any Debian-based system such as Debian and Ubuntu. diff --git a/docs/reference/setup/install/rpm.asciidoc b/docs/reference/setup/install/rpm.asciidoc index a30c8c313b26..60815d570ab3 100644 --- a/docs/reference/setup/install/rpm.asciidoc +++ b/docs/reference/setup/install/rpm.asciidoc @@ -1,6 +1,8 @@ [[rpm]] === Install Elasticsearch with RPM +:include-xpack: true + The RPM for Elasticsearch can be <> or from our <>. It can be used to install Elasticsearch on any RPM-based system such as OpenSuSE, SLES, Centos, Red Hat, @@ -131,6 +133,8 @@ include::xpack-indices.asciidoc[] endif::include-xpack[] +:include-xpack!: true + [id="{distro}-running-systemd"] include::systemd.asciidoc[] diff --git a/docs/reference/setup/install/targz-start.asciidoc b/docs/reference/setup/install/targz-start.asciidoc index 79c4131a5703..0539c74469e0 100644 --- a/docs/reference/setup/install/targz-start.asciidoc +++ b/docs/reference/setup/install/targz-start.asciidoc @@ -53,4 +53,4 @@ symbolic link. :slash: / -include::{es-repo-dir}/security/enroll-nodes.asciidoc[] \ No newline at end of file +include::{es-ref-dir}/security/enroll-nodes.asciidoc[] \ No newline at end of file diff --git a/docs/reference/setup/install/targz.asciidoc b/docs/reference/setup/install/targz.asciidoc index c3dd6ad354d9..470299abe9ac 100644 --- a/docs/reference/setup/install/targz.asciidoc +++ b/docs/reference/setup/install/targz.asciidoc @@ -1,6 +1,8 @@ [[targz]] === Install {es} from archive on Linux or MacOS +:include-xpack: true + {es} is available as a `.tar.gz` archive for Linux and MacOS. include::license.asciidoc[] diff --git a/docs/reference/setup/install/zip-windows-start.asciidoc b/docs/reference/setup/install/zip-windows-start.asciidoc index 60edbb9ec704..8eb4bfb9afeb 100644 --- a/docs/reference/setup/install/zip-windows-start.asciidoc +++ b/docs/reference/setup/install/zip-windows-start.asciidoc @@ -47,4 +47,4 @@ To stop {es}, press `Ctrl-C`. :slash: \ -include::{es-repo-dir}/security/enroll-nodes.asciidoc[] \ No newline at end of file +include::{es-ref-dir}/security/enroll-nodes.asciidoc[] \ No newline at end of file diff --git a/docs/reference/setup/install/zip-windows.asciidoc b/docs/reference/setup/install/zip-windows.asciidoc index 214feaab16ac..eb84ff149f8b 100644 --- a/docs/reference/setup/install/zip-windows.asciidoc +++ b/docs/reference/setup/install/zip-windows.asciidoc @@ -1,6 +1,8 @@ [[zip-windows]] === Install {es} with `.zip` on Windows +:include-xpack: true + {es} can be installed on Windows using the Windows `.zip` archive. This comes with a `elasticsearch-service.bat` command which will setup {es} to run as a service. diff --git a/docs/reference/setup/logging-config.asciidoc b/docs/reference/setup/logging-config.asciidoc index f6b0ceb2d6ae..7b36b6382c9b 100644 --- a/docs/reference/setup/logging-config.asciidoc +++ b/docs/reference/setup/logging-config.asciidoc @@ -5,7 +5,7 @@ You can use {es}'s application logs to monitor your cluster and diagnose issues. If you run {es} as a service, the default location of the logs varies based on your platform and installation method: -include::{es-repo-dir}/tab-widgets/logging-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/logging-widget.asciidoc[] If you run {es} from the command line, {es} prints logs to the standard output (`stdout`). diff --git a/docs/reference/setup/restart-cluster.asciidoc b/docs/reference/setup/restart-cluster.asciidoc index 899cc9f46545..9488c6632836 100644 --- a/docs/reference/setup/restart-cluster.asciidoc +++ b/docs/reference/setup/restart-cluster.asciidoc @@ -22,7 +22,7 @@ usage below the <> before to restar . *Disable shard allocation.* + -- -include::{es-repo-dir}/upgrade/disable-shard-alloc.asciidoc[] +include::{es-ref-dir}/upgrade/disable-shard-alloc.asciidoc[] -- // end::disable_shard_alloc[] @@ -72,7 +72,7 @@ or jobs with large model states. . *Shut down all nodes.* + -- -include::{es-repo-dir}/upgrade/shut-down-node.asciidoc[] +include::{es-ref-dir}/upgrade/shut-down-node.asciidoc[] -- . *Perform any needed changes.* @@ -176,7 +176,7 @@ the datafeeds from {kib} or with the <> and === Rolling restart -include::{es-repo-dir}/setup/restart-cluster.asciidoc[tag=disable_shard_alloc] +include::{es-ref-dir}/setup/restart-cluster.asciidoc[tag=disable_shard_alloc] . *Stop non-essential indexing and perform a flush.* (Optional) + @@ -191,7 +191,7 @@ POST /_flush -------------------------------------------------- -- -include::{es-repo-dir}/setup/restart-cluster.asciidoc[tag=stop_ml] +include::{es-ref-dir}/setup/restart-cluster.asciidoc[tag=stop_ml] + -- * If you perform a rolling restart, you can also leave your machine learning @@ -204,7 +204,7 @@ cluster. . *Shut down a single node in case of rolling restart.* + -- -include::{es-repo-dir}/upgrade/shut-down-node.asciidoc[] +include::{es-ref-dir}/upgrade/shut-down-node.asciidoc[] -- . *Perform any needed changes.* @@ -248,4 +248,4 @@ When the node has recovered and the cluster is stable, repeat these steps for each node that needs to be changed. -- -include::{es-repo-dir}/setup/restart-cluster.asciidoc[tag=restart_ml] +include::{es-ref-dir}/setup/restart-cluster.asciidoc[tag=restart_ml] diff --git a/docs/reference/shutdown/apis/shutdown-delete.asciidoc b/docs/reference/shutdown/apis/shutdown-delete.asciidoc index 5129b2f587be..133539adfaa3 100644 --- a/docs/reference/shutdown/apis/shutdown-delete.asciidoc +++ b/docs/reference/shutdown/apis/shutdown-delete.asciidoc @@ -40,7 +40,7 @@ The ID of a node that you prepared for shut down. [[delete-shutdown-api-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [[delete-shutdown-api-example]] ==== {api-examples-title} diff --git a/docs/reference/shutdown/apis/shutdown-get.asciidoc b/docs/reference/shutdown/apis/shutdown-get.asciidoc index 50fcb45b2f81..264a8dd7be18 100644 --- a/docs/reference/shutdown/apis/shutdown-get.asciidoc +++ b/docs/reference/shutdown/apis/shutdown-get.asciidoc @@ -40,7 +40,7 @@ If no ID is specified, returns the status of all nodes being prepared for shutdo [[get-shutdown-api-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [[get-shutdown-api-example]] ==== {api-examples-title} diff --git a/docs/reference/shutdown/apis/shutdown-put.asciidoc b/docs/reference/shutdown/apis/shutdown-put.asciidoc index b8af8fb8741d..236367f886ef 100644 --- a/docs/reference/shutdown/apis/shutdown-put.asciidoc +++ b/docs/reference/shutdown/apis/shutdown-put.asciidoc @@ -50,7 +50,7 @@ No error is thrown if you specify an invalid node ID. [[put-shutdown-api-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [role="child_attributes"] [[put-shutdown-api-request-body]] diff --git a/docs/reference/slm/apis/slm-get-status.asciidoc b/docs/reference/slm/apis/slm-get-status.asciidoc index fa9e2225f26e..d4afbaddb1be 100644 --- a/docs/reference/slm/apis/slm-get-status.asciidoc +++ b/docs/reference/slm/apis/slm-get-status.asciidoc @@ -25,7 +25,7 @@ You halt and restart the {slm-init} plugin with the ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [[slm-api-get-status-prereqs]] ==== {api-prereq-title} diff --git a/docs/reference/slm/apis/slm-put.asciidoc b/docs/reference/slm/apis/slm-put.asciidoc index 69dfc4af4e84..be265554deef 100644 --- a/docs/reference/slm/apis/slm-put.asciidoc +++ b/docs/reference/slm/apis/slm-put.asciidoc @@ -41,7 +41,7 @@ you want to create or update. [[slm-api-put-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [role="child_attributes"] [[slm-api-put-request-body]] @@ -55,7 +55,7 @@ Configuration for each snapshot created by the policy. [%collapsible%open] ==== :page-id: put-slm-api -include::{es-repo-dir}/snapshot-restore/apis/create-snapshot-api.asciidoc[tag=snapshot-config] +include::{es-ref-dir}/snapshot-restore/apis/create-snapshot-api.asciidoc[tag=snapshot-config] :!page-id: ==== diff --git a/docs/reference/slm/apis/slm-start.asciidoc b/docs/reference/slm/apis/slm-start.asciidoc index c17132908e41..9d9b8108cb57 100644 --- a/docs/reference/slm/apis/slm-start.asciidoc +++ b/docs/reference/slm/apis/slm-start.asciidoc @@ -30,7 +30,7 @@ Manually starting {slm-init} is only necessary if it has been stopped using the ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [[slm-api-start-example]] ==== {api-examples-title} diff --git a/docs/reference/slm/apis/slm-stop.asciidoc b/docs/reference/slm/apis/slm-stop.asciidoc index 82b3e1b849ee..253abec7b4d1 100644 --- a/docs/reference/slm/apis/slm-stop.asciidoc +++ b/docs/reference/slm/apis/slm-stop.asciidoc @@ -37,7 +37,7 @@ Use the <> to see if {slm-init} is running. ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [[slm-api-stop-example]] ==== {api-examples-title} diff --git a/docs/reference/snapshot-restore/apis/clean-up-repo-api.asciidoc b/docs/reference/snapshot-restore/apis/clean-up-repo-api.asciidoc index 2360698e0be8..249e192c0c58 100644 --- a/docs/reference/snapshot-restore/apis/clean-up-repo-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/clean-up-repo-api.asciidoc @@ -48,7 +48,7 @@ Name of the snapshot repository to review and clean up. [[clean-up-snapshot-repo-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] [role="child_attributes"] [[clean-up-snapshot-repo-api-response-body]] diff --git a/docs/reference/snapshot-restore/apis/clone-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/clone-snapshot-api.asciidoc index fab734e8413c..590bc7e7410f 100644 --- a/docs/reference/snapshot-restore/apis/clone-snapshot-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/clone-snapshot-api.asciidoc @@ -42,7 +42,7 @@ Name of the snapshot repository that both source and target snapshot belong to. [[clone-snapshot-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] `timeout`:: (Optional, <>) Specifies the period of time to wait for diff --git a/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc index 06693dadd387..baa28bb7b0a5 100644 --- a/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc @@ -56,7 +56,7 @@ unique within the snapshot repository. [[create-snapshot-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] `wait_for_completion`:: (Optional, Boolean) If `true`, the request returns a response when the snapshot diff --git a/docs/reference/snapshot-restore/apis/delete-repo-api.asciidoc b/docs/reference/snapshot-restore/apis/delete-repo-api.asciidoc index d3ebeeac3c03..2931faf49841 100644 --- a/docs/reference/snapshot-restore/apis/delete-repo-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/delete-repo-api.asciidoc @@ -51,7 +51,7 @@ supported. [[delete-snapshot-repo-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] `timeout`:: (Optional, <>) Specifies the period of time to wait for diff --git a/docs/reference/snapshot-restore/apis/delete-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/delete-snapshot-api.asciidoc index 5bc46f54ec13..d1431b8cb670 100644 --- a/docs/reference/snapshot-restore/apis/delete-snapshot-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/delete-snapshot-api.asciidoc @@ -56,7 +56,7 @@ Comma-separated list of snapshot names to delete. Also accepts wildcards (`*`). [[delete-snapshot-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] [[delete-snapshot-api-example]] ==== {api-example-title} diff --git a/docs/reference/snapshot-restore/apis/get-repo-api.asciidoc b/docs/reference/snapshot-restore/apis/get-repo-api.asciidoc index 1f03a44c5e49..cf1b9813c519 100644 --- a/docs/reference/snapshot-restore/apis/get-repo-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/get-repo-api.asciidoc @@ -59,7 +59,7 @@ cluster, omit this parameter or use `*` or `_all`. only. If `false`, the request gets information from the master node. Defaults to `false`. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] [role="child_attributes"] [[get-snapshot-repo-api-response-body]] diff --git a/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc index e685badc31d4..622e1ade024b 100644 --- a/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/get-snapshot-api.asciidoc @@ -83,7 +83,7 @@ Set <> to `true` to re [[get-snapshot-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] [[get-snapshot-api-ignore-unavailable]] `ignore_unavailable`:: diff --git a/docs/reference/snapshot-restore/apis/get-snapshot-status-api.asciidoc b/docs/reference/snapshot-restore/apis/get-snapshot-status-api.asciidoc index 150f4dfff48a..d8b03cbc0e88 100644 --- a/docs/reference/snapshot-restore/apis/get-snapshot-status-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/get-snapshot-status-api.asciidoc @@ -134,7 +134,7 @@ currently running snapshots. Wildcards (`*`) are not supported. [[get-snapshot-status-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] `ignore_unavailable`:: (Optional, Boolean) diff --git a/docs/reference/snapshot-restore/apis/put-repo-api.asciidoc b/docs/reference/snapshot-restore/apis/put-repo-api.asciidoc index 1154b970b907..c3e9c0a0904b 100644 --- a/docs/reference/snapshot-restore/apis/put-repo-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/put-repo-api.asciidoc @@ -52,7 +52,7 @@ IMPORTANT: Several options for this API can be specified using a query parameter or a request body parameter. If both parameters are specified, only the query parameter is used. -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] `timeout`:: (Optional, <>) Specifies the period of time to wait for diff --git a/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc index db8923505973..9fe06d73f1a6 100644 --- a/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/restore-snapshot-api.asciidoc @@ -113,7 +113,7 @@ Name of the snapshot to restore. [[restore-snapshot-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] `wait_for_completion`:: (Optional, Boolean) If `true`, the request returns a response when the restore diff --git a/docs/reference/snapshot-restore/apis/verify-repo-api.asciidoc b/docs/reference/snapshot-restore/apis/verify-repo-api.asciidoc index f0fc659df1a4..9d14e8a426e3 100644 --- a/docs/reference/snapshot-restore/apis/verify-repo-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/verify-repo-api.asciidoc @@ -47,7 +47,7 @@ Name of the snapshot repository to verify. [[verify-snapshot-repo-api-query-params]] ==== {api-query-parms-title} -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] `timeout`:: (Optional, <>) Specifies the period of time to wait for diff --git a/docs/reference/snapshot-restore/repository-read-only-url.asciidoc b/docs/reference/snapshot-restore/repository-read-only-url.asciidoc index 8f9cb7e198f8..53d0c340fcdc 100644 --- a/docs/reference/snapshot-restore/repository-read-only-url.asciidoc +++ b/docs/reference/snapshot-restore/repository-read-only-url.asciidoc @@ -1,7 +1,7 @@ [[snapshots-read-only-repository]] === Read-only URL repository -include::{es-repo-dir}/snapshot-restore/on-prem-repo-type.asciidoc[] +include::{es-ref-dir}/snapshot-restore/on-prem-repo-type.asciidoc[] You can use a URL repository to give a cluster read-only access to a shared file system. Since URL repositories are always read-only, they're a safer and more diff --git a/docs/reference/snapshot-restore/repository-shared-file-system.asciidoc b/docs/reference/snapshot-restore/repository-shared-file-system.asciidoc index 6be49d9d4422..be5347845a2f 100644 --- a/docs/reference/snapshot-restore/repository-shared-file-system.asciidoc +++ b/docs/reference/snapshot-restore/repository-shared-file-system.asciidoc @@ -1,7 +1,7 @@ [[snapshots-filesystem-repository]] === Shared file system repository -include::{es-repo-dir}/snapshot-restore/on-prem-repo-type.asciidoc[] +include::{es-ref-dir}/snapshot-restore/on-prem-repo-type.asciidoc[] Use a shared file system repository to store snapshots on a shared file system. @@ -13,7 +13,7 @@ master and data node. For running clusters, this requires a Supported `path.repo` values vary by platform: -include::{es-repo-dir}/tab-widgets/register-fs-repo-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/register-fs-repo-widget.asciidoc[] [[filesystem-repository-settings]] ==== Repository settings diff --git a/docs/reference/sql/apis/sql-search-api.asciidoc b/docs/reference/sql/apis/sql-search-api.asciidoc index 596378c5e439..118d7975aefd 100644 --- a/docs/reference/sql/apis/sql-search-api.asciidoc +++ b/docs/reference/sql/apis/sql-search-api.asciidoc @@ -123,7 +123,7 @@ the scroll request. Defaults to `45s` (45 seconds). (Optional, <>) Timeout before the request fails. Defaults to `90s` (90 seconds). -include::{es-repo-dir}/search/search.asciidoc[tag=runtime-mappings-def] +include::{es-ref-dir}/search/search.asciidoc[tag=runtime-mappings-def] [[sql-search-api-time-zone]] `time_zone`:: diff --git a/docs/reference/sql/index.asciidoc b/docs/reference/sql/index.asciidoc index 1d2b2e7fd3ec..2ae7ea78a803 100644 --- a/docs/reference/sql/index.asciidoc +++ b/docs/reference/sql/index.asciidoc @@ -2,7 +2,7 @@ [[xpack-sql]] = SQL -:sql-tests: {xes-repo-dir}/../../plugin/sql/qa/ +:sql-tests: {elasticsearch-root}/x-pack/docs/{lang}/../../plugin/sql/qa/ :sql-specs: {sql-tests}server/src/main/resources/ :jdbc-tests: {sql-tests}jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc :security-tests: {sql-tests}server/security/src/test/java/org/elasticsearch/xpack/sql/qa/security diff --git a/docs/reference/sql/security.asciidoc b/docs/reference/sql/security.asciidoc index b57179742700..a440ae69d8da 100644 --- a/docs/reference/sql/security.asciidoc +++ b/docs/reference/sql/security.asciidoc @@ -44,7 +44,7 @@ APIs to view or edit a role defined in `roles.yml`. This example configures a role that can run SQL in JDBC querying the `test` index: -include::{es-repo-dir}/rest-api/security/create-roles.asciidoc[tag=sql-queries-permission] +include::{es-ref-dir}/rest-api/security/create-roles.asciidoc[tag=sql-queries-permission] [discrete] [[sql-role-file-example]] diff --git a/docs/reference/tab-widgets/ilm.asciidoc b/docs/reference/tab-widgets/ilm.asciidoc index 60a92135d773..3682377d754e 100644 --- a/docs/reference/tab-widgets/ilm.asciidoc +++ b/docs/reference/tab-widgets/ilm.asciidoc @@ -85,5 +85,5 @@ Index Lifecycle Policies**. Click **Create policy**. You can also use the <>. -include::{es-repo-dir}/data-streams/set-up-a-data-stream.asciidoc[tag=ilm-policy-api-ex] +include::{es-ref-dir}/data-streams/set-up-a-data-stream.asciidoc[tag=ilm-policy-api-ex] // end::custom[] diff --git a/docs/reference/tab-widgets/register-fs-repo.asciidoc b/docs/reference/tab-widgets/register-fs-repo.asciidoc index 6ff0bf23ca9e..f222f87f5a3e 100644 --- a/docs/reference/tab-widgets/register-fs-repo.asciidoc +++ b/docs/reference/tab-widgets/register-fs-repo.asciidoc @@ -48,7 +48,7 @@ PUT _snapshot/my_fs_backup relative path, `my_fs_backup_location`, resolves to `/mount/backups/my_fs_backup_location`. -include::{es-repo-dir}/snapshot-restore/register-repository.asciidoc[tag=multi-cluster-repo] +include::{es-ref-dir}/snapshot-restore/register-repository.asciidoc[tag=multi-cluster-repo] // tag::fs-repo-read-only[] To register a file system repository as read-only using the create snapshot @@ -119,7 +119,7 @@ PUT _snapshot/my_fs_backup relative path, `My_fs_backup_location`, resolves to `E:\Mount\Backups\My_fs_backup_location`. -include::{es-repo-dir}/snapshot-restore/register-repository.asciidoc[tag=multi-cluster-repo] +include::{es-ref-dir}/snapshot-restore/register-repository.asciidoc[tag=multi-cluster-repo] include::register-fs-repo.asciidoc[tag=fs-repo-read-only] diff --git a/docs/reference/tab-widgets/snapshot-repo.asciidoc b/docs/reference/tab-widgets/snapshot-repo.asciidoc index e3871d84e483..b41ddbf128a3 100644 --- a/docs/reference/tab-widgets/snapshot-repo.asciidoc +++ b/docs/reference/tab-widgets/snapshot-repo.asciidoc @@ -15,5 +15,5 @@ You can also use any of the following custom repository types with {search-snaps // end::cloud[] // tag::self-managed[] -include::{es-repo-dir}/searchable-snapshots/index.asciidoc[tag=searchable-snapshot-repo-types] +include::{es-ref-dir}/searchable-snapshots/index.asciidoc[tag=searchable-snapshot-repo-types] // end::self-managed[] diff --git a/docs/reference/text-structure/apis/find-field-structure.asciidoc b/docs/reference/text-structure/apis/find-field-structure.asciidoc index 6788ddf7f42b..4fa108e92d4c 100644 --- a/docs/reference/text-structure/apis/find-field-structure.asciidoc +++ b/docs/reference/text-structure/apis/find-field-structure.asciidoc @@ -57,22 +57,22 @@ chosen. `field`:: (Required, string) The name of the field that's analyzed. -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-column-names] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-delimiter] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-column-names] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-delimiter] `documents_to_sample`:: (Optional, unsigned integer) The number of documents to include in the structural analysis. The minimum is 2; the default is 1000. -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-explain] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-format] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-grok-pattern] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-ecs-compatibility] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-quote] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-should-trim-fields] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-timeout] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-timestamp-field] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-timestamp-format] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-explain] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-format] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-grok-pattern] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-ecs-compatibility] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-quote] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-should-trim-fields] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-timeout] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-timestamp-field] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-timestamp-format] [discrete] [[find-field-structure-examples]] diff --git a/docs/reference/text-structure/apis/find-message-structure.asciidoc b/docs/reference/text-structure/apis/find-message-structure.asciidoc index 085f65b85212..6c1bf5089bed 100644 --- a/docs/reference/text-structure/apis/find-message-structure.asciidoc +++ b/docs/reference/text-structure/apis/find-message-structure.asciidoc @@ -52,17 +52,17 @@ chosen. [[find-message-structure-query-parms]] == {api-query-parms-title} -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-column-names] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-delimiter] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-explain] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-format] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-grok-pattern] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-ecs-compatibility] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-quote] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-should-trim-fields] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-timeout] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-timestamp-field] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-timestamp-format] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-column-names] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-delimiter] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-explain] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-format] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-grok-pattern] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-ecs-compatibility] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-quote] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-should-trim-fields] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-timeout] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-timestamp-field] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-timestamp-format] [discrete] [[find-message-structure-request-body]] diff --git a/docs/reference/text-structure/apis/find-structure.asciidoc b/docs/reference/text-structure/apis/find-structure.asciidoc index b49b0f352668..361560bace4e 100644 --- a/docs/reference/text-structure/apis/find-structure.asciidoc +++ b/docs/reference/text-structure/apis/find-structure.asciidoc @@ -55,21 +55,21 @@ chosen. [[find-structure-query-parms]] == {api-query-parms-title} -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-charset] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-column-names] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-delimiter] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-explain] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-format] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-grok-pattern] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-ecs-compatibility] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-has-header-row] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-line-merge-size-limit] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-lines-to-sample] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-quote] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-should-trim-fields] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-timeout] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-timestamp-field] -include::{es-repo-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-timestamp-format] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-charset] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-column-names] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-delimiter] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-explain] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-format] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-grok-pattern] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-ecs-compatibility] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-has-header-row] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-line-merge-size-limit] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-lines-to-sample] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-quote] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-should-trim-fields] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-timeout] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-timestamp-field] +include::{es-ref-dir}/text-structure/apis/find-structure-shared.asciidoc[tag=param-timestamp-format] [discrete] [[find-structure-request-body]] diff --git a/docs/reference/transform/apis/delete-transform.asciidoc b/docs/reference/transform/apis/delete-transform.asciidoc index 9a097407749a..111dda23690b 100644 --- a/docs/reference/transform/apis/delete-transform.asciidoc +++ b/docs/reference/transform/apis/delete-transform.asciidoc @@ -26,7 +26,7 @@ in the `transform_admin` built-in role. ``:: (Required, string) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-id] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-id] [[delete-transform-query-parms]] == {api-query-parms-title} diff --git a/docs/reference/transform/apis/get-transform-stats.asciidoc b/docs/reference/transform/apis/get-transform-stats.asciidoc index 2a7ed1913546..273b1d094979 100644 --- a/docs/reference/transform/apis/get-transform-stats.asciidoc +++ b/docs/reference/transform/apis/get-transform-stats.asciidoc @@ -49,7 +49,7 @@ specifying `*` as the ``, or by omitting the ``:: (Optional, string) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-id-wildcard] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-id-wildcard] [[get-transform-stats-query-parms]] @@ -57,15 +57,15 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-id-wildcard] `allow_no_match`:: (Optional, Boolean) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-match-transforms1] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-match-transforms1] `from`:: (Optional, integer) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=from-transforms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=from-transforms] `size`:: (Optional, integer) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=size-transforms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=size-transforms] [role="child_attributes"] [[get-transform-stats-response]] @@ -84,7 +84,7 @@ informational; you cannot update their values. ==== `changes_last_detected_at`::: (date) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=checkpointing-changes-last-detected-at] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=checkpointing-changes-last-detected-at] //Begin checkpointing.last `last`::: @@ -106,7 +106,7 @@ was created. `last_search_time`::: (date) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=checkpointing-last-search-time] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=checkpointing-last-search-time] //Begin checkpointing.next `next`::: @@ -191,7 +191,7 @@ that the {transform} is failing to keep up. `id`:: (string) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-id] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-id] //Begin node `node`:: @@ -221,11 +221,11 @@ example, `127.0.0.1:9300`. `reason`:: (string) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=state-transform-reason] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=state-transform-reason] `state`:: (string) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=state-transform] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=state-transform] //Begin stats `stats`:: @@ -237,71 +237,71 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=state-transform] `delete_time_in_ms`::: (long) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=delete-time-ms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=delete-time-ms] `documents_deleted`::: (long) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=docs-deleted-transform] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=docs-deleted-transform] `documents_indexed`::: (long) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=docs-indexed] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=docs-indexed] `documents_processed`::: (long) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=docs-processed] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=docs-processed] `exponential_avg_checkpoint_duration_ms`::: (double) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=exponential-avg-checkpoint-duration-ms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=exponential-avg-checkpoint-duration-ms] `exponential_avg_documents_indexed`::: (double) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=exponential-avg-documents-indexed] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=exponential-avg-documents-indexed] `exponential_avg_documents_processed`::: (double) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=exponential-avg-documents-processed] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=exponential-avg-documents-processed] `index_failures`::: (long) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-failures] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-failures] `index_time_in_ms`::: (long) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-time-ms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-time-ms] `index_total`::: (long) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-total] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=index-total] `pages_processed`::: (long) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=pages-processed] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=pages-processed] `processing_time_in_ms`::: (long) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=processing-time-ms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=processing-time-ms] `processing_total`::: (long) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=processing-total] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=processing-total] `search_failures`::: (long) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=search-failures] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=search-failures] `search_time_in_ms`::: (long) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=search-time-ms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=search-time-ms] `search_total`::: (long) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=search-total] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=search-total] `trigger_count`::: (long) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=trigger-count] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=trigger-count] ==== //End stats diff --git a/docs/reference/transform/apis/get-transform.asciidoc b/docs/reference/transform/apis/get-transform.asciidoc index c008e34e74fd..ece59138e289 100644 --- a/docs/reference/transform/apis/get-transform.asciidoc +++ b/docs/reference/transform/apis/get-transform.asciidoc @@ -40,14 +40,14 @@ specifying `*` as the ``, or by omitting the ``. ``:: (Optional, string) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-id-wildcard] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-id-wildcard] [[get-transform-query-parms]] == {api-query-parms-title} `allow_no_match`:: (Optional, Boolean) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-match-transforms1] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-match-transforms1] `exclude_generated`:: (Optional, Boolean) @@ -57,11 +57,11 @@ and then added to another cluster. Default is false. `from`:: (Optional, integer) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=from-transforms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=from-transforms] `size`:: (Optional, integer) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=size-transforms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=size-transforms] diff --git a/docs/reference/transform/apis/preview-transform.asciidoc b/docs/reference/transform/apis/preview-transform.asciidoc index 86ca91b32fdf..fa9ad0c0fc8f 100644 --- a/docs/reference/transform/apis/preview-transform.asciidoc +++ b/docs/reference/transform/apis/preview-transform.asciidoc @@ -91,29 +91,29 @@ expires, the request fails and returns an error. Defaults to `30s`. //Begin dest `dest`:: (Optional, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=dest] + .Properties of `dest` [%collapsible%open] ==== `index`::: (Optional, string) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-index] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=dest-index] `pipeline`::: (Optional, string) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-pipeline] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=dest-pipeline] ==== //End dest `frequency`:: (Optional, <>) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=frequency] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=frequency] //Begin latest `latest`:: (Required^*^, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-latest] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-latest] + .Properties of `latest` [%collapsible%open] @@ -121,11 +121,11 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-latest] `sort`::: (Required, string) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-sort] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-sort] `unique_key`::: (Required, array of strings) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-unique-key] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-unique-key] ==== //End latest @@ -133,43 +133,43 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-unique-key] //Begin pivot `pivot`:: (Required, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=pivot] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=pivot] + .Properties of `pivot` [%collapsible%open] ==== `aggregations` or `aggs`::: (Required, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=pivot-aggs] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=pivot-aggs] `group_by`::: (Required, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=pivot-group-by] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=pivot-group-by] ==== //End pivot //Begin retention policy `retention_policy`:: (Optional, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-retention] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-retention] + .Properties of `retention_policy` [%collapsible%open] ==== `time`::: (Required, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-retention-time] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-retention-time] + .Properties of `time` [%collapsible%open] ===== `field`::: (Required, string) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-retention-time-field] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-retention-time-field] `max_age`::: (Required, <>) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-retention-time-max-age] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-retention-time-max-age] ===== ==== //End retention policy @@ -177,29 +177,29 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-retention-ti //Begin source `source`:: (Required, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-transforms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=source-transforms] + .Properties of `source` [%collapsible%open] ==== `index`::: (Required, string or array) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-index-transforms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=source-index-transforms] `query`::: (Optional, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-query-transforms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=source-query-transforms] `runtime_mappings`::: (Optional, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-runtime-mappings-transforms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=source-runtime-mappings-transforms] ==== //End source //Begin sync `sync`:: (Optional, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=sync] + .Properties of `sync` [%collapsible%open] @@ -208,18 +208,18 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync] `time`::: (Optional, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync-time] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=sync-time] + .Properties of `time` [%collapsible%open] ===== `delay`:::: (Optional, <>) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync-time-delay] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=sync-time-delay] `field`:::: (Optional, string) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync-time-field] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=sync-time-field] ===== //End sync.time ==== @@ -228,29 +228,29 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync-time-field] //Begin settings `settings`:: (Optional, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-settings] + .Properties of `settings` [%collapsible%open] ==== `align_checkpoints`::: (Optional, boolean) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-align-checkpoints] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-align-checkpoints] `dates_as_epoch_millis`::: (Optional, boolean) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-dates-as-epoch-milli] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-dates-as-epoch-milli] `deduce_mappings`::: (Optional, boolean) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-deduce-mappings] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-deduce-mappings] `docs_per_second`::: (Optional, float) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-docs-per-second] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-docs-per-second] `max_page_search_size`::: (Optional, integer) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-max-page-search-size] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-max-page-search-size] `unattended`::: (Optional, boolean) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-unattended] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-unattended] ==== //End settings diff --git a/docs/reference/transform/apis/put-transform.asciidoc b/docs/reference/transform/apis/put-transform.asciidoc index a3f885795732..ed2ceba0a7a5 100644 --- a/docs/reference/transform/apis/put-transform.asciidoc +++ b/docs/reference/transform/apis/put-transform.asciidoc @@ -101,7 +101,7 @@ expires, the request fails and returns an error. Defaults to `30s`. //Begin dest `dest`:: (Required, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=dest] + .Properties of `dest` [%collapsible%open] @@ -109,12 +109,12 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest] `index`::: (Required, string) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-index] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=dest-index] //Begin aliases `aliases`::: (Optional, array of objects) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-aliases] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=dest-aliases] + .Properties of `aliases` [%collapsible%open] @@ -122,29 +122,29 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-aliases] `alias`:::: (Required, string) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-aliases-alias] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=dest-aliases-alias] `move_on_creation`:::: (Optional, boolean) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-aliases-move-on-creation] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=dest-aliases-move-on-creation] ===== //End aliases `pipeline`::: (Optional, string) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-pipeline] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=dest-pipeline] ==== //End dest `frequency`:: (Optional, <>) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=frequency] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=frequency] //Begin latest `latest`:: (Required^*^, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-latest] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-latest] + .Properties of `latest` [%collapsible%open] @@ -152,11 +152,11 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-latest] `sort`::: (Required, string) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-sort] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-sort] `unique_key`::: (Required, array of strings) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-unique-key] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-unique-key] ==== //End latest @@ -164,13 +164,13 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-unique-key] //Begin _meta `_meta`:: (Optional, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-metadata] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-metadata] //End _meta //Begin pivot `pivot`:: (Required^*^, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=pivot] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=pivot] + .Properties of `pivot` [%collapsible%open] @@ -178,36 +178,36 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=pivot] `aggregations` or `aggs`::: (Required, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=pivot-aggs] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=pivot-aggs] `group_by`::: (Required, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=pivot-group-by] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=pivot-group-by] ==== //End pivot //Begin retention policy `retention_policy`:: (Optional, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-retention] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-retention] + .Properties of `retention_policy` [%collapsible%open] ==== `time`::: (Required, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-retention-time] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-retention-time] + .Properties of `time` [%collapsible%open] ===== `field`::: (Required, string) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-retention-time-field] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-retention-time-field] `max_age`::: (Required, <>) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-retention-time-max-age] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-retention-time-max-age] ===== ==== //End retention policy @@ -215,39 +215,39 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-retention-ti //Begin settings `settings`:: (Optional, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-settings] + .Properties of `settings` [%collapsible%open] ==== `align_checkpoints`::: (Optional, boolean) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-align-checkpoints] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-align-checkpoints] `dates_as_epoch_millis`::: (Optional, boolean) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-dates-as-epoch-milli] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-dates-as-epoch-milli] `deduce_mappings`::: (Optional, boolean) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-deduce-mappings] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-deduce-mappings] `docs_per_second`::: (Optional, float) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-docs-per-second] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-docs-per-second] `max_page_search_size`::: (Optional, integer) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-max-page-search-size] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-max-page-search-size] `num_failure_retries`::: (Optional, integer) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-num-failure-retries] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-num-failure-retries] `unattended`::: (Optional, boolean) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-unattended] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-unattended] ==== //End settings //Begin source `source`:: (Required, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-transforms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=source-transforms] + .Properties of `source` [%collapsible%open] @@ -255,22 +255,22 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-transforms] `index`::: (Required, string or array) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-index-transforms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=source-index-transforms] `query`::: (Optional, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-query-transforms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=source-query-transforms] `runtime_mappings`::: (Optional, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-runtime-mappings-transforms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=source-runtime-mappings-transforms] ==== //End source //Begin sync `sync`:: (Optional, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=sync] + .Properties of `sync` [%collapsible%open] @@ -279,18 +279,18 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync] //Begin time `time`::: (Required, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync-time] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=sync-time] + .Properties of `time` [%collapsible%open] ===== `delay`:::: (Optional, <>) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync-time-delay] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=sync-time-delay] `field`:::: (Required, string) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync-time-field] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=sync-time-field] + -- TIP: It is strongly recommended to use a field that contains the diff --git a/docs/reference/transform/apis/reset-transform.asciidoc b/docs/reference/transform/apis/reset-transform.asciidoc index badbc0f02231..1194d3589275 100644 --- a/docs/reference/transform/apis/reset-transform.asciidoc +++ b/docs/reference/transform/apis/reset-transform.asciidoc @@ -38,7 +38,7 @@ to the latest format as if the <> API was used. The ``:: (Required, string) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-id] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-id] [[reset-transform-query-parms]] == {api-query-parms-title} diff --git a/docs/reference/transform/apis/schedule-now-transform.asciidoc b/docs/reference/transform/apis/schedule-now-transform.asciidoc index 202b2d61fc73..7a276edf0881 100644 --- a/docs/reference/transform/apis/schedule-now-transform.asciidoc +++ b/docs/reference/transform/apis/schedule-now-transform.asciidoc @@ -35,7 +35,7 @@ called again in the meantime. ``:: (Required, string) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-id] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-id] [[schedule-now-transform-query-parms]] == {api-query-parms-title} diff --git a/docs/reference/transform/apis/stop-transform.asciidoc b/docs/reference/transform/apis/stop-transform.asciidoc index 670e015c3314..e99fcbd413eb 100644 --- a/docs/reference/transform/apis/stop-transform.asciidoc +++ b/docs/reference/transform/apis/stop-transform.asciidoc @@ -41,7 +41,7 @@ comma-separated list or a wildcard expression. To stop all {transforms}, use `allow_no_match`:: (Optional, Boolean) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=allow-no-match-transforms2] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=allow-no-match-transforms2] `force`:: (Optional, Boolean) Set to `true` to stop a failed {transform} or to diff --git a/docs/reference/transform/apis/update-transform.asciidoc b/docs/reference/transform/apis/update-transform.asciidoc index 57a27b4efae7..1ac7d6d5410d 100644 --- a/docs/reference/transform/apis/update-transform.asciidoc +++ b/docs/reference/transform/apis/update-transform.asciidoc @@ -57,7 +57,7 @@ permanent failure. ``:: (Required, string) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-id] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-id] [[update-transform-query-parms]] == {api-query-parms-title} @@ -82,7 +82,7 @@ expires, the request fails and returns an error. Defaults to `30s`. //Begin dest `dest`:: (Optional, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=dest] + .Properties of `dest` [%collapsible%open] @@ -90,12 +90,12 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest] `index`::: (Required, string) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-index] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=dest-index] //Begin aliases `aliases`::: (Optional, array of objects) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-aliases] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=dest-aliases] + .Properties of `aliases` [%collapsible%open] @@ -103,53 +103,53 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-aliases] `alias`:::: (Required, string) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-aliases-alias] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=dest-aliases-alias] `move_on_creation`:::: (Optional, boolean) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-aliases-move-on-creation] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=dest-aliases-move-on-creation] ===== //End aliases `pipeline`::: (Optional, string) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-pipeline] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=dest-pipeline] ==== //End dest `frequency`:: (Optional, <>) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=frequency] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=frequency] //Begin _meta `_meta`:: (Optional, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-metadata] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-metadata] //End _meta //Begin retention policy `retention_policy`:: (Optional, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-retention] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-retention] + .Properties of `retention_policy` [%collapsible%open] ==== `time`::: (Required, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-retention-time] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-retention-time] + .Properties of `time` [%collapsible%open] ===== `field`::: (Required, string) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-retention-time-field] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-retention-time-field] `max_age`::: (Required, <>) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-retention-time-max-age] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-retention-time-max-age] ===== ==== //End retention policy @@ -157,39 +157,39 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-retention-ti //Begin settings `settings`:: (Optional, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-settings] + .Properties of `settings` [%collapsible%open] ==== `align_checkpoints`::: (Optional, boolean) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-align-checkpoints] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-align-checkpoints] `dates_as_epoch_millis`::: (Optional, boolean) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-dates-as-epoch-milli] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-dates-as-epoch-milli] `deduce_mappings`::: (Optional, boolean) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-deduce-mappings] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-deduce-mappings] `docs_per_second`::: (Optional, float) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-docs-per-second] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-docs-per-second] `max_page_search_size`::: (Optional, integer) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-max-page-search-size] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-max-page-search-size] `num_failure_retries`::: (Optional, integer) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-num-failure-retries] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-num-failure-retries] `unattended`::: (Optional, boolean) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-unattended] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-unattended] ==== //End settings //Begin source `source`:: (Optional, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-transforms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=source-transforms] + .Properties of `source` [%collapsible%open] @@ -197,18 +197,18 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-transforms] `index`::: (Required, string or array) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-index-transforms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=source-index-transforms] `query`::: (Optional, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-query-transforms] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=source-query-transforms] ==== //End source //Begin sync `sync`:: (Optional, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=sync] + -- NOTE: You can update these properties only if it is a continuous {transform}. You @@ -224,7 +224,7 @@ Instead, clone the {transform} in {kib} and add or remove the `sync` property. //Begin sync.time `time`::: (Required, object) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync-time] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=sync-time] + .Properties of `time` [%collapsible%open] @@ -232,11 +232,11 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync-time] `delay`:::: (Optional, <>) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync-time-delay] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=sync-time-delay] `field`:::: (Required, string) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync-time-field] +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=sync-time-field] + -- TIP: In general, it’s a good idea to use a field that contains the diff --git a/docs/reference/troubleshooting/common-issues/diagnose-unassigned-shards.asciidoc b/docs/reference/troubleshooting/common-issues/diagnose-unassigned-shards.asciidoc index 2d23763a7576..fe9422d6d4c5 100644 --- a/docs/reference/troubleshooting/common-issues/diagnose-unassigned-shards.asciidoc +++ b/docs/reference/troubleshooting/common-issues/diagnose-unassigned-shards.asciidoc @@ -6,7 +6,7 @@ allocation settings to lack of disk space. In order to diagnose the unassigned shards in your deployment use the following steps: -include::{es-repo-dir}/tab-widgets/troubleshooting/data/diagnose-unassigned-shards-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/troubleshooting/data/diagnose-unassigned-shards-widget.asciidoc[] diff --git a/docs/reference/troubleshooting/common-issues/high-cpu-usage.asciidoc b/docs/reference/troubleshooting/common-issues/high-cpu-usage.asciidoc index 536d18b65358..858683ef97a6 100644 --- a/docs/reference/troubleshooting/common-issues/high-cpu-usage.asciidoc +++ b/docs/reference/troubleshooting/common-issues/high-cpu-usage.asciidoc @@ -15,7 +15,7 @@ depleted, {es} will reject search requests until more threads are available. **Check CPU usage** -include::{es-repo-dir}/tab-widgets/cpu-usage-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/cpu-usage-widget.asciidoc[] **Check hot threads** diff --git a/docs/reference/troubleshooting/common-issues/high-jvm-memory-pressure.asciidoc b/docs/reference/troubleshooting/common-issues/high-jvm-memory-pressure.asciidoc index c71dda063dea..e88927f159f2 100644 --- a/docs/reference/troubleshooting/common-issues/high-jvm-memory-pressure.asciidoc +++ b/docs/reference/troubleshooting/common-issues/high-jvm-memory-pressure.asciidoc @@ -12,7 +12,7 @@ exceeds 85%. **Check JVM memory pressure** -include::{es-repo-dir}/tab-widgets/jvm-memory-pressure-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/jvm-memory-pressure-widget.asciidoc[] **Check garbage collection logs** diff --git a/docs/reference/troubleshooting/data/add-tier.asciidoc b/docs/reference/troubleshooting/data/add-tier.asciidoc index 830cb2bbe296..03077d7da1bb 100644 --- a/docs/reference/troubleshooting/data/add-tier.asciidoc +++ b/docs/reference/troubleshooting/data/add-tier.asciidoc @@ -6,7 +6,7 @@ The allocation of indices in an {es} deployment can be allocated on <> the indices expect to be allocated on to your deployment: -include::{es-repo-dir}/tab-widgets/troubleshooting/data/add-tier-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/troubleshooting/data/add-tier-widget.asciidoc[] diff --git a/docs/reference/troubleshooting/data/data-tiers-mixed-with-node-attr.asciidoc b/docs/reference/troubleshooting/data/data-tiers-mixed-with-node-attr.asciidoc index 7bf6ef91cce7..2f6b75b372fd 100644 --- a/docs/reference/troubleshooting/data/data-tiers-mixed-with-node-attr.asciidoc +++ b/docs/reference/troubleshooting/data/data-tiers-mixed-with-node-attr.asciidoc @@ -12,7 +12,7 @@ This could lead to unassigned shards or shards not transitioning to the desired In order to fix this follow the next steps: -include::{es-repo-dir}/tab-widgets/troubleshooting/data/migrate-to-data-tiers-routing-guide-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/troubleshooting/data/migrate-to-data-tiers-routing-guide-widget.asciidoc[] diff --git a/docs/reference/troubleshooting/data/enable-cluster-allocation.asciidoc b/docs/reference/troubleshooting/data/enable-cluster-allocation.asciidoc index 787588d2a165..e229d9016dfe 100644 --- a/docs/reference/troubleshooting/data/enable-cluster-allocation.asciidoc +++ b/docs/reference/troubleshooting/data/enable-cluster-allocation.asciidoc @@ -10,7 +10,7 @@ Forgetting to re-allow all data allocations can lead to unassigned shards. In order to (re)allow all data to be allocated follow these steps: -include::{es-repo-dir}/tab-widgets/troubleshooting/data/enable-cluster-allocation-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/troubleshooting/data/enable-cluster-allocation-widget.asciidoc[] diff --git a/docs/reference/troubleshooting/data/enable-index-allocation.asciidoc b/docs/reference/troubleshooting/data/enable-index-allocation.asciidoc index abdc3cade5c6..a4f266ad2d21 100644 --- a/docs/reference/troubleshooting/data/enable-index-allocation.asciidoc +++ b/docs/reference/troubleshooting/data/enable-index-allocation.asciidoc @@ -10,7 +10,7 @@ Forgetting to re-allow all data allocation can lead to unassigned shards. In order to (re)allow all data to be allocated follow these steps: -include::{es-repo-dir}/tab-widgets/troubleshooting/data/enable-index-allocation-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/troubleshooting/data/enable-index-allocation-widget.asciidoc[] diff --git a/docs/reference/troubleshooting/data/increase-cluster-shard-limit.asciidoc b/docs/reference/troubleshooting/data/increase-cluster-shard-limit.asciidoc index 916677329b2b..8b8703f9a9dc 100644 --- a/docs/reference/troubleshooting/data/increase-cluster-shard-limit.asciidoc +++ b/docs/reference/troubleshooting/data/increase-cluster-shard-limit.asciidoc @@ -14,7 +14,7 @@ satisfy the configuration. In order to fix this follow the next steps: -include::{es-repo-dir}/tab-widgets/troubleshooting/data/increase-cluster-shard-limit-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/troubleshooting/data/increase-cluster-shard-limit-widget.asciidoc[] diff --git a/docs/reference/troubleshooting/data/increase-shard-limit.asciidoc b/docs/reference/troubleshooting/data/increase-shard-limit.asciidoc index f97bff73bf10..121b5348ab36 100644 --- a/docs/reference/troubleshooting/data/increase-shard-limit.asciidoc +++ b/docs/reference/troubleshooting/data/increase-shard-limit.asciidoc @@ -12,7 +12,7 @@ satisfy the index configuration. In order to fix this follow the next steps: -include::{es-repo-dir}/tab-widgets/troubleshooting/data/total-shards-per-node-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/troubleshooting/data/total-shards-per-node-widget.asciidoc[] diff --git a/docs/reference/troubleshooting/data/increase-tier-capacity.asciidoc b/docs/reference/troubleshooting/data/increase-tier-capacity.asciidoc index a413918cee48..362a14c3874d 100644 --- a/docs/reference/troubleshooting/data/increase-tier-capacity.asciidoc +++ b/docs/reference/troubleshooting/data/increase-tier-capacity.asciidoc @@ -15,7 +15,7 @@ replicas, you can influence this behavior by adding more nodes to the cluster In order to fix this follow the next steps: -include::{es-repo-dir}/tab-widgets/troubleshooting/data/increase-tier-capacity-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/troubleshooting/data/increase-tier-capacity-widget.asciidoc[] diff --git a/docs/reference/troubleshooting/data/restore-from-snapshot.asciidoc b/docs/reference/troubleshooting/data/restore-from-snapshot.asciidoc index 95d8d8c4aff8..4b5de3937686 100644 --- a/docs/reference/troubleshooting/data/restore-from-snapshot.asciidoc +++ b/docs/reference/troubleshooting/data/restore-from-snapshot.asciidoc @@ -9,4 +9,4 @@ contain a copy of the data anymore. IMPORTANT: Restoring the missing data requires you to have a backup of the affected indices and data streams that is up-to-date enough for your use case. Please do not proceed without confirming this. -include::{es-repo-dir}/tab-widgets/troubleshooting/data/restore-from-snapshot-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/troubleshooting/data/restore-from-snapshot-widget.asciidoc[] diff --git a/docs/reference/troubleshooting/data/start-ilm.asciidoc b/docs/reference/troubleshooting/data/start-ilm.asciidoc index 18f1bd4f5353..1798af814e50 100644 --- a/docs/reference/troubleshooting/data/start-ilm.asciidoc +++ b/docs/reference/troubleshooting/data/start-ilm.asciidoc @@ -5,7 +5,7 @@ Automatic index lifecycle and data retention management is currently disabled. In order to start the automatic {ilm} service, follow these steps: -include::{es-repo-dir}/tab-widgets/troubleshooting/data/start-ilm-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/troubleshooting/data/start-ilm-widget.asciidoc[] diff --git a/docs/reference/troubleshooting/data/start-slm.asciidoc b/docs/reference/troubleshooting/data/start-slm.asciidoc index 7b5e57cfbf6c..66264e8ac1b2 100644 --- a/docs/reference/troubleshooting/data/start-slm.asciidoc +++ b/docs/reference/troubleshooting/data/start-slm.asciidoc @@ -6,7 +6,7 @@ snapshots will not be created automatically. In order to start the snapshot lifecycle management service, follow these steps: -include::{es-repo-dir}/tab-widgets/troubleshooting/data/start-slm-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/troubleshooting/data/start-slm-widget.asciidoc[] diff --git a/docs/reference/troubleshooting/disk/fix-data-node-out-of-disk.asciidoc b/docs/reference/troubleshooting/disk/fix-data-node-out-of-disk.asciidoc index b98b78582900..67541cb036bf 100644 --- a/docs/reference/troubleshooting/disk/fix-data-node-out-of-disk.asciidoc +++ b/docs/reference/troubleshooting/disk/fix-data-node-out-of-disk.asciidoc @@ -11,7 +11,7 @@ of two ways: [[increase-capacity-data-node]] === Increase the disk capacity of data nodes -include::{es-repo-dir}/tab-widgets/troubleshooting/disk/increase-data-node-capacity-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/troubleshooting/disk/increase-data-node-capacity-widget.asciidoc[] [[decrease-disk-usage-data-node]] === Decrease the disk usage of data nodes @@ -20,4 +20,4 @@ In order to decrease the disk usage in your cluster without losing any data, you NOTE: Reducing the replicas of an index can potentially reduce search throughput and data redundancy. However, it can quickly give the cluster breathing room until a more permanent solution is in place. -include::{es-repo-dir}/tab-widgets/troubleshooting/disk/decrease-data-node-disk-usage-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/troubleshooting/disk/decrease-data-node-disk-usage-widget.asciidoc[] diff --git a/docs/reference/troubleshooting/disk/fix-master-node-out-of-disk.asciidoc b/docs/reference/troubleshooting/disk/fix-master-node-out-of-disk.asciidoc index 6a32ab8aff37..73268a0f433e 100644 --- a/docs/reference/troubleshooting/disk/fix-master-node-out-of-disk.asciidoc +++ b/docs/reference/troubleshooting/disk/fix-master-node-out-of-disk.asciidoc @@ -5,4 +5,4 @@ out of space, you need to ensure that they have enough disk space to function. If the <> reports that your master node is out of space you need to increase the disk capacity of your master nodes. -include::{es-repo-dir}/tab-widgets/troubleshooting/disk/increase-master-node-capacity-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/troubleshooting/disk/increase-master-node-capacity-widget.asciidoc[] diff --git a/docs/reference/troubleshooting/disk/fix-other-node-out-of-disk.asciidoc b/docs/reference/troubleshooting/disk/fix-other-node-out-of-disk.asciidoc index d53e5337fad8..9334e0f17fd8 100644 --- a/docs/reference/troubleshooting/disk/fix-other-node-out-of-disk.asciidoc +++ b/docs/reference/troubleshooting/disk/fix-other-node-out-of-disk.asciidoc @@ -6,4 +6,4 @@ for example machine learning. If one or more of these nodes are running out of s enough disk space to function. If the <> reports that a node that is not a master and does not contain data is out of space you need to increase the disk capacity of this node. -include::{es-repo-dir}/tab-widgets/troubleshooting/disk/increase-other-node-capacity-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/troubleshooting/disk/increase-other-node-capacity-widget.asciidoc[] diff --git a/docs/reference/troubleshooting/snapshot/add-repository.asciidoc b/docs/reference/troubleshooting/snapshot/add-repository.asciidoc index 0de4667bd968..386c2561c03c 100644 --- a/docs/reference/troubleshooting/snapshot/add-repository.asciidoc +++ b/docs/reference/troubleshooting/snapshot/add-repository.asciidoc @@ -16,7 +16,7 @@ guide. To remedy the situation mark the repository as read-only or remove it from all the other deployments, and re-add (recreate) the repository in the current deployment: -include::{es-repo-dir}/tab-widgets/troubleshooting/snapshot/corrupt-repository-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/troubleshooting/snapshot/corrupt-repository-widget.asciidoc[] [[diagnosing-unknown-repositories]] diff --git a/docs/reference/troubleshooting/snapshot/repeated-snapshot-failures.asciidoc b/docs/reference/troubleshooting/snapshot/repeated-snapshot-failures.asciidoc index 4084fdc7ca42..2496781c0c8f 100644 --- a/docs/reference/troubleshooting/snapshot/repeated-snapshot-failures.asciidoc +++ b/docs/reference/troubleshooting/snapshot/repeated-snapshot-failures.asciidoc @@ -12,7 +12,7 @@ repeated failures before reporting a warning is controlled by the In the event that an automated {slm} policy execution is experiencing repeated failures, follow these steps to get more information about the problem: -include::{es-repo-dir}/tab-widgets/troubleshooting/snapshot/repeated-snapshot-failures-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/troubleshooting/snapshot/repeated-snapshot-failures-widget.asciidoc[] diff --git a/docs/reference/troubleshooting/troubleshooting-shards-capacity.asciidoc b/docs/reference/troubleshooting/troubleshooting-shards-capacity.asciidoc index 6a3a5bc06ce8..bc8fb7290f1e 100644 --- a/docs/reference/troubleshooting/troubleshooting-shards-capacity.asciidoc +++ b/docs/reference/troubleshooting/troubleshooting-shards-capacity.asciidoc @@ -7,4 +7,4 @@ The current shards capacity of the cluster is available in the <>. -include::{es-repo-dir}/tab-widgets/troubleshooting/troubleshooting-shards-capacity-widget.asciidoc[] +include::{es-ref-dir}/tab-widgets/troubleshooting/troubleshooting-shards-capacity-widget.asciidoc[] diff --git a/docs/reference/upgrade.asciidoc b/docs/reference/upgrade.asciidoc index 8f6d095971ff..d5057d9b87d8 100644 --- a/docs/reference/upgrade.asciidoc +++ b/docs/reference/upgrade.asciidoc @@ -52,7 +52,7 @@ the REST API. [[upgrade-fips-java17]] === FIPS Compliance and Java 17 -include::{es-repo-dir}/security/fips-java17.asciidoc[] +include::{es-ref-dir}/security/fips-java17.asciidoc[] include::upgrade/archived-settings.asciidoc[] From dbb700c67950c60372a268c8e228c0cefb5ca2b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Wed, 17 Apr 2024 14:39:50 +0200 Subject: [PATCH 26/43] Fix and unmute "Test frequent item sets unsupported types" yml test (#107350) --- .../rest-api-spec/test/ml/frequent_item_sets_agg.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/frequent_item_sets_agg.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/frequent_item_sets_agg.yml index db41e0d0efaa..bc44e8423178 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/frequent_item_sets_agg.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/frequent_item_sets_agg.yml @@ -7,6 +7,9 @@ setup: indices.create: index: store body: + settings: + number_of_shards: 1 + number_of_replicas: 0 mappings: properties: features: @@ -433,9 +436,6 @@ setup: --- "Test frequent item sets unsupported types": - - skip: - version: "all" - reason: "AwaitsFix https://github.com/elastic/elasticsearch/issues/106215" - do: catch: /Field \[geo_point\] of type \[geo_point\] is not supported for aggregation \[frequent_item_sets\]/ search: From adaa4763f3583ea32716a7d9066e347260e15ca4 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Wed, 17 Apr 2024 16:04:51 +0200 Subject: [PATCH 27/43] ESQL: Fix missing refs due to pruning renamed grouping columns (#107328) Sometimes, CombineProjections does not correctly update an aggregation's groupings when combining with a preceding projection. Fix this by resolving any aliases used in the groupings and de-duplicating them. --------- Co-authored-by: Andrei Stefan --- docs/changelog/107328.yaml | 7 +++ .../src/main/resources/stats.csv-spec | 17 ++++++ .../esql/optimizer/LogicalPlanOptimizer.java | 38 +++++++++++- .../optimizer/LogicalPlanOptimizerTests.java | 59 +++++++++++++------ 4 files changed, 103 insertions(+), 18 deletions(-) create mode 100644 docs/changelog/107328.yaml diff --git a/docs/changelog/107328.yaml b/docs/changelog/107328.yaml new file mode 100644 index 000000000000..a608d7567dde --- /dev/null +++ b/docs/changelog/107328.yaml @@ -0,0 +1,7 @@ +pr: 107328 +summary: "ESQL: Fix missing refs due to pruning renamed grouping columns" +area: ES|QL +type: bug +issues: + - 107083 + - 107166 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec index 749c44d1f6ec..113124d3a72a 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec @@ -1586,6 +1586,23 @@ c:l | k1:i | languages:i 10 | null | null ; +evalMultipleOverridingKeysWithAggregateExpr#[skip:-8.13.99,reason:supported in 8.14] +FROM employees +| EVAL k = languages, k1 = k +| STATS c = 3*COUNT() BY languages, k, k1, languages +| DROP k +| SORT languages +; + +c:l | k1:i | languages:i +45 | 1 | 1 +57 | 2 | 2 +51 | 3 | 3 +54 | 4 | 4 +63 | 5 | 5 +30 | null | null +; + minWithSortExpression1#[skip:-8.13.99,reason:supported in 8.14] FROM employees | STATS min = min(salary) by languages | SORT min + languages; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java index 2aaf34a1dd1d..6dd9bc1b74f5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java @@ -389,6 +389,7 @@ static class CombineProjections extends OptimizerRules.OptimizerRule } @Override + @SuppressWarnings("unchecked") protected LogicalPlan rule(UnaryPlan plan) { LogicalPlan child = plan.child(); @@ -419,7 +420,22 @@ protected LogicalPlan rule(UnaryPlan plan) { // Agg with underlying Project (group by on sub-queries) if (plan instanceof Aggregate a) { if (child instanceof Project p) { - plan = new Aggregate(a.source(), p.child(), a.groupings(), combineProjections(a.aggregates(), p.projections())); + var groupings = a.groupings(); + List groupingAttrs = new ArrayList<>(a.groupings().size()); + for (Expression grouping : groupings) { + if (grouping instanceof Attribute attribute) { + groupingAttrs.add(attribute); + } else { + // After applying ReplaceStatsNestedExpressionWithEval, groupings can only contain attributes. + throw new EsqlIllegalArgumentException("Expected an Attribute, got {}", grouping); + } + } + plan = new Aggregate( + a.source(), + p.child(), + combineUpperGroupingsAndLowerProjections(groupingAttrs, p.projections()), + combineProjections(a.aggregates(), p.projections()) + ); } } @@ -482,6 +498,26 @@ private static List combineProjections( return replaced; } + private static List combineUpperGroupingsAndLowerProjections( + List upperGroupings, + List lowerProjections + ) { + // Collect the alias map for resolving the source (f1 = 1, f2 = f1, etc..) + AttributeMap aliases = new AttributeMap<>(); + for (NamedExpression ne : lowerProjections) { + // Projections are just aliases for attributes, so casting is safe. + aliases.put(ne.toAttribute(), (Attribute) Alias.unwrap(ne)); + } + + // Replace any matching attribute directly with the aliased attribute from the projection. + AttributeSet replaced = new AttributeSet(); + for (Attribute attr : upperGroupings) { + // All substitutions happen before; groupings must be attributes at this point. + replaced.add(aliases.resolve(attr, attr)); + } + return new ArrayList<>(replaced); + } + /** * Replace grouping alias previously contained in the aggregations that might have been projected away. */ diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index 579a998755eb..ba4f15533d26 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -417,8 +417,8 @@ public void testCombineProjectionWithAggregation() { /** * Expects * Limit[1000[INTEGER]] - * \_Aggregate[[last_name{f}#23, first_name{f}#20, k{r}#4],[SUM(salary{f}#24) AS s, last_name{f}#23, first_name{f}#20, first_n - * ame{f}#20 AS k]] + * \_Aggregate[[last_name{f}#23, first_name{f}#20],[SUM(salary{f}#24) AS s, last_name{f}#23, first_name{f}#20, first_name{f}#2 + * 0 AS k]] * \_EsRelation[test][_meta_field{f}#25, emp_no{f}#19, first_name{f}#20, ..] */ public void testCombineProjectionWithAggregationAndEval() { @@ -432,7 +432,7 @@ public void testCombineProjectionWithAggregationAndEval() { var limit = as(plan, Limit.class); var agg = as(limit.child(), Aggregate.class); assertThat(Expressions.names(agg.aggregates()), contains("s", "last_name", "first_name", "k")); - assertThat(Expressions.names(agg.groupings()), contains("last_name", "first_name", "k")); + assertThat(Expressions.names(agg.groupings()), contains("last_name", "first_name")); } /** @@ -552,6 +552,12 @@ public void testCombineDisjunctionToInFromIn() { assertThat(condition.list(), equalTo(List.of(new Literal(EMPTY, 1, INTEGER), new Literal(EMPTY, 2, INTEGER)))); } + /** + * Expects + * Limit[1000[INTEGER]] + * \_Aggregate[[first_name{f}#12],[COUNT(salary{f}#16) AS count(salary), first_name{f}#12 AS x]] + * \_EsRelation[test][_meta_field{f}#17, emp_no{f}#11, first_name{f}#12, ..] + */ public void testCombineProjectionWithPruning() { var plan = plan(""" from test @@ -563,19 +569,17 @@ public void testCombineProjectionWithPruning() { var limit = as(plan, Limit.class); var agg = as(limit.child(), Aggregate.class); assertThat(Expressions.names(agg.aggregates()), contains("count(salary)", "x")); - assertThat(Expressions.names(agg.groupings()), contains("x")); + assertThat(Expressions.names(agg.groupings()), contains("first_name")); var alias = as(agg.aggregates().get(1), Alias.class); var field = as(alias.child(), FieldAttribute.class); assertThat(field.name(), is("first_name")); - var group = as(agg.groupings().get(0), Attribute.class); - assertThat(group, is(alias.toAttribute())); var from = as(agg.child(), EsRelation.class); } /** * Expects * Limit[1000[INTEGER]] - * \_Aggregate[[f{r}#7],[SUM(emp_no{f}#15) AS s, COUNT(first_name{f}#16) AS c, first_name{f}#16 AS f]] + * \_Aggregate[[first_name{f}#16],[SUM(emp_no{f}#15) AS s, COUNT(first_name{f}#16) AS c, first_name{f}#16 AS f]] * \_EsRelation[test][_meta_field{f}#21, emp_no{f}#15, first_name{f}#16, ..] */ public void testCombineProjectionWithAggregationFirstAndAliasedGroupingUsedInAgg() { @@ -599,13 +603,13 @@ public void testCombineProjectionWithAggregationFirstAndAliasedGroupingUsedInAgg as = as(aggs.get(2), Alias.class); assertThat(Expressions.name(as.child()), is("first_name")); - assertThat(Expressions.names(agg.groupings()), contains("f")); + assertThat(Expressions.names(agg.groupings()), contains("first_name")); } /** * Expects * Limit[1000[INTEGER]] - * \_Aggregate[[f{r}#7],[SUM(emp_no{f}#15) AS s, first_name{f}#16 AS f]] + * \_Aggregate[[first_name{f}#16],[SUM(emp_no{f}#15) AS s, first_name{f}#16 AS f]] * \_EsRelation[test][_meta_field{f}#21, emp_no{f}#15, first_name{f}#16, ..] */ public void testCombineProjectionWithAggregationFirstAndAliasedGroupingUnused() { @@ -625,7 +629,7 @@ public void testCombineProjectionWithAggregationFirstAndAliasedGroupingUnused() as = as(aggs.get(1), Alias.class); assertThat(Expressions.name(as.child()), is("first_name")); - assertThat(Expressions.names(agg.groupings()), contains("f")); + assertThat(Expressions.names(agg.groupings()), contains("first_name")); } /** @@ -2786,6 +2790,27 @@ public void testEliminateDuplicateAggsNonCount() { var source = as(agg.child(), EsRelation.class); } + /** + * Expects + * Limit[1000[INTEGER]] + * \_Aggregate[[salary{f}#12],[salary{f}#12, salary{f}#12 AS x]] + * \_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..] + */ + public void testEliminateDuplicateRenamedGroupings() { + var plan = plan(""" + from test + | eval x = salary + | stats by salary, x + """); + + var limit = as(plan, Limit.class); + var agg = as(limit.child(), Aggregate.class); + var relation = as(agg.child(), EsRelation.class); + + assertThat(Expressions.names(agg.groupings()), contains("salary")); + assertThat(Expressions.names(agg.aggregates()), contains("salary", "x")); + } + /** * Expected * Limit[2[INTEGER]] @@ -2832,7 +2857,7 @@ public void testRenameStatsDropGroup() { /** * Expected * Limit[1000[INTEGER]] - * \_Aggregate[[a{r}#2, bar{r}#8],[COUNT([2a][KEYWORD]) AS baz, b{r}#4 AS bar]] + * \_Aggregate[[a{r}#3, b{r}#5],[COUNT([2a][KEYWORD]) AS baz, b{r}#5 AS bar]] * \_Row[[1[INTEGER] AS a, 2[INTEGER] AS b]] */ public void testMultipleRenameStatsDropGroup() { @@ -2844,15 +2869,15 @@ public void testMultipleRenameStatsDropGroup() { var limit = as(plan, Limit.class); var agg = as(limit.child(), Aggregate.class); - assertThat(Expressions.names(agg.groupings()), contains("a", "bar")); + assertThat(Expressions.names(agg.groupings()), contains("a", "b")); var row = as(agg.child(), Row.class); } /** * Expected * Limit[1000[INTEGER]] - * \_Aggregate[[emp_no{f}#11, bar{r}#4],[MAX(salary{f}#16) AS baz, gender{f}#13 AS bar]] - * \_EsRelation[test][_meta_field{f}#17, emp_no{f}#11, first_name{f}#12, ..] + * \_Aggregate[[emp_no{f}#14, gender{f}#16],[MAX(salary{f}#19) AS baz, gender{f}#16 AS bar]] + * \_EsRelation[test][_meta_field{f}#20, emp_no{f}#14, first_name{f}#15, ..] */ public void testMultipleRenameStatsDropGroupMultirow() { LogicalPlan plan = optimizedPlan(""" @@ -2863,7 +2888,7 @@ public void testMultipleRenameStatsDropGroupMultirow() { var limit = as(plan, Limit.class); var agg = as(limit.child(), Aggregate.class); - assertThat(Expressions.names(agg.groupings()), contains("emp_no", "bar")); + assertThat(Expressions.names(agg.groupings()), contains("emp_no", "gender")); var row = as(agg.child(), EsRelation.class); } @@ -2944,7 +2969,7 @@ public void testIsNotNullConstraintForStatsWithAndOnGrouping() { /** * Expects * Limit[1000[INTEGER]] - * \_Aggregate[[x{r}#4],[SUM(salary{f}#13) AS sum(salary), salary{f}#13 AS x]] + * \_Aggregate[[salary{f}#13],[SUM(salary{f}#13) AS sum(salary), salary{f}#13 AS x]] * \_EsRelation[test][_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, ge..] */ public void testIsNotNullConstraintForStatsWithAndOnGroupingAlias() { @@ -2956,7 +2981,7 @@ public void testIsNotNullConstraintForStatsWithAndOnGroupingAlias() { var limit = as(plan, Limit.class); var agg = as(limit.child(), Aggregate.class); - assertThat(Expressions.names(agg.groupings()), contains("x")); + assertThat(Expressions.names(agg.groupings()), contains("salary")); assertThat(Expressions.names(agg.aggregates()), contains("sum(salary)", "x")); var from = as(agg.child(), EsRelation.class); } From 3df8afbafc7f13c523f37cc68a296b2be8a03b9f Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 17 Apr 2024 15:06:56 +0100 Subject: [PATCH 28/43] AwaitsFix for #107568 --- .../compute/operator/TimeSeriesAggregationOperatorTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/TimeSeriesAggregationOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/TimeSeriesAggregationOperatorTests.java index 15c2cb1c5721..6fe19d6eb0e6 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/TimeSeriesAggregationOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/TimeSeriesAggregationOperatorTests.java @@ -265,6 +265,7 @@ public void testBasicRateOrdinalBased() { // TODO: in a follow up add support for ordinal based time series grouping operator // (and then remove this test) // (ordinal based can only group by one field and never includes timestamp) + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/107568") public void testRandomRateOrdinalBased() { int numPods = between(1, 10); List pods = new ArrayList<>(); From eb6af0e6b549d0810d8b89791556bc791b766652 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Wed, 17 Apr 2024 15:29:22 +0100 Subject: [PATCH 29/43] Refactor PathTrie to tidy it up (#107542) --- .../elasticsearch/common/path/PathTrie.java | 138 +++++++----------- .../elasticsearch/rest/RestController.java | 2 +- .../org/elasticsearch/rest/RestUtils.java | 4 +- .../common/path/PathTrieTests.java | 93 ++++-------- 4 files changed, 88 insertions(+), 149 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/path/PathTrie.java b/server/src/main/java/org/elasticsearch/common/path/PathTrie.java index 306304ab016a..bec7dea64502 100644 --- a/server/src/main/java/org/elasticsearch/common/path/PathTrie.java +++ b/server/src/main/java/org/elasticsearch/common/path/PathTrie.java @@ -10,13 +10,15 @@ import org.elasticsearch.common.collect.Iterators; +import java.util.Arrays; import java.util.EnumSet; import java.util.HashMap; import java.util.Iterator; import java.util.Map; -import java.util.NoSuchElementException; -import java.util.function.BiFunction; +import java.util.function.BinaryOperator; import java.util.function.Supplier; +import java.util.function.UnaryOperator; +import java.util.stream.Stream; import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableMap; @@ -50,52 +52,43 @@ enum TrieMatchingMode { TrieMatchingMode.WILDCARD_ROOT_NODES_ALLOWED ); - public interface Decoder { - String decode(String value); - } - - private final Decoder decoder; + private final UnaryOperator decoder; private final TrieNode root; private T rootValue; private static final String SEPARATOR = "/"; private static final String WILDCARD = "*"; - public PathTrie(Decoder decoder) { + public PathTrie(UnaryOperator decoder) { this.decoder = decoder; - root = new TrieNode(SEPARATOR, null, WILDCARD); + root = new TrieNode(SEPARATOR, null); } - public class TrieNode { - private transient String key; - private transient T value; - private final String wildcard; - - private transient String namedWildcard; - + private class TrieNode { + private T value; + private String namedWildcard; private Map children; - private TrieNode(String key, T value, String wildcard) { - this.key = key; - this.wildcard = wildcard; + private TrieNode(String key, T value) { this.value = value; this.children = emptyMap(); if (isNamedWildcard(key)) { - namedWildcard = key.substring(key.indexOf('{') + 1, key.indexOf('}')); + updateNamedWildcard(key); } else { namedWildcard = null; } } - private void updateKeyWithNamedWildcard(String key) { - this.key = key; - String newNamedWildcard = key.substring(key.indexOf('{') + 1, key.indexOf('}')); - if (namedWildcard != null && newNamedWildcard.equals(namedWildcard) == false) { - throw new IllegalArgumentException( - "Trying to use conflicting wildcard names for same path: " + namedWildcard + " and " + newNamedWildcard - ); + private void updateNamedWildcard(String key) { + String newNamedWildcard = key.substring(1, key.length() - 1); + if (newNamedWildcard.equals(namedWildcard) == false) { + if (namedWildcard != null) { + throw new IllegalArgumentException( + "Trying to use conflicting wildcard names for same path: " + namedWildcard + " and " + newNamedWildcard + ); + } + namedWildcard = newNamedWildcard; } - namedWildcard = newNamedWildcard; } private void addInnerChild(String key, TrieNode child) { @@ -110,16 +103,17 @@ private synchronized void insert(String[] path, int index, T value) { String token = path[index]; String key = token; if (isNamedWildcard(token)) { - key = wildcard; + key = WILDCARD; } + TrieNode node = children.get(key); if (node == null) { T nodeValue = index == path.length - 1 ? value : null; - node = new TrieNode(token, nodeValue, wildcard); + node = new TrieNode(token, nodeValue); addInnerChild(key, node); } else { if (isNamedWildcard(token)) { - node.updateKeyWithNamedWildcard(token); + node.updateNamedWildcard(token); } /* * If the target node already exists, but is without a value, @@ -139,22 +133,23 @@ private synchronized void insert(String[] path, int index, T value) { node.insert(path, index + 1, value); } - private synchronized void insertOrUpdate(String[] path, int index, T value, BiFunction updater) { + private synchronized void insertOrUpdate(String[] path, int index, T value, BinaryOperator updater) { if (index >= path.length) return; String token = path[index]; String key = token; if (isNamedWildcard(token)) { - key = wildcard; + key = WILDCARD; } + TrieNode node = children.get(key); if (node == null) { T nodeValue = index == path.length - 1 ? value : null; - node = new TrieNode(token, nodeValue, wildcard); + node = new TrieNode(token, nodeValue); addInnerChild(key, node); } else { if (isNamedWildcard(token)) { - node.updateKeyWithNamedWildcard(token); + node.updateNamedWildcard(token); } /* * If the target node already exists, but is without a value, @@ -173,7 +168,7 @@ private synchronized void insertOrUpdate(String[] path, int index, T value, BiFu } private static boolean isNamedWildcard(String key) { - return key.indexOf('{') != -1 && key.indexOf('}') != -1; + return key.charAt(0) == '{' && key.charAt(key.length() - 1) == '}'; } private String namedWildcard() { @@ -184,7 +179,7 @@ private boolean isNamedWildcard() { return namedWildcard != null; } - public T retrieve(String[] path, int index, Map params, TrieMatchingMode trieMatchingMode) { + private T retrieve(String[] path, int index, Map params, TrieMatchingMode trieMatchingMode) { if (index >= path.length) return null; String token = path[index]; @@ -193,7 +188,7 @@ public T retrieve(String[] path, int index, Map params, TrieMatc if (node == null) { if (trieMatchingMode == TrieMatchingMode.WILDCARD_NODES_ALLOWED) { - node = children.get(wildcard); + node = children.get(WILDCARD); if (node == null) { return null; } @@ -202,7 +197,7 @@ public T retrieve(String[] path, int index, Map params, TrieMatc /* * Allow root node wildcard matches. */ - node = children.get(wildcard); + node = children.get(WILDCARD); if (node == null) { return null; } @@ -211,7 +206,7 @@ public T retrieve(String[] path, int index, Map params, TrieMatc /* * Allow leaf node wildcard matches. */ - node = children.get(wildcard); + node = children.get(WILDCARD); if (node == null) { return null; } @@ -220,32 +215,33 @@ public T retrieve(String[] path, int index, Map params, TrieMatc return null; } } else { + TrieNode wildcardNode; if (index + 1 == path.length && node.value == null - && children.get(wildcard) != null - && EXPLICIT_OR_ROOT_WILDCARD.contains(trieMatchingMode) == false) { + && EXPLICIT_OR_ROOT_WILDCARD.contains(trieMatchingMode) == false + && (wildcardNode = children.get(WILDCARD)) != null) { /* * If we are at the end of the path, the current node does not have a value but * there is a child wildcard node, use the child wildcard node. */ - node = children.get(wildcard); + node = wildcardNode; usedWildcard = true; } else if (index == 1 && node.value == null - && children.get(wildcard) != null - && trieMatchingMode == TrieMatchingMode.WILDCARD_ROOT_NODES_ALLOWED) { + && trieMatchingMode == TrieMatchingMode.WILDCARD_ROOT_NODES_ALLOWED + && (wildcardNode = children.get(WILDCARD)) != null) { /* * If we are at the root, and root wildcards are allowed, use the child wildcard * node. */ - node = children.get(wildcard); + node = wildcardNode; usedWildcard = true; } else { - usedWildcard = token.equals(wildcard); + usedWildcard = token.equals(WILDCARD); } } - put(params, node, token); + recordWildcardParam(params, node, token); if (index == (path.length - 1)) { return node.value; @@ -253,9 +249,9 @@ public T retrieve(String[] path, int index, Map params, TrieMatc T nodeValue = node.retrieve(path, index + 1, params, trieMatchingMode); if (nodeValue == null && usedWildcard == false && trieMatchingMode != TrieMatchingMode.EXPLICIT_NODES_ONLY) { - node = children.get(wildcard); + node = children.get(WILDCARD); if (node != null) { - put(params, node, token); + recordWildcardParam(params, node, token); nodeValue = node.retrieve(path, index + 1, params, trieMatchingMode); } } @@ -263,13 +259,13 @@ public T retrieve(String[] path, int index, Map params, TrieMatc return nodeValue; } - private void put(Map params, TrieNode node, String value) { + private void recordWildcardParam(Map params, TrieNode node, String value) { if (params != null && node.isNamedWildcard()) { - params.put(node.namedWildcard(), decoder.decode(value)); + params.put(node.namedWildcard(), decoder.apply(value)); } } - Iterator allNodeValues() { + private Iterator allNodeValues() { final Iterator childrenIterator = Iterators.flatMap(children.values().iterator(), TrieNode::allNodeValues); if (value == null) { return childrenIterator; @@ -277,11 +273,6 @@ Iterator allNodeValues() { return Iterators.concat(Iterators.single(value), childrenIterator); } } - - @Override - public String toString() { - return key; - } } public void insert(String path, T value) { @@ -308,7 +299,7 @@ public void insert(String path, T value) { * * allowing the value to be updated if desired. */ - public void insertOrUpdate(String path, T value, BiFunction updater) { + public void insertOrUpdate(String path, T value, BinaryOperator updater) { String[] strings = path.split(SEPARATOR); if (strings.length == 0) { if (rootValue != null) { @@ -334,8 +325,8 @@ public T retrieve(String path, Map params) { return retrieve(path, params, TrieMatchingMode.WILDCARD_NODES_ALLOWED); } - public T retrieve(String path, Map params, TrieMatchingMode trieMatchingMode) { - if (path.length() == 0) { + T retrieve(String path, Map params, TrieMatchingMode trieMatchingMode) { + if (path.isEmpty()) { return rootValue; } String[] strings = path.split(SEPARATOR); @@ -353,29 +344,12 @@ public T retrieve(String path, Map params, TrieMatchingMode trie } /** - * Returns an iterator of the objects stored in the {@code PathTrie}, using + * Returns a stream of the objects stored in the {@code PathTrie}, using * all possible {@code TrieMatchingMode} modes. The {@code paramSupplier} - * is called between each invocation of {@code next()} to supply a new map - * of parameters. + * is called for each mode to supply a new map of parameters. */ - public Iterator retrieveAll(String path, Supplier> paramSupplier) { - return new Iterator<>() { - - private int mode; - - @Override - public boolean hasNext() { - return mode < TrieMatchingMode.values().length; - } - - @Override - public T next() { - if (hasNext() == false) { - throw new NoSuchElementException("called next() without validating hasNext()! no more modes available"); - } - return retrieve(path, paramSupplier.get(), TrieMatchingMode.values()[mode++]); - } - }; + public Stream retrieveAll(String path, Supplier> paramSupplier) { + return Arrays.stream(TrieMatchingMode.values()).map(m -> retrieve(path, paramSupplier.get(), m)); } public Iterator allNodeValues() { diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java index d197fe50d60d..8ce9b08eba20 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestController.java +++ b/server/src/main/java/org/elasticsearch/rest/RestController.java @@ -644,7 +644,7 @@ Iterator getAllHandlers(@Nullable Map requestPar // we use rawPath since we don't want to decode it while processing the path resolution // so we can handle things like: // my_index/my_type/http%3A%2F%2Fwww.google.com - return handlers.retrieveAll(rawPath, paramsSupplier); + return handlers.retrieveAll(rawPath, paramsSupplier).iterator(); } /** diff --git a/server/src/main/java/org/elasticsearch/rest/RestUtils.java b/server/src/main/java/org/elasticsearch/rest/RestUtils.java index aa693f38a3e6..4aa82f5e4b7c 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestUtils.java +++ b/server/src/main/java/org/elasticsearch/rest/RestUtils.java @@ -9,7 +9,6 @@ package org.elasticsearch.rest; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.path.PathTrie; import org.elasticsearch.core.Booleans; import java.nio.charset.Charset; @@ -17,6 +16,7 @@ import java.util.Arrays; import java.util.Map; import java.util.Optional; +import java.util.function.UnaryOperator; import java.util.regex.Pattern; import static org.elasticsearch.rest.RestRequest.PATH_RESTRICTED; @@ -28,7 +28,7 @@ public class RestUtils { */ private static final boolean DECODE_PLUS_AS_SPACE = Booleans.parseBoolean(System.getProperty("es.rest.url_plus_as_space", "false")); - public static final PathTrie.Decoder REST_DECODER = RestUtils::decodeComponent; + public static final UnaryOperator REST_DECODER = RestUtils::decodeComponent; public static void decodeQueryString(String s, int fromIndex, Map params) { if (fromIndex < 0) { diff --git a/server/src/test/java/org/elasticsearch/common/path/PathTrieTests.java b/server/src/test/java/org/elasticsearch/common/path/PathTrieTests.java index 6b2e0122a599..39eaf0bb8c57 100644 --- a/server/src/test/java/org/elasticsearch/common/path/PathTrieTests.java +++ b/server/src/test/java/org/elasticsearch/common/path/PathTrieTests.java @@ -13,20 +13,16 @@ import org.elasticsearch.test.ESTestCase; import java.util.HashMap; -import java.util.Iterator; import java.util.Map; +import java.util.function.UnaryOperator; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; public class PathTrieTests extends ESTestCase { - public static final PathTrie.Decoder NO_DECODER = new PathTrie.Decoder() { - @Override - public String decode(String value) { - return value; - } - }; + public static final UnaryOperator NO_DECODER = UnaryOperator.identity(); public void testPath() { PathTrie trie = new PathTrie<>(NO_DECODER); @@ -50,9 +46,7 @@ public void testPath() { Map params = new HashMap<>(); assertThat(trie.retrieve("index1/insert/12", params), equalTo("bingo")); - assertThat(params.size(), equalTo(2)); - assertThat(params.get("index"), equalTo("index1")); - assertThat(params.get("docId"), equalTo("12")); + assertThat(params, equalTo(Map.of("index", "index1", "docId", "12"))); } public void testEmptyPath() { @@ -68,11 +62,11 @@ public void testDifferentNamesOnDifferentPath() { Map params = new HashMap<>(); assertThat(trie.retrieve("/a/test", params), equalTo("test1")); - assertThat(params.get("type"), equalTo("test")); + assertThat(params, equalTo(Map.of("type", "test"))); params.clear(); assertThat(trie.retrieve("/b/testX", params), equalTo("test2")); - assertThat(params.get("name"), equalTo("testX")); + assertThat(params, equalTo(Map.of("name", "testX"))); } public void testSameNameOnDifferentPath() { @@ -82,11 +76,11 @@ public void testSameNameOnDifferentPath() { Map params = new HashMap<>(); assertThat(trie.retrieve("/a/c/test", params), equalTo("test1")); - assertThat(params.get("name"), equalTo("test")); + assertThat(params, equalTo(Map.of("name", "test"))); params.clear(); assertThat(trie.retrieve("/b/testX", params), equalTo("test2")); - assertThat(params.get("name"), equalTo("testX")); + assertThat(params, equalTo(Map.of("name", "testX"))); } public void testPreferNonWildcardExecution() { @@ -125,56 +119,31 @@ public void testWildcardMatchingModes() { assertThat(trie.retrieve("/a", params, TrieMatchingMode.WILDCARD_ROOT_NODES_ALLOWED), equalTo("test1")); assertThat(trie.retrieve("/a", params, TrieMatchingMode.WILDCARD_LEAF_NODES_ALLOWED), equalTo("test1")); assertThat(trie.retrieve("/a", params, TrieMatchingMode.WILDCARD_NODES_ALLOWED), equalTo("test1")); - Iterator allPaths = trie.retrieveAll("/a", () -> params); - assertThat(allPaths.next(), equalTo(null)); - assertThat(allPaths.next(), equalTo("test1")); - assertThat(allPaths.next(), equalTo("test1")); - assertThat(allPaths.next(), equalTo("test1")); - assertFalse(allPaths.hasNext()); + assertThat(trie.retrieveAll("/a", () -> params).toList(), contains(null, "test1", "test1", "test1")); assertThat(trie.retrieve("/a/b", params, TrieMatchingMode.EXPLICIT_NODES_ONLY), nullValue()); assertThat(trie.retrieve("/a/b", params, TrieMatchingMode.WILDCARD_ROOT_NODES_ALLOWED), equalTo("test4")); assertThat(trie.retrieve("/a/b", params, TrieMatchingMode.WILDCARD_LEAF_NODES_ALLOWED), equalTo("test3")); assertThat(trie.retrieve("/a/b", params, TrieMatchingMode.WILDCARD_NODES_ALLOWED), equalTo("test3")); - allPaths = trie.retrieveAll("/a/b", () -> params); - assertThat(allPaths.next(), equalTo(null)); - assertThat(allPaths.next(), equalTo("test4")); - assertThat(allPaths.next(), equalTo("test3")); - assertThat(allPaths.next(), equalTo("test3")); - assertFalse(allPaths.hasNext()); + assertThat(trie.retrieveAll("/a/b", () -> params).toList(), contains(null, "test4", "test3", "test3")); assertThat(trie.retrieve("/a/b/c", params, TrieMatchingMode.EXPLICIT_NODES_ONLY), nullValue()); assertThat(trie.retrieve("/a/b/c", params, TrieMatchingMode.WILDCARD_ROOT_NODES_ALLOWED), equalTo("test5")); assertThat(trie.retrieve("/a/b/c", params, TrieMatchingMode.WILDCARD_LEAF_NODES_ALLOWED), equalTo("test7")); assertThat(trie.retrieve("/a/b/c", params, TrieMatchingMode.WILDCARD_NODES_ALLOWED), equalTo("test7")); - allPaths = trie.retrieveAll("/a/b/c", () -> params); - assertThat(allPaths.next(), equalTo(null)); - assertThat(allPaths.next(), equalTo("test5")); - assertThat(allPaths.next(), equalTo("test7")); - assertThat(allPaths.next(), equalTo("test7")); - assertFalse(allPaths.hasNext()); + assertThat(trie.retrieveAll("/a/b/c", () -> params).toList(), contains(null, "test5", "test7", "test7")); assertThat(trie.retrieve("/x/y/z", params, TrieMatchingMode.EXPLICIT_NODES_ONLY), nullValue()); assertThat(trie.retrieve("/x/y/z", params, TrieMatchingMode.WILDCARD_ROOT_NODES_ALLOWED), nullValue()); assertThat(trie.retrieve("/x/y/z", params, TrieMatchingMode.WILDCARD_LEAF_NODES_ALLOWED), nullValue()); assertThat(trie.retrieve("/x/y/z", params, TrieMatchingMode.WILDCARD_NODES_ALLOWED), equalTo("test9")); - allPaths = trie.retrieveAll("/x/y/z", () -> params); - assertThat(allPaths.next(), equalTo(null)); - assertThat(allPaths.next(), equalTo(null)); - assertThat(allPaths.next(), equalTo(null)); - assertThat(allPaths.next(), equalTo("test9")); - assertFalse(allPaths.hasNext()); + assertThat(trie.retrieveAll("/x/y/z", () -> params).toList(), contains(null, null, null, "test9")); assertThat(trie.retrieve("/d/e/f", params, TrieMatchingMode.EXPLICIT_NODES_ONLY), nullValue()); assertThat(trie.retrieve("/d/e/f", params, TrieMatchingMode.WILDCARD_ROOT_NODES_ALLOWED), nullValue()); assertThat(trie.retrieve("/d/e/f", params, TrieMatchingMode.WILDCARD_LEAF_NODES_ALLOWED), nullValue()); assertThat(trie.retrieve("/d/e/f", params, TrieMatchingMode.WILDCARD_NODES_ALLOWED), equalTo("test10")); - allPaths = trie.retrieveAll("/d/e/f", () -> params); - assertThat(allPaths.next(), equalTo(null)); - assertThat(allPaths.next(), equalTo(null)); - assertThat(allPaths.next(), equalTo(null)); - assertThat(allPaths.next(), equalTo("test10")); - assertFalse(allPaths.hasNext()); + assertThat(trie.retrieveAll("/d/e/f", () -> params).toList(), contains(null, null, null, "test10")); } // https://github.com/elastic/elasticsearch/pull/17916 @@ -208,13 +177,11 @@ public void testSamePathConcreteResolution() { Map params = new HashMap<>(); assertThat(trie.retrieve("/a/b/c", params), equalTo("test1")); - assertThat(params.get("x"), equalTo("a")); - assertThat(params.get("y"), equalTo("b")); - assertThat(params.get("z"), equalTo("c")); + assertThat(params, equalTo(Map.of("x", "a", "y", "b", "z", "c"))); + params.clear(); assertThat(trie.retrieve("/a/_y/c", params), equalTo("test2")); - assertThat(params.get("x"), equalTo("a")); - assertThat(params.get("k"), equalTo("c")); + assertThat(params, equalTo(Map.of("x", "a", "k", "c"))); } public void testNamedWildcardAndLookupWithWildcard() { @@ -227,23 +194,23 @@ public void testNamedWildcardAndLookupWithWildcard() { Map params = new HashMap<>(); assertThat(trie.retrieve("/x/*", params), equalTo("test1")); - assertThat(params.get("test"), equalTo("*")); + assertThat(params, equalTo(Map.of("test", "*"))); - params = new HashMap<>(); + params.clear(); assertThat(trie.retrieve("/b/a", params), equalTo("test2")); - assertThat(params.get("test"), equalTo("b")); + assertThat(params, equalTo(Map.of("test", "b"))); - params = new HashMap<>(); + params.clear(); assertThat(trie.retrieve("/*", params), equalTo("test3")); - assertThat(params.get("test"), equalTo("*")); + assertThat(params, equalTo(Map.of("test", "*"))); - params = new HashMap<>(); + params.clear(); assertThat(trie.retrieve("/*/_endpoint", params), equalTo("test4")); - assertThat(params.get("test"), equalTo("*")); + assertThat(params, equalTo(Map.of("test", "*"))); - params = new HashMap<>(); + params.clear(); assertThat(trie.retrieve("a/*/_endpoint", params), equalTo("test5")); - assertThat(params.get("test"), equalTo("*")); + assertThat(params, equalTo(Map.of("test", "*"))); } // https://github.com/elastic/elasticsearch/issues/14177 @@ -252,14 +219,12 @@ public void testEscapedSlashWithinUrl() { PathTrie pathTrie = new PathTrie<>(RestUtils.REST_DECODER); pathTrie.insert("/{index}/{type}/{id}", "test"); HashMap params = new HashMap<>(); + assertThat(pathTrie.retrieve("/index/type/a%2Fe", params), equalTo("test")); - assertThat(params.get("index"), equalTo("index")); - assertThat(params.get("type"), equalTo("type")); - assertThat(params.get("id"), equalTo("a/e")); + assertThat(params, equalTo(Map.of("index", "index", "type", "type", "id", "a/e"))); + params.clear(); assertThat(pathTrie.retrieve("//type/id", params), equalTo("test")); - assertThat(params.get("index"), equalTo("")); - assertThat(params.get("type"), equalTo("type")); - assertThat(params.get("id"), equalTo("id")); + assertThat(params, equalTo(Map.of("index", "", "type", "type", "id", "id"))); } } From 5f5947799b94299b9f9f46483a81641be9400206 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Wed, 17 Apr 2024 11:08:51 -0500 Subject: [PATCH 30/43] Adding a putIfAbsent() method on EnrichCache (#107499) --- .../xpack/enrich/EnrichCache.java | 29 +++++ .../xpack/enrich/EnrichProcessorFactory.java | 19 ++- .../xpack/enrich/EnrichCacheTests.java | 118 ++++++++++++++++++ 3 files changed, 156 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichCache.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichCache.java index 749bce59a4bb..722328b6b76d 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichCache.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichCache.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.enrich; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.metadata.IndexAbstraction; @@ -24,6 +25,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.BiConsumer; /** * A simple cache for enrich that uses {@link Cache}. There is one instance of this cache and @@ -52,6 +54,32 @@ public final class EnrichCache { this.cache = CacheBuilder.>>builder().setMaximumWeight(maxSize).build(); } + /** + * This method notifies the given listener of the value in this cache for the given searchRequest. If there is no value in the cache + * for the searchRequest, then the new cache value is computed using searchResponseFetcher. + * @param searchRequest The key for the cache request + * @param searchResponseFetcher The function used to compute the value to be put in the cache, if there is no value in the cache already + * @param listener A listener to be notified of the value in the cache + */ + public void computeIfAbsent( + SearchRequest searchRequest, + BiConsumer> searchResponseFetcher, + ActionListener>> listener + ) { + // intentionally non-locking for simplicity...it's OK if we re-put the same key/value in the cache during a race condition. + List> response = get(searchRequest); + if (response != null) { + listener.onResponse(response); + } else { + searchResponseFetcher.accept(searchRequest, ActionListener.wrap(resp -> { + List> value = toCacheValue(resp); + put(searchRequest, value); + listener.onResponse(deepCopy(value, false)); + }, listener::onFailure)); + } + } + + // non-private for unit testing only List> get(SearchRequest searchRequest) { String enrichIndex = getEnrichIndexKey(searchRequest); CacheKey cacheKey = new CacheKey(enrichIndex, searchRequest); @@ -64,6 +92,7 @@ public final class EnrichCache { } } + // non-private for unit testing only void put(SearchRequest searchRequest, List> response) { String enrichIndex = getEnrichIndexKey(searchRequest); CacheKey cacheKey = new CacheKey(enrichIndex, searchRequest); diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java index 907ebb0c9ce3..9890a96aae82 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java @@ -131,16 +131,15 @@ public void accept(ClusterState state) { Client originClient = new OriginSettingClient(client, ENRICH_ORIGIN); return (req, handler) -> { // intentionally non-locking for simplicity...it's OK if we re-put the same key/value in the cache during a race condition. - List> response = enrichCache.get(req); - if (response != null) { - handler.accept(response, null); - } else { - originClient.execute(EnrichCoordinatorProxyAction.INSTANCE, req, ActionListener.wrap(resp -> { - List> value = EnrichCache.toCacheValue(resp); - enrichCache.put(req, value); - handler.accept(EnrichCache.deepCopy(value, false), null); - }, e -> { handler.accept(null, e); })); - } + enrichCache.computeIfAbsent( + req, + (searchRequest, searchResponseActionListener) -> originClient.execute( + EnrichCoordinatorProxyAction.INSTANCE, + searchRequest, + searchResponseActionListener + ), + ActionListener.wrap(resp -> handler.accept(resp, null), e -> handler.accept(null, e)) + ); }; } } diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichCacheTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichCacheTests.java index 735d68f61416..fe3c3b3e467e 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichCacheTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichCacheTests.java @@ -6,21 +6,31 @@ */ package org.elasticsearch.xpack.enrich; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.xpack.core.enrich.EnrichPolicy; +import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -138,6 +148,114 @@ public void testCaching() { assertThat(cacheStats.getEvictions(), equalTo(4L)); } + public void testPutIfAbsent() throws InterruptedException { + // Emulate cluster metadata: + // (two enrich indices with corresponding alias entries) + var metadata = Metadata.builder() + .put( + IndexMetadata.builder(EnrichPolicy.getBaseName("policy1") + "-1") + .settings(settings(IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + .putAlias(AliasMetadata.builder(EnrichPolicy.getBaseName("policy1")).build()) + ) + .put( + IndexMetadata.builder(EnrichPolicy.getBaseName("policy2") + "-1") + .settings(settings(IndexVersion.current())) + .numberOfShards(1) + .numberOfReplicas(0) + .putAlias(AliasMetadata.builder(EnrichPolicy.getBaseName("policy2")).build()) + ) + .build(); + + // Emulated search requests that an enrich processor could generate: + // (two unique searches for two enrich policies) + var searchRequest1 = new SearchRequest(EnrichPolicy.getBaseName("policy1")).source( + new SearchSourceBuilder().query(new MatchQueryBuilder("match_field", "1")) + ); + final List> searchResponseMap = List.of( + Map.of("key1", "value1", "key2", "value2"), + Map.of("key3", "value3", "key4", "value4") + ); + EnrichCache enrichCache = new EnrichCache(3); + enrichCache.setMetadata(metadata); + + { + CountDownLatch queriedDatabaseLatch = new CountDownLatch(1); + CountDownLatch notifiedOfResultLatch = new CountDownLatch(1); + enrichCache.computeIfAbsent(searchRequest1, (searchRequest, searchResponseActionListener) -> { + SearchResponse searchResponse = convertToSearchResponse(searchResponseMap); + searchResponseActionListener.onResponse(searchResponse); + searchResponse.decRef(); + queriedDatabaseLatch.countDown(); + }, new ActionListener<>() { + @Override + public void onResponse(List> response) { + assertThat(response, equalTo(searchResponseMap)); + notifiedOfResultLatch.countDown(); + } + + @Override + public void onFailure(Exception e) { + fail(e); + } + }); + assertThat(queriedDatabaseLatch.await(5, TimeUnit.SECONDS), equalTo(true)); + assertThat(notifiedOfResultLatch.await(5, TimeUnit.SECONDS), equalTo(true)); + } + + { + CountDownLatch notifiedOfResultLatch = new CountDownLatch(1); + enrichCache.computeIfAbsent(searchRequest1, (searchRequest, searchResponseActionListener) -> { + fail("Expected no call to the database because item should have been in the cache"); + }, new ActionListener<>() { + @Override + public void onResponse(List> maps) { + notifiedOfResultLatch.countDown(); + } + + @Override + public void onFailure(Exception e) { + fail(e); + } + }); + assertThat(notifiedOfResultLatch.await(5, TimeUnit.SECONDS), equalTo(true)); + } + } + + private SearchResponse convertToSearchResponse(List> searchResponseList) { + SearchHit[] hitArray = searchResponseList.stream().map(map -> { + try { + return SearchHit.unpooled(0, "id").sourceRef(convertMapToJson(map)); + } catch (IOException e) { + throw new RuntimeException(e); + } + }).toArray(SearchHit[]::new); + SearchHits hits = SearchHits.unpooled(hitArray, null, 0); + return new SearchResponse( + hits, + null, + null, + false, + false, + null, + 1, + null, + 5, + 4, + 0, + randomLong(), + null, + SearchResponse.Clusters.EMPTY + ); + } + + private BytesReference convertMapToJson(Map simpleMap) throws IOException { + try (XContentBuilder builder = JsonXContent.contentBuilder().map(simpleMap)) { + return BytesReference.bytes(builder); + } + } + public void testDeepCopy() { Map original = new HashMap<>(); { From f28529d237f1fc75214e02ebdbecad57797ee120 Mon Sep 17 00:00:00 2001 From: Oleksandr Kolomiiets Date: Wed, 17 Apr 2024 09:33:21 -0700 Subject: [PATCH 31/43] Handle infinity during synthetic source construction for scaled float field (#107494) For really large values, rounding error is enough to push the reconstructed value for synthetic source into infinity. Existing code didn't take it into account. This PR adds a check to detect infinity and just proceed with returning it as is in synthetic source. Closes #107101. --- docs/changelog/107494.yaml | 6 ++++ .../mapper/extras/ScaledFloatFieldMapper.java | 14 ++++++++ .../extras/ScaledFloatFieldMapperTests.java | 35 ++++++++++++++++++- 3 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/107494.yaml diff --git a/docs/changelog/107494.yaml b/docs/changelog/107494.yaml new file mode 100644 index 000000000000..1d71ce284a4a --- /dev/null +++ b/docs/changelog/107494.yaml @@ -0,0 +1,6 @@ +pr: 107494 +summary: Handle infinity during synthetic source construction for scaled float field +area: Mapping +type: bug +issues: + - 107101 diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index 09507ae926f4..cb17503579e3 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -749,6 +749,20 @@ protected void writeValue(XContentBuilder b, long value) throws IOException { */ static double decodeForSyntheticSource(long scaledValue, double scalingFactor) { double v = scaledValue / scalingFactor; + + // If original double value is close to MAX_VALUE + // and rounding is performed in the direction of the same infinity + // it is possible to "overshoot" infinity during reconstruction. + // E.g. for a value close to Double.MAX_VALUE "true" scaled value is 10.5 + // and with rounding it becomes 11. + // Now, because of that rounding difference, 11 divided by scaling factor goes into infinity. + // There is nothing we can do about it so we'll return the closest finite value to infinity + // which is MAX_VALUE. + if (Double.isInfinite(v)) { + var sign = v == Double.POSITIVE_INFINITY ? 1 : -1; + return sign * Double.MAX_VALUE; + } + long reenc = Math.round(v * scalingFactor); if (reenc != scaledValue) { if (reenc > scaledValue) { diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java index d6eb55dfb23e..253df4de999d 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java @@ -395,6 +395,12 @@ private Tuple generateValue() { private double round(double d) { long encoded = Math.round(d * scalingFactor); double decoded = encoded / scalingFactor; + // Special case due to rounding, see implementation. + if (Double.isInfinite(decoded)) { + var sign = decoded == Double.POSITIVE_INFINITY ? 1 : -1; + return sign * Double.MAX_VALUE; + } + long reencoded = Math.round(decoded * scalingFactor); if (encoded != reencoded) { if (encoded > reencoded) { @@ -406,6 +412,11 @@ private double round(double d) { } private double roundDocValues(double d) { + // Special case due to rounding, see implementation. + if (Math.abs(d) == Double.MAX_VALUE) { + return d; + } + long encoded = Math.round(d * scalingFactor); return encoded * (1 / scalingFactor); } @@ -526,7 +537,7 @@ public void testEncodeDecodeSaturatedLow() { } /** - * Tests that numbers whose encoded value is {@code Long.MIN_VALUE} can be round + * Tests that numbers whose encoded value is {@code Long.MAX_VALUE} can be round * tripped through synthetic source. */ public void testEncodeDecodeSaturatedHigh() { @@ -580,6 +591,28 @@ public void testDecodeEncode() { ); } + /** + * Tests the case when decoded value is infinite due to rounding. + */ + public void testDecodeHandlingInfinity() { + for (var sign : new long[] { 1, -1 }) { + long encoded = 101; + double encodedNoRounding = 100.5; + assertEquals(encoded, Math.round(encodedNoRounding)); + + var signedMax = sign * Double.MAX_VALUE; + // We need a scaling factor that will + // 1. make encoded long small resulting in significant loss of precision due to rounding + // 2. result in long value being rounded in correct direction. + // + // So we take a scaling factor that would put us right at MAX_VALUE + // without rounding and hence go beyond MAX_VALUE with rounding. + double scalingFactor = (encodedNoRounding / signedMax); + + assertThat(ScaledFloatFieldMapper.decodeForSyntheticSource(encoded, scalingFactor), equalTo(signedMax)); + } + } + private double encodeDecode(double value, double scalingFactor) { return ScaledFloatFieldMapper.decodeForSyntheticSource(ScaledFloatFieldMapper.encode(value, scalingFactor), scalingFactor); } From cc753389c1da13a44c064718989dfcc70d92ed6c Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 17 Apr 2024 17:04:59 +0000 Subject: [PATCH 32/43] Bump to version 8.15.0 --- .backportrc.json | 4 +-- .buildkite/pipelines/intake.yml | 2 +- .buildkite/pipelines/periodic-packaging.yml | 16 ++++++++++++ .buildkite/pipelines/periodic.yml | 14 ++++++++-- .ci/bwcVersions | 1 + .ci/snapshotBwcVersions | 1 + build-tools-internal/version.properties | 2 +- docs/reference/migration/index.asciidoc | 2 ++ .../reference/migration/migrate_8_15.asciidoc | 20 ++++++++++++++ docs/reference/release-notes.asciidoc | 2 ++ docs/reference/release-notes/8.15.0.asciidoc | 8 ++++++ .../release-notes/highlights.asciidoc | 26 ++++++------------- .../main/java/org/elasticsearch/Version.java | 3 ++- 13 files changed, 76 insertions(+), 25 deletions(-) create mode 100644 docs/reference/migration/migrate_8_15.asciidoc create mode 100644 docs/reference/release-notes/8.15.0.asciidoc diff --git a/.backportrc.json b/.backportrc.json index cb8aa183f7bf..59843f4d5f13 100644 --- a/.backportrc.json +++ b/.backportrc.json @@ -1,9 +1,9 @@ { "upstream" : "elastic/elasticsearch", - "targetBranchChoices" : [ "main", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ], + "targetBranchChoices" : [ "main", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ], "targetPRLabels" : [ "backport" ], "branchLabelMapping" : { - "^v8.14.0$" : "main", + "^v8.15.0$" : "main", "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2" } } \ No newline at end of file diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml index 18a93c9b63a3..8103b40cbaff 100644 --- a/.buildkite/pipelines/intake.yml +++ b/.buildkite/pipelines/intake.yml @@ -48,7 +48,7 @@ steps: timeout_in_minutes: 300 matrix: setup: - BWC_VERSION: ["7.17.21", "8.13.3", "8.14.0"] + BWC_VERSION: ["7.17.21", "8.13.3", "8.14.0", "8.15.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index c306e1d9f63c..347b7ddde752 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -561,6 +561,22 @@ steps: env: BWC_VERSION: 8.14.0 + - label: "{{matrix.image}} / 8.15.0 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.0 + timeout_in_minutes: 300 + matrix: + setup: + image: + - rocky-8 + - ubuntu-2004 + agents: + provider: gcp + image: family/elasticsearch-{{matrix.image}} + machineType: custom-16-32768 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: 8.15.0 + - group: packaging-tests-windows steps: - label: "{{matrix.image}} / packaging-tests-windows" diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index 3410436eda2b..9291ec2efcbd 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -332,6 +332,16 @@ steps: buildDirectory: /dev/shm/bk env: BWC_VERSION: 8.14.0 + - label: 8.15.0 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.0#bwcTest + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: n1-standard-32 + buildDirectory: /dev/shm/bk + env: + BWC_VERSION: 8.15.0 - label: concurrent-search-tests command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true functionalTests timeout_in_minutes: 420 @@ -396,7 +406,7 @@ steps: setup: ES_RUNTIME_JAVA: - openjdk17 - BWC_VERSION: ["7.17.21", "8.13.3", "8.14.0"] + BWC_VERSION: ["7.17.21", "8.13.3", "8.14.0", "8.15.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 @@ -438,7 +448,7 @@ steps: - graalvm-ce17 - openjdk17 - openjdk21 - BWC_VERSION: ["7.17.21", "8.13.3", "8.14.0"] + BWC_VERSION: ["7.17.21", "8.13.3", "8.14.0", "8.15.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 46165da472e7..32a5ef8f8d1e 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -32,3 +32,4 @@ BWC_VERSION: - "8.12.2" - "8.13.3" - "8.14.0" + - "8.15.0" diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions index dfd238a041b1..6ee9691a9e5e 100644 --- a/.ci/snapshotBwcVersions +++ b/.ci/snapshotBwcVersions @@ -2,3 +2,4 @@ BWC_VERSION: - "7.17.21" - "8.13.3" - "8.14.0" + - "8.15.0" diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index 0883097e75aa..d3d528cbff49 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -1,4 +1,4 @@ -elasticsearch = 8.14.0 +elasticsearch = 8.15.0 lucene = 9.10.0 bundled_jdk_vendor = openjdk diff --git a/docs/reference/migration/index.asciidoc b/docs/reference/migration/index.asciidoc index c52438054783..51a2898b5d59 100644 --- a/docs/reference/migration/index.asciidoc +++ b/docs/reference/migration/index.asciidoc @@ -1,5 +1,6 @@ include::migration_intro.asciidoc[] +* <> * <> * <> * <> @@ -16,6 +17,7 @@ include::migration_intro.asciidoc[] * <> * <> +include::migrate_8_15.asciidoc[] include::migrate_8_14.asciidoc[] include::migrate_8_13.asciidoc[] include::migrate_8_12.asciidoc[] diff --git a/docs/reference/migration/migrate_8_15.asciidoc b/docs/reference/migration/migrate_8_15.asciidoc new file mode 100644 index 000000000000..a183e68a5069 --- /dev/null +++ b/docs/reference/migration/migrate_8_15.asciidoc @@ -0,0 +1,20 @@ +[[migrating-8.15]] +== Migrating to 8.15 +++++ +8.15 +++++ + +This section discusses the changes that you need to be aware of when migrating +your application to {es} 8.15. + +See also <> and <>. + +coming::[8.15.0] + + +[discrete] +[[breaking-changes-8.15]] +=== Breaking changes + +There are no breaking changes in {es} 8.15. + diff --git a/docs/reference/release-notes.asciidoc b/docs/reference/release-notes.asciidoc index 05c97d51a38e..3cef5cc88bbb 100644 --- a/docs/reference/release-notes.asciidoc +++ b/docs/reference/release-notes.asciidoc @@ -6,6 +6,7 @@ This section summarizes the changes in each release. +* <> * <> * <> * <> @@ -64,6 +65,7 @@ This section summarizes the changes in each release. -- +include::release-notes/8.15.0.asciidoc[] include::release-notes/8.14.0.asciidoc[] include::release-notes/8.13.2.asciidoc[] include::release-notes/8.13.1.asciidoc[] diff --git a/docs/reference/release-notes/8.15.0.asciidoc b/docs/reference/release-notes/8.15.0.asciidoc new file mode 100644 index 000000000000..97f4a51a1142 --- /dev/null +++ b/docs/reference/release-notes/8.15.0.asciidoc @@ -0,0 +1,8 @@ +[[release-notes-8.15.0]] +== {es} version 8.15.0 + +coming[8.15.0] + +Also see <>. + + diff --git a/docs/reference/release-notes/highlights.asciidoc b/docs/reference/release-notes/highlights.asciidoc index 8d9d743a239f..8c1590d17288 100644 --- a/docs/reference/release-notes/highlights.asciidoc +++ b/docs/reference/release-notes/highlights.asciidoc @@ -11,7 +11,8 @@ For detailed information about this release, see the <> and // Add previous release to the list Other versions: -{ref-bare}/8.13/release-highlights.html[8.13] +{ref-bare}/8.14/release-highlights.html[8.14] +| {ref-bare}/8.13/release-highlights.html[8.13] | {ref-bare}/8.12/release-highlights.html[8.12] | {ref-bare}/8.11/release-highlights.html[8.11] | {ref-bare}/8.10/release-highlights.html[8.10] @@ -28,24 +29,13 @@ Other versions: endif::[] +// The notable-highlights tag marks entries that +// should be featured in the Stack Installation and Upgrade Guide: // tag::notable-highlights[] - -[discrete] -[[add_global_retention_in_data_stream_lifecycle]] -=== Add global retention in data stream lifecycle -Data stream lifecycle now supports configuring retention on a cluster level, namely global retention. Global retention -allows us to configure two different retentions: - -- `default_retention` is applied to all data streams managed by the data stream lifecycle that do not have retention -defined on the data stream level. -- `max_retention` is applied to all data streams managed by the data stream lifecycle and it allows any data stream -data to be deleted after the `max_retention` has passed. - -Furthermore, we introduce the term `effective_retention` which is the retention applied at a certain moment to a data -stream considering all the available retention configurations. - -{es-pull}105682[#105682] - +// [discrete] +// === Heading +// +// Description. // end::notable-highlights[] diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index 88a1049a4255..ab7b26570a66 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -173,7 +173,8 @@ public class Version implements VersionId, ToXContentFragment { public static final Version V_8_13_2 = new Version(8_13_02_99); public static final Version V_8_13_3 = new Version(8_13_03_99); public static final Version V_8_14_0 = new Version(8_14_00_99); - public static final Version CURRENT = V_8_14_0; + public static final Version V_8_15_0 = new Version(8_15_00_99); + public static final Version CURRENT = V_8_15_0; private static final NavigableMap VERSION_IDS; private static final Map VERSION_STRINGS; From 1aa182a2733359c1feafc82fdc78b00bc1d24a12 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com> Date: Wed, 17 Apr 2024 13:28:00 -0400 Subject: [PATCH 33/43] [ML] Inference API time to reserve tokens for rate limiter (#107571) * Refactoring tests * Adding time to reserve tests --- .../xpack/inference/common/RateLimiter.java | 58 +++++-- ...erTests.java => BaseRateLimiterTests.java} | 100 ++++++------ .../common/RateLimiterAcquireTests.java | 32 ++++ .../common/RateLimiterReserveTests.java | 30 ++++ .../common/RateLimiterTimeToReserveTests.java | 142 ++++++++++++++++++ 5 files changed, 307 insertions(+), 55 deletions(-) rename x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/{RateLimiterTests.java => BaseRateLimiterTests.java} (67%) create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/RateLimiterAcquireTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/RateLimiterReserveTests.java create mode 100644 x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/RateLimiterTimeToReserveTests.java diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/RateLimiter.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/RateLimiter.java index ac28aa87f554..bbc5082d4500 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/RateLimiter.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/RateLimiter.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.inference.common; import org.elasticsearch.common.Strings; +import org.elasticsearch.core.TimeValue; import java.time.Clock; import java.time.Instant; @@ -92,24 +93,59 @@ public final synchronized void setRate(double newAccumulatedTokensLimit, double * @throws InterruptedException _ */ public void acquire(int tokens) throws InterruptedException { + sleeper.sleep(reserveInternal(tokens)); + } + + /** + * Returns the amount of time to wait for the tokens to become available but does not reserve them in advance. + * A caller will need to call {@link #reserve(int)} or {@link #acquire(int)} after this call. + * @param tokens the number of items of work that should be throttled, typically you'd pass a value of 1 here. Must be greater than 0. + * @return the amount of time to wait + */ + public TimeValue timeToReserve(int tokens) { + var timeToReserveRes = timeToReserveInternal(tokens); + + return new TimeValue((long) timeToReserveRes.microsToWait, TimeUnit.MICROSECONDS); + } + + private TimeToReserve timeToReserveInternal(int tokens) { + validateTokenRequest(tokens); + + double microsToWait; + accumulateTokens(); + var accumulatedTokensToUse = Math.min(tokens, accumulatedTokens); + var additionalTokensRequired = tokens - accumulatedTokensToUse; + microsToWait = additionalTokensRequired / tokensPerMicros; + + return new TimeToReserve(microsToWait, accumulatedTokensToUse); + } + + private record TimeToReserve(double microsToWait, double accumulatedTokensToUse) {} + + private static void validateTokenRequest(int tokens) { if (tokens <= 0) { throw new IllegalArgumentException("Requested tokens must be positive"); } + } - double microsToWait; - synchronized (this) { - accumulateTokens(); - var accumulatedTokensToUse = Math.min(tokens, accumulatedTokens); - var additionalTokensRequired = tokens - accumulatedTokensToUse; - microsToWait = additionalTokensRequired / tokensPerMicros; - accumulatedTokens -= accumulatedTokensToUse; - nextTokenAvailability = nextTokenAvailability.plus((long) microsToWait, ChronoUnit.MICROS); - } + /** + * Returns the amount of time to wait for the tokens to become available. + * @param tokens the number of items of work that should be throttled, typically you'd pass a value of 1 here. Must be greater than 0. + * @return the amount of time to wait + */ + public TimeValue reserve(int tokens) { + return new TimeValue(reserveInternal(tokens), TimeUnit.MICROSECONDS); + } + + private synchronized long reserveInternal(int tokens) { + var timeToReserveRes = timeToReserveInternal(tokens); + accumulatedTokens -= timeToReserveRes.accumulatedTokensToUse; + nextTokenAvailability = nextTokenAvailability.plus((long) timeToReserveRes.microsToWait, ChronoUnit.MICROS); - sleeper.sleep((long) microsToWait); + return (long) timeToReserveRes.microsToWait; } - private void accumulateTokens() { + private synchronized void accumulateTokens() { var now = Instant.now(clock); if (now.isAfter(nextTokenAvailability)) { var elapsedTimeMicros = microsBetweenExact(nextTokenAvailability, now); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/RateLimiterTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/BaseRateLimiterTests.java similarity index 67% rename from x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/RateLimiterTests.java rename to x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/BaseRateLimiterTests.java index 46931f12aaf4..d012f135839c 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/RateLimiterTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/BaseRateLimiterTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.inference.common; import org.elasticsearch.common.Strings; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.test.ESTestCase; import java.time.Clock; @@ -17,11 +18,19 @@ import static org.hamcrest.Matchers.is; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -public class RateLimiterTests extends ESTestCase { +public abstract class BaseRateLimiterTests extends ESTestCase { + + protected abstract TimeValue tokenMethod(RateLimiter limiter, int tokens) throws InterruptedException; + + protected abstract void sleepValidationMethod( + TimeValue result, + RateLimiter.Sleeper mockSleeper, + int numberOfClassToExpect, + long expectedMicrosecondsToSleep + ) throws InterruptedException; + public void testThrows_WhenAccumulatedTokensLimit_IsNegative() { var exception = expectThrows( IllegalArgumentException.class, @@ -65,19 +74,19 @@ public void testThrows_WhenTokensPerTimeUnit_IsNegative() { assertThat(exception.getMessage(), is("Tokens per time unit must be greater than 0")); } - public void testAcquire_Throws_WhenTokens_IsZero() { + public void testMethod_Throws_WhenTokens_IsZero() { var limiter = new RateLimiter(0, 1, TimeUnit.SECONDS, new RateLimiter.TimeUnitSleeper(), Clock.systemUTC()); var exception = expectThrows(IllegalArgumentException.class, () -> limiter.acquire(0)); assertThat(exception.getMessage(), is("Requested tokens must be positive")); } - public void testAcquire_Throws_WhenTokens_IsNegative() { + public void testMethod_Throws_WhenTokens_IsNegative() { var limiter = new RateLimiter(0, 1, TimeUnit.SECONDS, new RateLimiter.TimeUnitSleeper(), Clock.systemUTC()); var exception = expectThrows(IllegalArgumentException.class, () -> limiter.acquire(-1)); assertThat(exception.getMessage(), is("Requested tokens must be positive")); } - public void testAcquire_First_CallDoesNotSleep() throws InterruptedException { + public void testMethod_First_CallDoesNotSleep() throws InterruptedException { var now = Clock.systemUTC().instant(); var clock = mock(Clock.class); when(clock.instant()).thenReturn(now); @@ -85,11 +94,11 @@ public void testAcquire_First_CallDoesNotSleep() throws InterruptedException { var sleeper = mock(RateLimiter.Sleeper.class); var limiter = new RateLimiter(1, 1, TimeUnit.MINUTES, sleeper, clock); - limiter.acquire(1); - verify(sleeper, times(1)).sleep(0); + var res = tokenMethod(limiter, 1); + sleepValidationMethod(res, sleeper, 1, 0); } - public void testAcquire_DoesNotSleep_WhenTokenRateIsHigh() throws InterruptedException { + public void testMethod_DoesNotSleep_WhenTokenRateIsHigh() throws InterruptedException { var now = Clock.systemUTC().instant(); var clock = mock(Clock.class); when(clock.instant()).thenReturn(now); @@ -97,11 +106,11 @@ public void testAcquire_DoesNotSleep_WhenTokenRateIsHigh() throws InterruptedExc var sleeper = mock(RateLimiter.Sleeper.class); var limiter = new RateLimiter(0, Double.MAX_VALUE, TimeUnit.MICROSECONDS, sleeper, clock); - limiter.acquire(1); - verify(sleeper, times(1)).sleep(0); + var res = tokenMethod(limiter, 1); + sleepValidationMethod(res, sleeper, 1, 0); } - public void testAcquire_AcceptsMaxIntValue_WhenTokenRateIsHigh() throws InterruptedException { + public void testMethod_AcceptsMaxIntValue_WhenTokenRateIsHigh() throws InterruptedException { var now = Clock.systemUTC().instant(); var clock = mock(Clock.class); when(clock.instant()).thenReturn(now); @@ -109,11 +118,11 @@ public void testAcquire_AcceptsMaxIntValue_WhenTokenRateIsHigh() throws Interrup var sleeper = mock(RateLimiter.Sleeper.class); var limiter = new RateLimiter(0, Double.MAX_VALUE, TimeUnit.MICROSECONDS, sleeper, clock); - limiter.acquire(Integer.MAX_VALUE); - verify(sleeper, times(1)).sleep(0); + var res = tokenMethod(limiter, Integer.MAX_VALUE); + sleepValidationMethod(res, sleeper, 1, 0); } - public void testAcquire_AcceptsMaxIntValue_WhenTokenRateIsLow() throws InterruptedException { + public void testMethod_AcceptsMaxIntValue_WhenTokenRateIsLow() throws InterruptedException { var now = Clock.systemUTC().instant(); var clock = mock(Clock.class); when(clock.instant()).thenReturn(now); @@ -122,13 +131,13 @@ public void testAcquire_AcceptsMaxIntValue_WhenTokenRateIsLow() throws Interrupt double tokensPerDay = 1; var limiter = new RateLimiter(0, tokensPerDay, TimeUnit.DAYS, sleeper, clock); - limiter.acquire(Integer.MAX_VALUE); + var res = tokenMethod(limiter, Integer.MAX_VALUE); double tokensPerMicro = tokensPerDay / TimeUnit.DAYS.toMicros(1); - verify(sleeper, times(1)).sleep((long) ((double) Integer.MAX_VALUE / tokensPerMicro)); + sleepValidationMethod(res, sleeper, 1, (long) ((double) Integer.MAX_VALUE / tokensPerMicro)); } - public void testAcquire_SleepsForOneMinute_WhenRequestingOneUnavailableToken() throws InterruptedException { + public void testMethod_SleepsForOneMinute_WhenRequestingOneUnavailableToken() throws InterruptedException { var now = Clock.systemUTC().instant(); var clock = mock(Clock.class); when(clock.instant()).thenReturn(now); @@ -136,11 +145,11 @@ public void testAcquire_SleepsForOneMinute_WhenRequestingOneUnavailableToken() t var sleeper = mock(RateLimiter.Sleeper.class); var limiter = new RateLimiter(1, 1, TimeUnit.MINUTES, sleeper, clock); - limiter.acquire(2); - verify(sleeper, times(1)).sleep(TimeUnit.MINUTES.toMicros(1)); + var res = tokenMethod(limiter, 2); + sleepValidationMethod(res, sleeper, 1, TimeUnit.MINUTES.toMicros(1)); } - public void testAcquire_SleepsForOneMinute_WhenRequestingOneUnavailableToken_NoAccumulated() throws InterruptedException { + public void testMethod_SleepsForOneMinute_WhenRequestingOneUnavailableToken_NoAccumulated() throws InterruptedException { var now = Clock.systemUTC().instant(); var clock = mock(Clock.class); when(clock.instant()).thenReturn(now); @@ -148,11 +157,11 @@ public void testAcquire_SleepsForOneMinute_WhenRequestingOneUnavailableToken_NoA var sleeper = mock(RateLimiter.Sleeper.class); var limiter = new RateLimiter(0, 1, TimeUnit.MINUTES, sleeper, clock); - limiter.acquire(1); - verify(sleeper, times(1)).sleep(TimeUnit.MINUTES.toMicros(1)); + var res = tokenMethod(limiter, 1); + sleepValidationMethod(res, sleeper, 1, TimeUnit.MINUTES.toMicros(1)); } - public void testAcquire_SleepsFor10Minute_WhenRequesting10UnavailableToken_NoAccumulated() throws InterruptedException { + public void testMethod_SleepsFor10Minute_WhenRequesting10UnavailableToken_NoAccumulated() throws InterruptedException { var now = Clock.systemUTC().instant(); var clock = mock(Clock.class); when(clock.instant()).thenReturn(now); @@ -160,11 +169,11 @@ public void testAcquire_SleepsFor10Minute_WhenRequesting10UnavailableToken_NoAcc var sleeper = mock(RateLimiter.Sleeper.class); var limiter = new RateLimiter(0, 1, TimeUnit.MINUTES, sleeper, clock); - limiter.acquire(10); - verify(sleeper, times(1)).sleep(TimeUnit.MINUTES.toMicros(10)); + var res = tokenMethod(limiter, 10); + sleepValidationMethod(res, sleeper, 1, TimeUnit.MINUTES.toMicros(10)); } - public void testAcquire_IncrementsNextTokenAvailabilityInstant_ByOneMinute() throws InterruptedException { + public void testMethod_IncrementsNextTokenAvailabilityInstant_ByOneMinute() throws InterruptedException { var now = Clock.systemUTC().instant(); var clock = mock(Clock.class); when(clock.instant()).thenReturn(now); @@ -172,12 +181,12 @@ public void testAcquire_IncrementsNextTokenAvailabilityInstant_ByOneMinute() thr var sleeper = mock(RateLimiter.Sleeper.class); var limiter = new RateLimiter(0, 1, TimeUnit.MINUTES, sleeper, clock); - limiter.acquire(1); - verify(sleeper, times(1)).sleep(TimeUnit.MINUTES.toMicros(1)); + var res = tokenMethod(limiter, 1); + sleepValidationMethod(res, sleeper, 1, TimeUnit.MINUTES.toMicros(1)); assertThat(limiter.getNextTokenAvailability(), is(now.plus(1, ChronoUnit.MINUTES))); } - public void testAcquire_SecondCallToAcquire_ShouldWait_WhenAccumulatedTokensAreDepleted() throws InterruptedException { + public void testMethod_SecondCallToAcquire_ShouldWait_WhenAccumulatedTokensAreDepleted() throws InterruptedException { var now = Clock.systemUTC().instant(); var clock = mock(Clock.class); when(clock.instant()).thenReturn(now); @@ -185,13 +194,14 @@ public void testAcquire_SecondCallToAcquire_ShouldWait_WhenAccumulatedTokensAreD var sleeper = mock(RateLimiter.Sleeper.class); var limiter = new RateLimiter(1, 1, TimeUnit.MINUTES, sleeper, clock); - limiter.acquire(1); - verify(sleeper, times(1)).sleep(0); - limiter.acquire(1); - verify(sleeper, times(1)).sleep(TimeUnit.MINUTES.toMicros(1)); + + var res = tokenMethod(limiter, 1); + sleepValidationMethod(res, sleeper, 1, 0); + res = tokenMethod(limiter, 1); + sleepValidationMethod(res, sleeper, 1, TimeUnit.MINUTES.toMicros(1)); } - public void testAcquire_SecondCallToAcquire_ShouldWaitForHalfDuration_WhenElapsedTimeIsHalfRequiredDuration() + public void testMethod_SecondCallToAcquire_ShouldWaitForHalfDuration_WhenElapsedTimeIsHalfRequiredDuration() throws InterruptedException { var now = Clock.systemUTC().instant(); var clock = mock(Clock.class); @@ -200,14 +210,15 @@ public void testAcquire_SecondCallToAcquire_ShouldWaitForHalfDuration_WhenElapse var sleeper = mock(RateLimiter.Sleeper.class); var limiter = new RateLimiter(1, 1, TimeUnit.MINUTES, sleeper, clock); - limiter.acquire(1); - verify(sleeper, times(1)).sleep(0); + + var res = tokenMethod(limiter, 1); + sleepValidationMethod(res, sleeper, 1, 0); when(clock.instant()).thenReturn(now.plus(Duration.ofSeconds(30))); - limiter.acquire(1); - verify(sleeper, times(1)).sleep(TimeUnit.SECONDS.toMicros(30)); + res = tokenMethod(limiter, 1); + sleepValidationMethod(res, sleeper, 1, TimeUnit.SECONDS.toMicros(30)); } - public void testAcquire_ShouldAccumulateTokens() throws InterruptedException { + public void testMethod_ShouldAccumulateTokens() throws InterruptedException { var now = Clock.systemUTC().instant(); var clock = mock(Clock.class); when(clock.instant()).thenReturn(now); @@ -215,11 +226,12 @@ public void testAcquire_ShouldAccumulateTokens() throws InterruptedException { var sleeper = mock(RateLimiter.Sleeper.class); var limiter = new RateLimiter(10, 10, TimeUnit.MINUTES, sleeper, clock); - limiter.acquire(5); - verify(sleeper, times(1)).sleep(0); + + var res = tokenMethod(limiter, 5); + sleepValidationMethod(res, sleeper, 1, 0); // it should accumulate 5 tokens when(clock.instant()).thenReturn(now.plus(Duration.ofSeconds(30))); - limiter.acquire(10); - verify(sleeper, times(2)).sleep(0); + res = tokenMethod(limiter, 10); + sleepValidationMethod(res, sleeper, 2, 0); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/RateLimiterAcquireTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/RateLimiterAcquireTests.java new file mode 100644 index 000000000000..1f59fa7bb5ba --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/RateLimiterAcquireTests.java @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.common; + +import org.elasticsearch.core.TimeValue; + +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class RateLimiterAcquireTests extends BaseRateLimiterTests { + + @Override + protected TimeValue tokenMethod(RateLimiter limiter, int tokens) throws InterruptedException { + limiter.acquire(tokens); + return null; + } + + @Override + protected void sleepValidationMethod( + TimeValue result, + RateLimiter.Sleeper mockSleeper, + int numberOfClassToExpect, + long expectedMicrosecondsToSleep + ) throws InterruptedException { + verify(mockSleeper, times(numberOfClassToExpect)).sleep(expectedMicrosecondsToSleep); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/RateLimiterReserveTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/RateLimiterReserveTests.java new file mode 100644 index 000000000000..5c32c6c560e7 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/RateLimiterReserveTests.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.common; + +import org.elasticsearch.core.TimeValue; + +import static org.hamcrest.Matchers.is; + +public class RateLimiterReserveTests extends BaseRateLimiterTests { + + @Override + protected TimeValue tokenMethod(RateLimiter limiter, int tokens) { + return limiter.reserve(tokens); + } + + @Override + protected void sleepValidationMethod( + TimeValue result, + RateLimiter.Sleeper mockSleeper, + int numberOfClassToExpect, + long expectedMicrosecondsToSleep + ) { + assertThat(result.getMicros(), is(expectedMicrosecondsToSleep)); + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/RateLimiterTimeToReserveTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/RateLimiterTimeToReserveTests.java new file mode 100644 index 000000000000..a69846d67c4e --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/RateLimiterTimeToReserveTests.java @@ -0,0 +1,142 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.common; + +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.test.ESTestCase; + +import java.time.Clock; +import java.time.Duration; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class RateLimiterTimeToReserveTests extends ESTestCase { + public void testTimeToReserve_Returns_1Second() { + var now = Clock.systemUTC().instant(); + var clock = mock(Clock.class); + when(clock.instant()).thenReturn(now); + + var sleeper = mock(RateLimiter.Sleeper.class); + + var limiter = new RateLimiter(0, 1, TimeUnit.SECONDS, sleeper, clock); + var timeToWait = limiter.timeToReserve(1); + assertThat(timeToWait, is(TimeValue.timeValueSeconds(1))); + } + + public void testTimeToReserve_Returns_1Second_WithoutReservingToken() { + var now = Clock.systemUTC().instant(); + var clock = mock(Clock.class); + when(clock.instant()).thenReturn(now); + + var sleeper = mock(RateLimiter.Sleeper.class); + + var limiter = new RateLimiter(0, 1, TimeUnit.SECONDS, sleeper, clock); + var timeToWait = limiter.timeToReserve(1); + assertThat(timeToWait, is(TimeValue.timeValueSeconds(1))); + + timeToWait = limiter.timeToReserve(1); + assertThat(timeToWait, is(TimeValue.timeValueSeconds(1))); + } + + public void testTimeToReserve_Returns_0Seconds_WhenTokenIsAlreadyAvailable() { + var now = Clock.systemUTC().instant(); + var clock = mock(Clock.class); + when(clock.instant()).thenReturn(now); + + var sleeper = mock(RateLimiter.Sleeper.class); + + var limiter = new RateLimiter(1, 1, TimeUnit.SECONDS, sleeper, clock); + var timeToWait = limiter.timeToReserve(1); + assertThat(timeToWait, is(TimeValue.timeValueSeconds(0))); + } + + public void testTimeToReserve_Returns_0Seconds_WhenTokenIsAlreadyAvailable_WithoutReservingToken() { + var now = Clock.systemUTC().instant(); + var clock = mock(Clock.class); + when(clock.instant()).thenReturn(now); + + var sleeper = mock(RateLimiter.Sleeper.class); + + var limiter = new RateLimiter(1, 1, TimeUnit.SECONDS, sleeper, clock); + var timeToWait = limiter.timeToReserve(1); + assertThat(timeToWait, is(TimeValue.timeValueSeconds(0))); + + timeToWait = limiter.timeToReserve(1); + assertThat(timeToWait, is(TimeValue.timeValueSeconds(0))); + } + + public void testTimeToReserve_Returns_1Seconds_When1TokenIsAlreadyAvailable_ButRequires2Tokens() { + var now = Clock.systemUTC().instant(); + var clock = mock(Clock.class); + when(clock.instant()).thenReturn(now); + + var sleeper = mock(RateLimiter.Sleeper.class); + + var limiter = new RateLimiter(1, 1, TimeUnit.SECONDS, sleeper, clock); + var timeToWait = limiter.timeToReserve(2); + assertThat(timeToWait, is(TimeValue.timeValueSeconds(1))); + } + + public void testTimeToReserve_Returns_1Seconds_When1TokenIsAlreadyAvailable_ButRequires2Tokens_WithoutReservingToken() { + var now = Clock.systemUTC().instant(); + var clock = mock(Clock.class); + when(clock.instant()).thenReturn(now); + + var sleeper = mock(RateLimiter.Sleeper.class); + + var limiter = new RateLimiter(1, 1, TimeUnit.SECONDS, sleeper, clock); + var timeToWait = limiter.timeToReserve(2); + assertThat(timeToWait, is(TimeValue.timeValueSeconds(1))); + + timeToWait = limiter.timeToReserve(2); + assertThat(timeToWait, is(TimeValue.timeValueSeconds(1))); + } + + public void testTimeToReserve_Returns_0Seconds_WhenTimeAdvancesToAccumulate2Tokens() { + var now = Clock.systemUTC().instant(); + var clock = mock(Clock.class); + when(clock.instant()).thenReturn(now); + + var sleeper = mock(RateLimiter.Sleeper.class); + + var limiter = new RateLimiter(2, 1, TimeUnit.SECONDS, sleeper, clock); + // drain the accumulated tokens + var drainedTokensTime = limiter.reserve(2); + assertThat(drainedTokensTime, is(TimeValue.timeValueSeconds(0))); + + when(clock.instant()).thenReturn(now.plus(Duration.ofSeconds(2))); + // 2 tokens should now be available + var timeToWait = limiter.timeToReserve(2); + assertThat(timeToWait, is(TimeValue.timeValueSeconds(0))); + } + + public void testTimeToReserve_Returns_0Seconds_WhenTimeAdvancesToAccumulate2Tokens_MethodCallDoesNotReserveTokens() { + var now = Clock.systemUTC().instant(); + var clock = mock(Clock.class); + when(clock.instant()).thenReturn(now); + + var sleeper = mock(RateLimiter.Sleeper.class); + + var limiter = new RateLimiter(2, 1, TimeUnit.SECONDS, sleeper, clock); + // drain the accumulated tokens + var drainedTokensTime = limiter.reserve(2); + assertThat(drainedTokensTime, is(TimeValue.timeValueSeconds(0))); + + when(clock.instant()).thenReturn(now.plus(Duration.ofSeconds(2))); + // 2 tokens should now be available + var timeToWait = limiter.timeToReserve(2); + assertThat(timeToWait, is(TimeValue.timeValueSeconds(0))); + + // 2 tokens should still be available + timeToWait = limiter.timeToReserve(2); + assertThat(timeToWait, is(TimeValue.timeValueSeconds(0))); + } +} From 53d012bcb931bb6d5f2a0b558157a28550f4799d Mon Sep 17 00:00:00 2001 From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com> Date: Wed, 17 Apr 2024 13:29:53 -0400 Subject: [PATCH 34/43] Adding poll peek (#107576) --- .../AdjustableCapacityBlockingQueue.java | 34 +++++++++++ .../AdjustableCapacityBlockingQueueTests.java | 59 +++++++++++++++++++ 2 files changed, 93 insertions(+) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/AdjustableCapacityBlockingQueue.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/AdjustableCapacityBlockingQueue.java index e73151b44a3e..ea600488ea8f 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/AdjustableCapacityBlockingQueue.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/AdjustableCapacityBlockingQueue.java @@ -145,6 +145,40 @@ public E take() throws InterruptedException { } } + public E peek() { + final ReentrantReadWriteLock.ReadLock readLock = lock.readLock(); + + readLock.lock(); + try { + var oldItem = prioritizedReadingQueue.peek(); + + if (oldItem != null) { + return oldItem; + } + + return currentQueue.peek(); + } finally { + readLock.unlock(); + } + } + + public E poll() { + final ReentrantReadWriteLock.ReadLock readLock = lock.readLock(); + + readLock.lock(); + try { + var oldItem = prioritizedReadingQueue.poll(); + + if (oldItem != null) { + return oldItem; + } + + return currentQueue.poll(); + } finally { + readLock.unlock(); + } + } + /** * Returns the number of elements stored in the queue. If the capacity was recently changed, the value returned could be * greater than the capacity. This occurs when the capacity was reduced and there were more elements in the queue than the diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/AdjustableCapacityBlockingQueueTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/AdjustableCapacityBlockingQueueTests.java index 09cd065ce3cd..5a70b98313f7 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/AdjustableCapacityBlockingQueueTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/AdjustableCapacityBlockingQueueTests.java @@ -233,6 +233,65 @@ public void testTake_RemovesItemFromQueue() throws InterruptedException { assertThat(queue.size(), is(0)); } + public void testPeek_ReturnsItemWithoutRemoving() { + var queue = new AdjustableCapacityBlockingQueue<>(QUEUE_CREATOR, 1); + assertThat(queue.size(), is(0)); + + queue.offer(0); + assertThat(queue.size(), is(1)); + assertThat(queue.peek(), is(0)); + assertThat(queue.size(), is(1)); + assertThat(queue.peek(), is(0)); + } + + public void testPeek_ExistingItem_RemainsAtFront_AfterCapacityChange() throws InterruptedException { + var queue = new AdjustableCapacityBlockingQueue<>(QUEUE_CREATOR, 1); + queue.offer(0); + assertThat(queue.size(), is(1)); + assertThat(queue.remainingCapacity(), is(0)); + assertThat(queue.peek(), is(0)); + + queue.setCapacity(2); + assertThat(queue.remainingCapacity(), is(1)); + assertThat(queue.peek(), is(0)); + + queue.offer(1); + assertThat(queue.peek(), is(0)); + assertThat(queue.take(), is(0)); + assertThat(queue.peek(), is(1)); + } + + public void testPoll_ReturnsNull_WhenNoItemsAreAvailable() { + var queue = new AdjustableCapacityBlockingQueue<>(QUEUE_CREATOR, 1); + assertNull(queue.poll()); + } + + public void testPoll_ReturnsFirstElement() { + var queue = new AdjustableCapacityBlockingQueue<>(QUEUE_CREATOR, 1); + queue.offer(0); + assertThat(queue.poll(), is(0)); + assertThat(queue.size(), is(0)); + assertThat(queue.remainingCapacity(), is(1)); + } + + public void testPoll_ReturnsFirstElement_AfterCapacityIncrease() { + var queue = new AdjustableCapacityBlockingQueue<>(QUEUE_CREATOR, 1); + queue.offer(0); + queue.setCapacity(2); + queue.offer(1); + + assertThat(queue.remainingCapacity(), is(0)); + assertThat(queue.size(), is(2)); + + assertThat(queue.poll(), is(0)); + assertThat(queue.size(), is(1)); + assertThat(queue.remainingCapacity(), is(1)); + + assertThat(queue.poll(), is(1)); + assertThat(queue.size(), is(0)); + assertThat(queue.remainingCapacity(), is(2)); + } + public static AdjustableCapacityBlockingQueue.QueueCreator mockQueueCreator(BlockingQueue backingQueue) { return new AdjustableCapacityBlockingQueue.QueueCreator<>() { @Override From 223e7f829bdc1b262344d98ed9deb565086475e3 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Wed, 17 Apr 2024 19:37:04 +0200 Subject: [PATCH 35/43] Avoid attempting to load the same empty field twice in fetch phase (#107551) During the fetch phase, there's a number of stored fields that are requested explicitly or loaded by default. That information is included in `StoredFieldsSpec` that each fetch sub phase exposes. We attempt to provide stored fields that are already loaded to the fields lookup that scripts as well as value fetchers use to load field values (via `SearchLookup`). This is done in `PreloadedFieldLookupProvider.` The current logic makes available values for fields that have been found, so that scripts or value fetchers that request them don't load them again ad-hoc. What happens though for stored fields that don't have a value for a specific doc, is that they are treated like any other field that was not requested, and loaded again, although they will not be found, which causes overhead. This change makes available to `PreloadedFieldLookupProvider` the list of required stored fields, so that it can better distinguish between fields that we already attempted to load (although we may not have found a value for them) and those that need to be loaded ad-hoc (for instance because a script is requesting them for the first time). This is an existing issue, that has become evident as we moved fetching of metadata fields to `FetchFieldsPhase`, that relies on value fetchers, and hence on `SearchLookup`. We end up attempting to load default metadata fields (`_ignored` and `_routing`) twice when they are not present in a document, which makes us call `LeafReader#storedFields` additional times for the same document providing a `SingleFieldVisitor` that will never find a value. Another existing issue that this PR fixes is for the `FetchFieldsPhase` to extend the `StoredFieldsSpec` that it exposes to include the metadata fields that the phase is now responsible for loading. That results in `_ignored` being included in the output of the debug stored fields section when profiling is enabled. The fact that it was previously missing is an existing bug (it was missing in `StoredFieldLoader#fieldsToLoad`). Yet another existing issues that this PR fixes is that `_id` has been until now always loaded on demand when requested via fetch fields or script. That is because it is not part of the preloaded stored fields that the fetch phase passes over to the `PreloadedFieldLookupProvider`. That causes overhead as the field has already been loaded, and should not be loaded once again when explicitly requested. --- docs/changelog/107551.yaml | 5 ++ docs/reference/search/profile.asciidoc | 4 +- .../rest-api-spec/test/30_inner_hits.yml | 2 +- .../rest-api-spec/test/search/370_profile.yml | 6 +- .../search/source/MetadataFetchingIT.java | 14 +++++ .../search/fetch/FetchPhase.java | 8 ++- .../fetch/PreloadedFieldLookupProvider.java | 39 +++++++++++-- .../fetch/subphase/FetchFieldsPhase.java | 5 +- .../search/lookup/FieldLookup.java | 4 +- .../PreloadedFieldLookupProviderTests.java | 16 +++++- .../fetch/subphase/FetchFieldsPhaseTests.java | 57 +++++++++++++++++++ 11 files changed, 141 insertions(+), 19 deletions(-) create mode 100644 docs/changelog/107551.yaml diff --git a/docs/changelog/107551.yaml b/docs/changelog/107551.yaml new file mode 100644 index 000000000000..78e64cc52663 --- /dev/null +++ b/docs/changelog/107551.yaml @@ -0,0 +1,5 @@ +pr: 107551 +summary: Avoid attempting to load the same empty field twice in fetch phase +area: Search +type: bug +issues: [] diff --git a/docs/reference/search/profile.asciidoc b/docs/reference/search/profile.asciidoc index 3fed14231808..48c65ed0abc7 100644 --- a/docs/reference/search/profile.asciidoc +++ b/docs/reference/search/profile.asciidoc @@ -194,7 +194,7 @@ The API returns the following result: "load_source_count": 5 }, "debug": { - "stored_fields": ["_id", "_routing", "_source"] + "stored_fields": ["_id", "_ignored", "_routing", "_source"] }, "children": [ { @@ -1051,7 +1051,7 @@ And here is the fetch profile: "load_source_count": 5 }, "debug": { - "stored_fields": ["_id", "_routing", "_source"] + "stored_fields": ["_id", "_ignored", "_routing", "_source"] }, "children": [ { diff --git a/modules/parent-join/src/yamlRestTest/resources/rest-api-spec/test/30_inner_hits.yml b/modules/parent-join/src/yamlRestTest/resources/rest-api-spec/test/30_inner_hits.yml index a561ebbae00e..eff9a9beb35b 100644 --- a/modules/parent-join/src/yamlRestTest/resources/rest-api-spec/test/30_inner_hits.yml +++ b/modules/parent-join/src/yamlRestTest/resources/rest-api-spec/test/30_inner_hits.yml @@ -140,7 +140,7 @@ profile fetch: - gt: { profile.shards.0.fetch.breakdown.next_reader: 0 } - gt: { profile.shards.0.fetch.breakdown.load_stored_fields_count: 0 } - gt: { profile.shards.0.fetch.breakdown.load_stored_fields: 0 } - - match: { profile.shards.0.fetch.debug.stored_fields: [_id, _routing, _source] } + - match: { profile.shards.0.fetch.debug.stored_fields: [_id, _ignored, _routing, _source] } - length: { profile.shards.0.fetch.children: 4 } - match: { profile.shards.0.fetch.children.0.type: FetchFieldsPhase } - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml index 200f7292291b..dda3d14a5ae1 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml @@ -41,7 +41,7 @@ fetch fields: - gt: { profile.shards.0.fetch.breakdown.next_reader: 0 } - gt: { profile.shards.0.fetch.breakdown.load_stored_fields_count: 0 } - gt: { profile.shards.0.fetch.breakdown.load_stored_fields: 0 } - - match: { profile.shards.0.fetch.debug.stored_fields: [_id, _routing, _source] } + - match: { profile.shards.0.fetch.debug.stored_fields: [_id, _ignored, _routing, _source] } - length: { profile.shards.0.fetch.children: 2 } - match: { profile.shards.0.fetch.children.0.type: FetchFieldsPhase } - gt: { profile.shards.0.fetch.children.0.breakdown.next_reader_count: 0 } @@ -74,7 +74,7 @@ fetch source: - gt: { profile.shards.0.fetch.breakdown.next_reader: 0 } - gt: { profile.shards.0.fetch.breakdown.load_stored_fields_count: 0 } - gt: { profile.shards.0.fetch.breakdown.load_stored_fields: 0 } - - match: { profile.shards.0.fetch.debug.stored_fields: [_id, _routing, _source] } + - match: { profile.shards.0.fetch.debug.stored_fields: [_id, _ignored, _routing, _source] } - length: { profile.shards.0.fetch.children: 3 } - match: { profile.shards.0.fetch.children.0.type: FetchFieldsPhase } - match: { profile.shards.0.fetch.children.1.type: FetchSourcePhase } @@ -139,7 +139,7 @@ fetch nested source: - gt: { profile.shards.0.fetch.breakdown.next_reader: 0 } - gt: { profile.shards.0.fetch.breakdown.load_stored_fields_count: 0 } - gt: { profile.shards.0.fetch.breakdown.load_stored_fields: 0 } - - match: { profile.shards.0.fetch.debug.stored_fields: [_id, _routing, _source] } + - match: { profile.shards.0.fetch.debug.stored_fields: [_id, _ignored, _routing, _source] } - length: { profile.shards.0.fetch.children: 4 } - match: { profile.shards.0.fetch.children.0.type: FetchFieldsPhase } - match: { profile.shards.0.fetch.children.1.type: FetchSourcePhase } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/source/MetadataFetchingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/source/MetadataFetchingIT.java index b8d1d45a6f85..9e0dd984c9a2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/source/MetadataFetchingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/source/MetadataFetchingIT.java @@ -168,4 +168,18 @@ public void testInvalid() { assertThat(exc.getMessage(), equalTo("cannot combine _none_ with other fields")); } } + + public void testFetchId() { + assertAcked(prepareCreate("test")); + ensureGreen(); + + prepareIndex("test").setId("1").setSource("field", "value").get(); + refresh(); + + assertResponse(prepareSearch("test").addFetchField("_id"), response -> { + assertEquals(1, response.getHits().getHits().length); + assertEquals("1", response.getHits().getAt(0).getId()); + assertEquals("1", response.getHits().getAt(0).field("_id").getValue()); + }); + } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java index 2fa3e903a007..4b5c647da0c9 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java @@ -112,9 +112,13 @@ private SearchHits buildSearchHits(SearchContext context, int[] docIdsToLoad, Pr context.getSearchExecutionContext().setLookupProviders(sourceProvider, ctx -> fieldLookupProvider); List processors = getProcessors(context.shardTarget(), fetchContext, profiler); - StoredFieldsSpec storedFieldsSpec = StoredFieldsSpec.build(processors, FetchSubPhaseProcessor::storedFieldsSpec); storedFieldsSpec = storedFieldsSpec.merge(new StoredFieldsSpec(false, false, sourceLoader.requiredStoredFields())); + // Ideally the required stored fields would be provided as constructor argument a few lines above, but that requires moving + // the getProcessors call to before the setLookupProviders call, which causes weird issues in InnerHitsPhase. + // setLookupProviders resets the SearchLookup used throughout the rest of the fetch phase, which StoredValueFetchers rely on + // to retrieve stored fields, and InnerHitsPhase is the last sub-fetch phase and re-runs the entire fetch phase. + fieldLookupProvider.setPreloadedStoredFieldNames(storedFieldsSpec.requiredStoredFields()); StoredFieldLoader storedFieldLoader = profiler.storedFields(StoredFieldLoader.fromSpec(storedFieldsSpec)); IdLoader idLoader = context.newIdLoader(); @@ -164,7 +168,7 @@ protected SearchHit nextDoc(int doc) throws IOException { leafIdLoader ); sourceProvider.source = hit.source(); - fieldLookupProvider.storedFields = hit.loadedFields(); + fieldLookupProvider.setPreloadedStoredFieldValues(hit.hit().getId(), hit.loadedFields()); for (FetchSubPhaseProcessor processor : processors) { processor.process(hit); } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/PreloadedFieldLookupProvider.java b/server/src/main/java/org/elasticsearch/search/fetch/PreloadedFieldLookupProvider.java index 31cd74c878a0..b335ce4aa280 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/PreloadedFieldLookupProvider.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/PreloadedFieldLookupProvider.java @@ -9,12 +9,16 @@ package org.elasticsearch.search.fetch; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.search.lookup.FieldLookup; import org.elasticsearch.search.lookup.LeafFieldLookupProvider; import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Supplier; /** @@ -26,15 +30,22 @@ */ class PreloadedFieldLookupProvider implements LeafFieldLookupProvider { - Map> storedFields; - LeafFieldLookupProvider backUpLoader; - Supplier loaderSupplier; + private final SetOnce> preloadedStoredFieldNames = new SetOnce<>(); + private Map> preloadedStoredFieldValues; + private String id; + private LeafFieldLookupProvider backUpLoader; + private Supplier loaderSupplier; @Override public void populateFieldLookup(FieldLookup fieldLookup, int doc) throws IOException { String field = fieldLookup.fieldType().name(); - if (storedFields.containsKey(field)) { - fieldLookup.setValues(storedFields.get(field)); + + if (field.equals(IdFieldMapper.NAME)) { + fieldLookup.setValues(Collections.singletonList(id)); + return; + } + if (preloadedStoredFieldNames.get().contains(field)) { + fieldLookup.setValues(preloadedStoredFieldValues.get(field)); return; } // stored field not preloaded, go and get it directly @@ -44,8 +55,26 @@ public void populateFieldLookup(FieldLookup fieldLookup, int doc) throws IOExcep backUpLoader.populateFieldLookup(fieldLookup, doc); } + void setPreloadedStoredFieldNames(Set preloadedStoredFieldNames) { + this.preloadedStoredFieldNames.set(preloadedStoredFieldNames); + } + + void setPreloadedStoredFieldValues(String id, Map> preloadedStoredFieldValues) { + assert preloadedStoredFieldNames.get().containsAll(preloadedStoredFieldValues.keySet()) + : "Provided stored field that was not expected to be preloaded? " + + preloadedStoredFieldValues.keySet() + + " - " + + preloadedStoredFieldNames; + this.preloadedStoredFieldValues = preloadedStoredFieldValues; + this.id = id; + } + void setNextReader(LeafReaderContext ctx) { backUpLoader = null; loaderSupplier = () -> LeafFieldLookupProvider.fromStoredFields().apply(ctx); } + + LeafFieldLookupProvider getBackUpLoader() { + return backUpLoader; + } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java index 882eb1cf9c75..287c47505bf3 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java @@ -40,6 +40,7 @@ public final class FetchFieldsPhase implements FetchSubPhase { private static final List DEFAULT_METADATA_FIELDS = List.of( new FieldAndFormat(IgnoredFieldMapper.NAME, null), new FieldAndFormat(RoutingFieldMapper.NAME, null), + // will only be fetched when mapped (older archived indices) new FieldAndFormat(LegacyTypeFieldMapper.NAME, null) ); @@ -95,9 +96,9 @@ public void setNextReader(LeafReaderContext readerContext) { @Override public StoredFieldsSpec storedFieldsSpec() { if (fieldFetcher != null) { - return fieldFetcher.storedFieldsSpec(); + return metadataFieldFetcher.storedFieldsSpec().merge(fieldFetcher.storedFieldsSpec()); } - return StoredFieldsSpec.NO_REQUIREMENTS; + return metadataFieldFetcher.storedFieldsSpec(); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/lookup/FieldLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/FieldLookup.java index fa4eb8f21f78..97ff1862c785 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/FieldLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/FieldLookup.java @@ -31,7 +31,9 @@ public MappedFieldType fieldType() { */ public void setValues(List values) { assert valuesLoaded == false : "Call clear() before calling setValues()"; - values.stream().map(fieldType::valueForDisplay).forEach(this.values::add); + if (values != null) { + values.stream().map(fieldType::valueForDisplay).forEach(this.values::add); + } this.valuesLoaded = true; } diff --git a/server/src/test/java/org/elasticsearch/search/fetch/PreloadedFieldLookupProviderTests.java b/server/src/test/java/org/elasticsearch/search/fetch/PreloadedFieldLookupProviderTests.java index 85d9c32a1ee5..13cdb01156f0 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/PreloadedFieldLookupProviderTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/PreloadedFieldLookupProviderTests.java @@ -13,11 +13,13 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.memory.MemoryIndex; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.search.lookup.FieldLookup; import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Map; @@ -30,7 +32,16 @@ public class PreloadedFieldLookupProviderTests extends ESTestCase { public void testFallback() throws IOException { PreloadedFieldLookupProvider lookup = new PreloadedFieldLookupProvider(); - lookup.storedFields = Map.of("foo", List.of("bar")); + lookup.setPreloadedStoredFieldNames(Collections.singleton("foo")); + lookup.setPreloadedStoredFieldValues("id", Map.of("foo", List.of("bar"))); + + MappedFieldType idFieldType = mock(MappedFieldType.class); + when(idFieldType.name()).thenReturn(IdFieldMapper.NAME); + when(idFieldType.valueForDisplay(any())).then(invocation -> (invocation.getArguments()[0])); + FieldLookup idFieldLookup = new FieldLookup(idFieldType); + lookup.populateFieldLookup(idFieldLookup, 0); + assertEquals("id", idFieldLookup.getValue()); + assertNull(lookup.getBackUpLoader()); // fallback didn't get used because 'foo' is in the list MappedFieldType fieldType = mock(MappedFieldType.class); when(fieldType.name()).thenReturn("foo"); @@ -39,7 +50,7 @@ public void testFallback() throws IOException { lookup.populateFieldLookup(fieldLookup, 0); assertEquals("BAR", fieldLookup.getValue()); - assertNull(lookup.backUpLoader); // fallback didn't get used because 'foo' is in the list + assertNull(lookup.getBackUpLoader()); // fallback didn't get used because 'foo' is in the list MappedFieldType unloadedFieldType = mock(MappedFieldType.class); when(unloadedFieldType.name()).thenReturn("unloaded"); @@ -56,5 +67,4 @@ public void testFallback() throws IOException { lookup.populateFieldLookup(unloadedFieldLookup, 0); assertEquals("VALUE", unloadedFieldLookup.getValue()); } - } diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhaseTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhaseTests.java index 39e73837c83e..3a7460c05ca8 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhaseTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.index.mapper.DocValueFetcher; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NestedLookup; +import org.elasticsearch.index.mapper.StoredValueFetcher; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchHit; @@ -26,6 +27,9 @@ import org.elasticsearch.search.fetch.FetchContext; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.FetchSubPhaseProcessor; +import org.elasticsearch.search.fetch.StoredFieldsContext; +import org.elasticsearch.search.fetch.StoredFieldsSpec; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.Source; import org.elasticsearch.test.ESTestCase; @@ -35,6 +39,7 @@ import java.util.Set; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -90,6 +95,58 @@ public void testDocValueFetcher() throws IOException { reader.close(); dir.close(); + } + + public void testStoredFieldsSpec() { + StoredFieldsContext storedFieldsContext = StoredFieldsContext.fromList(List.of("stored", "_metadata")); + FetchFieldsContext ffc = new FetchFieldsContext(List.of(new FieldAndFormat("field", null))); + + SearchLookup searchLookup = mock(SearchLookup.class); + + SearchExecutionContext sec = mock(SearchExecutionContext.class); + when(sec.isMetadataField(any())).then(invocation -> invocation.getArguments()[0].toString().startsWith("_")); + + MappedFieldType routingFt = mock(MappedFieldType.class); + when(routingFt.valueFetcher(any(), any())).thenReturn(new StoredValueFetcher(searchLookup, "_routing")); + when(sec.getFieldType(eq("_routing"))).thenReturn(routingFt); + + // this would normally not be mapped -> getMatchingFieldsNames would not resolve it (unless for older archive indices) + MappedFieldType typeFt = mock(MappedFieldType.class); + when(typeFt.valueFetcher(any(), any())).thenReturn(new StoredValueFetcher(searchLookup, "_type")); + when(sec.getFieldType(eq("_type"))).thenReturn(typeFt); + + MappedFieldType ignoredFt = mock(MappedFieldType.class); + when(ignoredFt.valueFetcher(any(), any())).thenReturn(new StoredValueFetcher(searchLookup, "_ignored")); + when(sec.getFieldType(eq("_ignored"))).thenReturn(ignoredFt); + + // Ideally we would test that explicitly requested stored fields are included in stored fields spec, but isStored is final hence it + // can't be mocked. In reality, _metadata would be included but stored would not. + MappedFieldType storedFt = mock(MappedFieldType.class); + when(sec.getFieldType(eq("stored"))).thenReturn(storedFt); + MappedFieldType metadataFt = mock(MappedFieldType.class); + when(sec.getFieldType(eq("_metadata"))).thenReturn(metadataFt); + + MappedFieldType fieldType = mock(MappedFieldType.class); + when(fieldType.valueFetcher(any(), any())).thenReturn( + new DocValueFetcher( + DocValueFormat.RAW, + new SortedNumericIndexFieldData("field", IndexNumericFieldData.NumericType.LONG, CoreValuesSourceType.NUMERIC, null, false) + ) + ); + when(sec.getFieldType(eq("field"))).thenReturn(fieldType); + when(sec.getMatchingFieldNames(any())).then(invocation -> Set.of(invocation.getArguments()[0])); + when(sec.nestedLookup()).thenReturn(NestedLookup.EMPTY); + FetchContext fetchContext = mock(FetchContext.class); + when(fetchContext.fetchFieldsContext()).thenReturn(ffc); + when(fetchContext.storedFieldsContext()).thenReturn(storedFieldsContext); + when(fetchContext.getSearchExecutionContext()).thenReturn(sec); + FetchFieldsPhase fetchFieldsPhase = new FetchFieldsPhase(); + FetchSubPhaseProcessor processor = fetchFieldsPhase.getProcessor(fetchContext); + StoredFieldsSpec storedFieldsSpec = processor.storedFieldsSpec(); + assertEquals(3, storedFieldsSpec.requiredStoredFields().size()); + assertTrue(storedFieldsSpec.requiredStoredFields().contains("_routing")); + assertTrue(storedFieldsSpec.requiredStoredFields().contains("_ignored")); + assertTrue(storedFieldsSpec.requiredStoredFields().contains("_type")); } } From 19db4903d10689b736da8a60d96530424c1978d7 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Wed, 17 Apr 2024 20:22:53 +0200 Subject: [PATCH 36/43] Update skip for profile yaml tests following #107551 --- .../rest-api-spec/test/search/370_profile.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml index dda3d14a5ae1..817c62dbdd12 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/370_profile.yml @@ -22,8 +22,8 @@ setup: --- fetch fields: - skip: - version: ' - 8.13.99' - reason: fetch fields and stored_fields using ValueFetcher + version: ' - 8.14.99' + reason: _ignored is returned only from 8.15 on - do: search: @@ -57,8 +57,8 @@ fetch fields: --- fetch source: - skip: - version: ' - 8.13.99' - reason: fetch fields and stored_fields using ValueFetcher + version: ' - 8.14.99' + reason: _ignored is returned only from 8.15 on - do: search: @@ -88,8 +88,8 @@ fetch source: --- fetch nested source: - skip: - version: ' - 8.13.99' - reason: fetch fields and stored_fields using ValueFetcher + version: ' - 8.14.99' + reason: _ignored is returned only from 8.15 on - do: indices.create: From 732c7c4c30e8ac068d267a393315695e5d7573cb Mon Sep 17 00:00:00 2001 From: Mary Gouseti Date: Wed, 17 Apr 2024 21:36:26 +0300 Subject: [PATCH 37/43] [DSL] Remove REST APIs for global retention (#107565) --- docs/changelog/105682.yaml | 20 -- .../data-streams/data-stream-apis.asciidoc | 12 - .../apis/delete-global-retention.asciidoc | 121 ---------- .../apis/get-global-retention.asciidoc | 90 -------- .../lifecycle/apis/get-lifecycle.asciidoc | 8 +- .../apis/put-global-retention.asciidoc | 131 ----------- .../data-streams/lifecycle/index.asciidoc | 12 +- ...rial-manage-data-stream-retention.asciidoc | 183 --------------- ...orial-manage-existing-data-stream.asciidoc | 4 +- .../tutorial-manage-new-data-stream.asciidoc | 17 +- ...grate-data-stream-from-ilm-to-dsl.asciidoc | 16 +- .../DataStreamGlobalRetentionIT.java | 147 ------------ ...treamGlobalRetentionPermissionsRestIT.java | 213 ------------------ .../datastreams/DataStreamsPlugin.java | 6 - .../ExplainDataStreamLifecycleAction.java | 7 +- .../action/GetDataStreamLifecycleAction.java | 27 ++- ...DeleteDataStreamGlobalRetentionAction.java | 49 ---- ...estGetDataStreamGlobalRetentionAction.java | 47 ---- ...estPutDataStreamGlobalRetentionAction.java | 53 ----- ...plainDataStreamLifecycleResponseTests.java | 18 -- .../lifecycle/10_explain_lifecycle.yml | 16 +- .../190_create_data_stream_with_lifecycle.yml | 6 +- .../test/data_stream/lifecycle/20_basic.yml | 15 +- .../data_stream/lifecycle/30_not_found.yml | 23 +- .../lifecycle/40_global_retention.yml | 139 ------------ .../data_streams.delete_global_retention.json | 35 --- .../data_streams.get_global_retention.json | 29 --- .../data_streams.put_global_retention.json | 39 ---- .../cluster.component_template/10_basic.yml | 12 +- .../indices.get_index_template/10_basic.yml | 14 +- .../10_basic.yml | 6 +- .../indices.simulate_template/10_basic.yml | 30 ++- .../get/GetComponentTemplateAction.java | 4 +- .../get/GetComposableIndexTemplateAction.java | 4 +- .../post/SimulateIndexTemplateResponse.java | 4 +- .../datastreams/GetDataStreamAction.java | 4 +- .../ExplainIndexDataStreamLifecycle.java | 6 +- .../GetComponentTemplateResponseTests.java | 4 - 38 files changed, 79 insertions(+), 1492 deletions(-) delete mode 100644 docs/changelog/105682.yaml delete mode 100644 docs/reference/data-streams/lifecycle/apis/delete-global-retention.asciidoc delete mode 100644 docs/reference/data-streams/lifecycle/apis/get-global-retention.asciidoc delete mode 100644 docs/reference/data-streams/lifecycle/apis/put-global-retention.asciidoc delete mode 100644 docs/reference/data-streams/lifecycle/tutorial-manage-data-stream-retention.asciidoc delete mode 100644 modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamGlobalRetentionIT.java delete mode 100644 modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamGlobalRetentionPermissionsRestIT.java delete mode 100644 modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDeleteDataStreamGlobalRetentionAction.java delete mode 100644 modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestGetDataStreamGlobalRetentionAction.java delete mode 100644 modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestPutDataStreamGlobalRetentionAction.java delete mode 100644 modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/40_global_retention.yml delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/data_streams.delete_global_retention.json delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/data_streams.get_global_retention.json delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/data_streams.put_global_retention.json diff --git a/docs/changelog/105682.yaml b/docs/changelog/105682.yaml deleted file mode 100644 index f1713357ace8..000000000000 --- a/docs/changelog/105682.yaml +++ /dev/null @@ -1,20 +0,0 @@ -pr: 105682 -summary: Introduce global retention in data stream lifecycle. -area: Data streams -type: feature -issues: - - 106169 -highlight: - title: Add global retention in data stream lifecycle - body: |- - Data stream lifecycle now supports configuring retention on a cluster level, namely global retention. Global retention - allows us to configure two different retentions: - - - `default_retention` is applied to all data streams managed by the data stream lifecycle that do not have retention - defined on the data stream level. - - `max_retention` is applied to all data streams managed by the data stream lifecycle and it allows any data stream - data to be deleted after the `max_retention` has passed. - - Furthermore, we introduce the term `effective_retention` which is the retention applied at a certain moment to a data - stream considering all the available retention configurations. - notable: true \ No newline at end of file diff --git a/docs/reference/data-streams/data-stream-apis.asciidoc b/docs/reference/data-streams/data-stream-apis.asciidoc index d525f0d8a788..c13703ab2a6e 100644 --- a/docs/reference/data-streams/data-stream-apis.asciidoc +++ b/docs/reference/data-streams/data-stream-apis.asciidoc @@ -27,12 +27,6 @@ preview:[] preview:[] * <> preview:[] -* <> -preview:[] -* <> -preview:[] -* <> -preview:[] The following API is available for <>: @@ -65,10 +59,4 @@ include::{es-ref-dir}/data-streams/lifecycle/apis/explain-lifecycle.asciidoc[] include::{es-ref-dir}/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc[] -include::{es-ref-dir}/data-streams/lifecycle/apis/put-global-retention.asciidoc[] - -include::{es-ref-dir}/data-streams/lifecycle/apis/get-global-retention.asciidoc[] - -include::{es-ref-dir}/data-streams/lifecycle/apis/delete-global-retention.asciidoc[] - include::{es-ref-dir}/indices/downsample-data-stream.asciidoc[] diff --git a/docs/reference/data-streams/lifecycle/apis/delete-global-retention.asciidoc b/docs/reference/data-streams/lifecycle/apis/delete-global-retention.asciidoc deleted file mode 100644 index 5b211eaf09e1..000000000000 --- a/docs/reference/data-streams/lifecycle/apis/delete-global-retention.asciidoc +++ /dev/null @@ -1,121 +0,0 @@ -[[data-streams-delete-global-retention]] -=== Delete the global retention of data streams -++++ -Delete Data Stream Global Retention -++++ - -preview::[] - -Deletes the global retention configuration that applies on every data stream managed by <>. - -[[delete-global-retention-api-prereqs]] -==== {api-prereq-title} - -** If the {es} {security-features} are enabled, you must have the `manage_data_stream_global_retention` <> to use this API. - -[[data-streams-delete-global-retention-request]] -==== {api-request-title} - -`DELETE _data_stream/_global_retention` - -[[data-streams-delete-global-retention-desc]] -==== {api-description-title} - -Deletes the global retention configuration that is applied on data streams managed by data stream lifecycle. - -[role="child_attributes"] -[[delete-global-retention-api-query-parms]] -==== {api-query-parms-title} - -`dry_run`:: -(Boolean) Signals that the request should determine the effect of the removal of the existing without updating -the global retention. The default value is `false`, which means the removal will happen. - -[[delete-global-retention-api-response-body]] -==== {api-response-body-title} - -`acknowledged`:: -(boolean) -True, if the global retention has been removed. False, if it fails or if it was a dry run. - -`dry_run`:: -(boolean) -True, if this was a dry run, false otherwise. - -`affected_data_streams`:: -(array of objects) -Contains information about the data streams affected by the change. -+ -.Properties of objects in `affected_data_streams` -[%collapsible%open] -==== -`name`:: -(string) -Name of the data stream. -`previous_effective_retention`:: -(string) -The retention that was effective before the change of this request. `infinite` if there was no retention applicable. -`new_effective_retention`:: -(string) -The retention that is or would be effective after this request. `infinite` if there is no retention applicable. -==== - -[[data-streams-delete-global-retention-example]] -==== {api-examples-title} - -//// - -[source,console] --------------------------------------------------- -PUT _data_stream/_global_retention -{ - "default_retention": "7d", - "max_retention": "90d" -} - -PUT /_index_template/template -{ - "index_patterns": ["my-data-stream*"], - "template": { - "lifecycle": {} - }, - "data_stream": { } -} - -PUT /_data_stream/my-data-stream ----- -// TESTSETUP -//// - -//// -[source,console] ----- -DELETE /_data_stream/my-data-stream* -DELETE /_index_template/template -DELETE /_data_stream/_global_retention ----- -// TEARDOWN -//// - -Let's update the global retention: -[source,console] --------------------------------------------------- -DELETE _data_stream/_global_retention --------------------------------------------------- - -The response will look like the following: - -[source,console-result] --------------------------------------------------- -{ - "acknowledged": true, - "dry_run": false, - "affected_data_streams": [ - { - "name": "my-data-stream", - "previous_effective_retention": "7d", - "new_effective_retention": "infinite" - } - ] -} --------------------------------------------------- diff --git a/docs/reference/data-streams/lifecycle/apis/get-global-retention.asciidoc b/docs/reference/data-streams/lifecycle/apis/get-global-retention.asciidoc deleted file mode 100644 index 0997c2d84ece..000000000000 --- a/docs/reference/data-streams/lifecycle/apis/get-global-retention.asciidoc +++ /dev/null @@ -1,90 +0,0 @@ -[[data-streams-get-global-retention]] -=== Get the global retention of data streams -++++ -Get Data Stream Global Retention -++++ - -preview::[] - -Gets the global retention that applies on every data stream managed by <>. - -[[get-global-retention-api-prereqs]] -==== {api-prereq-title} - -** If the {es} {security-features} are enabled, you must have the `monitor_data_stream_global_retention` or -`manage_data_stream_global_retention` <> to use this API. - -[[data-streams-get-global-retention-request]] -==== {api-request-title} - -`GET _data_stream/_global_retention` - -[[data-streams-get-global-retention-desc]] -==== {api-description-title} - -Gets the global retention configuration that is applied on data streams managed by data stream lifecycle. - -[role="child_attributes"] -[[get-global-retention-api-query-parms]] -==== {api-query-parms-title} - -include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local] - -include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] - -[[get-global-retention-api-response-body]] -==== {api-response-body-title} - -`default_retention`:: -(Optional, string) -The default retention that will apply to any data stream managed by data stream lifecycle that does not have a retention -defined on the data stream level. - -`max_retention`:: -(Optional, string) -The max retention that will apply to all data streams managed by data stream lifecycle. The max retention will override the -retention of a data stream whose retention exceeds the max retention. - - -[[data-streams-get-global-retention-example]] -==== {api-examples-title} - -//// - -[source,console] --------------------------------------------------- -PUT _data_stream/_global_retention -{ - "default_retention": "7d", - "max_retention": "90d" -} --------------------------------------------------- -// TESTSETUP - -[source,console] --------------------------------------------------- -DELETE _data_stream/_global_retention --------------------------------------------------- -// TEARDOWN - -//// - -Let's retrieve the global retention: - -[source,console] --------------------------------------------------- -GET _data_stream/_global_retention --------------------------------------------------- - -The response will look like the following: - -[source,console-result] --------------------------------------------------- -{ - "default_retention": "7d", <1> - "max_retention": "90d" <2> -} --------------------------------------------------- -<1> 7 days retention will be applied to any data stream that does not have retention set in its lifecycle. -<2> 90 days retention will be applied to all data streams that have retention that exceeds the 90 days, this -applies to data streams that have infinite retention too. \ No newline at end of file diff --git a/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc b/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc index 83955417abd0..0d80a31bd4f5 100644 --- a/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc +++ b/docs/reference/data-streams/lifecycle/apis/get-lifecycle.asciidoc @@ -130,18 +130,14 @@ The response will look like the following: "name": "my-data-stream-1", "lifecycle": { "enabled": true, - "data_retention": "7d", - "effective_retention": "7d", - "retention_determined_by": "data_stream_configuration" + "data_retention": "7d" } }, { "name": "my-data-stream-2", "lifecycle": { "enabled": true, - "data_retention": "7d", - "effective_retention": "7d", - "retention_determined_by": "data_stream_configuration" + "data_retention": "7d" } } ] diff --git a/docs/reference/data-streams/lifecycle/apis/put-global-retention.asciidoc b/docs/reference/data-streams/lifecycle/apis/put-global-retention.asciidoc deleted file mode 100644 index c9bc804c1340..000000000000 --- a/docs/reference/data-streams/lifecycle/apis/put-global-retention.asciidoc +++ /dev/null @@ -1,131 +0,0 @@ -[[data-streams-put-global-retention]] -=== Update the global retention of data streams -++++ -Update Data Stream Global Retention -++++ - -preview::[] - -Updates the global retention configuration that applies on every data stream managed by <>. - -[[put-global-retention-api-prereqs]] -==== {api-prereq-title} - -** If the {es} {security-features} are enabled, you must have the `manage_data_stream_global_retention` <> to use this API. - -[[data-streams-put-global-retention-request]] -==== {api-request-title} - -`PUT _data_stream/_global_retention` - -[[data-streams-put-global-retention-desc]] -==== {api-description-title} - -Updates the global retention configuration that is applied on data streams managed by data stream lifecycle. - -[role="child_attributes"] -[[put-global-retention-api-query-parms]] -==== {api-query-parms-title} - -`dry_run`:: -(Boolean) Signals that the request should determine the effect of the provided configuration without updating the -global retention settings. The default value is `false`, which means the configuration provided will be applied. - -[[put-global-retention-api-request-body]] -==== {api-request-body-title} - -`default_retention`:: -(Optional, string) -The default retention that will apply to any data stream managed by data stream lifecycle that does not have a retention -defined on the data stream level. - -`max_retention`:: -(Optional, string) -The max retention that will apply to all data streams managed by data stream lifecycle. The max retention will override the -retention of a data stream which retention exceeds the max retention. - -[[put-global-retention-api-response-body]] -==== {api-response-body-title} - -`acknowledged`:: -(boolean) -True, if the global retention has been updated to the provided values. False, if it fails or if it was a dry run. - -`dry_run`:: -(boolean) -True, if this was a dry run, false otherwise. - -`affected_data_streams`:: -(array of objects) -Contains information about the data streams affected by the change. -+ -.Properties of objects in `affected_data_streams` -[%collapsible%open] -==== -`name`:: -(string) -Name of the data stream. -`previous_effective_retention`:: -(string) -The retention that was effective before the change of this request. `infinite` if there was no retention applicable. -`new_effective_retention`:: -(string) -The retention that is or would be effective after this request. `infinite` if there is no retention applicable. -==== - -[[data-streams-put-global-retention-example]] -==== {api-examples-title} - -//// -[source,console] ----- -PUT /_index_template/template -{ - "index_patterns": ["my-data-stream*"], - "template": { - "lifecycle": {} - }, - "data_stream": { } -} - -PUT /_data_stream/my-data-stream ----- -// TESTSETUP -//// - -//// -[source,console] ----- -DELETE /_data_stream/my-data-stream* -DELETE /_index_template/template -DELETE /_data_stream/_global_retention ----- -// TEARDOWN -//// - -Let's update the global retention: -[source,console] --------------------------------------------------- -PUT _data_stream/_global_retention -{ - "default_retention": "7d", - "max_retention": "90d" -} --------------------------------------------------- - -The response will look like the following: - -[source,console-result] --------------------------------------------------- -{ - "acknowledged": true, - "dry_run": false, - "affected_data_streams": [ - { - "name": "my-data-stream", - "previous_effective_retention": "infinite", - "new_effective_retention": "7d" - } - ] -} --------------------------------------------------- diff --git a/docs/reference/data-streams/lifecycle/index.asciidoc b/docs/reference/data-streams/lifecycle/index.asciidoc index dff3dae22f8e..bf861df7c80d 100644 --- a/docs/reference/data-streams/lifecycle/index.asciidoc +++ b/docs/reference/data-streams/lifecycle/index.asciidoc @@ -16,8 +16,7 @@ To achieve that, it supports: * Automatic <>, which chunks your incoming data in smaller pieces to facilitate better performance and backwards incompatible mapping changes. * Configurable retention, which allows you to configure the time period for which your data is guaranteed to be stored. -{es} is allowed at a later time to delete data older than this time period. Retention can be configured on the data stream level -or on a global level. Read more about the different options in this <>. +{es} is allowed at a later time to delete data older than this time period. A data stream lifecycle also supports downsampling the data stream backing indices. See <> for @@ -43,10 +42,9 @@ data that is most likely to keep being queried. 4. If <> is configured it will execute all the configured downsampling rounds. 5. Applies retention to the remaining backing indices. This means deleting the backing indices whose -`generation_time` is longer than the effective retention period (read more about the -<>). The `generation_time` is only applicable to rolled -over backing indices and it is either the time since the backing index got rolled over, or the time optionally configured -in the <> setting. +`generation_time` is longer than the configured retention period. The `generation_time` is only applicable to rolled over backing +indices and it is either the time since the backing index got rolled over, or the time optionally configured in the +<> setting. IMPORTANT: We use the `generation_time` instead of the creation time because this ensures that all data in the backing index have passed the retention period. As a result, the retention period is not the exact time data gets deleted, but @@ -79,6 +77,4 @@ include::tutorial-manage-new-data-stream.asciidoc[] include::tutorial-manage-existing-data-stream.asciidoc[] -include::tutorial-manage-data-stream-retention.asciidoc[] - include::tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc[] diff --git a/docs/reference/data-streams/lifecycle/tutorial-manage-data-stream-retention.asciidoc b/docs/reference/data-streams/lifecycle/tutorial-manage-data-stream-retention.asciidoc deleted file mode 100644 index 7b84cd238ce4..000000000000 --- a/docs/reference/data-streams/lifecycle/tutorial-manage-data-stream-retention.asciidoc +++ /dev/null @@ -1,183 +0,0 @@ -[role="xpack"] -[[tutorial-manage-data-stream-retention]] -=== Tutorial: Data stream retention - -preview::[] - -In this tutorial, we are going to go over the data stream lifecycle retention, define it, go over how it can be configured and how -it can be applied. Keep in mind, the following options apply only to data streams that are managed by the data stream lifecycle. - -. <> -. <> -. <> -. <> - -You can verify if a data steam is managed by the data stream lifecycle via the <>: - -//// -[source,console] ----- -PUT /_index_template/template -{ - "index_patterns": ["my-data-stream*"], - "template": { - "lifecycle": {} - }, - "data_stream": { } -} - -PUT /_data_stream/my-data-stream ----- -// TESTSETUP -//// - -//// -[source,console] ----- -DELETE /_data_stream/my-data-stream* -DELETE /_index_template/template -DELETE /_data_stream/_global_retention ----- -// TEARDOWN -//// - -[source,console] --------------------------------------------------- -GET _data_stream/my-data-stream/_lifecycle --------------------------------------------------- - -The result should look like this: - -[source,console-result] --------------------------------------------------- -{ - "data_streams": [ - { - "name": "my-data-stream", <1> - "lifecycle": { - "enabled": true <2> - } - } - ] -} --------------------------------------------------- -// TESTRESPONSE[skip:the result is for illustrating purposes only] -<1> The name of your data stream. -<2> Ensure that the lifecycle is enabled, meaning this should be `true`. - -[discrete] -[[what-is-retention]] -==== What is data stream retention? - -We define retention as the least amount of time the data of a data stream are going to be kept in {es}. After this time period -has passed, {es} is allowed to remove these data to free up space and/or manage costs. - -NOTE: Retention does not define the period that the data will be removed, but the minimum time period they will be kept. - -We define 4 different types of retention: - -* The data stream retention, or `data_retention`, which is the retention configured on the data stream level. It can be -set via an <> for future data streams or via the <> for an existing data stream. When the data stream retention is not set, it implies that the data -need to be kept forever. -* The global default retention, or `default_retention`, which is a retention configured on a cluster level and will be -applied to all data streams managed by data stream lifecycle that do not have `data_retention` configured. Effectively, -it ensures that there will be no data streams keeping their data forever. This can be set via the -<>. -* The global max retention, or `max_retention`, which is a retention configured on a cluster level and will be applied to -all data streams managed by data stream lifecycle. Effectively, it ensures that there will be no data streams whose retention -will exceed this time period. This can be set via the <>. -* The effective retention, or `effective_retention`, which is the retention applied at a data stream on a given moment. -Effective retention cannot be set, it is derived by taking into account all the configured retention listed above and is -calculated as it is described <>. - -[discrete] -[[retention-configuration]] -==== How to configure retention? - -- By setting the `data_retention` on the data stream level. This retention can be configured in two ways: -+ --- For new data streams, it can be defined in the index template that would be applied during the data stream's creation. -You can use the <>, for example: -+ -[source,console] --------------------------------------------------- -PUT _index_template/template -{ - "index_patterns": ["my-data-stream*"], - "data_stream": { }, - "priority": 500, - "template": { - "lifecycle": { - "data_retention": "7d" - } - }, - "_meta": { - "description": "Template with data stream lifecycle" - } -} --------------------------------------------------- --- For an existing data stream, it can be set via the <>. -+ -[source,console] ----- -PUT _data_stream/my-data-stream/_lifecycle -{ - "data_retention": "30d" <1> -} ----- -// TEST[continued] -<1> The retention period of this data stream is set to 30 days. - -- By setting the global retention via the `default_retention` and `max_retention` that are set on a cluster level. You -can set them via the <>. For example: -+ -[source,console] --------------------------------------------------- -PUT _data_stream/_global_retention -{ - "default_retention": "7d", - "max_retention": "90d" -} --------------------------------------------------- -// TEST[continued] - -[discrete] -[[effective-retention-calculation]] -==== How is the effective retention calculated? -The effective is calculated in the following way: - -- The `effective_retention` is the `default_retention`, when `default_retention` is defined and the data stream does not -have `data_retention`. -- The `effective_retention` is the `data_retention`, when `data_retention` is defined and if `max_retention` is defined, -it is less than the `max_retention`. -- The `effective_retention` is the `max_retention`, when `max_retention` is defined, and the data stream has either no -`data_retention` or its `data_retention` is greater than the `max_retention`. - -The above is demonstrated in the examples below: - -|=== -|`default_retention` |`max_retention` |`data_retention` |`effective_retention` |Retention determined by - -|Not set |Not set |Not set |Infinite |N/A -|Not relevant |12 months |**30 days** |30 days |`data_retention` -|Not relevant |Not set |**30 days** |30 days |`data_retention` -|**30 days** |12 months |Not set |30 days |`default_retention` -|**30 days** |30 days |Not set |30 days |`default_retention` -|Not relevant |**30 days** |12 months |30 days |`max_retention` -|Not set |**30 days** |Not set |30 days |`max_retention` -|=== - -[discrete] -[[effective-retention-application]] -==== How is the effective retention applied? - -Retention is applied to the remaining backing indices of a data stream as the last step of -<>. Data stream lifecycle will retrieve the backing indices -whose `generation_time` is longer than the effective retention period and delete them. The `generation_time` is only -applicable to rolled over backing indices and it is either the time since the backing index got rolled over, or the time -optionally configured in the <> setting. - -IMPORTANT: We use the `generation_time` instead of the creation time because this ensures that all data in the backing -index have passed the retention period. As a result, the retention period is not the exact time data get deleted, but -the minimum time data will be stored. \ No newline at end of file diff --git a/docs/reference/data-streams/lifecycle/tutorial-manage-existing-data-stream.asciidoc b/docs/reference/data-streams/lifecycle/tutorial-manage-existing-data-stream.asciidoc index 7be2b30b9b83..5670faaade3c 100644 --- a/docs/reference/data-streams/lifecycle/tutorial-manage-existing-data-stream.asciidoc +++ b/docs/reference/data-streams/lifecycle/tutorial-manage-existing-data-stream.asciidoc @@ -74,9 +74,7 @@ The response will look like: "generation_time": "6.84s", <9> "lifecycle": { "enabled": true, - "data_retention": "30d", - "effective_retention": "30d" <10> - "retention_determined_by": "data_stream_configuration" + "data_retention": "30d" <10> } } } diff --git a/docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc b/docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc index ecfdc1688408..6f1d81ab6ead 100644 --- a/docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc +++ b/docs/reference/data-streams/lifecycle/tutorial-manage-new-data-stream.asciidoc @@ -93,12 +93,10 @@ The result will look like this: { "data_streams": [ { - "name": "my-data-stream", <1> + "name": "my-data-stream",<1> "lifecycle": { - "enabled": true, <2> - "data_retention": "7d", <3> - "effective_retention": "7d", <4> - "retention_determined_by": "data_stream_configuration" <5> + "enabled": true, <2> + "data_retention": "7d" <3> } } ] @@ -106,11 +104,8 @@ The result will look like this: -------------------------------------------------- <1> The name of your data stream. <2> Shows if the data stream lifecycle is enabled for this data stream. -<3> The desired retention period of the data indexed in this data stream, this means that if there are no other limitations -the data for this data stream will be preserved for at least 7 days. -<4> The effective retention, this means that the data in this data stream will +<3> The retention period of the data indexed in this data stream, this means that the data in this data stream will be kept at least for 7 days. After that {es} can delete it at its own discretion. -<5> The configuration that determined the effective retention. If you want to see more information about how the data stream lifecycle is applied on individual backing indices use the <>: @@ -133,9 +128,7 @@ The result will look like this: "time_since_index_creation": "1.6m", <3> "lifecycle": { <4> "enabled": true, - "data_retention": "7d", - "effective_retention": "7d", - "retention_determined_by": "data_stream_configuration" + "data_retention": "7d" } } } diff --git a/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc b/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc index 65eaf472890f..3125c82120d8 100644 --- a/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc +++ b/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc @@ -200,10 +200,10 @@ PUT _index_template/dsl-data-stream-template "template": { "settings": { "index.lifecycle.name": "pre-dsl-ilm-policy", - "index.lifecycle.prefer_ilm": false <1> + "index.lifecycle.prefer_ilm": false <1> }, - "lifecycle": { <2> - "data_retention": "7d" <3> + "lifecycle": { + "data_retention": "7d" <2> } } } @@ -215,8 +215,6 @@ PUT _index_template/dsl-data-stream-template precedence over data stream lifecycle. <2> We're configuring the data stream lifecycle so _new_ data streams will be managed by data stream lifecycle. -<3> The desired retention, meaning that this data stream should keep the data for at least 7 days, -if this retention is possible. We've now made sure that new data streams will be managed by data stream lifecycle. @@ -270,9 +268,7 @@ GET _data_stream/dsl-data-stream "template": "dsl-data-stream-template", "lifecycle": { "enabled": true, - "data_retention": "7d", - "effective_retention": "7d", - "retention_determined_by": "data_stream_configuration" + "data_retention": "7d" }, "ilm_policy": "pre-dsl-ilm-policy", "next_generation_managed_by": "Data stream lifecycle", <3> @@ -350,9 +346,7 @@ GET _data_stream/dsl-data-stream "template": "dsl-data-stream-template", "lifecycle": { "enabled": true, - "data_retention": "7d", - "effective_retention": "7d", - "retention_determined_by": "data_stream_configuration" + "data_retention": "7d" }, "ilm_policy": "pre-dsl-ilm-policy", "next_generation_managed_by": "Data stream lifecycle", diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamGlobalRetentionIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamGlobalRetentionIT.java deleted file mode 100644 index 557e70ba65e9..000000000000 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamGlobalRetentionIT.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -package org.elasticsearch.datastreams.lifecycle; - -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.datastreams.DisabledSecurityDataStreamTestCase; -import org.junit.After; -import org.junit.Before; - -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.nullValue; - -public class DataStreamGlobalRetentionIT extends DisabledSecurityDataStreamTestCase { - - @Before - public void setup() throws IOException { - updateClusterSettings( - Settings.builder() - .put("data_streams.lifecycle.poll_interval", "1s") - .put("cluster.lifecycle.default.rollover", "min_docs=1,max_docs=1") - .build() - ); - // Create a template with the default lifecycle - Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/1"); - putComposableIndexTemplateRequest.setJsonEntity(""" - { - "index_patterns": ["my-data-stream*"], - "data_stream": {}, - "template": { - "lifecycle": {} - } - } - """); - assertOK(client().performRequest(putComposableIndexTemplateRequest)); - - // Create a data streams with one doc - Request createDocRequest = new Request("POST", "/my-data-stream/_doc?refresh=true"); - createDocRequest.setJsonEntity("{ \"@timestamp\": \"2022-12-12\"}"); - assertOK(client().performRequest(createDocRequest)); - } - - @After - public void cleanUp() throws IOException { - adminClient().performRequest(new Request("DELETE", "_data_stream/*")); - } - - @SuppressWarnings("unchecked") - public void testDefaultRetention() throws Exception { - { - // Set global retention - Request request = new Request("PUT", "_data_stream/_global_retention"); - request.setJsonEntity(""" - { - "default_retention": "10s" - }"""); - assertAcknowledged(client().performRequest(request)); - } - - // Verify that the effective retention matches the default retention - { - Request request = new Request("GET", "/_data_stream/my-data-stream"); - Response response = client().performRequest(request); - List dataStreams = (List) entityAsMap(response).get("data_streams"); - assertThat(dataStreams.size(), is(1)); - Map dataStream = (Map) dataStreams.get(0); - assertThat(dataStream.get("name"), is("my-data-stream")); - Map lifecycle = (Map) dataStream.get("lifecycle"); - assertThat(lifecycle.get("effective_retention"), is("10s")); - assertThat(lifecycle.get("retention_determined_by"), is("default_global_retention")); - assertThat(lifecycle.get("data_retention"), nullValue()); - } - - // Verify that the first generation index was removed - assertBusy(() -> { - Response response = client().performRequest(new Request("GET", "/_data_stream/my-data-stream")); - Map dataStream = ((List>) entityAsMap(response).get("data_streams")).get(0); - assertThat(dataStream.get("name"), is("my-data-stream")); - List backingIndices = (List) dataStream.get("indices"); - assertThat(backingIndices.size(), is(1)); - // 2 backing indices created + 1 for the deleted index - assertThat(dataStream.get("generation"), is(3)); - }, 20, TimeUnit.SECONDS); - } - - @SuppressWarnings("unchecked") - public void testMaxRetention() throws Exception { - { - // Set global retention - Request request = new Request("PUT", "_data_stream/_global_retention"); - request.setJsonEntity(""" - { - "max_retention": "10s" - }"""); - assertAcknowledged(client().performRequest(request)); - } - boolean withDataStreamLevelRetention = randomBoolean(); - if (withDataStreamLevelRetention) { - Request request = new Request("PUT", "_data_stream/my-data-stream/_lifecycle"); - request.setJsonEntity(""" - { - "data_retention": "30d" - }"""); - assertAcknowledged(client().performRequest(request)); - } - - // Verify that the effective retention matches the max retention - { - Request request = new Request("GET", "/_data_stream/my-data-stream"); - Response response = client().performRequest(request); - List dataStreams = (List) entityAsMap(response).get("data_streams"); - assertThat(dataStreams.size(), is(1)); - Map dataStream = (Map) dataStreams.get(0); - assertThat(dataStream.get("name"), is("my-data-stream")); - Map lifecycle = (Map) dataStream.get("lifecycle"); - assertThat(lifecycle.get("effective_retention"), is("10s")); - assertThat(lifecycle.get("retention_determined_by"), is("max_global_retention")); - if (withDataStreamLevelRetention) { - assertThat(lifecycle.get("data_retention"), is("30d")); - } else { - assertThat(lifecycle.get("data_retention"), nullValue()); - } - } - - // Verify that the first generation index was removed - assertBusy(() -> { - Response response = client().performRequest(new Request("GET", "/_data_stream/my-data-stream")); - Map dataStream = ((List>) entityAsMap(response).get("data_streams")).get(0); - assertThat(dataStream.get("name"), is("my-data-stream")); - List backingIndices = (List) dataStream.get("indices"); - assertThat(backingIndices.size(), is(1)); - // 2 backing indices created + 1 for the deleted index - assertThat(dataStream.get("generation"), is(3)); - }, 20, TimeUnit.SECONDS); - } -} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamGlobalRetentionPermissionsRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamGlobalRetentionPermissionsRestIT.java deleted file mode 100644 index e2e82b343fc5..000000000000 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamGlobalRetentionPermissionsRestIT.java +++ /dev/null @@ -1,213 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.datastreams.lifecycle; - -import org.apache.http.HttpHost; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.ResponseException; -import org.elasticsearch.common.settings.SecureString; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.test.cluster.ElasticsearchCluster; -import org.elasticsearch.test.cluster.FeatureFlag; -import org.elasticsearch.test.cluster.local.distribution.DistributionType; -import org.elasticsearch.test.cluster.util.resource.Resource; -import org.elasticsearch.test.rest.ESRestTestCase; -import org.junit.ClassRule; - -import java.util.Map; - -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; - -public class DataStreamGlobalRetentionPermissionsRestIT extends ESRestTestCase { - - private static final String PASSWORD = "secret-test-password"; - - @ClassRule - public static ElasticsearchCluster cluster = ElasticsearchCluster.local() - .distribution(DistributionType.DEFAULT) - .feature(FeatureFlag.FAILURE_STORE_ENABLED) - .setting("xpack.watcher.enabled", "false") - .setting("xpack.ml.enabled", "false") - .setting("xpack.security.enabled", "true") - .setting("xpack.security.transport.ssl.enabled", "false") - .setting("xpack.security.http.ssl.enabled", "false") - .user("test_admin", PASSWORD, "superuser", false) - .user("test_manage_global_retention", PASSWORD, "manage_data_stream_global_retention", false) - .user("test_monitor_global_retention", PASSWORD, "monitor_data_stream_global_retention", false) - .user("test_monitor", PASSWORD, "manage_data_stream_lifecycle", false) - .user("test_no_privilege", PASSWORD, "no_privilege", false) - .rolesFile(Resource.fromClasspath("roles.yml")) - .build(); - - @Override - protected String getTestRestCluster() { - return cluster.getHttpAddresses(); - } - - @Override - protected Settings restClientSettings() { - // If this test is running in a test framework that handles its own authorization, we don't want to overwrite it. - if (super.restClientSettings().keySet().contains(ThreadContext.PREFIX + ".Authorization")) { - return super.restClientSettings(); - } else { - // Note: This user is assigned the role "manage_data_stream_lifecycle". That role is defined in roles.yml. - String token = basicAuthHeaderValue("test_data_stream_lifecycle", new SecureString(PASSWORD.toCharArray())); - return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); - } - } - - @Override - protected Settings restAdminSettings() { - // If this test is running in a test framework that handles its own authorization, we don't want to overwrite it. - if (super.restClientSettings().keySet().contains(ThreadContext.PREFIX + ".Authorization")) { - return super.restClientSettings(); - } else { - // Note: We use the admin user because the other one is too unprivileged, so it breaks the initialization of the test - String token = basicAuthHeaderValue("test_admin", new SecureString(PASSWORD.toCharArray())); - return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); - } - } - - private Settings restManageGlobalRetentionClientSettings() { - String token = basicAuthHeaderValue("test_manage_global_retention", new SecureString(PASSWORD.toCharArray())); - return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); - } - - private Settings restMonitorGlobalRetentionClientSettings() { - String token = basicAuthHeaderValue("test_monitor_global_retention", new SecureString(PASSWORD.toCharArray())); - return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); - } - - private Settings restOnlyManageLifecycleClientSettings() { - String token = basicAuthHeaderValue("test_monitor", new SecureString(PASSWORD.toCharArray())); - return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); - } - - private Settings restNoPrivilegeClientSettings() { - String token = basicAuthHeaderValue("test_no_privilege", new SecureString(PASSWORD.toCharArray())); - return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); - } - - public void testManageGlobalRetentionPrivileges() throws Exception { - try (var client = buildClient(restManageGlobalRetentionClientSettings(), getClusterHosts().toArray(new HttpHost[0]))) { - Request request = new Request("PUT", "_data_stream/_global_retention"); - request.setJsonEntity(""" - { - "default_retention": "1d", - "max_retention": "7d" - }"""); - assertAcknowledged(client.performRequest(request)); - Map response = entityAsMap(client.performRequest(new Request("GET", "/_data_stream/_global_retention"))); - assertThat(response.get("default_retention"), equalTo("1d")); - assertThat(response.get("max_retention"), equalTo("7d")); - assertAcknowledged(client.performRequest(new Request("DELETE", "/_data_stream/_global_retention"))); - } - } - - public void testMonitorGlobalRetentionPrivileges() throws Exception { - { - Request request = new Request("PUT", "_data_stream/_global_retention"); - request.setJsonEntity(""" - { - "default_retention": "1d", - "max_retention": "7d" - }"""); - assertAcknowledged(adminClient().performRequest(request)); - } - try (var client = buildClient(restMonitorGlobalRetentionClientSettings(), getClusterHosts().toArray(new HttpHost[0]))) { - Request request = new Request("PUT", "_data_stream/_global_retention"); - request.setJsonEntity(""" - { - "default_retention": "1d", - "max_retention": "7d" - }"""); - ResponseException responseException = expectThrows(ResponseException.class, () -> client.performRequest(request)); - assertThat(responseException.getResponse().getStatusLine().getStatusCode(), is(403)); - assertThat( - responseException.getMessage(), - containsString( - "action [cluster:admin/data_stream/global_retention/put] is unauthorized for user [test_monitor_global_retention]" - ) - ); - responseException = expectThrows( - ResponseException.class, - () -> client.performRequest(new Request("DELETE", "/_data_stream/_global_retention")) - ); - assertThat(responseException.getResponse().getStatusLine().getStatusCode(), is(403)); - assertThat( - responseException.getMessage(), - containsString( - "action [cluster:admin/data_stream/global_retention/delete] is unauthorized for user [test_monitor_global_retention]" - ) - ); - Map response = entityAsMap(client.performRequest(new Request("GET", "/_data_stream/_global_retention"))); - assertThat(response.get("default_retention"), equalTo("1d")); - assertThat(response.get("max_retention"), equalTo("7d")); - } - } - - public void testManageLifecyclePrivileges() throws Exception { - try (var client = buildClient(restOnlyManageLifecycleClientSettings(), getClusterHosts().toArray(new HttpHost[0]))) { - Request request = new Request("PUT", "_data_stream/_global_retention"); - request.setJsonEntity(""" - { - "default_retention": "1d", - "max_retention": "7d" - }"""); - ResponseException responseException = expectThrows(ResponseException.class, () -> client.performRequest(request)); - assertThat(responseException.getResponse().getStatusLine().getStatusCode(), is(403)); - assertThat( - responseException.getMessage(), - containsString("action [cluster:admin/data_stream/global_retention/put] is unauthorized for user [test_monitor]") - ); - // This use has the monitor privilege which includes the monitor_data_stream_global_retention - Response response = client.performRequest(new Request("GET", "/_data_stream/_global_retention")); - assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); - } - } - - public void testNoPrivileges() throws Exception { - try (var client = buildClient(restNoPrivilegeClientSettings(), getClusterHosts().toArray(new HttpHost[0]))) { - Request request = new Request("PUT", "_data_stream/_global_retention"); - request.setJsonEntity(""" - { - "default_retention": "1d", - "max_retention": "7d" - }"""); - ResponseException responseException = expectThrows(ResponseException.class, () -> client.performRequest(request)); - assertThat(responseException.getResponse().getStatusLine().getStatusCode(), is(403)); - assertThat( - responseException.getMessage(), - containsString("action [cluster:admin/data_stream/global_retention/put] is unauthorized for user [test_no_privilege]") - ); - responseException = expectThrows( - ResponseException.class, - () -> client.performRequest(new Request("DELETE", "/_data_stream/_global_retention")) - ); - assertThat(responseException.getResponse().getStatusLine().getStatusCode(), is(403)); - assertThat( - responseException.getMessage(), - containsString("action [cluster:admin/data_stream/global_retention/delete] is unauthorized for user [test_no_privilege]") - ); - responseException = expectThrows( - ResponseException.class, - () -> client.performRequest(new Request("GET", "/_data_stream/_global_retention")) - ); - assertThat(responseException.getResponse().getStatusLine().getStatusCode(), is(403)); - assertThat( - responseException.getMessage(), - containsString("action [cluster:monitor/data_stream/global_retention/get] is unauthorized for user [test_no_privilege]") - ); - } - } -} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java index 53e0bc287d3e..c18da970ca26 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java @@ -58,12 +58,9 @@ import org.elasticsearch.datastreams.lifecycle.health.DataStreamLifecycleHealthIndicatorService; import org.elasticsearch.datastreams.lifecycle.health.DataStreamLifecycleHealthInfoPublisher; import org.elasticsearch.datastreams.lifecycle.rest.RestDataStreamLifecycleStatsAction; -import org.elasticsearch.datastreams.lifecycle.rest.RestDeleteDataStreamGlobalRetentionAction; import org.elasticsearch.datastreams.lifecycle.rest.RestDeleteDataStreamLifecycleAction; import org.elasticsearch.datastreams.lifecycle.rest.RestExplainDataStreamLifecycleAction; -import org.elasticsearch.datastreams.lifecycle.rest.RestGetDataStreamGlobalRetentionAction; import org.elasticsearch.datastreams.lifecycle.rest.RestGetDataStreamLifecycleAction; -import org.elasticsearch.datastreams.lifecycle.rest.RestPutDataStreamGlobalRetentionAction; import org.elasticsearch.datastreams.lifecycle.rest.RestPutDataStreamLifecycleAction; import org.elasticsearch.datastreams.rest.RestCreateDataStreamAction; import org.elasticsearch.datastreams.rest.RestDataStreamsStatsAction; @@ -290,9 +287,6 @@ public List getRestHandlers( handlers.add(new RestDeleteDataStreamLifecycleAction()); handlers.add(new RestExplainDataStreamLifecycleAction()); handlers.add(new RestDataStreamLifecycleStatsAction()); - handlers.add(new RestPutDataStreamGlobalRetentionAction()); - handlers.add(new RestGetDataStreamGlobalRetentionAction()); - handlers.add(new RestDeleteDataStreamGlobalRetentionAction()); return handlers; } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/ExplainDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/ExplainDataStreamLifecycleAction.java index 5bfdf2d38200..7af70f98e8ce 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/ExplainDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/ExplainDataStreamLifecycleAction.java @@ -18,7 +18,6 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.MasterNodeReadRequest; import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; -import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -216,11 +215,7 @@ public Iterator toXContentChunked(ToXContent.Params outerP return builder; }), Iterators.map(indices.iterator(), explainIndexDataLifecycle -> (builder, params) -> { builder.field(explainIndexDataLifecycle.getIndex()); - ToXContent.Params withEffectiveRetentionParams = new ToXContent.DelegatingMapParams( - DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS, - params - ); - explainIndexDataLifecycle.toXContent(builder, withEffectiveRetentionParams, rolloverConfiguration, globalRetention); + explainIndexDataLifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention); return builder; }), Iterators.single((builder, params) -> { builder.endObject(); diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleAction.java index 79e1b7177155..0a5bda11f2a2 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/action/GetDataStreamLifecycleAction.java @@ -237,17 +237,22 @@ public Iterator toXContentChunked(ToXContent.Params outerParams) { builder.startObject(); builder.startArray(DATA_STREAMS_FIELD.getPreferredName()); return builder; - }), Iterators.map(dataStreamLifecycles.iterator(), dataStreamLifecycle -> (builder, params) -> { - ToXContent.Params withEffectiveRetentionParams = new ToXContent.DelegatingMapParams( - org.elasticsearch.cluster.metadata.DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS, - params - ); - return dataStreamLifecycle.toXContent(builder, withEffectiveRetentionParams, rolloverConfiguration, globalRetention); - }), Iterators.single((builder, params) -> { - builder.endArray(); - builder.endObject(); - return builder; - })); + }), + Iterators.map( + dataStreamLifecycles.iterator(), + dataStreamLifecycle -> (builder, params) -> dataStreamLifecycle.toXContent( + builder, + params, + rolloverConfiguration, + globalRetention + ) + ), + Iterators.single((builder, params) -> { + builder.endArray(); + builder.endObject(); + return builder; + }) + ); } @Override diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDeleteDataStreamGlobalRetentionAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDeleteDataStreamGlobalRetentionAction.java deleted file mode 100644 index 1ac12c918605..000000000000 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestDeleteDataStreamGlobalRetentionAction.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -package org.elasticsearch.datastreams.lifecycle.rest; - -import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.datastreams.lifecycle.action.DeleteDataStreamGlobalRetentionAction; -import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.Scope; -import org.elasticsearch.rest.ServerlessScope; -import org.elasticsearch.rest.action.RestChunkedToXContentListener; - -import java.io.IOException; -import java.util.List; - -import static org.elasticsearch.rest.RestRequest.Method.DELETE; - -/** - * Removes the data stream global retention configuration - */ -@ServerlessScope(Scope.PUBLIC) -public class RestDeleteDataStreamGlobalRetentionAction extends BaseRestHandler { - - @Override - public String getName() { - return "delete_data_stream_global_retention_action"; - } - - @Override - public List routes() { - return List.of(new Route(DELETE, "/_data_stream/_global_retention")); - } - - @Override - protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { - DeleteDataStreamGlobalRetentionAction.Request request = new DeleteDataStreamGlobalRetentionAction.Request(); - request.dryRun(restRequest.paramAsBoolean("dry_run", false)); - return channel -> client.execute( - DeleteDataStreamGlobalRetentionAction.INSTANCE, - request, - new RestChunkedToXContentListener<>(channel) - ); - } -} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestGetDataStreamGlobalRetentionAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestGetDataStreamGlobalRetentionAction.java deleted file mode 100644 index cbe403af35f7..000000000000 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestGetDataStreamGlobalRetentionAction.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -package org.elasticsearch.datastreams.lifecycle.rest; - -import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.datastreams.lifecycle.action.GetDataStreamGlobalRetentionAction; -import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.Scope; -import org.elasticsearch.rest.ServerlessScope; -import org.elasticsearch.rest.action.RestToXContentListener; - -import java.io.IOException; -import java.util.List; - -import static org.elasticsearch.rest.RestRequest.Method.GET; - -/** - * Retrieves the data stream global retention configuration. - */ -@ServerlessScope(Scope.PUBLIC) -public class RestGetDataStreamGlobalRetentionAction extends BaseRestHandler { - - @Override - public String getName() { - return "get_data_stream_global_retention_action"; - } - - @Override - public List routes() { - return List.of(new Route(GET, "/_data_stream/_global_retention")); - } - - @Override - protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { - GetDataStreamGlobalRetentionAction.Request request = new GetDataStreamGlobalRetentionAction.Request(); - request.local(restRequest.paramAsBoolean("local", request.local())); - request.masterNodeTimeout(restRequest.paramAsTime("master_timeout", request.masterNodeTimeout())); - - return channel -> client.execute(GetDataStreamGlobalRetentionAction.INSTANCE, request, new RestToXContentListener<>(channel)); - } -} diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestPutDataStreamGlobalRetentionAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestPutDataStreamGlobalRetentionAction.java deleted file mode 100644 index 5331c4df16db..000000000000 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestPutDataStreamGlobalRetentionAction.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -package org.elasticsearch.datastreams.lifecycle.rest; - -import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.datastreams.lifecycle.action.PutDataStreamGlobalRetentionAction; -import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.Scope; -import org.elasticsearch.rest.ServerlessScope; -import org.elasticsearch.rest.action.RestChunkedToXContentListener; -import org.elasticsearch.xcontent.XContentParser; - -import java.io.IOException; -import java.util.List; - -import static org.elasticsearch.rest.RestRequest.Method.PUT; - -/** - * Updates the default_retention and the max_retention of the data stream global retention configuration. It - * does not accept an empty payload. - */ -@ServerlessScope(Scope.PUBLIC) -public class RestPutDataStreamGlobalRetentionAction extends BaseRestHandler { - - @Override - public String getName() { - return "put_data_stream_global_retention_action"; - } - - @Override - public List routes() { - return List.of(new Route(PUT, "/_data_stream/_global_retention")); - } - - @Override - protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { - try (XContentParser parser = restRequest.contentParser()) { - PutDataStreamGlobalRetentionAction.Request request = PutDataStreamGlobalRetentionAction.Request.parseRequest(parser); - request.dryRun(restRequest.paramAsBoolean("dry_run", false)); - return channel -> client.execute( - PutDataStreamGlobalRetentionAction.INSTANCE, - request, - new RestChunkedToXContentListener<>(channel) - ); - } - } -} diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/ExplainDataStreamLifecycleResponseTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/ExplainDataStreamLifecycleResponseTests.java index 462c0626c629..6b9184ea09e0 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/ExplainDataStreamLifecycleResponseTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/action/ExplainDataStreamLifecycleResponseTests.java @@ -37,7 +37,6 @@ import static org.elasticsearch.datastreams.lifecycle.action.ExplainDataStreamLifecycleAction.Response; import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS; -import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; @@ -196,23 +195,6 @@ public void testToXContent() throws IOException { Map lifecycleMap = (Map) explainIndexMap.get("lifecycle"); assertThat(lifecycleMap.get("data_retention"), nullValue()); - if (response.getGlobalRetention() == null) { - assertThat(lifecycleMap.get("effective_retention"), nullValue()); - assertThat(lifecycleMap.get("retention_determined_by"), nullValue()); - } else if (response.getGlobalRetention().getDefaultRetention() != null) { - assertThat( - lifecycleMap.get("effective_retention"), - equalTo(response.getGlobalRetention().getDefaultRetention().getStringRep()) - ); - assertThat(lifecycleMap.get("retention_determined_by"), equalTo("default_global_retention")); - } else { - assertThat( - lifecycleMap.get("effective_retention"), - equalTo(response.getGlobalRetention().getMaxRetention().getStringRep()) - ); - assertThat(lifecycleMap.get("retention_determined_by"), equalTo("max_global_retention")); - } - Map lifecycleRollover = (Map) lifecycleMap.get("rollover"); assertThat(lifecycleRollover.get("min_primary_shard_docs"), is(4)); assertThat(lifecycleRollover.get("max_primary_shard_docs"), is(9)); diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/10_explain_lifecycle.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/10_explain_lifecycle.yml index c79775c51c39..d03174b448ff 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/10_explain_lifecycle.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/10_explain_lifecycle.yml @@ -1,9 +1,9 @@ --- "Explain backing index lifecycle": - - skip: - cluster_features: ["data_stream.lifecycle.global_retention"] - reason: "Data stream lifecycle with effective retention was released in 8.14" - features: allowed_warnings + - requires: + cluster_features: [ "gte_v8.11.0" ] + reason: "Data stream lifecycle was released as tech preview in 8.11" + test_runner_features: allowed_warnings - do: allowed_warnings: - "index template [template-with-lifecycle] has index patterns [managed-data-stream] matching patterns from existing older templates [global] with patterns (global => [*]); this template [template-with-lifecycle] will take precedence during new index creation" @@ -36,9 +36,7 @@ indices.explain_data_lifecycle: index: $backing_index - match: { indices.$backing_index.managed_by_lifecycle: true } - - match: { indices.$backing_index.lifecycle.data_retention: "30d" } - - match: { indices.$backing_index.lifecycle.effective_retention: "30d"} - - match: { indices.$backing_index.lifecycle.retention_determined_by: "data_stream_configuration"} + - match: { indices.$backing_index.lifecycle.data_retention: '30d' } - match: { indices.$backing_index.lifecycle.enabled: true } - is_false: indices.$backing_index.lifecycle.rollover @@ -48,9 +46,7 @@ index: $backing_index include_defaults: true - match: { indices.$backing_index.managed_by_lifecycle: true } - - match: { indices.$backing_index.lifecycle.data_retention: "30d" } - - match: { indices.$backing_index.lifecycle.effective_retention: "30d"} - - match: { indices.$backing_index.lifecycle.retention_determined_by: "data_stream_configuration"} + - match: { indices.$backing_index.lifecycle.data_retention: '30d' } - is_true: indices.$backing_index.lifecycle.rollover diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/190_create_data_stream_with_lifecycle.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/190_create_data_stream_with_lifecycle.yml index 745fd342d3a4..e13f245855f8 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/190_create_data_stream_with_lifecycle.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/190_create_data_stream_with_lifecycle.yml @@ -1,8 +1,8 @@ --- "Create data stream with lifecycle": - requires: - cluster_features: ["gte_v8.14.0"] - reason: "Data stream lifecycle with effective retention was released in 8.14" + cluster_features: [ "gte_v8.11.0" ] + reason: "Data stream lifecycle was GA in 8.11" test_runner_features: allowed_warnings - do: allowed_warnings: @@ -35,7 +35,5 @@ - match: { data_streams.0.template: 'template-with-lifecycle' } - match: { data_streams.0.hidden: false } - match: { data_streams.0.lifecycle.data_retention: '30d' } - - match: { data_streams.0.lifecycle.effective_retention: '30d'} - - match: { data_streams.0.lifecycle.retention_determined_by: 'data_stream_configuration'} - match: { data_streams.0.lifecycle.enabled: true } - is_true: data_streams.0.lifecycle.rollover diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml index ea34c6880d1f..18aee1bf7723 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/20_basic.yml @@ -1,8 +1,8 @@ setup: - skip: features: allowed_warnings - cluster_features: ["data_stream.lifecycle.global_retention"] - reason: "Data stream lifecycles with global retention are only supported in 8.14+" + cluster_features: ["gte_v8.11.0"] + reason: "Data stream lifecycles only supported in 8.11+" - do: allowed_warnings: - "index template [my-lifecycle] has index patterns [data-stream-with-lifecycle] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-lifecycle] will take precedence during new index creation" @@ -47,8 +47,6 @@ setup: - length: { data_streams: 1} - match: { data_streams.0.name: data-stream-with-lifecycle } - match: { data_streams.0.lifecycle.data_retention: '10d' } - - match: { data_streams.0.lifecycle.effective_retention: '10d' } - - match: { data_streams.0.lifecycle.retention_determined_by: 'data_stream_configuration' } - match: { data_streams.0.lifecycle.enabled: true} --- @@ -63,7 +61,6 @@ setup: - length: { data_streams: 1} - match: { data_streams.0.name: simple-data-stream1 } - match: { data_streams.0.lifecycle.enabled: true} - - is_false: data_streams.0.lifecycle.effective_retention --- "Put data stream lifecycle": @@ -95,7 +92,6 @@ setup: - length: { data_streams: 2 } - match: { data_streams.0.name: data-stream-with-lifecycle } - match: { data_streams.0.lifecycle.data_retention: "30d" } - - is_false: data_streams.0.lifecycle.effective_retention - match: { data_streams.0.lifecycle.enabled: false} - match: { data_streams.0.lifecycle.downsampling.0.after: '10d'} - match: { data_streams.0.lifecycle.downsampling.0.fixed_interval: '1h'} @@ -103,7 +99,6 @@ setup: - match: { data_streams.0.lifecycle.downsampling.1.fixed_interval: '10h'} - match: { data_streams.1.name: simple-data-stream1 } - match: { data_streams.1.lifecycle.data_retention: "30d" } - - is_false: data_streams.0.lifecycle.effective_retention - match: { data_streams.1.lifecycle.enabled: false} - match: { data_streams.1.lifecycle.downsampling.0.after: '10d'} - match: { data_streams.1.lifecycle.downsampling.0.fixed_interval: '1h'} @@ -129,8 +124,6 @@ setup: - match: { data_streams.0.lifecycle.enabled: true} - match: { data_streams.1.name: simple-data-stream1 } - match: { data_streams.1.lifecycle.data_retention: "30d" } - - match: { data_streams.1.lifecycle.effective_retention: "30d"} - - match: { data_streams.1.lifecycle.retention_determined_by: "data_stream_configuration"} - match: { data_streams.1.lifecycle.enabled: true} @@ -144,8 +137,6 @@ setup: - length: { data_streams: 1} - match: { data_streams.0.name: data-stream-with-lifecycle } - match: { data_streams.0.lifecycle.data_retention: "10d" } - - match: { data_streams.0.lifecycle.effective_retention: "10d"} - - match: { data_streams.0.lifecycle.retention_determined_by: "data_stream_configuration"} - is_true: data_streams.0.lifecycle.rollover --- @@ -163,8 +154,6 @@ setup: - length: { data_streams: 1 } - match: { data_streams.0.name: simple-data-stream1 } - match: { data_streams.0.lifecycle.data_retention: "30d" } - - match: { data_streams.0.lifecycle.effective_retention: "30d"} - - match: { data_streams.0.lifecycle.retention_determined_by: "data_stream_configuration"} - match: { data_streams.0.lifecycle.enabled: true } - do: diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/30_not_found.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/30_not_found.yml index 303fbddd6c19..24d0a5649a61 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/30_not_found.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/30_not_found.yml @@ -23,18 +23,13 @@ setup: --- "Get data stream lifecycle": - - skip: - cluster_features: ["data_stream.lifecycle.global_retention"] - reason: "Data stream lifecycle with effective retention was released in 8.14" - do: indices.get_data_lifecycle: name: "*" - length: { data_streams: 1} - match: { data_streams.0.name: my-data-stream-1 } - - match: { data_streams.0.lifecycle.data_retention: "10d" } - - match: { data_streams.0.lifecycle.effective_retention: "10d"} - - match: { data_streams.0.lifecycle.retention_determined_by: "data_stream_configuration"} + - match: { data_streams.0.lifecycle.data_retention: '10d' } - match: { data_streams.0.lifecycle.enabled: true} --- @@ -48,9 +43,7 @@ setup: --- "Put data stream lifecycle does not succeed when at lease one data stream does not exist": - - skip: - cluster_features: ["data_stream.lifecycle.global_retention"] - reason: "Data stream lifecycle with effective retention was released in 8.14" + - do: catch: missing indices.put_data_lifecycle: @@ -64,16 +57,12 @@ setup: name: "*" - length: { data_streams: 1 } - match: { data_streams.0.name: my-data-stream-1 } - - match: { data_streams.0.lifecycle.data_retention: "10d" } - - match: { data_streams.0.lifecycle.effective_retention: "10d"} - - match: { data_streams.0.lifecycle.retention_determined_by: "data_stream_configuration"} + - match: { data_streams.0.lifecycle.data_retention: '10d' } - match: { data_streams.0.lifecycle.enabled: true } --- "Delete data stream lifecycle does not succeed when at lease one data stream does not exist": - - skip: - cluster_features: ["data_stream.lifecycle.global_retention"] - reason: "Data stream lifecycle with effective retention was released in 8.14" + - do: catch: missing indices.delete_data_lifecycle: @@ -85,7 +74,5 @@ setup: name: "*" - length: { data_streams: 1 } - match: { data_streams.0.name: my-data-stream-1 } - - match: { data_streams.0.lifecycle.data_retention: "10d" } - - match: { data_streams.0.lifecycle.effective_retention: "10d"} - - match: { data_streams.0.lifecycle.retention_determined_by: "data_stream_configuration"} + - match: { data_streams.0.lifecycle.data_retention: '10d' } - match: { data_streams.0.lifecycle.enabled: true } diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/40_global_retention.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/40_global_retention.yml deleted file mode 100644 index 93df045e4568..000000000000 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/lifecycle/40_global_retention.yml +++ /dev/null @@ -1,139 +0,0 @@ -setup: - - skip: - features: allowed_warnings - cluster_features: ["data_stream.lifecycle.global_retention"] - reason: "Global retention was added in 8.14" - - do: - allowed_warnings: - - "index template [my-lifecycle] has index patterns [my-data-stream-1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-lifecycle] will take precedence during new index creation" - indices.put_index_template: - name: my-lifecycle - body: - index_patterns: [my-data-stream-*] - template: - settings: - index.number_of_replicas: 0 - lifecycle: {} - data_stream: {} - ---- -"CRUD global retention": - - do: - indices.create_data_stream: - name: my-data-stream-1 - - do: - cluster.health: - index: my-data-stream-1 - wait_for_status: green - - do: - data_streams.put_global_retention: - body: - default_retention: "7d" - max_retention: "90d" - - is_true: acknowledged - - is_false: dry_run - - match: {affected_data_streams.0.name: "my-data-stream-1"} - - match: {affected_data_streams.0.previous_effective_retention: "infinite"} - - match: {affected_data_streams.0.new_effective_retention: "7d"} - - - do: - data_streams.get_global_retention: { } - - match: { default_retention: "7d" } - - match: { max_retention: "90d" } - - - do: - data_streams.delete_global_retention: { } - - is_true: acknowledged - - is_false: dry_run - - match: { affected_data_streams.0.name: "my-data-stream-1" } - - match: { affected_data_streams.0.previous_effective_retention: "7d" } - - match: { affected_data_streams.0.new_effective_retention: "infinite" } - - - do: - data_streams.get_global_retention: { } - - is_false: default_retention - - is_false: max_retention - - - do: - indices.delete_data_stream: - name: my-data-stream-1 ---- -"Dry run global retention": - - do: - indices.create_data_stream: - name: my-data-stream-2 - - do: - indices.put_data_lifecycle: - name: "my-data-stream-2" - body: > - { - "data_retention": "90d" - } - - is_true: acknowledged - - - do: - data_streams.put_global_retention: - dry_run: true - body: - default_retention: "7d" - max_retention: "30d" - - is_false: acknowledged - - is_true: dry_run - - match: {affected_data_streams.0.name: "my-data-stream-2"} - - match: {affected_data_streams.0.previous_effective_retention: "90d"} - - match: {affected_data_streams.0.new_effective_retention: "30d"} - - - do: - indices.get_data_stream: - name: "my-data-stream-2" - include_defaults: true - - match: { data_streams.0.name: my-data-stream-2 } - - match: { data_streams.0.lifecycle.effective_retention: '90d' } - - match: { data_streams.0.lifecycle.retention_determined_by: 'data_stream_configuration' } - - do: - indices.delete_data_stream: - name: my-data-stream-2 ---- -"Default global retention is retrieved by data stream and index templates": - - do: - indices.create_data_stream: - name: my-data-stream-3 - - - do: - data_streams.put_global_retention: - body: - default_retention: "7d" - max_retention: "90d" - - is_true: acknowledged - - is_false: dry_run - - match: {affected_data_streams.0.name: "my-data-stream-3"} - - match: {affected_data_streams.0.previous_effective_retention: "infinite"} - - match: {affected_data_streams.0.new_effective_retention: "7d"} - - - do: - data_streams.get_global_retention: { } - - match: { default_retention: "7d" } - - match: { max_retention: "90d" } - - - do: - indices.get_data_stream: - name: "my-data-stream-3" - - match: { data_streams.0.name: my-data-stream-3 } - - match: { data_streams.0.lifecycle.effective_retention: '7d' } - - match: { data_streams.0.lifecycle.retention_determined_by: 'default_global_retention' } - - match: { data_streams.0.lifecycle.enabled: true } - - - do: - indices.get_index_template: - name: my-lifecycle - - - match: { index_templates.0.name: my-lifecycle } - - match: { index_templates.0.index_template.template.lifecycle.enabled: true } - - match: { index_templates.0.index_template.template.lifecycle.effective_retention: "7d" } - - match: { index_templates.0.index_template.template.lifecycle.retention_determined_by: "default_global_retention" } - - - do: - data_streams.delete_global_retention: { } - - do: - indices.delete_data_stream: - name: my-data-stream-3 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/data_streams.delete_global_retention.json b/rest-api-spec/src/main/resources/rest-api-spec/api/data_streams.delete_global_retention.json deleted file mode 100644 index 1eb4621a7b05..000000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/data_streams.delete_global_retention.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "data_streams.delete_global_retention":{ - "documentation":{ - "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams-delete-global-retention.html", - "description":"Deletes the global retention configuration that applies to all data streams managed by the data stream lifecycle." - }, - "stability":"experimental", - "visibility":"public", - "headers":{ - "accept": [ "application/json"], - "content_type": ["application/json"] - }, - "url":{ - "paths":[ - { - "path":"/_data_stream/_global_retention", - "methods":[ - "DELETE" - ] - } - ] - }, - "params":{ - "dry_run":{ - "type":"boolean", - "description":"Determines whether the global retention provided should be applied or only the impact should be determined.", - "default":false - }, - "master_timeout":{ - "type":"time", - "description":"Specify timeout for connection to master." - } - } - } -} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/data_streams.get_global_retention.json b/rest-api-spec/src/main/resources/rest-api-spec/api/data_streams.get_global_retention.json deleted file mode 100644 index 9084db36d7d9..000000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/data_streams.get_global_retention.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "data_streams.get_global_retention":{ - "documentation":{ - "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams-get-global-retention.html", - "description":"Returns global retention configuration that applies to all data streams managed by the data stream lifecycle." - }, - "stability":"experimental", - "visibility":"public", - "headers":{ - "accept": [ "application/json"] - }, - "url":{ - "paths":[ - { - "path":"/_data_stream/_global_retention", - "methods":[ - "GET" - ] - } - ] - }, - "params":{ - "local":{ - "type":"boolean", - "description":"Return the global retention retrieved from the node that received the request." - } - } - } -} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/data_streams.put_global_retention.json b/rest-api-spec/src/main/resources/rest-api-spec/api/data_streams.put_global_retention.json deleted file mode 100644 index 9f369f4c7616..000000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/data_streams.put_global_retention.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "data_streams.put_global_retention":{ - "documentation":{ - "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams-put-global-retention.html", - "description":"Updates the global retention configuration that applies to all data streams managed by the data stream lifecycle." - }, - "stability":"experimental", - "visibility":"public", - "headers":{ - "accept": [ "application/json"], - "content_type": ["application/json"] - }, - "url":{ - "paths":[ - { - "path":"/_data_stream/_global_retention", - "methods":[ - "PUT" - ] - } - ] - }, - "params":{ - "dry_run":{ - "type":"boolean", - "description":"Determines whether the global retention provided should be applied or only the impact should be determined.", - "default":false - }, - "master_timeout":{ - "type":"time", - "description":"Specify timeout for connection to master" - } - }, - "body":{ - "description":"The global retention configuration including optional values for default and max retention.", - "required":true - } - } -} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/10_basic.yml index af2d6f946d2f..f698d3399f27 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.component_template/10_basic.yml @@ -117,8 +117,8 @@ --- "Add data stream lifecycle": - requires: - cluster_features: ["data_stream.lifecycle.global_retention"] - reason: "Data stream lifecycle with global retention was available from 8.14" + cluster_features: ["gte_v8.11.0"] + reason: "Data stream lifecycle was available from 8.11" - do: cluster.put_component_template: @@ -141,14 +141,12 @@ - match: {component_templates.0.component_template.version: 1} - match: {component_templates.0.component_template.template.lifecycle.enabled: true} - match: {component_templates.0.component_template.template.lifecycle.data_retention: "10d"} - - match: {component_templates.0.component_template.template.lifecycle.effective_retention: "10d"} - - match: {component_templates.0.component_template.template.lifecycle.retention_determined_by: "data_stream_configuration"} --- "Get data stream lifecycle with default rollover": - requires: - cluster_features: ["data_stream.lifecycle.global_retention"] - reason: "Data stream lifecycle with effective retention was available from 8.14" + cluster_features: ["gte_v8.11.0"] + reason: "Data stream lifecycle was available from 8.11" - do: cluster.put_component_template: @@ -172,6 +170,4 @@ - match: {component_templates.0.component_template.version: 1} - match: {component_templates.0.component_template.template.lifecycle.enabled: true} - match: {component_templates.0.component_template.template.lifecycle.data_retention: "10d"} - - match: {component_templates.0.component_template.template.lifecycle.effective_retention: "10d"} - - match: {component_templates.0.component_template.template.lifecycle.retention_determined_by: "data_stream_configuration"} - is_true: component_templates.0.component_template.template.lifecycle.rollover diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_index_template/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_index_template/10_basic.yml index dc3361fefab6..2079c01079ce 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_index_template/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_index_template/10_basic.yml @@ -93,8 +93,8 @@ setup: --- "Add data stream lifecycle": - skip: - cluster_features: ["data_stream.lifecycle.global_retention"] - reason: "Data stream lifecycle with effective retention was released in 8.14" + cluster_features: ["gte_v8.11.0"] + reason: "Data stream lifecycle in index templates was updated after 8.10" features: allowed_warnings - do: @@ -124,14 +124,12 @@ setup: - match: {index_templates.0.index_template.template.mappings: {properties: {field: {type: keyword}}}} - match: {index_templates.0.index_template.template.lifecycle.enabled: true} - match: {index_templates.0.index_template.template.lifecycle.data_retention: "30d"} - - match: {index_templates.0.index_template.template.lifecycle.effective_retention: "30d"} - - match: {index_templates.0.index_template.template.lifecycle.retention_determined_by: "data_stream_configuration"} --- "Get data stream lifecycle with default rollover": - skip: - cluster_features: ["data_stream.lifecycle.global_retention"] - reason: "Data stream lifecycle with effective retention was released in 8.14" + cluster_features: ["gte_v8.11.0"] + reason: "Data stream lifecycle in index templates was updated after 8.10" features: allowed_warnings - do: @@ -154,13 +152,11 @@ setup: - match: {index_templates.0.index_template.index_patterns: ["data-stream-with-lifecycle-*"]} - match: {index_templates.0.index_template.template.lifecycle.enabled: true} - match: {index_templates.0.index_template.template.lifecycle.data_retention: "30d"} - - match: {index_templates.0.index_template.template.lifecycle.effective_retention: "30d"} - - match: {index_templates.0.index_template.template.lifecycle.retention_determined_by: "data_stream_configuration"} - is_true: index_templates.0.index_template.template.lifecycle.rollover --- "Reject data stream lifecycle without data stream configuration": - - skip: + - requires: cluster_features: ["gte_v8.11.0"] reason: "Data stream lifecycle in index templates was updated after 8.10" - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml index 6790014be995..81c8cf64169e 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml @@ -227,8 +227,8 @@ --- "Simulate index template with lifecycle and include defaults": - requires: - cluster_features: ["gte_v8.14.0"] - reason: "Data stream lifecycle with effective retention was released in 8.14" + cluster_features: ["gte_v8.11.0"] + reason: "Lifecycle is only available in 8.11+" test_runner_features: ["default_shards"] - do: @@ -248,7 +248,5 @@ - match: {template.lifecycle.enabled: true} - match: {template.lifecycle.data_retention: "7d"} - - match: {template.lifecycle.effective_retention: "7d"} - - match: {template.lifecycle.retention_determined_by: "data_stream_configuration"} - is_true: template.lifecycle.rollover - match: {overlapping: []} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.simulate_template/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.simulate_template/10_basic.yml index ff53a762b75e..236653b7ca9a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.simulate_template/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.simulate_template/10_basic.yml @@ -1,9 +1,9 @@ --- "Simulate template without a template in the body": - - requires: - cluster_features: ["gte_v7.9.0"] + - skip: + version: " - 7.8.99" reason: "only available in 7.9+" - test_runner_features: ["default_shards"] + features: ["default_shards"] - do: indices.put_index_template: @@ -30,10 +30,10 @@ --- "Simulate index template specifying a new template": - - requires: - cluster_features: ["gte_v7.9.0"] + - skip: + version: " - 7.8.99" reason: "only available in 7.9+" - test_runner_features: ["default_shards"] + features: ["default_shards"] - do: indices.put_index_template: @@ -84,10 +84,10 @@ --- "Simulate template matches overlapping legacy and composable templates": - - requires: - cluster_features: ["gte_v7.9.0"] + - skip: + version: " - 7.8.99" reason: "only available in 7.9+" - test_runner_features: ["allowed_warnings", "default_shards"] + features: ["allowed_warnings", "default_shards"] - do: indices.put_template: @@ -147,10 +147,10 @@ --- "Simulate replacing a template with a newer version": - - requires: - cluster_features: ["gte_v8.0.0"] + - skip: + version: " - 7.99.99" reason: "not yet backported" - test_runner_features: ["allowed_warnings", "default_shards"] + features: ["allowed_warnings", "default_shards"] - do: indices.put_index_template: @@ -202,8 +202,8 @@ --- "Simulate template with lifecycle and include defaults": - requires: - cluster_features: ["gte_v8.14.0"] - reason: "Data stream lifecycle with effective retention was released in 8.14" + cluster_features: [ "gte_v8.11.0" ] + reason: "Lifecycle is only available in 8.11+" test_runner_features: ["default_shards"] - do: @@ -223,6 +223,4 @@ - match: {template.lifecycle.enabled: true} - match: {template.lifecycle.data_retention: "7d"} - - match: {template.lifecycle.effective_retention: "7d"} - - match: {template.lifecycle.retention_determined_by: "data_stream_configuration"} - is_true: template.lifecycle.rollover diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java index d352f1be5e65..3bf9c3715b29 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateAction.java @@ -16,7 +16,6 @@ import org.elasticsearch.action.support.master.MasterNodeReadRequest; import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; -import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.Nullable; @@ -191,14 +190,13 @@ public int hashCode() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - Params withEffectiveRetentionParams = new DelegatingMapParams(DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS, params); builder.startObject(); builder.startArray(COMPONENT_TEMPLATES.getPreferredName()); for (Map.Entry componentTemplate : this.componentTemplates.entrySet()) { builder.startObject(); builder.field(NAME.getPreferredName(), componentTemplate.getKey()); builder.field(COMPONENT_TEMPLATE.getPreferredName()); - componentTemplate.getValue().toXContent(builder, withEffectiveRetentionParams, rolloverConfiguration, globalRetention); + componentTemplate.getValue().toXContent(builder, params, rolloverConfiguration, globalRetention); builder.endObject(); } builder.endArray(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java index 668e3f8e7c10..240fdd2ae819 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/get/GetComposableIndexTemplateAction.java @@ -16,7 +16,6 @@ import org.elasticsearch.action.support.master.MasterNodeReadRequest; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; -import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.Nullable; @@ -190,14 +189,13 @@ public int hashCode() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - Params withEffectiveRetentionParams = new DelegatingMapParams(DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS, params); builder.startObject(); builder.startArray(INDEX_TEMPLATES.getPreferredName()); for (Map.Entry indexTemplate : this.indexTemplates.entrySet()) { builder.startObject(); builder.field(NAME.getPreferredName(), indexTemplate.getKey()); builder.field(INDEX_TEMPLATE.getPreferredName()); - indexTemplate.getValue().toXContent(builder, withEffectiveRetentionParams, rolloverConfiguration, globalRetention); + indexTemplate.getValue().toXContent(builder, params, rolloverConfiguration, globalRetention); builder.endObject(); } builder.endArray(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java index 378df2d7d53e..4ff38222ccc9 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java @@ -12,7 +12,6 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration; import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; -import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -114,11 +113,10 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - Params withEffectiveRetentionParams = new DelegatingMapParams(DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS, params); builder.startObject(); if (this.resolvedTemplate != null) { builder.field(TEMPLATE.getPreferredName()); - this.resolvedTemplate.toXContent(builder, withEffectiveRetentionParams, rolloverConfiguration, globalRetention); + this.resolvedTemplate.toXContent(builder, params, rolloverConfiguration, globalRetention); } if (this.overlappingTemplates != null) { builder.startArray(OVERLAPPING.getPreferredName()); diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java index 1a2103d665b3..f2a581472303 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java @@ -20,7 +20,6 @@ import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamAutoShardingEvent; import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; -import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -544,11 +543,10 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - Params withEffectiveRetentionParams = new DelegatingMapParams(DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS, params); builder.startObject(); builder.startArray(DATA_STREAMS_FIELD.getPreferredName()); for (DataStreamInfo dataStream : dataStreams) { - dataStream.toXContent(builder, withEffectiveRetentionParams, rolloverConfiguration, globalRetention); + dataStream.toXContent(builder, params, rolloverConfiguration, globalRetention); } builder.endArray(); builder.endObject(); diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java index 2b79377fb71e..bb6c3f90f1b0 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/ExplainIndexDataStreamLifecycle.java @@ -132,11 +132,7 @@ public XContentBuilder toXContent( } if (this.lifecycle != null) { builder.field(LIFECYCLE_FIELD.getPreferredName()); - Params withEffectiveRetentionParams = new DelegatingMapParams( - DataStreamLifecycle.INCLUDE_EFFECTIVE_RETENTION_PARAMS, - params - ); - lifecycle.toXContent(builder, withEffectiveRetentionParams, rolloverConfiguration, globalRetention); + lifecycle.toXContent(builder, params, rolloverConfiguration, globalRetention); } if (this.error != null) { if (error.firstOccurrenceTimestamp() != -1L && error.recordedTimestamp() != -1L && error.retryCount() != -1) { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java index 025f51b7df99..d31c9fddf271 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/get/GetComponentTemplateResponseTests.java @@ -34,7 +34,6 @@ import static org.elasticsearch.cluster.metadata.ComponentTemplateTests.randomSettings; import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.not; public class GetComponentTemplateResponseTests extends AbstractWireSerializingTestCase { @Override @@ -103,9 +102,6 @@ public void testXContentSerializationWithRolloverAndEffectiveRetention() throws .keySet()) { assertThat(serialized, containsString(label)); } - // We check that even if there was no retention provided by the user, the global retention applies - assertThat(serialized, not(containsString("data_retention"))); - assertThat(serialized, containsString("effective_retention")); } } From a8096411792ed06f0417ff129edffbb3c4c761ea Mon Sep 17 00:00:00 2001 From: Kathleen DeRusso Date: Wed, 17 Apr 2024 20:40:53 +0200 Subject: [PATCH 38/43] Fix typo in text_expansion query docs example (#107572) * Fix typo in docs example * fix indentation --- .../query-dsl/text-expansion-query.asciidoc | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/reference/query-dsl/text-expansion-query.asciidoc b/docs/reference/query-dsl/text-expansion-query.asciidoc index 27fca2bb5637..5b3f98b5e1ea 100644 --- a/docs/reference/query-dsl/text-expansion-query.asciidoc +++ b/docs/reference/query-dsl/text-expansion-query.asciidoc @@ -232,12 +232,12 @@ GET my-index/_search "text_expansion":{ "ml.tokens":{ "model_id":".elser_model_2", - "model_text":"How is the weather in Jamaica?" - }, - "pruning_config": { - "tokens_freq_ratio_threshold": 5, - "tokens_weight_threshold": 0.4, - "only_score_pruned_tokens": false + "model_text":"How is the weather in Jamaica?", + "pruning_config": { + "tokens_freq_ratio_threshold": 5, + "tokens_weight_threshold": 0.4, + "only_score_pruned_tokens": false + } } } }, @@ -248,12 +248,12 @@ GET my-index/_search "text_expansion": { "ml.tokens": { "model_id": ".elser_model_2", - "model_text": "How is the weather in Jamaica?" - }, - "pruning_config": { - "tokens_freq_ratio_threshold": 5, - "tokens_weight_threshold": 0.4, - "only_score_pruned_tokens": true + "model_text": "How is the weather in Jamaica?", + "pruning_config": { + "tokens_freq_ratio_threshold": 5, + "tokens_weight_threshold": 0.4, + "only_score_pruned_tokens": true + } } } } From faf696557e93eccb55f62c007eabbb2fc24e6097 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Wed, 17 Apr 2024 14:52:41 -0400 Subject: [PATCH 39/43] Add 8.14 to branches.json --- branches.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/branches.json b/branches.json index 772693505b9e..daf6d249f726 100644 --- a/branches.json +++ b/branches.json @@ -4,6 +4,9 @@ { "branch": "main" }, + { + "branch": "8.14" + }, { "branch": "8.13" }, From 91d3bb026fa8272bb847e84f85cc18f6f93ebc2a Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Wed, 17 Apr 2024 14:58:20 -0400 Subject: [PATCH 40/43] [ci] Add checkPart4 to missing locations (#107552) --- .buildkite/pipelines/lucene-snapshot/run-tests.yml | 8 ++++++++ .buildkite/pipelines/periodic-platform-support.yml | 2 ++ .buildkite/pipelines/periodic.template.yml | 2 ++ .buildkite/pipelines/periodic.yml | 2 ++ 4 files changed, 14 insertions(+) diff --git a/.buildkite/pipelines/lucene-snapshot/run-tests.yml b/.buildkite/pipelines/lucene-snapshot/run-tests.yml index 15d78f8495ca..a5d3c4e5f793 100644 --- a/.buildkite/pipelines/lucene-snapshot/run-tests.yml +++ b/.buildkite/pipelines/lucene-snapshot/run-tests.yml @@ -32,6 +32,14 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + - label: part4 + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart4 + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk - group: bwc-snapshots steps: - label: "{{matrix.BWC_VERSION}} / bwc-snapshots" diff --git a/.buildkite/pipelines/periodic-platform-support.yml b/.buildkite/pipelines/periodic-platform-support.yml index 0240fd03f4a8..a3922d822692 100644 --- a/.buildkite/pipelines/periodic-platform-support.yml +++ b/.buildkite/pipelines/periodic-platform-support.yml @@ -47,6 +47,7 @@ steps: - checkPart1 - checkPart2 - checkPart3 + - checkPart4 - checkRestCompat agents: provider: gcp @@ -70,6 +71,7 @@ steps: - checkPart1 - checkPart2 - checkPart3 + - checkPart4 - checkRestCompat agents: provider: aws diff --git a/.buildkite/pipelines/periodic.template.yml b/.buildkite/pipelines/periodic.template.yml index 05d516992a7f..7315dc9de260 100644 --- a/.buildkite/pipelines/periodic.template.yml +++ b/.buildkite/pipelines/periodic.template.yml @@ -49,6 +49,7 @@ steps: - checkPart1 - checkPart2 - checkPart3 + - checkPart4 - checkRestCompat agents: provider: gcp @@ -89,6 +90,7 @@ steps: - checkPart1 - checkPart2 - checkPart3 + - checkPart4 - checkRestCompat agents: provider: gcp diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index 9291ec2efcbd..64ea30266bf3 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -390,6 +390,7 @@ steps: - checkPart1 - checkPart2 - checkPart3 + - checkPart4 - checkRestCompat agents: provider: gcp @@ -430,6 +431,7 @@ steps: - checkPart1 - checkPart2 - checkPart3 + - checkPart4 - checkRestCompat agents: provider: gcp From d9b8245a10a22db941eddf239ee58199e295307a Mon Sep 17 00:00:00 2001 From: Sander Philipse <94373878+sphilipse@users.noreply.github.com> Date: Thu, 18 Apr 2024 11:43:46 +0200 Subject: [PATCH 41/43] [Search] Add default ingest pipeline (#107558) * [Search] Add default ingest pipeline * Spotless apply --- .../entsearch/search_default_pipeline.json | 130 ++++++++++++++++++ .../connector/ConnectorTemplateRegistry.java | 9 ++ 2 files changed, 139 insertions(+) create mode 100644 x-pack/plugin/core/template-resources/src/main/resources/entsearch/search_default_pipeline.json diff --git a/x-pack/plugin/core/template-resources/src/main/resources/entsearch/search_default_pipeline.json b/x-pack/plugin/core/template-resources/src/main/resources/entsearch/search_default_pipeline.json new file mode 100644 index 000000000000..bd556900a42e --- /dev/null +++ b/x-pack/plugin/core/template-resources/src/main/resources/entsearch/search_default_pipeline.json @@ -0,0 +1,130 @@ +{ + "version": ${xpack.application.connector.template.version}, + "description": "Default search ingest pipeline", + "_meta": { + "managed_by": "Search", + "managed": true + }, + "processors": [ + { + "attachment": { + "description": "Extract text from binary attachments", + "field": "_attachment", + "target_field": "_extracted_attachment", + "ignore_missing": true, + "indexed_chars_field": "_attachment_indexed_chars", + "if": "ctx?._extract_binary_content == true", + "on_failure": [ + { + "append": { + "description": "Record error information", + "field": "_ingestion_errors", + "value": "Processor 'attachment' in pipeline '{{ _ingest.on_failure_pipeline }}' failed with message '{{ _ingest.on_failure_message }}'" + } + } + ], + "remove_binary": false + } + }, + { + "set": { + "tag": "set_body", + "description": "Set any extracted text on the 'body' field", + "field": "body", + "copy_from": "_extracted_attachment.content", + "ignore_empty_value": true, + "if": "ctx?._extract_binary_content == true", + "on_failure": [ + { + "append": { + "description": "Record error information", + "field": "_ingestion_errors", + "value": "Processor 'set' with tag 'set_body' in pipeline '{{ _ingest.on_failure_pipeline }}' failed with message '{{ _ingest.on_failure_message }}'" + } + } + ] + } + }, + { + "gsub": { + "tag": "remove_replacement_chars", + "description": "Remove unicode 'replacement' characters", + "field": "body", + "pattern": "�", + "replacement": "", + "ignore_missing": true, + "if": "ctx?._extract_binary_content == true", + "on_failure": [ + { + "append": { + "description": "Record error information", + "field": "_ingestion_errors", + "value": "Processor 'gsub' with tag 'remove_replacement_chars' in pipeline '{{ _ingest.on_failure_pipeline }}' failed with message '{{ _ingest.on_failure_message }}'" + } + } + ] + } + }, + { + "gsub": { + "tag": "remove_extra_whitespace", + "description": "Squish whitespace", + "field": "body", + "pattern": "\\s+", + "replacement": " ", + "ignore_missing": true, + "if": "ctx?._reduce_whitespace == true", + "on_failure": [ + { + "append": { + "description": "Record error information", + "field": "_ingestion_errors", + "value": "Processor 'gsub' with tag 'remove_extra_whitespace' in pipeline '{{ _ingest.on_failure_pipeline }}' failed with message '{{ _ingest.on_failure_message }}'" + } + } + ] + } + }, + { + "trim": { + "description": "Trim leading and trailing whitespace", + "field": "body", + "ignore_missing": true, + "if": "ctx?._reduce_whitespace == true", + "on_failure": [ + { + "append": { + "description": "Record error information", + "field": "_ingestion_errors", + "value": "Processor 'trim' in pipeline '{{ _ingest.on_failure_pipeline }}' failed with message '{{ _ingest.on_failure_message }}'" + } + } + ] + } + }, + { + "remove": { + "tag": "remove_meta_fields", + "description": "Remove meta fields", + "field": [ + "_attachment", + "_attachment_indexed_chars", + "_extracted_attachment", + "_extract_binary_content", + "_reduce_whitespace", + "_run_ml_inference" + ], + "ignore_missing": true, + "on_failure": [ + { + "append": { + "description": "Record error information", + "field": "_ingestion_errors", + "value": "Processor 'remove' with tag 'remove_meta_fields' in pipeline '{{ _ingest.on_failure_pipeline }}' failed with message '{{ _ingest.on_failure_message }}'" + } + } + ] + } + } + ] +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java index c57650541b41..e4ce4d8181fd 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java @@ -54,6 +54,9 @@ public class ConnectorTemplateRegistry extends IndexTemplateRegistry { public static final String ENT_SEARCH_GENERIC_PIPELINE_NAME = "ent-search-generic-ingestion"; public static final String ENT_SEARCH_GENERIC_PIPELINE_FILE = "generic_ingestion_pipeline"; + public static final String SEARCH_DEFAULT_PIPELINE_NAME = "search-default-ingestion"; + public static final String SEARCH_DEFAULT_PIPELINE_FILE = "search_default_pipeline"; + // Resource config public static final String ROOT_RESOURCE_PATH = "/entsearch/"; public static final String ROOT_TEMPLATE_RESOURCE_PATH = ROOT_RESOURCE_PATH + "connector/"; @@ -115,6 +118,12 @@ protected List getIngestPipelines() { ROOT_RESOURCE_PATH + ENT_SEARCH_GENERIC_PIPELINE_FILE + ".json", REGISTRY_VERSION, TEMPLATE_VERSION_VARIABLE + ), + new JsonIngestPipelineConfig( + SEARCH_DEFAULT_PIPELINE_NAME, + ROOT_RESOURCE_PATH + SEARCH_DEFAULT_PIPELINE_FILE + ".json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE ) ); } From 61a2c52f18b4e38d8be695a677a166e0cac5212d Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 18 Apr 2024 12:24:17 +0100 Subject: [PATCH 42/43] Fix `CONCURRENT_REPOSITORY_WRITERS` link (#107603) This page was split up in #104614 but the `ReferenceDocs` symbol links to the top-level page still rather than the correct subpage. This fixes the link. --- .../org/elasticsearch/common/reference-docs-links.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json index ead7387b0e1a..503f02b25eb8 100644 --- a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json +++ b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json @@ -4,7 +4,7 @@ "UNSTABLE_CLUSTER_TROUBLESHOOTING": "troubleshooting-unstable-cluster.html", "LAGGING_NODE_TROUBLESHOOTING": "troubleshooting-unstable-cluster.html#_diagnosing_lagging_nodes_2", "SHARD_LOCK_TROUBLESHOOTING": "troubleshooting-unstable-cluster.html#_diagnosing_shardlockobtainfailedexception_failures_2", - "CONCURRENT_REPOSITORY_WRITERS": "add-repository.html", + "CONCURRENT_REPOSITORY_WRITERS": "diagnosing-corrupted-repositories.html", "ARCHIVE_INDICES": "archive-indices.html", "HTTP_TRACER": "modules-network.html#http-rest-request-tracer", "LOGGING": "logging.html", From 9adf2422dfeec594eb6ac9cff81152084d01c2dc Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 18 Apr 2024 12:24:45 +0100 Subject: [PATCH 43/43] Add links to repo troubleshooting sub-pages (#107604) Since #104614 the top-level repo troubleshooting page is just a short paragraph which talks about "this page" but in fact refers to information spread across a number of subsequent pages. It's not obvious to the reader that they need to use the navigation menu to get to the information they seek. Moreover we link to this page from an exception message today so there's a reasonable chance that users will find it when trying to troubleshoot a genuine problem. This commit rewords things slightly and adds links to the subsequent pages to the body of the page to avoid this confusion. --- .../troubleshooting/snapshot/add-repository.asciidoc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/reference/troubleshooting/snapshot/add-repository.asciidoc b/docs/reference/troubleshooting/snapshot/add-repository.asciidoc index 386c2561c03c..e70bd244df3a 100644 --- a/docs/reference/troubleshooting/snapshot/add-repository.asciidoc +++ b/docs/reference/troubleshooting/snapshot/add-repository.asciidoc @@ -2,8 +2,12 @@ == Troubleshooting broken repositories There are several situations where the <> might report an issue -regarding the integrity of snapshot repositories in the cluster. This page explains -the recommended actions for diagnosing corrupted, unknown, and invalid repositories. +regarding the integrity of snapshot repositories in the cluster. The following pages explain +the recommended actions for diagnosing corrupted, unknown, and invalid repositories: + +- <> +- <> +- <> [[diagnosing-corrupted-repositories]] === Diagnosing corrupted repositories