From a765f89c212c436ff36f554020365ea913728ac8 Mon Sep 17 00:00:00 2001 From: Ioana Tagirta Date: Fri, 13 Dec 2024 12:59:14 +0100 Subject: [PATCH 01/35] Make search functions translation aware (#118355) * Introduce TranslationAware interface * Serialize query builder * Fix EsqlNodeSubclassTests * Add javadoc * Address review comments * Revert changes on making constructors private --- .../org/elasticsearch/TransportVersions.java | 1 + .../core/expression/TranslationAware.java | 20 ++++++++ .../TranslationAwareExpressionQuery.java | 35 ++++++++++++++ .../function/EsqlFunctionRegistry.java | 6 +-- .../function/fulltext/FullTextFunction.java | 46 ++++++++++++++++++- .../expression/function/fulltext/Kql.java | 38 ++++++++++++--- .../expression/function/fulltext/Match.java | 33 +++++++++++-- .../function/fulltext/QueryString.java | 42 ++++++++++++++--- .../expression/function/fulltext/Term.java | 33 +++++++++++-- .../planner/EsqlExpressionTranslators.java | 5 ++ 10 files changed, 234 insertions(+), 25 deletions(-) create mode 100644 x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TranslationAware.java create mode 100644 x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TranslationAwareExpressionQuery.java diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 4135b1f0b8e9a..388123e86c882 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -138,6 +138,7 @@ static TransportVersion def(int id) { public static final TransportVersion ADD_DATA_STREAM_OPTIONS_TO_TEMPLATES = def(8_805_00_0); public static final TransportVersion KNN_QUERY_RESCORE_OVERSAMPLE = def(8_806_00_0); public static final TransportVersion SEMANTIC_QUERY_LENIENT = def(8_807_00_0); + public static final TransportVersion ESQL_QUERY_BUILDER_IN_SEARCH_FUNCTIONS = def(8_808_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TranslationAware.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TranslationAware.java new file mode 100644 index 0000000000000..b1ac2b36314fa --- /dev/null +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TranslationAware.java @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.core.expression; + +import org.elasticsearch.xpack.esql.core.planner.TranslatorHandler; +import org.elasticsearch.xpack.esql.core.querydsl.query.Query; + +/** + * Expressions can implement this interface to control how they would be translated and pushed down as Lucene queries. + * When an expression implements {@link TranslationAware}, we call {@link #asQuery(TranslatorHandler)} to get the + * {@link Query} translation, instead of relying on the registered translators from EsqlExpressionTranslators. + */ +public interface TranslationAware { + Query asQuery(TranslatorHandler translatorHandler); +} diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TranslationAwareExpressionQuery.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TranslationAwareExpressionQuery.java new file mode 100644 index 0000000000000..92a42d3053b68 --- /dev/null +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/querydsl/query/TranslationAwareExpressionQuery.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.core.querydsl.query; + +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * Expressions that store their own {@link QueryBuilder} and implement + * {@link org.elasticsearch.xpack.esql.core.expression.TranslationAware} can use {@link TranslationAwareExpressionQuery} + * to wrap their {@link QueryBuilder}, instead of using the other existing {@link Query} implementations. + */ +public class TranslationAwareExpressionQuery extends Query { + private final QueryBuilder queryBuilder; + + public TranslationAwareExpressionQuery(Source source, QueryBuilder queryBuilder) { + super(source); + this.queryBuilder = queryBuilder; + } + + @Override + public QueryBuilder asBuilder() { + return queryBuilder; + } + + @Override + protected String innerToString() { + return queryBuilder.toString(); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 50d0d2438d8a1..1ccc22eb3a6a4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -416,7 +416,7 @@ private static FunctionDefinition[][] functions() { def(MvSum.class, MvSum::new, "mv_sum"), def(Split.class, Split::new, "split") }, // fulltext functions - new FunctionDefinition[] { def(Match.class, Match::new, "match"), def(QueryString.class, QueryString::new, "qstr") } }; + new FunctionDefinition[] { def(Match.class, bi(Match::new), "match"), def(QueryString.class, uni(QueryString::new), "qstr") } }; } @@ -426,9 +426,9 @@ private static FunctionDefinition[][] snapshotFunctions() { // The delay() function is for debug/snapshot environments only and should never be enabled in a non-snapshot build. // This is an experimental function and can be removed without notice. def(Delay.class, Delay::new, "delay"), - def(Kql.class, Kql::new, "kql"), + def(Kql.class, uni(Kql::new), "kql"), def(Rate.class, Rate::withUnresolvedTimestamp, "rate"), - def(Term.class, Term::new, "term") } }; + def(Term.class, bi(Term::new), "term") } }; } public EsqlFunctionRegistry snapshotRegistry() { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java index 78dc05af8f342..432d2d5f07429 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java @@ -8,14 +8,21 @@ package org.elasticsearch.xpack.esql.expression.function.fulltext; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Nullability; +import org.elasticsearch.xpack.esql.core.expression.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.expression.function.Function; +import org.elasticsearch.xpack.esql.core.planner.ExpressionTranslator; +import org.elasticsearch.xpack.esql.core.planner.TranslatorHandler; +import org.elasticsearch.xpack.esql.core.querydsl.query.Query; +import org.elasticsearch.xpack.esql.core.querydsl.query.TranslationAwareExpressionQuery; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import java.util.List; +import java.util.Objects; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNotNullAndFoldable; @@ -26,13 +33,15 @@ * These functions needs to be pushed down to Lucene queries to be executed - there's no Evaluator for them, but depend on * {@link org.elasticsearch.xpack.esql.optimizer.LocalPhysicalPlanOptimizer} to rewrite them into Lucene queries. */ -public abstract class FullTextFunction extends Function { +public abstract class FullTextFunction extends Function implements TranslationAware { private final Expression query; + private final QueryBuilder queryBuilder; - protected FullTextFunction(Source source, Expression query, List children) { + protected FullTextFunction(Source source, Expression query, List children, QueryBuilder queryBuilder) { super(source, children); this.query = query; + this.queryBuilder = queryBuilder; } @Override @@ -116,4 +125,37 @@ public Nullability nullable() { public String functionType() { return "function"; } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), queryBuilder); + } + + @Override + public boolean equals(Object obj) { + if (false == super.equals(obj)) { + return false; + } + + return Objects.equals(queryBuilder, ((FullTextFunction) obj).queryBuilder); + } + + @Override + public Query asQuery(TranslatorHandler translatorHandler) { + if (queryBuilder != null) { + return new TranslationAwareExpressionQuery(source(), queryBuilder); + } + + ExpressionTranslator translator = translator(); + return translator.translate(this, translatorHandler); + } + + public QueryBuilder queryBuilder() { + return queryBuilder; + } + + @SuppressWarnings("rawtypes") + protected abstract ExpressionTranslator translator(); + + public abstract Expression replaceQueryBuilder(QueryBuilder queryBuilder); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java index c03902373c02e..1f7bcadd259a0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Kql.java @@ -7,16 +7,20 @@ package org.elasticsearch.xpack.esql.expression.function.fulltext; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.planner.ExpressionTranslator; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.planner.EsqlExpressionTranslators; import org.elasticsearch.xpack.esql.querydsl.query.KqlQuery; import java.io.IOException; @@ -26,7 +30,7 @@ * Full text function that performs a {@link KqlQuery} . */ public class Kql extends FullTextFunction { - public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Kql", Kql::new); + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Kql", Kql::readFrom); @FunctionInfo( returnType = "boolean", @@ -42,17 +46,30 @@ public Kql( description = "Query string in KQL query string format." ) Expression queryString ) { - super(source, queryString, List.of(queryString)); + super(source, queryString, List.of(queryString), null); } - private Kql(StreamInput in) throws IOException { - this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class)); + public Kql(Source source, Expression queryString, QueryBuilder queryBuilder) { + super(source, queryString, List.of(queryString), queryBuilder); + } + + private static Kql readFrom(StreamInput in) throws IOException { + Source source = Source.readFrom((PlanStreamInput) in); + Expression query = in.readNamedWriteable(Expression.class); + QueryBuilder queryBuilder = null; + if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_QUERY_BUILDER_IN_SEARCH_FUNCTIONS)) { + queryBuilder = in.readOptionalNamedWriteable(QueryBuilder.class); + } + return new Kql(source, query, queryBuilder); } @Override public void writeTo(StreamOutput out) throws IOException { source().writeTo(out); out.writeNamedWriteable(query()); + if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_QUERY_BUILDER_IN_SEARCH_FUNCTIONS)) { + out.writeOptionalNamedWriteable(queryBuilder()); + } } @Override @@ -62,12 +79,21 @@ public String getWriteableName() { @Override public Expression replaceChildren(List newChildren) { - return new Kql(source(), newChildren.get(0)); + return new Kql(source(), newChildren.get(0), queryBuilder()); } @Override protected NodeInfo info() { - return NodeInfo.create(this, Kql::new, query()); + return NodeInfo.create(this, Kql::new, query(), queryBuilder()); } + @Override + protected ExpressionTranslator translator() { + return new EsqlExpressionTranslators.KqlFunctionTranslator(); + } + + @Override + public Expression replaceQueryBuilder(QueryBuilder queryBuilder) { + return new Kql(source(), query(), queryBuilder); + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java index 2b9a7c73a5853..0b2268fe1b022 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java @@ -8,15 +8,18 @@ package org.elasticsearch.xpack.esql.expression.function.fulltext; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.xpack.esql.capabilities.Validatable; import org.elasticsearch.xpack.esql.common.Failure; import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.planner.ExpressionTranslator; import org.elasticsearch.xpack.esql.core.querydsl.query.QueryStringQuery; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -27,6 +30,7 @@ import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.planner.EsqlExpressionTranslators; import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter; import java.io.IOException; @@ -109,7 +113,11 @@ public Match( description = "Value to find in the provided field." ) Expression matchQuery ) { - super(source, matchQuery, List.of(field, matchQuery)); + this(source, field, matchQuery, null); + } + + public Match(Source source, Expression field, Expression matchQuery, QueryBuilder queryBuilder) { + super(source, matchQuery, List.of(field, matchQuery), queryBuilder); this.field = field; } @@ -117,7 +125,11 @@ private static Match readFrom(StreamInput in) throws IOException { Source source = Source.readFrom((PlanStreamInput) in); Expression field = in.readNamedWriteable(Expression.class); Expression query = in.readNamedWriteable(Expression.class); - return new Match(source, field, query); + QueryBuilder queryBuilder = null; + if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_QUERY_BUILDER_IN_SEARCH_FUNCTIONS)) { + queryBuilder = in.readOptionalNamedWriteable(QueryBuilder.class); + } + return new Match(source, field, query, queryBuilder); } @Override @@ -125,6 +137,9 @@ public void writeTo(StreamOutput out) throws IOException { source().writeTo(out); out.writeNamedWriteable(field()); out.writeNamedWriteable(query()); + if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_QUERY_BUILDER_IN_SEARCH_FUNCTIONS)) { + out.writeOptionalNamedWriteable(queryBuilder()); + } } @Override @@ -224,12 +239,12 @@ public Object queryAsObject() { @Override public Expression replaceChildren(List newChildren) { - return new Match(source(), newChildren.get(0), newChildren.get(1)); + return new Match(source(), newChildren.get(0), newChildren.get(1), queryBuilder()); } @Override protected NodeInfo info() { - return NodeInfo.create(this, Match::new, field, query()); + return NodeInfo.create(this, Match::new, field, query(), queryBuilder()); } protected TypeResolutions.ParamOrdinal queryParamOrdinal() { @@ -245,6 +260,16 @@ public String functionType() { return isOperator() ? "operator" : super.functionType(); } + @Override + protected ExpressionTranslator translator() { + return new EsqlExpressionTranslators.MatchFunctionTranslator(); + } + + @Override + public Expression replaceQueryBuilder(QueryBuilder queryBuilder) { + return new Match(source(), field, query(), queryBuilder); + } + @Override public String functionName() { return isOperator() ? ":" : super.functionName(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryString.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryString.java index bd79661534b76..ea21411d09173 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryString.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/QueryString.java @@ -7,10 +7,13 @@ package org.elasticsearch.xpack.esql.expression.function.fulltext; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.planner.ExpressionTranslator; import org.elasticsearch.xpack.esql.core.querydsl.query.QueryStringQuery; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -18,6 +21,7 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.planner.EsqlExpressionTranslators; import java.io.IOException; import java.util.List; @@ -27,7 +31,11 @@ */ public class QueryString extends FullTextFunction { - public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "QStr", QueryString::new); + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "QStr", + QueryString::readFrom + ); @FunctionInfo( returnType = "boolean", @@ -44,17 +52,30 @@ public QueryString( description = "Query string in Lucene query string format." ) Expression queryString ) { - super(source, queryString, List.of(queryString)); + super(source, queryString, List.of(queryString), null); } - private QueryString(StreamInput in) throws IOException { - this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class)); + public QueryString(Source source, Expression queryString, QueryBuilder queryBuilder) { + super(source, queryString, List.of(queryString), queryBuilder); + } + + private static QueryString readFrom(StreamInput in) throws IOException { + Source source = Source.readFrom((PlanStreamInput) in); + Expression query = in.readNamedWriteable(Expression.class); + QueryBuilder queryBuilder = null; + if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_QUERY_BUILDER_IN_SEARCH_FUNCTIONS)) { + queryBuilder = in.readOptionalNamedWriteable(QueryBuilder.class); + } + return new QueryString(source, query, queryBuilder); } @Override public void writeTo(StreamOutput out) throws IOException { source().writeTo(out); out.writeNamedWriteable(query()); + if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_QUERY_BUILDER_IN_SEARCH_FUNCTIONS)) { + out.writeOptionalNamedWriteable(queryBuilder()); + } } @Override @@ -69,12 +90,21 @@ public String functionName() { @Override public Expression replaceChildren(List newChildren) { - return new QueryString(source(), newChildren.get(0)); + return new QueryString(source(), newChildren.get(0), queryBuilder()); } @Override protected NodeInfo info() { - return NodeInfo.create(this, QueryString::new, query()); + return NodeInfo.create(this, QueryString::new, query(), queryBuilder()); } + @Override + protected ExpressionTranslator translator() { + return new EsqlExpressionTranslators.QueryStringFunctionTranslator(); + } + + @Override + public Expression replaceQueryBuilder(QueryBuilder queryBuilder) { + return new QueryString(source(), query(), queryBuilder); + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Term.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Term.java index 125a5b02b6e1c..ff8085cd1b44b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Term.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Term.java @@ -7,15 +7,18 @@ package org.elasticsearch.xpack.esql.expression.function.fulltext; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.xpack.esql.capabilities.Validatable; import org.elasticsearch.xpack.esql.common.Failure; import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.planner.ExpressionTranslator; import org.elasticsearch.xpack.esql.core.querydsl.query.TermQuery; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -23,6 +26,7 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.planner.EsqlExpressionTranslators; import java.io.IOException; import java.util.List; @@ -56,7 +60,11 @@ public Term( description = "Term you wish to find in the provided field." ) Expression termQuery ) { - super(source, termQuery, List.of(field, termQuery)); + this(source, field, termQuery, null); + } + + public Term(Source source, Expression field, Expression termQuery, QueryBuilder queryBuilder) { + super(source, termQuery, List.of(field, termQuery), queryBuilder); this.field = field; } @@ -64,7 +72,11 @@ private static Term readFrom(StreamInput in) throws IOException { Source source = Source.readFrom((PlanStreamInput) in); Expression field = in.readNamedWriteable(Expression.class); Expression query = in.readNamedWriteable(Expression.class); - return new Term(source, field, query); + QueryBuilder queryBuilder = null; + if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_QUERY_BUILDER_IN_SEARCH_FUNCTIONS)) { + queryBuilder = in.readOptionalNamedWriteable(QueryBuilder.class); + } + return new Term(source, field, query, queryBuilder); } @Override @@ -72,6 +84,9 @@ public void writeTo(StreamOutput out) throws IOException { source().writeTo(out); out.writeNamedWriteable(field()); out.writeNamedWriteable(query()); + if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_QUERY_BUILDER_IN_SEARCH_FUNCTIONS)) { + out.writeOptionalNamedWriteable(queryBuilder()); + } } @Override @@ -101,18 +116,28 @@ public void validate(Failures failures) { @Override public Expression replaceChildren(List newChildren) { - return new Term(source(), newChildren.get(0), newChildren.get(1)); + return new Term(source(), newChildren.get(0), newChildren.get(1), queryBuilder()); } @Override protected NodeInfo info() { - return NodeInfo.create(this, Term::new, field, query()); + return NodeInfo.create(this, Term::new, field, query(), queryBuilder()); } protected TypeResolutions.ParamOrdinal queryParamOrdinal() { return SECOND; } + @Override + protected ExpressionTranslator translator() { + return new EsqlExpressionTranslators.TermFunctionTranslator(); + } + + @Override + public Expression replaceQueryBuilder(QueryBuilder queryBuilder) { + return new Term(source(), field, query(), queryBuilder); + } + public Expression field() { return field; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java index 7820f0f657f7f..43bbf9a5f4ff1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java @@ -16,6 +16,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.TypedAttribute; import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction; import org.elasticsearch.xpack.esql.core.expression.predicate.Range; @@ -100,7 +101,11 @@ public final class EsqlExpressionTranslators { ); public static Query toQuery(Expression e, TranslatorHandler handler) { + if (e instanceof TranslationAware ta) { + return ta.asQuery(handler); + } Query translation = null; + for (ExpressionTranslator translator : QUERY_TRANSLATORS) { translation = translator.translate(e, handler); if (translation != null) { From ccdea4a21cd1fe195e5005d81f0a3a49e1f91c67 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Fri, 13 Dec 2024 13:41:46 +0100 Subject: [PATCH 02/35] ESQL: tests for LOOKUP JOIN with non-unique join keys (#118471) Add a csv dataset and tests for `LOOKUP JOIN` where the join keys are not unique. In particular, add tests that include MVs and nulls to see how `LOOKUP JOIN` treats these. --- .../xpack/esql/CsvTestsDataLoader.java | 3 + .../resources/languages_non_unique_key.csv | 10 +++ .../src/main/resources/lookup-join.csv-spec | 78 ++++++++++++++++++- 3 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages_non_unique_key.csv diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index dbeb54996733a..3b656ded94dd7 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -63,6 +63,8 @@ public class CsvTestsDataLoader { private static final TestsDataset LANGUAGES = new TestsDataset("languages"); private static final TestsDataset LANGUAGES_LOOKUP = LANGUAGES.withIndex("languages_lookup") .withSetting("languages_lookup-settings.json"); + private static final TestsDataset LANGUAGES_LOOKUP_NON_UNIQUE_KEY = LANGUAGES_LOOKUP.withIndex("languages_lookup_non_unique_key") + .withData("languages_non_unique_key.csv"); private static final TestsDataset ALERTS = new TestsDataset("alerts"); private static final TestsDataset UL_LOGS = new TestsDataset("ul_logs"); private static final TestsDataset SAMPLE_DATA = new TestsDataset("sample_data"); @@ -114,6 +116,7 @@ public class CsvTestsDataLoader { Map.entry(APPS_SHORT.indexName, APPS_SHORT), Map.entry(LANGUAGES.indexName, LANGUAGES), Map.entry(LANGUAGES_LOOKUP.indexName, LANGUAGES_LOOKUP), + Map.entry(LANGUAGES_LOOKUP_NON_UNIQUE_KEY.indexName, LANGUAGES_LOOKUP_NON_UNIQUE_KEY), Map.entry(UL_LOGS.indexName, UL_LOGS), Map.entry(SAMPLE_DATA.indexName, SAMPLE_DATA), Map.entry(MV_SAMPLE_DATA.indexName, MV_SAMPLE_DATA), diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages_non_unique_key.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages_non_unique_key.csv new file mode 100644 index 0000000000000..1578762f8d1cb --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages_non_unique_key.csv @@ -0,0 +1,10 @@ +language_code:integer,language_name:keyword,country:keyword +1,English,Canada +1,English, +1,,United Kingdom +1,English,United States of America +2,German,[Germany,Austria] +2,German,Switzerland +2,German, +4,Quenya, +5,,Atlantis diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index 12e333c0ed9f2..f6704d33934af 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -3,7 +3,6 @@ // Reuses the sample dataset and commands from enrich.csv-spec // -//TODO: this sometimes returns null instead of the looked up value (likely related to the execution order) basicOnTheDataNode required_capability: join_lookup_v5 @@ -102,6 +101,83 @@ emp_no:integer | language_code:integer | language_name:keyword 10003 | 4 | German ; +nonUniqueLeftKeyOnTheDataNode +required_capability: join_lookup_v5 + +FROM employees +| WHERE emp_no <= 10030 +| EVAL language_code = emp_no % 10 +| WHERE language_code < 3 +| LOOKUP JOIN languages_lookup ON language_code +| SORT emp_no +| KEEP emp_no, language_code, language_name +; + +emp_no:integer | language_code:integer | language_name:keyword +10001 |1 | English +10002 |2 | French +10010 |0 | null +10011 |1 | English +10012 |2 | French +10020 |0 | null +10021 |1 | English +10022 |2 | French +10030 |0 | null +; + +nonUniqueRightKeyOnTheDataNode +required_capability: join_lookup_v5 + +FROM employees +| EVAL language_code = emp_no % 10 +| LOOKUP JOIN languages_lookup_non_unique_key ON language_code +| WHERE emp_no > 10090 AND emp_no < 10096 +| SORT emp_no +| EVAL country = MV_SORT(country) +| KEEP emp_no, language_code, language_name, country +; + +emp_no:integer | language_code:integer | language_name:keyword | country:keyword +10091 | 1 | [English, English, English] | [Canada, United Kingdom, United States of America] +10092 | 2 | [German, German, German] | [Austria, Germany, Switzerland] +10093 | 3 | null | null +10094 | 4 | Quenya | null +10095 | 5 | null | Atlantis +; + +nonUniqueRightKeyOnTheCoordinator +required_capability: join_lookup_v5 + +FROM employees +| SORT emp_no +| LIMIT 5 +| EVAL language_code = emp_no % 10 +| LOOKUP JOIN languages_lookup_non_unique_key ON language_code +| EVAL country = MV_SORT(country) +| KEEP emp_no, language_code, language_name, country +; + +emp_no:integer | language_code:integer | language_name:keyword | country:keyword +10001 | 1 | [English, English, English] | [Canada, United Kingdom, United States of America] +10002 | 2 | [German, German, German] | [Austria, Germany, Switzerland] +10003 | 3 | null | null +10004 | 4 | Quenya | null +10005 | 5 | null | Atlantis +; + +nonUniqueRightKeyFromRow +required_capability: join_lookup_v5 + +ROW language_code = 2 +| LOOKUP JOIN languages_lookup_non_unique_key ON language_code +| DROP country.keyword +| EVAL country = MV_SORT(country) +; + +language_code:integer | language_name:keyword | country:keyword +2 | [German, German, German] | [Austria, Germany, Switzerland] +; + lookupIPFromRow required_capability: join_lookup_v5 From 71f98221cd8e0813e24d04f68cb03f1cc7e3f639 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Fri, 13 Dec 2024 14:09:06 +0100 Subject: [PATCH 03/35] Fix bug in InferenceUpgradeTestCase.getConfigsWithBreakingChangeHandling (#118624) We need to load the two fields from the same response. Otherwise, we can have a sort of race where we load "endpoints" from pre-8.15 as empty and then load "models" from a post-8.15 node also empty, resulting in an empty list because we took the wrong info from either response. closes #118163 --- .../xpack/application/InferenceUpgradeTestCase.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/InferenceUpgradeTestCase.java b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/InferenceUpgradeTestCase.java index 58335eb53b366..d38503a884092 100644 --- a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/InferenceUpgradeTestCase.java +++ b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/InferenceUpgradeTestCase.java @@ -19,7 +19,6 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Objects; import static org.elasticsearch.core.Strings.format; @@ -112,13 +111,10 @@ protected void put(String inferenceId, String modelConfig, TaskType taskType) th @SuppressWarnings("unchecked") // in version 8.15, there was a breaking change where "models" was renamed to "endpoints" LinkedList> getConfigsWithBreakingChangeHandling(TaskType testTaskType, String oldClusterId) throws IOException { - + var response = get(testTaskType, oldClusterId); LinkedList> configs; - configs = new LinkedList<>( - (List>) Objects.requireNonNullElse((get(testTaskType, oldClusterId).get("endpoints")), List.of()) - ); - configs.addAll(Objects.requireNonNullElse((List>) get(testTaskType, oldClusterId).get("models"), List.of())); - + configs = new LinkedList<>((List>) response.getOrDefault("endpoints", List.of())); + configs.addAll((List>) response.getOrDefault("models", List.of())); return configs; } } From 21b7afd6692105fc8f79b9c02ef4999c6e9a0b18 Mon Sep 17 00:00:00 2001 From: "elastic-renovate-prod[bot]" <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 13:13:52 +0000 Subject: [PATCH 04/35] Update docker.elastic.co/wolfi/chainguard-base:latest Docker digest to 1b51ff6 (#117902) Co-authored-by: elastic-renovate-prod[bot] <174716857+elastic-renovate-prod[bot]@users.noreply.github.com> Co-authored-by: Rene Groeschke --- .../main/java/org/elasticsearch/gradle/internal/DockerBase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java index 0d7bcea168df8..d54eb798ce783 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java @@ -22,7 +22,7 @@ public enum DockerBase { // Chainguard based wolfi image with latest jdk // This is usually updated via renovatebot // spotless:off - WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:32f06b169bb4b0f257fbb10e8c8379f06d3ee1355c89b3327cb623781a29590e", + WOLFI("docker.elastic.co/wolfi/chainguard-base:latest@sha256:1b51ff6dba78c98d3e02b0cd64a8ce3238c7a40408d21e3af12a329d44db6f23", "-wolfi", "apk" ), From 528593b55f26005fec910bfb89f9468ec7798d1f Mon Sep 17 00:00:00 2001 From: Liam Thompson <32779855+leemthompo@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:00:29 +0100 Subject: [PATCH 05/35] [DOCS] Link to Elastic Rerank model landing page (#118574) - Add link to Python notebook - Fix heading level --- docs/reference/inference/service-elasticsearch.asciidoc | 7 ++++++- docs/reference/search/retriever.asciidoc | 7 ++++++- .../search/search-your-data/retrievers-examples.asciidoc | 3 +-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/reference/inference/service-elasticsearch.asciidoc b/docs/reference/inference/service-elasticsearch.asciidoc index cd06e6d7b2f64..bf7e2976bbe63 100644 --- a/docs/reference/inference/service-elasticsearch.asciidoc +++ b/docs/reference/inference/service-elasticsearch.asciidoc @@ -153,7 +153,12 @@ For further details, refer to the {ml-docs}/ml-nlp-elser.html[ELSER model docume [[inference-example-elastic-reranker]] ==== Elastic Rerank via the `elasticsearch` service -The following example shows how to create an {infer} endpoint called `my-elastic-rerank` to perform a `rerank` task type using the built-in Elastic Rerank cross-encoder model. +The following example shows how to create an {infer} endpoint called `my-elastic-rerank` to perform a `rerank` task type using the built-in {ml-docs}/ml-nlp-rerank.html[Elastic Rerank] cross-encoder model. + +[TIP] +==== +Refer to this https://github.com/elastic/elasticsearch-labs/blob/main/notebooks/search/12-semantic-reranking-elastic-rerank.ipynb[Python notebook] for an end-to-end example using Elastic Rerank. +==== The API request below will automatically download the Elastic Rerank model if it isn't already downloaded and then deploy the model. Once deployed, the model can be used for semantic re-ranking with a <>. diff --git a/docs/reference/search/retriever.asciidoc b/docs/reference/search/retriever.asciidoc index cb04d4fb6fbf1..f20e9148bf5e7 100644 --- a/docs/reference/search/retriever.asciidoc +++ b/docs/reference/search/retriever.asciidoc @@ -442,7 +442,12 @@ If the child retriever already specifies any filters, then this top-level filter [[text-similarity-reranker-retriever-example-elastic-rerank]] ==== Example: Elastic Rerank -This examples demonstrates how to deploy the Elastic Rerank model and use it to re-rank search results using the `text_similarity_reranker` retriever. +[TIP] +==== +Refer to this https://github.com/elastic/elasticsearch-labs/blob/main/notebooks/search/12-semantic-reranking-elastic-rerank.ipynb[Python notebook] for an end-to-end example using Elastic Rerank. +==== + +This example demonstrates how to deploy the {ml-docs}/ml-nlp-rerank.html[Elastic Rerank] model and use it to re-rank search results using the `text_similarity_reranker` retriever. Follow these steps: diff --git a/docs/reference/search/search-your-data/retrievers-examples.asciidoc b/docs/reference/search/search-your-data/retrievers-examples.asciidoc index ad1cc32dcee01..5cada8960aeab 100644 --- a/docs/reference/search/search-your-data/retrievers-examples.asciidoc +++ b/docs/reference/search/search-your-data/retrievers-examples.asciidoc @@ -1,9 +1,8 @@ [[retrievers-examples]] +=== Retrievers examples Learn how to combine different retrievers in these hands-on examples. -=== Retrievers examples - [discrete] [[retrievers-examples-setup]] ==== Add example data From 1e26791515184a965444ea45a300691d17cffb8d Mon Sep 17 00:00:00 2001 From: Mark Tozzi Date: Fri, 13 Dec 2024 09:25:52 -0500 Subject: [PATCH 06/35] Esql bucket function for date nanos (#118474) This adds support for running the bucket function over a date nanos field. Code wise, this just delegates to DateTrunc, which already supports date nanos, so most of the PR is just tests and the auto-generated docs. Resolves #118031 --- docs/changelog/118474.yaml | 6 + .../functions/kibana/definition/bucket.json | 306 ++++++++++++++++++ .../esql/functions/types/bucket.asciidoc | 11 + .../src/main/resources/date_nanos.csv-spec | 74 +++++ .../xpack/esql/action/EsqlCapabilities.java | 5 + .../expression/function/grouping/Bucket.java | 12 +- .../function/grouping/BucketTests.java | 91 +++++- 7 files changed, 498 insertions(+), 7 deletions(-) create mode 100644 docs/changelog/118474.yaml diff --git a/docs/changelog/118474.yaml b/docs/changelog/118474.yaml new file mode 100644 index 0000000000000..1b0c6942eb323 --- /dev/null +++ b/docs/changelog/118474.yaml @@ -0,0 +1,6 @@ +pr: 118474 +summary: Esql bucket function for date nanos +area: ES|QL +type: enhancement +issues: + - 118031 diff --git a/docs/reference/esql/functions/kibana/definition/bucket.json b/docs/reference/esql/functions/kibana/definition/bucket.json index 660e1be49fda9..18802f5ff8fef 100644 --- a/docs/reference/esql/functions/kibana/definition/bucket.json +++ b/docs/reference/esql/functions/kibana/definition/bucket.json @@ -310,6 +310,312 @@ "variadic" : false, "returnType" : "date" }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "date_period", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + }, + { + "name" : "from", + "type" : "date", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "date", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + }, + { + "name" : "from", + "type" : "date", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "keyword", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + }, + { + "name" : "from", + "type" : "date", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "text", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + }, + { + "name" : "from", + "type" : "keyword", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "date", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + }, + { + "name" : "from", + "type" : "keyword", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "keyword", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + }, + { + "name" : "from", + "type" : "keyword", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "text", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + }, + { + "name" : "from", + "type" : "text", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "date", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + }, + { + "name" : "from", + "type" : "text", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "keyword", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "integer", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + }, + { + "name" : "from", + "type" : "text", + "optional" : true, + "description" : "Start of the range. Can be a number, a date or a date expressed as a string." + }, + { + "name" : "to", + "type" : "text", + "optional" : true, + "description" : "End of the range. Can be a number, a date or a date expressed as a string." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, + { + "params" : [ + { + "name" : "field", + "type" : "date_nanos", + "optional" : false, + "description" : "Numeric or date expression from which to derive buckets." + }, + { + "name" : "buckets", + "type" : "time_duration", + "optional" : false, + "description" : "Target number of buckets, or desired bucket size if `from` and `to` parameters are omitted." + } + ], + "variadic" : false, + "returnType" : "date_nanos" + }, { "params" : [ { diff --git a/docs/reference/esql/functions/types/bucket.asciidoc b/docs/reference/esql/functions/types/bucket.asciidoc index 172e84b6f7860..2e6985e6bc4ed 100644 --- a/docs/reference/esql/functions/types/bucket.asciidoc +++ b/docs/reference/esql/functions/types/bucket.asciidoc @@ -16,6 +16,17 @@ date | integer | text | date | date date | integer | text | keyword | date date | integer | text | text | date date | time_duration | | | date +date_nanos | date_period | | | date_nanos +date_nanos | integer | date | date | date_nanos +date_nanos | integer | date | keyword | date_nanos +date_nanos | integer | date | text | date_nanos +date_nanos | integer | keyword | date | date_nanos +date_nanos | integer | keyword | keyword | date_nanos +date_nanos | integer | keyword | text | date_nanos +date_nanos | integer | text | date | date_nanos +date_nanos | integer | text | keyword | date_nanos +date_nanos | integer | text | text | date_nanos +date_nanos | time_duration | | | date_nanos double | double | | | double double | integer | double | double | double double | integer | double | integer | double diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec index 0d113c0422562..f4b5c98d596ae 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec @@ -549,6 +549,80 @@ yr:date_nanos | mo:date_nanos | mn:date_nanos 2023-01-01T00:00:00.000000000Z | 2023-10-01T00:00:00.000000000Z | 2023-10-23T12:10:00.000000000Z | 2023-10-23T12:15:03.360000000Z ; +Bucket Date nanos by Year +required_capability: date_trunc_date_nanos +required_capability: date_nanos_bucket + +FROM date_nanos +| WHERE millis > "2020-01-01" +| STATS ct = count(*) BY yr = BUCKET(nanos, 1 year); + +ct:long | yr:date_nanos +8 | 2023-01-01T00:00:00.000000000Z +; + +Bucket Date nanos by Year, range version +required_capability: date_trunc_date_nanos +required_capability: date_nanos_bucket + +FROM date_nanos +| WHERE millis > "2020-01-01" +| STATS ct = count(*) BY yr = BUCKET(nanos, 5, "1999-01-01", NOW()); + +ct:long | yr:date_nanos +8 | 2023-01-01T00:00:00.000000000Z +; + +Bucket Date nanos by Month +required_capability: date_trunc_date_nanos +required_capability: date_nanos_bucket + +FROM date_nanos +| WHERE millis > "2020-01-01" +| STATS ct = count(*) BY mo = BUCKET(nanos, 1 month); + +ct:long | mo:date_nanos +8 | 2023-10-01T00:00:00.000000000Z +; + +Bucket Date nanos by Month, range version +required_capability: date_trunc_date_nanos +required_capability: date_nanos_bucket + +FROM date_nanos +| WHERE millis > "2020-01-01" +| STATS ct = count(*) BY mo = BUCKET(nanos, 20, "2023-01-01", "2023-12-31"); + +ct:long | mo:date_nanos +8 | 2023-10-01T00:00:00.000000000Z +; + +Bucket Date nanos by Week, range version +required_capability: date_trunc_date_nanos +required_capability: date_nanos_bucket + +FROM date_nanos +| WHERE millis > "2020-01-01" +| STATS ct = count(*) BY mo = BUCKET(nanos, 55, "2023-01-01", "2023-12-31"); + +ct:long | mo:date_nanos +8 | 2023-10-23T00:00:00.000000000Z +; +Bucket Date nanos by 10 minutes +required_capability: date_trunc_date_nanos +required_capability: date_nanos_bucket + +FROM date_nanos +| WHERE millis > "2020-01-01" +| STATS ct = count(*) BY mn = BUCKET(nanos, 10 minutes); + +ct:long | mn:date_nanos +4 | 2023-10-23T13:50:00.000000000Z +1 | 2023-10-23T13:30:00.000000000Z +1 | 2023-10-23T12:20:00.000000000Z +2 | 2023-10-23T12:10:00.000000000Z +; + Add date nanos required_capability: date_nanos_add_subtract diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 4cf3162fcca3b..649ec1eba9785 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -367,6 +367,11 @@ public enum Cap { */ DATE_TRUNC_DATE_NANOS(), + /** + * Support date nanos values as the field argument to bucket + */ + DATE_NANOS_BUCKET(), + /** * support aggregations on date nanos */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java index 9e40b85fd6590..347d542f5212d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java @@ -90,7 +90,7 @@ public class Bucket extends GroupingFunction implements Validatable, TwoOptional private final Expression to; @FunctionInfo( - returnType = { "double", "date" }, + returnType = { "double", "date", "date_nanos" }, description = """ Creates groups of values - buckets - out of a datetime or numeric input. The size of the buckets can either be provided directly, or chosen based on a recommended count and values range.""", @@ -169,7 +169,7 @@ public Bucket( Source source, @Param( name = "field", - type = { "integer", "long", "double", "date" }, + type = { "integer", "long", "double", "date", "date_nanos" }, description = "Numeric or date expression from which to derive buckets." ) Expression field, @Param( @@ -241,7 +241,7 @@ public boolean foldable() { @Override public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { - if (field.dataType() == DataType.DATETIME) { + if (field.dataType() == DataType.DATETIME || field.dataType() == DataType.DATE_NANOS) { Rounding.Prepared preparedRounding; if (buckets.dataType().isWholeNumber()) { int b = ((Number) buckets.fold()).intValue(); @@ -314,8 +314,8 @@ private double pickRounding(int buckets, double from, double to) { } // supported parameter type combinations (1st, 2nd, 3rd, 4th): - // datetime, integer, string/datetime, string/datetime - // datetime, rounding/duration, -, - + // datetime/date_nanos, integer, string/datetime, string/datetime + // datetime/date_nanos, rounding/duration, -, - // numeric, integer, numeric, numeric // numeric, numeric, -, - @Override @@ -329,7 +329,7 @@ protected TypeResolution resolveType() { return TypeResolution.TYPE_RESOLVED; } - if (fieldType == DataType.DATETIME) { + if (fieldType == DataType.DATETIME || fieldType == DataType.DATE_NANOS) { TypeResolution resolution = isType( buckets, dt -> dt.isWholeNumber() || DataType.isTemporalAmount(dt), diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/BucketTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/BucketTests.java index 7e7d91cdf76f4..f01b06c23e8a8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/BucketTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/grouping/BucketTests.java @@ -12,15 +12,19 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Rounding; +import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.logging.LogManager; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.hamcrest.Matcher; +import org.hamcrest.Matchers; import java.time.Duration; +import java.time.Instant; import java.time.Period; import java.util.ArrayList; import java.util.List; @@ -38,6 +42,7 @@ public BucketTests(@Name("TestCase") Supplier testCas public static Iterable parameters() { List suppliers = new ArrayList<>(); dateCases(suppliers, "fixed date", () -> DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-02-17T09:00:00.00Z")); + dateNanosCases(suppliers, "fixed date nanos", () -> DateUtils.toLong(Instant.parse("2023-02-17T09:00:00.00Z"))); dateCasesWithSpan( suppliers, "fixed date with period", @@ -54,6 +59,22 @@ public static Iterable parameters() { Duration.ofDays(1L), "[86400000 in Z][fixed]" ); + dateNanosCasesWithSpan( + suppliers, + "fixed date nanos with period", + () -> DateUtils.toLong(Instant.parse("2023-01-01T00:00:00.00Z")), + DataType.DATE_PERIOD, + Period.ofYears(1), + "[YEAR_OF_CENTURY in Z][fixed to midnight]" + ); + dateNanosCasesWithSpan( + suppliers, + "fixed date nanos with duration", + () -> DateUtils.toLong(Instant.parse("2023-02-17T09:00:00.00Z")), + DataType.TIME_DURATION, + Duration.ofDays(1L), + "[86400000 in Z][fixed]" + ); numberCases(suppliers, "fixed long", DataType.LONG, () -> 100L); numberCasesWithSpan(suppliers, "fixed long with span", DataType.LONG, () -> 100L); numberCases(suppliers, "fixed int", DataType.INTEGER, () -> 100); @@ -142,6 +163,62 @@ private static void dateCasesWithSpan( })); } + private static void dateNanosCasesWithSpan( + List suppliers, + String name, + LongSupplier date, + DataType spanType, + Object span, + String spanStr + ) { + suppliers.add(new TestCaseSupplier(name, List.of(DataType.DATE_NANOS, spanType), () -> { + List args = new ArrayList<>(); + args.add(new TestCaseSupplier.TypedData(date.getAsLong(), DataType.DATE_NANOS, "field")); + args.add(new TestCaseSupplier.TypedData(span, spanType, "buckets").forceLiteral()); + return new TestCaseSupplier.TestCase( + args, + Matchers.startsWith("DateTruncDateNanosEvaluator[fieldVal=Attribute[channel=0], rounding=Rounding["), + DataType.DATE_NANOS, + resultsMatcher(args) + ); + })); + } + + private static void dateNanosCases(List suppliers, String name, LongSupplier date) { + for (DataType fromType : DATE_BOUNDS_TYPE) { + for (DataType toType : DATE_BOUNDS_TYPE) { + suppliers.add(new TestCaseSupplier(name, List.of(DataType.DATE_NANOS, DataType.INTEGER, fromType, toType), () -> { + List args = new ArrayList<>(); + args.add(new TestCaseSupplier.TypedData(date.getAsLong(), DataType.DATE_NANOS, "field")); + // TODO more "from" and "to" and "buckets" + args.add(new TestCaseSupplier.TypedData(50, DataType.INTEGER, "buckets").forceLiteral()); + args.add(dateBound("from", fromType, "2023-02-01T00:00:00.00Z")); + args.add(dateBound("to", toType, "2023-03-01T09:00:00.00Z")); + return new TestCaseSupplier.TestCase( + args, + Matchers.startsWith("DateTruncDateNanosEvaluator[fieldVal=Attribute[channel=0], rounding=Rounding["), + DataType.DATE_NANOS, + resultsMatcher(args) + ); + })); + // same as above, but a low bucket count and datetime bounds that match it (at hour span) + suppliers.add(new TestCaseSupplier(name, List.of(DataType.DATE_NANOS, DataType.INTEGER, fromType, toType), () -> { + List args = new ArrayList<>(); + args.add(new TestCaseSupplier.TypedData(date.getAsLong(), DataType.DATE_NANOS, "field")); + args.add(new TestCaseSupplier.TypedData(4, DataType.INTEGER, "buckets").forceLiteral()); + args.add(dateBound("from", fromType, "2023-02-17T09:00:00Z")); + args.add(dateBound("to", toType, "2023-02-17T12:00:00Z")); + return new TestCaseSupplier.TestCase( + args, + Matchers.startsWith("DateTruncDateNanosEvaluator[fieldVal=Attribute[channel=0], rounding=Rounding["), + DataType.DATE_NANOS, + equalTo(Rounding.builder(Rounding.DateTimeUnit.HOUR_OF_DAY).build().prepareForUnknown().round(date.getAsLong())) + ); + })); + } + } + } + private static final DataType[] NUMBER_BOUNDS_TYPES = new DataType[] { DataType.INTEGER, DataType.LONG, DataType.DOUBLE }; private static void numberCases(List suppliers, String name, DataType numberType, Supplier number) { @@ -221,7 +298,19 @@ private static TestCaseSupplier.TypedData keywordDateLiteral(String name, DataTy private static Matcher resultsMatcher(List typedData) { if (typedData.get(0).type() == DataType.DATETIME) { long millis = ((Number) typedData.get(0).data()).longValue(); - return equalTo(Rounding.builder(Rounding.DateTimeUnit.DAY_OF_MONTH).build().prepareForUnknown().round(millis)); + long expected = Rounding.builder(Rounding.DateTimeUnit.DAY_OF_MONTH).build().prepareForUnknown().round(millis); + LogManager.getLogger(getTestClass()).info("Expected: " + Instant.ofEpochMilli(expected)); + LogManager.getLogger(getTestClass()).info("Input: " + Instant.ofEpochMilli(millis)); + return equalTo(expected); + } + if (typedData.get(0).type() == DataType.DATE_NANOS) { + long nanos = ((Number) typedData.get(0).data()).longValue(); + long expected = DateUtils.toNanoSeconds( + Rounding.builder(Rounding.DateTimeUnit.DAY_OF_MONTH).build().prepareForUnknown().round(DateUtils.toMilliSeconds(nanos)) + ); + LogManager.getLogger(getTestClass()).info("Expected: " + DateUtils.toInstant(expected)); + LogManager.getLogger(getTestClass()).info("Input: " + DateUtils.toInstant(nanos)); + return equalTo(expected); } return equalTo(((Number) typedData.get(0).data()).doubleValue()); } From a583a38518f321d6092637e66151fc0bd1fe4d1d Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Fri, 13 Dec 2024 15:38:52 +0100 Subject: [PATCH 07/35] fix typo in muted CSV test for scoring in ES|QL (#118665) --- muted-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/muted-tests.yml b/muted-tests.yml index d5b933b96d73b..36dfc306b0147 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -174,7 +174,7 @@ tests: - class: "org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT" method: "test {scoring.*}" issue: https://github.com/elastic/elasticsearch/issues/117641 -- class: "org.elasticsearch.xpack.esql.qa.mixed.MultilusterEsqlSpecIT" +- class: "org.elasticsearch.xpack.esql.qa.mixed.MultiClusterEsqlSpecIT" method: "test {scoring.*}" issue: https://github.com/elastic/elasticsearch/issues/118460 - class: "org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT" From 44a231acf2471f39497b7740ddc349306c97cd6a Mon Sep 17 00:00:00 2001 From: Max Hniebergall <137079448+maxhniebergall@users.noreply.github.com> Date: Fri, 13 Dec 2024 09:53:33 -0500 Subject: [PATCH 08/35] [Inference API] Replace ElasticsearchTimeoutException with ElasticsearchStatusException (#118618) * Replace ElasticsearchTimeoutException with ElasticsearchTimeoutException with 408 status to avoid cuasing 503s --- .../xpack/inference/external/http/sender/RequestTask.java | 8 ++++++-- .../external/http/sender/HttpRequestSenderTests.java | 8 +++++--- .../external/http/sender/RequestExecutorServiceTests.java | 5 +++-- .../inference/external/http/sender/RequestTaskTests.java | 5 +++-- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTask.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTask.java index 9ccb93a0858ae..e5c29adeb9176 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTask.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTask.java @@ -7,13 +7,14 @@ package org.elasticsearch.xpack.inference.external.http.sender; -import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ListenerTimeouts; import org.elasticsearch.common.Strings; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.threadpool.ThreadPool; import java.util.Objects; @@ -64,7 +65,10 @@ private ActionListener getListener( threadPool.executor(UTILITY_THREAD_POOL_NAME), notificationListener, (ignored) -> notificationListener.onFailure( - new ElasticsearchTimeoutException(Strings.format("Request timed out waiting to be sent after [%s]", timeout)) + new ElasticsearchStatusException( + Strings.format("Request timed out waiting to be sent after [%s]", timeout), + RestStatus.REQUEST_TIMEOUT + ) ) ); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSenderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSenderTests.java index 79f6aa8164b75..b3e7db6009204 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSenderTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/HttpRequestSenderTests.java @@ -8,7 +8,7 @@ package org.elasticsearch.xpack.inference.external.http.sender; import org.apache.http.HttpHeaders; -import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; @@ -162,12 +162,13 @@ public void testHttpRequestSender_Throws_WhenATimeoutOccurs() throws Exception { PlainActionFuture listener = new PlainActionFuture<>(); sender.send(RequestManagerTests.createMock(), new DocumentsOnlyInput(List.of()), TimeValue.timeValueNanos(1), listener); - var thrownException = expectThrows(ElasticsearchTimeoutException.class, () -> listener.actionGet(TIMEOUT)); + var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT)); assertThat( thrownException.getMessage(), is(format("Request timed out waiting to be sent after [%s]", TimeValue.timeValueNanos(1))) ); + assertThat(thrownException.status().getStatus(), is(408)); } } @@ -187,12 +188,13 @@ public void testHttpRequestSenderWithTimeout_Throws_WhenATimeoutOccurs() throws PlainActionFuture listener = new PlainActionFuture<>(); sender.send(RequestManagerTests.createMock(), new DocumentsOnlyInput(List.of()), TimeValue.timeValueNanos(1), listener); - var thrownException = expectThrows(ElasticsearchTimeoutException.class, () -> listener.actionGet(TIMEOUT)); + var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT)); assertThat( thrownException.getMessage(), is(format("Request timed out waiting to be sent after [%s]", TimeValue.timeValueNanos(1))) ); + assertThat(thrownException.status().getStatus(), is(408)); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceTests.java index e09e4968571e5..7e29fad56812d 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestExecutorServiceTests.java @@ -8,7 +8,7 @@ package org.elasticsearch.xpack.inference.external.http.sender; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.common.Strings; @@ -238,12 +238,13 @@ public void testExecute_CallsOnFailure_WhenRequestTimesOut() { var listener = new PlainActionFuture(); service.execute(RequestManagerTests.createMock(), new DocumentsOnlyInput(List.of()), TimeValue.timeValueNanos(1), listener); - var thrownException = expectThrows(ElasticsearchTimeoutException.class, () -> listener.actionGet(TIMEOUT)); + var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT)); assertThat( thrownException.getMessage(), is(format("Request timed out waiting to be sent after [%s]", TimeValue.timeValueNanos(1))) ); + assertThat(thrownException.status().getStatus(), is(408)); } public void testExecute_PreservesThreadContext() throws InterruptedException, ExecutionException, TimeoutException { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTaskTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTaskTests.java index c839c266e9320..e37a1a213569e 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTaskTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/http/sender/RequestTaskTests.java @@ -7,7 +7,7 @@ package org.elasticsearch.xpack.inference.external.http.sender; -import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.core.TimeValue; @@ -86,13 +86,14 @@ public void testRequest_ReturnsTimeoutException() { listener ); - var thrownException = expectThrows(ElasticsearchTimeoutException.class, () -> listener.actionGet(TIMEOUT)); + var thrownException = expectThrows(ElasticsearchStatusException.class, () -> listener.actionGet(TIMEOUT)); assertThat( thrownException.getMessage(), is(format("Request timed out waiting to be sent after [%s]", TimeValue.timeValueMillis(1))) ); assertTrue(requestTask.hasCompleted()); assertTrue(requestTask.getRequestCompletedFunction().get()); + assertThat(thrownException.status().getStatus(), is(408)); } public void testRequest_DoesNotCallOnFailureTwiceWhenTimingOut() throws Exception { From 1bad1cf6b2256f594361059a8090d2707aa54001 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Fri, 13 Dec 2024 16:21:42 +0100 Subject: [PATCH 09/35] Implementing the correct exit functions (Runtime) (#118657) --- .../entitlement/bridge/EntitlementChecker.java | 6 +++++- .../qa/common/RestEntitlementsCheckAction.java | 17 ++++++++++++----- .../api/ElasticsearchEntitlementChecker.java | 7 ++++++- .../Java23ElasticsearchEntitlementChecker.java | 4 ++-- 4 files changed, 25 insertions(+), 9 deletions(-) diff --git a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java index ad0f14bcf4478..a6b8a31fc3894 100644 --- a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java +++ b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java @@ -13,7 +13,11 @@ import java.net.URLStreamHandlerFactory; public interface EntitlementChecker { - void check$java_lang_System$exit(Class callerClass, int status); + + // Exit the JVM process + void check$$exit(Class callerClass, Runtime runtime, int status); + + void check$$halt(Class callerClass, Runtime runtime, int status); // URLClassLoader ctor void check$java_net_URLClassLoader$(Class callerClass, URL[] urls); diff --git a/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java b/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java index e63fa4f3b726b..1ac4a7506eacb 100644 --- a/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java +++ b/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java @@ -47,14 +47,21 @@ static CheckAction serverAndPlugin(Runnable action) { } private static final Map checkActions = Map.ofEntries( - entry("system_exit", CheckAction.serverOnly(RestEntitlementsCheckAction::systemExit)), + entry("runtime_exit", CheckAction.serverOnly(RestEntitlementsCheckAction::runtimeExit)), + entry("runtime_halt", CheckAction.serverOnly(RestEntitlementsCheckAction::runtimeHalt)), entry("create_classloader", CheckAction.serverAndPlugin(RestEntitlementsCheckAction::createClassLoader)) ); - @SuppressForbidden(reason = "Specifically testing System.exit") - private static void systemExit() { - logger.info("Calling System.exit(123);"); - System.exit(123); + @SuppressForbidden(reason = "Specifically testing Runtime.exit") + private static void runtimeExit() { + logger.info("Calling Runtime.exit;"); + Runtime.getRuntime().exit(123); + } + + @SuppressForbidden(reason = "Specifically testing Runtime.halt") + private static void runtimeHalt() { + logger.info("Calling Runtime.halt;"); + Runtime.getRuntime().halt(123); } private static void createClassLoader() { diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java index aa63b630ed7cd..a5ca0543ad15a 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java @@ -28,7 +28,12 @@ public ElasticsearchEntitlementChecker(PolicyManager policyManager) { } @Override - public void check$java_lang_System$exit(Class callerClass, int status) { + public void check$$exit(Class callerClass, Runtime runtime, int status) { + policyManager.checkExitVM(callerClass); + } + + @Override + public void check$$halt(Class callerClass, Runtime runtime, int status) { policyManager.checkExitVM(callerClass); } diff --git a/libs/entitlement/src/main23/java/org/elasticsearch/entitlement/runtime/api/Java23ElasticsearchEntitlementChecker.java b/libs/entitlement/src/main23/java/org/elasticsearch/entitlement/runtime/api/Java23ElasticsearchEntitlementChecker.java index d0f9f4f48609c..912d76ecfc01a 100644 --- a/libs/entitlement/src/main23/java/org/elasticsearch/entitlement/runtime/api/Java23ElasticsearchEntitlementChecker.java +++ b/libs/entitlement/src/main23/java/org/elasticsearch/entitlement/runtime/api/Java23ElasticsearchEntitlementChecker.java @@ -19,8 +19,8 @@ public Java23ElasticsearchEntitlementChecker(PolicyManager policyManager) { } @Override - public void check$java_lang_System$exit(Class callerClass, int status) { + public void check$$exit(Class callerClass, Runtime runtime, int status) { // TODO: this is just an example, we shouldn't really override a method implemented in the superclass - super.check$java_lang_System$exit(callerClass, status); + super.check$$exit(callerClass, runtime, status); } } From 5411b93d493ddc81682b49cf6cb9bac2607c4f2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Fri, 13 Dec 2024 16:24:54 +0100 Subject: [PATCH 10/35] Entitlements tools: public callers finder (#116257) * WIP: Tool to find all public caller from a starting list of (JDK) methods. * Add public-callers-finder tool, extract common stuff to common module * Adjustments to visibility/functions and classes and modules to print out * Spotless * Missing gradle configuration * Add details in README as requested in PR * Update ASM version * Including protected methods --- .../entitlement/tools/ExternalAccess.java | 68 ++++++ .../entitlement/tools/Utils.java | 37 +++- .../tools/public-callers-finder/README.md | 50 +++++ .../tools/public-callers-finder/build.gradle | 61 ++++++ .../licenses/asm-LICENSE.txt | 26 +++ .../licenses/asm-NOTICE.txt | 1 + .../FindUsagesClassVisitor.java | 141 +++++++++++++ .../tools/publiccallersfinder/Main.java | 197 ++++++++++++++++++ .../{src => }/README.md | 0 .../tools/securitymanager/scanner/Main.java | 50 ++--- .../scanner/SecurityCheckClassVisitor.java | 22 +- 11 files changed, 603 insertions(+), 50 deletions(-) create mode 100644 libs/entitlement/tools/common/src/main/java/org/elasticsearch/entitlement/tools/ExternalAccess.java create mode 100644 libs/entitlement/tools/public-callers-finder/README.md create mode 100644 libs/entitlement/tools/public-callers-finder/build.gradle create mode 100644 libs/entitlement/tools/public-callers-finder/licenses/asm-LICENSE.txt create mode 100644 libs/entitlement/tools/public-callers-finder/licenses/asm-NOTICE.txt create mode 100644 libs/entitlement/tools/public-callers-finder/src/main/java/org/elasticsearch/entitlement/tools/publiccallersfinder/FindUsagesClassVisitor.java create mode 100644 libs/entitlement/tools/public-callers-finder/src/main/java/org/elasticsearch/entitlement/tools/publiccallersfinder/Main.java rename libs/entitlement/tools/securitymanager-scanner/{src => }/README.md (100%) diff --git a/libs/entitlement/tools/common/src/main/java/org/elasticsearch/entitlement/tools/ExternalAccess.java b/libs/entitlement/tools/common/src/main/java/org/elasticsearch/entitlement/tools/ExternalAccess.java new file mode 100644 index 0000000000000..cd049a91fa4da --- /dev/null +++ b/libs/entitlement/tools/common/src/main/java/org/elasticsearch/entitlement/tools/ExternalAccess.java @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.tools; + +import java.util.Arrays; +import java.util.EnumSet; +import java.util.stream.Collectors; + +public enum ExternalAccess { + PUBLIC_CLASS, + PUBLIC_METHOD, + PROTECTED_METHOD; + + private static final String DELIMITER = ":"; + + public static String toString(EnumSet externalAccesses) { + return externalAccesses.stream().map(Enum::toString).collect(Collectors.joining(DELIMITER)); + } + + public static EnumSet fromPermissions( + boolean packageExported, + boolean publicClass, + boolean publicMethod, + boolean protectedMethod + ) { + if (publicMethod && protectedMethod) { + throw new IllegalArgumentException(); + } + + EnumSet externalAccesses = EnumSet.noneOf(ExternalAccess.class); + if (publicMethod) { + externalAccesses.add(ExternalAccess.PUBLIC_METHOD); + } else if (protectedMethod) { + externalAccesses.add(ExternalAccess.PROTECTED_METHOD); + } + + if (packageExported && publicClass) { + externalAccesses.add(ExternalAccess.PUBLIC_CLASS); + } + return externalAccesses; + } + + public static boolean isExternallyAccessible(EnumSet access) { + return access.contains(ExternalAccess.PUBLIC_CLASS) + && (access.contains(ExternalAccess.PUBLIC_METHOD) || access.contains(ExternalAccess.PROTECTED_METHOD)); + } + + public static EnumSet fromString(String accessAsString) { + if ("PUBLIC".equals(accessAsString)) { + return EnumSet.of(ExternalAccess.PUBLIC_CLASS, ExternalAccess.PUBLIC_METHOD); + } + if ("PUBLIC-METHOD".equals(accessAsString)) { + return EnumSet.of(ExternalAccess.PUBLIC_METHOD); + } + if ("PRIVATE".equals(accessAsString)) { + return EnumSet.noneOf(ExternalAccess.class); + } + + return EnumSet.copyOf(Arrays.stream(accessAsString.split(DELIMITER)).map(ExternalAccess::valueOf).toList()); + } +} diff --git a/libs/entitlement/tools/common/src/main/java/org/elasticsearch/entitlement/tools/Utils.java b/libs/entitlement/tools/common/src/main/java/org/elasticsearch/entitlement/tools/Utils.java index c72e550a529cd..c6a71f55db4c6 100644 --- a/libs/entitlement/tools/common/src/main/java/org/elasticsearch/entitlement/tools/Utils.java +++ b/libs/entitlement/tools/common/src/main/java/org/elasticsearch/entitlement/tools/Utils.java @@ -11,16 +11,28 @@ import java.io.IOException; import java.lang.module.ModuleDescriptor; +import java.net.URI; import java.nio.file.FileSystem; +import java.nio.file.FileSystems; import java.nio.file.Files; +import java.nio.file.Path; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; public class Utils { - public static Map> findModuleExports(FileSystem fs) throws IOException { + private static final Set EXCLUDED_MODULES = Set.of( + "java.desktop", + "jdk.jartool", + "jdk.jdi", + "java.security.jgss", + "jdk.jshell" + ); + + private static Map> findModuleExports(FileSystem fs) throws IOException { var modulesExports = new HashMap>(); try (var stream = Files.walk(fs.getPath("modules"))) { stream.filter(p -> p.getFileName().toString().equals("module-info.class")).forEach(x -> { @@ -42,4 +54,27 @@ public static Map> findModuleExports(FileSystem fs) throws I return modulesExports; } + public interface JdkModuleConsumer { + void accept(String moduleName, List moduleClasses, Set moduleExports); + } + + public static void walkJdkModules(JdkModuleConsumer c) throws IOException { + + FileSystem fs = FileSystems.getFileSystem(URI.create("jrt:/")); + + var moduleExports = Utils.findModuleExports(fs); + + try (var stream = Files.walk(fs.getPath("modules"))) { + var modules = stream.filter(x -> x.toString().endsWith(".class")) + .collect(Collectors.groupingBy(x -> x.subpath(1, 2).toString())); + + for (var kv : modules.entrySet()) { + var moduleName = kv.getKey(); + if (Utils.EXCLUDED_MODULES.contains(moduleName) == false) { + var thisModuleExports = moduleExports.get(moduleName); + c.accept(moduleName, kv.getValue(), thisModuleExports); + } + } + } + } } diff --git a/libs/entitlement/tools/public-callers-finder/README.md b/libs/entitlement/tools/public-callers-finder/README.md new file mode 100644 index 0000000000000..794576b3409a8 --- /dev/null +++ b/libs/entitlement/tools/public-callers-finder/README.md @@ -0,0 +1,50 @@ +This tool scans the JDK on which it is running. It takes a list of methods (compatible with the output of the `securitymanager-scanner` tool), and looks for the "public surface" of these methods (i.e. any class/method accessible from regular Java code that calls into the original list, directly or transitively). + +It acts basically as a recursive "Find Usages" in Intellij, stopping at the first fully accessible point (public method on a public class). +The tool scans every method in every class inside the same java module; e.g. +if you have a private method `File#normalizedList`, it will scan `java.base` to find +public methods like `File#list(String)`, `File#list(FilenameFilter, String)` and +`File#listFiles(File)`. + +The tool considers implemented interfaces (directly); e.g. if we're looking at a +method `C.m`, where `C implements I`, it will look for calls to `I.m`. It will +also consider (indirectly) calls to `S.m` (where `S` is a supertype of `C`), as +it treats calls to `super` in `S.m` as regular calls (e.g. `example() -> S.m() -> C.m()`). + + +In order to run the tool, use: +```shell +./gradlew :libs:entitlement:tools:public-callers-finder:run [] +``` +Where `input-file` is a CSV file (columns separated by `TAB`) that contains the following columns: +Module name +1. unused +2. unused +3. unused +4. Fully qualified class name (ASM style, with `/` separators) +5. Method name +6. Method descriptor (ASM signature) +7. Visibility (PUBLIC/PUBLIC-METHOD/PRIVATE) + +And `bubble-up-from-public` is a boolean (`true|false`) indicating if the code should stop at the first public method (`false`: default, recommended) or continue to find usages recursively even after reaching the "public surface". + +The output of the tool is another CSV file, with one line for each entry-point, columns separated by `TAB` + +1. Module name +2. File name (from source root) +3. Line number +4. Fully qualified class name (ASM style, with `/` separators) +5. Method name +6. Method descriptor (ASM signature) +7. Visibility (PUBLIC/PUBLIC-METHOD/PRIVATE) +8. Original caller Module name +9. Original caller Class name (ASM style, with `/` separators) +10. Original caller Method name +11. Original caller Visibility + +Examples: +``` +java.base DeleteOnExitHook.java 50 java/io/DeleteOnExitHook$1 run ()V PUBLIC java.base java/io/File delete PUBLIC +java.base ZipFile.java 254 java/util/zip/ZipFile (Ljava/io/File;ILjava/nio/charset/Charset;)V PUBLIC java.base java/io/File delete PUBLIC +java.logging FileHandler.java 279 java/util/logging/FileHandler ()V PUBLIC java.base java/io/File delete PUBLIC +``` diff --git a/libs/entitlement/tools/public-callers-finder/build.gradle b/libs/entitlement/tools/public-callers-finder/build.gradle new file mode 100644 index 0000000000000..083b1a43b9794 --- /dev/null +++ b/libs/entitlement/tools/public-callers-finder/build.gradle @@ -0,0 +1,61 @@ +plugins { + id 'application' +} + +apply plugin: 'elasticsearch.build' +apply plugin: 'elasticsearch.publish' + +tasks.named("dependencyLicenses").configure { + mapping from: /asm-.*/, to: 'asm' +} + +group = 'org.elasticsearch.entitlement.tools' + +ext { + javaMainClass = "org.elasticsearch.entitlement.tools.publiccallersfinder.Main" +} + +application { + mainClass.set(javaMainClass) + applicationDefaultJvmArgs = [ + '--add-exports', 'java.base/sun.security.util=ALL-UNNAMED', + '--add-opens', 'java.base/java.lang=ALL-UNNAMED', + '--add-opens', 'java.base/java.net=ALL-UNNAMED', + '--add-opens', 'java.base/java.net.spi=ALL-UNNAMED', + '--add-opens', 'java.base/java.util.concurrent=ALL-UNNAMED', + '--add-opens', 'java.base/javax.crypto=ALL-UNNAMED', + '--add-opens', 'java.base/javax.security.auth=ALL-UNNAMED', + '--add-opens', 'java.base/jdk.internal.logger=ALL-UNNAMED', + '--add-opens', 'java.base/sun.nio.ch=ALL-UNNAMED', + '--add-opens', 'jdk.management.jfr/jdk.management.jfr=ALL-UNNAMED', + '--add-opens', 'java.logging/java.util.logging=ALL-UNNAMED', + '--add-opens', 'java.logging/sun.util.logging.internal=ALL-UNNAMED', + '--add-opens', 'java.naming/javax.naming.ldap.spi=ALL-UNNAMED', + '--add-opens', 'java.rmi/sun.rmi.runtime=ALL-UNNAMED', + '--add-opens', 'jdk.dynalink/jdk.dynalink=ALL-UNNAMED', + '--add-opens', 'jdk.dynalink/jdk.dynalink.linker=ALL-UNNAMED', + '--add-opens', 'java.desktop/sun.awt=ALL-UNNAMED', + '--add-opens', 'java.sql.rowset/javax.sql.rowset.spi=ALL-UNNAMED', + '--add-opens', 'java.sql/java.sql=ALL-UNNAMED', + '--add-opens', 'java.xml.crypto/com.sun.org.apache.xml.internal.security.utils=ALL-UNNAMED' + ] +} + +repositories { + mavenCentral() +} + +dependencies { + compileOnly(project(':libs:core')) + implementation 'org.ow2.asm:asm:9.7.1' + implementation 'org.ow2.asm:asm-util:9.7.1' + implementation(project(':libs:entitlement:tools:common')) +} + +tasks.named('forbiddenApisMain').configure { + replaceSignatureFiles 'jdk-signatures' +} + +tasks.named("thirdPartyAudit").configure { + ignoreMissingClasses() +} diff --git a/libs/entitlement/tools/public-callers-finder/licenses/asm-LICENSE.txt b/libs/entitlement/tools/public-callers-finder/licenses/asm-LICENSE.txt new file mode 100644 index 0000000000000..afb064f2f2666 --- /dev/null +++ b/libs/entitlement/tools/public-callers-finder/licenses/asm-LICENSE.txt @@ -0,0 +1,26 @@ +Copyright (c) 2012 France Télécom +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. diff --git a/libs/entitlement/tools/public-callers-finder/licenses/asm-NOTICE.txt b/libs/entitlement/tools/public-callers-finder/licenses/asm-NOTICE.txt new file mode 100644 index 0000000000000..8d1c8b69c3fce --- /dev/null +++ b/libs/entitlement/tools/public-callers-finder/licenses/asm-NOTICE.txt @@ -0,0 +1 @@ + diff --git a/libs/entitlement/tools/public-callers-finder/src/main/java/org/elasticsearch/entitlement/tools/publiccallersfinder/FindUsagesClassVisitor.java b/libs/entitlement/tools/public-callers-finder/src/main/java/org/elasticsearch/entitlement/tools/publiccallersfinder/FindUsagesClassVisitor.java new file mode 100644 index 0000000000000..6f136d0977e3f --- /dev/null +++ b/libs/entitlement/tools/public-callers-finder/src/main/java/org/elasticsearch/entitlement/tools/publiccallersfinder/FindUsagesClassVisitor.java @@ -0,0 +1,141 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.tools.publiccallersfinder; + +import org.elasticsearch.entitlement.tools.ExternalAccess; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.Label; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Type; + +import java.lang.constant.ClassDesc; +import java.lang.reflect.AccessFlag; +import java.util.EnumSet; +import java.util.Set; + +import static org.objectweb.asm.Opcodes.ACC_PROTECTED; +import static org.objectweb.asm.Opcodes.ACC_PUBLIC; +import static org.objectweb.asm.Opcodes.ASM9; + +class FindUsagesClassVisitor extends ClassVisitor { + + private int classAccess; + private boolean accessibleViaInterfaces; + + record MethodDescriptor(String className, String methodName, String methodDescriptor) {} + + record EntryPoint( + String moduleName, + String source, + int line, + String className, + String methodName, + String methodDescriptor, + EnumSet access + ) {} + + interface CallerConsumer { + void accept(String source, int line, String className, String methodName, String methodDescriptor, EnumSet access); + } + + private final Set moduleExports; + private final MethodDescriptor methodToFind; + private final CallerConsumer callers; + private String className; + private String source; + + protected FindUsagesClassVisitor(Set moduleExports, MethodDescriptor methodToFind, CallerConsumer callers) { + super(ASM9); + this.moduleExports = moduleExports; + this.methodToFind = methodToFind; + this.callers = callers; + } + + @Override + public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) { + super.visit(version, access, name, signature, superName, interfaces); + this.className = name; + this.classAccess = access; + if (interfaces.length > 0) { + this.accessibleViaInterfaces = findAccessibility(interfaces, moduleExports); + } + } + + private static boolean findAccessibility(String[] interfaces, Set moduleExports) { + var accessibleViaInterfaces = false; + for (var interfaceName : interfaces) { + if (moduleExports.contains(getPackageName(interfaceName))) { + var interfaceType = Type.getObjectType(interfaceName); + try { + var clazz = Class.forName(interfaceType.getClassName()); + if (clazz.accessFlags().contains(AccessFlag.PUBLIC)) { + accessibleViaInterfaces = true; + } + } catch (ClassNotFoundException ignored) {} + } + } + return accessibleViaInterfaces; + } + + @Override + public void visitSource(String source, String debug) { + super.visitSource(source, debug); + this.source = source; + } + + @Override + public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) { + return new FindUsagesMethodVisitor(super.visitMethod(access, name, descriptor, signature, exceptions), name, descriptor, access); + } + + private static String getPackageName(String className) { + return ClassDesc.ofInternalName(className).packageName(); + } + + private class FindUsagesMethodVisitor extends MethodVisitor { + + private final String methodName; + private int line; + private final String methodDescriptor; + private final int methodAccess; + + protected FindUsagesMethodVisitor(MethodVisitor mv, String methodName, String methodDescriptor, int methodAccess) { + super(ASM9, mv); + this.methodName = methodName; + this.methodDescriptor = methodDescriptor; + this.methodAccess = methodAccess; + } + + @Override + public void visitMethodInsn(int opcode, String owner, String name, String descriptor, boolean isInterface) { + super.visitMethodInsn(opcode, owner, name, descriptor, isInterface); + + if (methodToFind.className.equals(owner)) { + if (methodToFind.methodName.equals(name)) { + if (methodToFind.methodDescriptor == null || methodToFind.methodDescriptor.equals(descriptor)) { + EnumSet externalAccess = ExternalAccess.fromPermissions( + moduleExports.contains(getPackageName(className)), + accessibleViaInterfaces || (classAccess & ACC_PUBLIC) != 0, + (methodAccess & ACC_PUBLIC) != 0, + (methodAccess & ACC_PROTECTED) != 0 + ); + callers.accept(source, line, className, methodName, methodDescriptor, externalAccess); + } + } + } + } + + @Override + public void visitLineNumber(int line, Label start) { + super.visitLineNumber(line, start); + this.line = line; + } + } +} diff --git a/libs/entitlement/tools/public-callers-finder/src/main/java/org/elasticsearch/entitlement/tools/publiccallersfinder/Main.java b/libs/entitlement/tools/public-callers-finder/src/main/java/org/elasticsearch/entitlement/tools/publiccallersfinder/Main.java new file mode 100644 index 0000000000000..60b3a3c9f3c8e --- /dev/null +++ b/libs/entitlement/tools/public-callers-finder/src/main/java/org/elasticsearch/entitlement/tools/publiccallersfinder/Main.java @@ -0,0 +1,197 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.tools.publiccallersfinder; + +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.entitlement.tools.ExternalAccess; +import org.elasticsearch.entitlement.tools.Utils; +import org.objectweb.asm.ClassReader; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collection; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class Main { + + private static final String SEPARATOR = "\t"; + + record CallChain(FindUsagesClassVisitor.EntryPoint entryPoint, CallChain next) {} + + interface UsageConsumer { + void usageFound(CallChain originalEntryPoint, CallChain newMethod); + } + + private static void findTransitiveUsages( + Collection firstLevelCallers, + List classesToScan, + Set moduleExports, + boolean bubbleUpFromPublic, + UsageConsumer usageConsumer + ) { + for (var caller : firstLevelCallers) { + var methodsToCheck = new ArrayDeque<>(Set.of(caller)); + var methodsSeen = new HashSet(); + + while (methodsToCheck.isEmpty() == false) { + var methodToCheck = methodsToCheck.removeFirst(); + var m = methodToCheck.entryPoint(); + var visitor2 = new FindUsagesClassVisitor( + moduleExports, + new FindUsagesClassVisitor.MethodDescriptor(m.className(), m.methodName(), m.methodDescriptor()), + (source, line, className, methodName, methodDescriptor, access) -> { + var newMethod = new CallChain( + new FindUsagesClassVisitor.EntryPoint( + m.moduleName(), + source, + line, + className, + methodName, + methodDescriptor, + access + ), + methodToCheck + ); + + var notSeenBefore = methodsSeen.add(newMethod.entryPoint()); + if (notSeenBefore) { + if (ExternalAccess.isExternallyAccessible(access)) { + usageConsumer.usageFound(caller.next(), newMethod); + } + if (access.contains(ExternalAccess.PUBLIC_METHOD) == false || bubbleUpFromPublic) { + methodsToCheck.add(newMethod); + } + } + } + ); + + for (var classFile : classesToScan) { + try { + ClassReader cr = new ClassReader(Files.newInputStream(classFile)); + cr.accept(visitor2, 0); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + } + } + + private static void identifyTopLevelEntryPoints( + FindUsagesClassVisitor.MethodDescriptor methodToFind, + String methodToFindModule, + EnumSet methodToFindAccess, + boolean bubbleUpFromPublic + ) throws IOException { + + Utils.walkJdkModules((moduleName, moduleClasses, moduleExports) -> { + var originalCallers = new ArrayList(); + var visitor = new FindUsagesClassVisitor( + moduleExports, + methodToFind, + (source, line, className, methodName, methodDescriptor, access) -> originalCallers.add( + new CallChain( + new FindUsagesClassVisitor.EntryPoint(moduleName, source, line, className, methodName, methodDescriptor, access), + new CallChain( + new FindUsagesClassVisitor.EntryPoint( + methodToFindModule, + "", + 0, + methodToFind.className(), + methodToFind.methodName(), + methodToFind.methodDescriptor(), + methodToFindAccess + ), + null + ) + ) + ) + ); + + for (var classFile : moduleClasses) { + try { + ClassReader cr = new ClassReader(Files.newInputStream(classFile)); + cr.accept(visitor, 0); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + originalCallers.stream().filter(c -> ExternalAccess.isExternallyAccessible(c.entryPoint().access())).forEach(c -> { + var originalCaller = c.next(); + printRow(getEntryPointString(c.entryPoint().moduleName(), c.entryPoint()), getOriginalEntryPointString(originalCaller)); + }); + var firstLevelCallers = bubbleUpFromPublic ? originalCallers : originalCallers.stream().filter(Main::isNotFullyPublic).toList(); + + if (firstLevelCallers.isEmpty() == false) { + findTransitiveUsages( + firstLevelCallers, + moduleClasses, + moduleExports, + bubbleUpFromPublic, + (originalEntryPoint, newMethod) -> printRow( + getEntryPointString(moduleName, newMethod.entryPoint()), + getOriginalEntryPointString(originalEntryPoint) + ) + ); + } + }); + } + + private static boolean isNotFullyPublic(CallChain c) { + return (c.entryPoint().access().contains(ExternalAccess.PUBLIC_CLASS) + && c.entryPoint().access().contains(ExternalAccess.PUBLIC_METHOD)) == false; + } + + @SuppressForbidden(reason = "This tool prints the CSV to stdout") + private static void printRow(String entryPointString, String originalEntryPoint) { + System.out.println(entryPointString + SEPARATOR + originalEntryPoint); + } + + private static String getEntryPointString(String moduleName, FindUsagesClassVisitor.EntryPoint e) { + return moduleName + SEPARATOR + e.source() + SEPARATOR + e.line() + SEPARATOR + e.className() + SEPARATOR + e.methodName() + + SEPARATOR + e.methodDescriptor() + SEPARATOR + ExternalAccess.toString(e.access()); + } + + private static String getOriginalEntryPointString(CallChain originalCallChain) { + return originalCallChain.entryPoint().moduleName() + SEPARATOR + originalCallChain.entryPoint().className() + SEPARATOR + + originalCallChain.entryPoint().methodName() + SEPARATOR + ExternalAccess.toString(originalCallChain.entryPoint().access()); + } + + interface MethodDescriptorConsumer { + void accept(FindUsagesClassVisitor.MethodDescriptor methodDescriptor, String moduleName, EnumSet access) + throws IOException; + } + + private static void parseCsv(Path csvPath, MethodDescriptorConsumer methodConsumer) throws IOException { + var lines = Files.readAllLines(csvPath); + for (var l : lines) { + var tokens = l.split(SEPARATOR); + var moduleName = tokens[0]; + var className = tokens[3]; + var methodName = tokens[4]; + var methodDescriptor = tokens[5]; + var access = ExternalAccess.fromString(tokens[6]); + methodConsumer.accept(new FindUsagesClassVisitor.MethodDescriptor(className, methodName, methodDescriptor), moduleName, access); + } + } + + public static void main(String[] args) throws IOException { + var csvFilePath = Path.of(args[0]); + boolean bubbleUpFromPublic = args.length >= 2 && Boolean.parseBoolean(args[1]); + parseCsv(csvFilePath, (method, module, access) -> identifyTopLevelEntryPoints(method, module, access, bubbleUpFromPublic)); + } +} diff --git a/libs/entitlement/tools/securitymanager-scanner/src/README.md b/libs/entitlement/tools/securitymanager-scanner/README.md similarity index 100% rename from libs/entitlement/tools/securitymanager-scanner/src/README.md rename to libs/entitlement/tools/securitymanager-scanner/README.md diff --git a/libs/entitlement/tools/securitymanager-scanner/src/main/java/org/elasticsearch/entitlement/tools/securitymanager/scanner/Main.java b/libs/entitlement/tools/securitymanager-scanner/src/main/java/org/elasticsearch/entitlement/tools/securitymanager/scanner/Main.java index bea49e0296e67..7c2dd69d60f0c 100644 --- a/libs/entitlement/tools/securitymanager-scanner/src/main/java/org/elasticsearch/entitlement/tools/securitymanager/scanner/Main.java +++ b/libs/entitlement/tools/securitymanager-scanner/src/main/java/org/elasticsearch/entitlement/tools/securitymanager/scanner/Main.java @@ -10,47 +10,35 @@ package org.elasticsearch.entitlement.tools.securitymanager.scanner; import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.entitlement.tools.ExternalAccess; import org.elasticsearch.entitlement.tools.Utils; import org.objectweb.asm.ClassReader; import java.io.IOException; -import java.net.URI; -import java.nio.file.FileSystem; -import java.nio.file.FileSystems; import java.nio.file.Files; import java.util.HashMap; import java.util.List; -import java.util.Set; public class Main { - static final Set excludedModules = Set.of("java.desktop"); - private static void identifySMChecksEntryPoints() throws IOException { - FileSystem fs = FileSystems.getFileSystem(URI.create("jrt:/")); - - var moduleExports = Utils.findModuleExports(fs); - var callers = new HashMap>(); var visitor = new SecurityCheckClassVisitor(callers); - try (var stream = Files.walk(fs.getPath("modules"))) { - stream.filter(x -> x.toString().endsWith(".class")).forEach(x -> { - var moduleName = x.subpath(1, 2).toString(); - if (excludedModules.contains(moduleName) == false) { - try { - ClassReader cr = new ClassReader(Files.newInputStream(x)); - visitor.setCurrentModule(moduleName, moduleExports.get(moduleName)); - var path = x.getNameCount() > 3 ? x.subpath(2, x.getNameCount() - 1).toString() : ""; - visitor.setCurrentSourcePath(path); - cr.accept(visitor, 0); - } catch (IOException e) { - throw new RuntimeException(e); - } + Utils.walkJdkModules((moduleName, moduleClasses, moduleExports) -> { + for (var classFile : moduleClasses) { + try { + ClassReader cr = new ClassReader(Files.newInputStream(classFile)); + visitor.setCurrentModule(moduleName, moduleExports); + var path = classFile.getNameCount() > 3 ? classFile.subpath(2, classFile.getNameCount() - 1).toString() : ""; + visitor.setCurrentSourcePath(path); + cr.accept(visitor, 0); + } catch (IOException e) { + throw new RuntimeException(e); } - }); - } + } + }); printToStdout(callers); } @@ -68,16 +56,8 @@ private static void printToStdout(HashMap excludedClasses = Set.of(SECURITY_MANAGER_INTERNAL_NAME); - enum ExternalAccess { - CLASS, - METHOD - } - record CallerInfo( String moduleName, String source, @@ -208,15 +205,12 @@ public void visitMethodInsn(int opcode, String owner, String name, String descri || opcode == INVOKEDYNAMIC) { if (SECURITY_MANAGER_INTERNAL_NAME.equals(owner)) { - EnumSet externalAccesses = EnumSet.noneOf(ExternalAccess.class); - if (moduleExports.contains(getPackageName(className))) { - if ((methodAccess & ACC_PUBLIC) != 0) { - externalAccesses.add(ExternalAccess.METHOD); - } - if ((classAccess & ACC_PUBLIC) != 0) { - externalAccesses.add(ExternalAccess.CLASS); - } - } + EnumSet externalAccesses = ExternalAccess.fromPermissions( + moduleExports.contains(getPackageName(className)), + (classAccess & ACC_PUBLIC) != 0, + (methodAccess & ACC_PUBLIC) != 0, + (methodAccess & ACC_PROTECTED) != 0 + ); if (name.equals("checkPermission")) { var callers = callerInfoByMethod.computeIfAbsent(name, ignored -> new ArrayList<>()); From 0441555503593cf40fcb04b6edc49da3734a0738 Mon Sep 17 00:00:00 2001 From: Bogdan Pintea Date: Fri, 13 Dec 2024 16:46:25 +0100 Subject: [PATCH 11/35] ESQL: Disable grok.OverwriteName* on pre-8.13 BWC tests (#118655) This prevents two tests in `grok` and `dissect` suites - `overwriteName` and `overwriteNameWhere` and one in the `stats` suite - `byStringAndLongWithAlias` - to run against pre-8.13.0 versions. Reason being that coordinators prior to that version can generate invalid node plans, that'd fail (verification) on 8.18+ nodes. --- .../qa/testFixtures/src/main/resources/dissect.csv-spec | 6 ++++-- .../esql/qa/testFixtures/src/main/resources/grok.csv-spec | 6 ++++-- .../esql/qa/testFixtures/src/main/resources/stats.csv-spec | 3 ++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dissect.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dissect.csv-spec index 38f09d2e3c56e..cde5427bf37d6 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dissect.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dissect.csv-spec @@ -223,7 +223,8 @@ null | null | null ; -overwriteName +// the query is incorrectly physically plan (fails the verification) in pre-8.13.0 versions +overwriteName#[skip:-8.12.99] from employees | sort emp_no asc | eval full_name = concat(first_name, " ", last_name) | dissect full_name "%{emp_no} %{b}" | keep full_name, emp_no, b | limit 3; full_name:keyword | emp_no:keyword | b:keyword @@ -244,7 +245,8 @@ emp_no:integer | first_name:keyword | rest:keyword ; -overwriteNameWhere +// the query is incorrectly physically plan (fails the verification) in pre-8.13.0 versions +overwriteNameWhere#[skip:-8.12.99] from employees | sort emp_no asc | eval full_name = concat(first_name, " ", last_name) | dissect full_name "%{emp_no} %{b}" | where emp_no == "Bezalel" | keep full_name, emp_no, b | limit 3; full_name:keyword | emp_no:keyword | b:keyword diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/grok.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/grok.csv-spec index 98c88d06caa75..eece1bdfbffa4 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/grok.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/grok.csv-spec @@ -199,7 +199,8 @@ null | null | null ; -overwriteName +// the query is incorrectly physically plan (fails the verification) in pre-8.13.0 versions +overwriteName#[skip:-8.12.99] from employees | sort emp_no asc | eval full_name = concat(first_name, " ", last_name) | grok full_name "%{WORD:emp_no} %{WORD:b}" | keep full_name, emp_no, b | limit 3; full_name:keyword | emp_no:keyword | b:keyword @@ -209,7 +210,8 @@ Parto Bamford | Parto | Bamford ; -overwriteNameWhere +// the query is incorrectly physically plan (fails the verification) in pre-8.13.0 versions +overwriteNameWhere#[skip:-8.12.99] from employees | sort emp_no asc | eval full_name = concat(first_name, " ", last_name) | grok full_name "%{WORD:emp_no} %{WORD:b}" | where emp_no == "Bezalel" | keep full_name, emp_no, b | limit 3; full_name:keyword | emp_no:keyword | b:keyword diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec index d76f4c05d955f..100c0d716d65c 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec @@ -564,7 +564,8 @@ c:long | gender:keyword | trunk_worked_seconds:long 0 | null | 200000000 ; -byStringAndLongWithAlias +// the query is incorrectly physically plan (fails the verification) in pre-8.13.0 versions +byStringAndLongWithAlias#[skip:-8.12.99] FROM employees | EVAL trunk_worked_seconds = avg_worked_seconds / 100000000 * 100000000 | RENAME gender as g, trunk_worked_seconds as tws From cf7cb4bbb95996136a49340cc72acb850a703b44 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 13 Dec 2024 15:58:16 +0000 Subject: [PATCH 12/35] Bump versions after 8.17.0 release --- .buildkite/pipelines/intake.yml | 2 +- .buildkite/pipelines/periodic-packaging.yml | 6 +++--- .buildkite/pipelines/periodic.yml | 10 +++++----- .ci/bwcVersions | 2 +- .ci/snapshotBwcVersions | 2 +- server/src/main/java/org/elasticsearch/Version.java | 1 + .../resources/org/elasticsearch/TransportVersions.csv | 1 + .../org/elasticsearch/index/IndexVersions.csv | 1 + 8 files changed, 14 insertions(+), 11 deletions(-) diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml index 6c8b8edfcbac1..6e15d64154960 100644 --- a/.buildkite/pipelines/intake.yml +++ b/.buildkite/pipelines/intake.yml @@ -56,7 +56,7 @@ steps: timeout_in_minutes: 300 matrix: setup: - BWC_VERSION: ["8.16.2", "8.17.0", "8.18.0", "9.0.0"] + BWC_VERSION: ["8.16.2", "8.17.1", "8.18.0", "9.0.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index 2fbcd075b9719..9619de3c2c98b 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -301,8 +301,8 @@ steps: env: BWC_VERSION: 8.16.2 - - label: "{{matrix.image}} / 8.17.0 / packaging-tests-upgrade" - command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.0 + - label: "{{matrix.image}} / 8.17.1 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.1 timeout_in_minutes: 300 matrix: setup: @@ -315,7 +315,7 @@ steps: machineType: custom-16-32768 buildDirectory: /dev/shm/bk env: - BWC_VERSION: 8.17.0 + BWC_VERSION: 8.17.1 - label: "{{matrix.image}} / 8.18.0 / packaging-tests-upgrade" command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.0 diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index 94c9020a794a2..f2d169cd2b30d 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -325,8 +325,8 @@ steps: - signal_reason: agent_stop limit: 3 - - label: 8.17.0 / bwc - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.0#bwcTest + - label: 8.17.1 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.1#bwcTest timeout_in_minutes: 300 agents: provider: gcp @@ -335,7 +335,7 @@ steps: buildDirectory: /dev/shm/bk preemptible: true env: - BWC_VERSION: 8.17.0 + BWC_VERSION: 8.17.1 retry: automatic: - exit_status: "-1" @@ -448,7 +448,7 @@ steps: setup: ES_RUNTIME_JAVA: - openjdk21 - BWC_VERSION: ["8.16.2", "8.17.0", "8.18.0", "9.0.0"] + BWC_VERSION: ["8.16.2", "8.17.1", "8.18.0", "9.0.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 @@ -490,7 +490,7 @@ steps: ES_RUNTIME_JAVA: - openjdk21 - openjdk23 - BWC_VERSION: ["8.16.2", "8.17.0", "8.18.0", "9.0.0"] + BWC_VERSION: ["8.16.2", "8.17.1", "8.18.0", "9.0.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 79de891452117..3cb983373138f 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -16,6 +16,6 @@ BWC_VERSION: - "8.14.3" - "8.15.5" - "8.16.2" - - "8.17.0" + - "8.17.1" - "8.18.0" - "9.0.0" diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions index 5514fc376a285..e05c0774c9819 100644 --- a/.ci/snapshotBwcVersions +++ b/.ci/snapshotBwcVersions @@ -1,5 +1,5 @@ BWC_VERSION: - "8.16.2" - - "8.17.0" + - "8.17.1" - "8.18.0" - "9.0.0" diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index f03505de310d5..47c43eadcfb03 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -192,6 +192,7 @@ public class Version implements VersionId, ToXContentFragment { public static final Version V_8_16_1 = new Version(8_16_01_99); public static final Version V_8_16_2 = new Version(8_16_02_99); public static final Version V_8_17_0 = new Version(8_17_00_99); + public static final Version V_8_17_1 = new Version(8_17_01_99); public static final Version V_8_18_0 = new Version(8_18_00_99); public static final Version V_9_0_0 = new Version(9_00_00_99); public static final Version CURRENT = V_9_0_0; diff --git a/server/src/main/resources/org/elasticsearch/TransportVersions.csv b/server/src/main/resources/org/elasticsearch/TransportVersions.csv index faeb7fe848159..08db0822dfef5 100644 --- a/server/src/main/resources/org/elasticsearch/TransportVersions.csv +++ b/server/src/main/resources/org/elasticsearch/TransportVersions.csv @@ -135,3 +135,4 @@ 8.15.5,8702003 8.16.0,8772001 8.16.1,8772004 +8.17.0,8797002 diff --git a/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv b/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv index 1fc8bd8648ad6..afe696f31d323 100644 --- a/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv +++ b/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv @@ -135,3 +135,4 @@ 8.15.5,8512000 8.16.0,8518000 8.16.1,8518000 +8.17.0,8521000 From a68269b1763b9ed843d4e778373f38233c426aa5 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 13 Dec 2024 15:59:29 +0000 Subject: [PATCH 13/35] Prune changelogs after 8.17.0 release --- docs/changelog/104683.yaml | 5 ----- docs/changelog/112881.yaml | 5 ----- docs/changelog/112989.yaml | 5 ----- docs/changelog/113194.yaml | 5 ----- docs/changelog/113713.yaml | 5 ----- docs/changelog/113920.yaml | 5 ----- docs/changelog/114334.yaml | 7 ------- docs/changelog/114482.yaml | 5 ----- docs/changelog/114484.yaml | 6 ------ docs/changelog/114620.yaml | 5 ----- docs/changelog/114665.yaml | 6 ------ docs/changelog/114681.yaml | 6 ------ docs/changelog/114742.yaml | 5 ----- docs/changelog/114819.yaml | 6 ------ docs/changelog/114855.yaml | 5 ----- docs/changelog/114862.yaml | 5 ----- docs/changelog/114869.yaml | 5 ----- docs/changelog/114899.yaml | 5 ----- docs/changelog/114924.yaml | 5 ----- docs/changelog/114934.yaml | 6 ------ docs/changelog/114964.yaml | 6 ------ docs/changelog/115041.yaml | 6 ------ docs/changelog/115091.yaml | 7 ------- docs/changelog/115102.yaml | 6 ------ docs/changelog/115142.yaml | 6 ------ docs/changelog/115266.yaml | 6 ------ docs/changelog/115359.yaml | 6 ------ docs/changelog/115414.yaml | 9 --------- docs/changelog/115585.yaml | 6 ------ docs/changelog/115640.yaml | 6 ------ docs/changelog/115655.yaml | 5 ----- docs/changelog/115678.yaml | 5 ----- docs/changelog/115687.yaml | 5 ----- docs/changelog/115744.yaml | 6 ------ docs/changelog/115792.yaml | 5 ----- docs/changelog/115797.yaml | 6 ------ docs/changelog/115807.yaml | 5 ----- docs/changelog/115812.yaml | 5 ----- docs/changelog/115814.yaml | 6 ------ docs/changelog/115858.yaml | 5 ----- docs/changelog/115994.yaml | 5 ----- docs/changelog/116021.yaml | 6 ------ docs/changelog/116082.yaml | 5 ----- docs/changelog/116128.yaml | 5 ----- docs/changelog/116211.yaml | 5 ----- docs/changelog/116325.yaml | 5 ----- docs/changelog/116346.yaml | 5 ----- docs/changelog/116348.yaml | 5 ----- docs/changelog/116431.yaml | 5 ----- docs/changelog/116437.yaml | 5 ----- docs/changelog/116447.yaml | 5 ----- docs/changelog/116515.yaml | 5 ----- docs/changelog/116583.yaml | 7 ------- docs/changelog/116591.yaml | 5 ----- docs/changelog/116656.yaml | 6 ------ docs/changelog/116664.yaml | 6 ------ docs/changelog/116689.yaml | 10 ---------- docs/changelog/116809.yaml | 5 ----- docs/changelog/116819.yaml | 5 ----- docs/changelog/116931.yaml | 5 ----- docs/changelog/116953.yaml | 6 ------ docs/changelog/116957.yaml | 5 ----- docs/changelog/116962.yaml | 5 ----- docs/changelog/116980.yaml | 6 ------ docs/changelog/117080.yaml | 5 ----- docs/changelog/117105.yaml | 6 ------ docs/changelog/117189.yaml | 5 ----- docs/changelog/117213.yaml | 6 ------ docs/changelog/117271.yaml | 5 ----- docs/changelog/117294.yaml | 5 ----- docs/changelog/117297.yaml | 5 ----- docs/changelog/117312.yaml | 5 ----- docs/changelog/117316.yaml | 5 ----- docs/changelog/117350.yaml | 5 ----- docs/changelog/117404.yaml | 5 ----- docs/changelog/117503.yaml | 6 ------ docs/changelog/117551.yaml | 5 ----- docs/changelog/117575.yaml | 5 ----- docs/changelog/117595.yaml | 5 ----- docs/changelog/117657.yaml | 5 ----- docs/changelog/117762.yaml | 6 ------ docs/changelog/117792.yaml | 6 ------ docs/changelog/117842.yaml | 5 ----- docs/changelog/117865.yaml | 5 ----- docs/changelog/117914.yaml | 5 ----- docs/changelog/117920.yaml | 6 ------ docs/changelog/117953.yaml | 5 ----- docs/changelog/118354.yaml | 5 ----- docs/changelog/118370.yaml | 6 ------ docs/changelog/118378.yaml | 5 ----- 90 files changed, 493 deletions(-) delete mode 100644 docs/changelog/104683.yaml delete mode 100644 docs/changelog/112881.yaml delete mode 100644 docs/changelog/112989.yaml delete mode 100644 docs/changelog/113194.yaml delete mode 100644 docs/changelog/113713.yaml delete mode 100644 docs/changelog/113920.yaml delete mode 100644 docs/changelog/114334.yaml delete mode 100644 docs/changelog/114482.yaml delete mode 100644 docs/changelog/114484.yaml delete mode 100644 docs/changelog/114620.yaml delete mode 100644 docs/changelog/114665.yaml delete mode 100644 docs/changelog/114681.yaml delete mode 100644 docs/changelog/114742.yaml delete mode 100644 docs/changelog/114819.yaml delete mode 100644 docs/changelog/114855.yaml delete mode 100644 docs/changelog/114862.yaml delete mode 100644 docs/changelog/114869.yaml delete mode 100644 docs/changelog/114899.yaml delete mode 100644 docs/changelog/114924.yaml delete mode 100644 docs/changelog/114934.yaml delete mode 100644 docs/changelog/114964.yaml delete mode 100644 docs/changelog/115041.yaml delete mode 100644 docs/changelog/115091.yaml delete mode 100644 docs/changelog/115102.yaml delete mode 100644 docs/changelog/115142.yaml delete mode 100644 docs/changelog/115266.yaml delete mode 100644 docs/changelog/115359.yaml delete mode 100644 docs/changelog/115414.yaml delete mode 100644 docs/changelog/115585.yaml delete mode 100644 docs/changelog/115640.yaml delete mode 100644 docs/changelog/115655.yaml delete mode 100644 docs/changelog/115678.yaml delete mode 100644 docs/changelog/115687.yaml delete mode 100644 docs/changelog/115744.yaml delete mode 100644 docs/changelog/115792.yaml delete mode 100644 docs/changelog/115797.yaml delete mode 100644 docs/changelog/115807.yaml delete mode 100644 docs/changelog/115812.yaml delete mode 100644 docs/changelog/115814.yaml delete mode 100644 docs/changelog/115858.yaml delete mode 100644 docs/changelog/115994.yaml delete mode 100644 docs/changelog/116021.yaml delete mode 100644 docs/changelog/116082.yaml delete mode 100644 docs/changelog/116128.yaml delete mode 100644 docs/changelog/116211.yaml delete mode 100644 docs/changelog/116325.yaml delete mode 100644 docs/changelog/116346.yaml delete mode 100644 docs/changelog/116348.yaml delete mode 100644 docs/changelog/116431.yaml delete mode 100644 docs/changelog/116437.yaml delete mode 100644 docs/changelog/116447.yaml delete mode 100644 docs/changelog/116515.yaml delete mode 100644 docs/changelog/116583.yaml delete mode 100644 docs/changelog/116591.yaml delete mode 100644 docs/changelog/116656.yaml delete mode 100644 docs/changelog/116664.yaml delete mode 100644 docs/changelog/116689.yaml delete mode 100644 docs/changelog/116809.yaml delete mode 100644 docs/changelog/116819.yaml delete mode 100644 docs/changelog/116931.yaml delete mode 100644 docs/changelog/116953.yaml delete mode 100644 docs/changelog/116957.yaml delete mode 100644 docs/changelog/116962.yaml delete mode 100644 docs/changelog/116980.yaml delete mode 100644 docs/changelog/117080.yaml delete mode 100644 docs/changelog/117105.yaml delete mode 100644 docs/changelog/117189.yaml delete mode 100644 docs/changelog/117213.yaml delete mode 100644 docs/changelog/117271.yaml delete mode 100644 docs/changelog/117294.yaml delete mode 100644 docs/changelog/117297.yaml delete mode 100644 docs/changelog/117312.yaml delete mode 100644 docs/changelog/117316.yaml delete mode 100644 docs/changelog/117350.yaml delete mode 100644 docs/changelog/117404.yaml delete mode 100644 docs/changelog/117503.yaml delete mode 100644 docs/changelog/117551.yaml delete mode 100644 docs/changelog/117575.yaml delete mode 100644 docs/changelog/117595.yaml delete mode 100644 docs/changelog/117657.yaml delete mode 100644 docs/changelog/117762.yaml delete mode 100644 docs/changelog/117792.yaml delete mode 100644 docs/changelog/117842.yaml delete mode 100644 docs/changelog/117865.yaml delete mode 100644 docs/changelog/117914.yaml delete mode 100644 docs/changelog/117920.yaml delete mode 100644 docs/changelog/117953.yaml delete mode 100644 docs/changelog/118354.yaml delete mode 100644 docs/changelog/118370.yaml delete mode 100644 docs/changelog/118378.yaml diff --git a/docs/changelog/104683.yaml b/docs/changelog/104683.yaml deleted file mode 100644 index d4f40b59cfd91..0000000000000 --- a/docs/changelog/104683.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 104683 -summary: "Feature: re-structure document ID generation favoring _id inverted index compression" -area: Logs -type: enhancement -issues: [] diff --git a/docs/changelog/112881.yaml b/docs/changelog/112881.yaml deleted file mode 100644 index a8a0d542f8201..0000000000000 --- a/docs/changelog/112881.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112881 -summary: "ESQL: Remove parent from `FieldAttribute`" -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/112989.yaml b/docs/changelog/112989.yaml deleted file mode 100644 index 364f012f94420..0000000000000 --- a/docs/changelog/112989.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 112989 -summary: Upgrade Bouncy Castle FIPS dependencies -area: Security -type: upgrade -issues: [] diff --git a/docs/changelog/113194.yaml b/docs/changelog/113194.yaml deleted file mode 100644 index 132659321c65e..0000000000000 --- a/docs/changelog/113194.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 113194 -summary: Add Search Phase APM metrics -area: Search -type: enhancement -issues: [] diff --git a/docs/changelog/113713.yaml b/docs/changelog/113713.yaml deleted file mode 100644 index c5478c95e464d..0000000000000 --- a/docs/changelog/113713.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 113713 -summary: Adding inference endpoint validation for `AzureAiStudioService` -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/113920.yaml b/docs/changelog/113920.yaml deleted file mode 100644 index 4699ae6d7dd65..0000000000000 --- a/docs/changelog/113920.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 113920 -summary: Add initial support for `semantic_text` field type -area: Search -type: enhancement -issues: [] diff --git a/docs/changelog/114334.yaml b/docs/changelog/114334.yaml deleted file mode 100644 index d0fefe40c6970..0000000000000 --- a/docs/changelog/114334.yaml +++ /dev/null @@ -1,7 +0,0 @@ -pr: 114334 -summary: Don't return TEXT type for functions that take TEXT -area: ES|QL -type: bug -issues: - - 111537 - - 114333 diff --git a/docs/changelog/114482.yaml b/docs/changelog/114482.yaml deleted file mode 100644 index a5e2e981f7adc..0000000000000 --- a/docs/changelog/114482.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 114482 -summary: Remove snapshot build restriction for match and qstr functions -area: ES|QL -type: feature -issues: [] diff --git a/docs/changelog/114484.yaml b/docs/changelog/114484.yaml deleted file mode 100644 index 48f54ad0218bb..0000000000000 --- a/docs/changelog/114484.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 114484 -summary: Add `docvalue_fields` Support for `dense_vector` Fields -area: Search -type: enhancement -issues: - - 108470 diff --git a/docs/changelog/114620.yaml b/docs/changelog/114620.yaml deleted file mode 100644 index 92498db92061f..0000000000000 --- a/docs/changelog/114620.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 114620 -summary: "ES|QL: add metrics for functions" -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/114665.yaml b/docs/changelog/114665.yaml deleted file mode 100644 index b90bb799bd896..0000000000000 --- a/docs/changelog/114665.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 114665 -summary: Fixing remote ENRICH by pushing the Enrich inside `FragmentExec` -area: ES|QL -type: bug -issues: - - 105095 diff --git a/docs/changelog/114681.yaml b/docs/changelog/114681.yaml deleted file mode 100644 index 2a9901114e56f..0000000000000 --- a/docs/changelog/114681.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 114681 -summary: "Support for unsigned 64 bit numbers in Cpu stats" -area: Infra/Core -type: enhancement -issues: - - 112274 diff --git a/docs/changelog/114742.yaml b/docs/changelog/114742.yaml deleted file mode 100644 index 5bd3dad4400b8..0000000000000 --- a/docs/changelog/114742.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 114742 -summary: Adding support for additional mapping to simulate ingest API -area: Ingest Node -type: enhancement -issues: [] diff --git a/docs/changelog/114819.yaml b/docs/changelog/114819.yaml deleted file mode 100644 index f8d03f7024801..0000000000000 --- a/docs/changelog/114819.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 114819 -summary: Don't use a `BytesStreamOutput` to copy keys in `BytesRefBlockHash` -area: EQL -type: bug -issues: - - 114599 diff --git a/docs/changelog/114855.yaml b/docs/changelog/114855.yaml deleted file mode 100644 index daa6b985a14cf..0000000000000 --- a/docs/changelog/114855.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 114855 -summary: Add query rules retriever -area: Relevance -type: enhancement -issues: [ ] diff --git a/docs/changelog/114862.yaml b/docs/changelog/114862.yaml deleted file mode 100644 index fb5f05fb8e2f9..0000000000000 --- a/docs/changelog/114862.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 114862 -summary: "[Inference API] Add API to get configuration of inference services" -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/114869.yaml b/docs/changelog/114869.yaml deleted file mode 100644 index 755418e7ce4d9..0000000000000 --- a/docs/changelog/114869.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 114869 -summary: Standardize error code when bulk body is invalid -area: CRUD -type: bug -issues: [] diff --git a/docs/changelog/114899.yaml b/docs/changelog/114899.yaml deleted file mode 100644 index 399aa5cf35409..0000000000000 --- a/docs/changelog/114899.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 114899 -summary: "ES|QL: Fix stats by constant expression" -area: ES|QL -type: bug -issues: [] diff --git a/docs/changelog/114924.yaml b/docs/changelog/114924.yaml deleted file mode 100644 index 536f446ef790d..0000000000000 --- a/docs/changelog/114924.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 114924 -summary: Reducing error-level stack trace logging for normal events in `GeoIpDownloader` -area: Ingest Node -type: bug -issues: [] diff --git a/docs/changelog/114934.yaml b/docs/changelog/114934.yaml deleted file mode 100644 index 68628993b1c80..0000000000000 --- a/docs/changelog/114934.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 114934 -summary: "[ES|QL] To_DatePeriod and To_TimeDuration return better error messages on\ - \ `union_type` fields" -area: ES|QL -type: bug -issues: [] diff --git a/docs/changelog/114964.yaml b/docs/changelog/114964.yaml deleted file mode 100644 index 8274aeb76a937..0000000000000 --- a/docs/changelog/114964.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 114964 -summary: Add a `monitor_stats` privilege and allow that privilege for remote cluster - privileges -area: Authorization -type: enhancement -issues: [] diff --git a/docs/changelog/115041.yaml b/docs/changelog/115041.yaml deleted file mode 100644 index f4c047c1569ec..0000000000000 --- a/docs/changelog/115041.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 115041 -summary: Increase default `queue_capacity` to 10_000 and decrease max `queue_capacity` - to 100_000 -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/115091.yaml b/docs/changelog/115091.yaml deleted file mode 100644 index 762bcca5e8c52..0000000000000 --- a/docs/changelog/115091.yaml +++ /dev/null @@ -1,7 +0,0 @@ -pr: 115091 -summary: Added stricter range type checks and runtime warnings for ENRICH -area: ES|QL -type: bug -issues: - - 107357 - - 116799 diff --git a/docs/changelog/115102.yaml b/docs/changelog/115102.yaml deleted file mode 100644 index f679bb6c223a6..0000000000000 --- a/docs/changelog/115102.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 115102 -summary: Watch Next Run Interval Resets On Shard Move or Node Restart -area: Watcher -type: bug -issues: - - 111433 diff --git a/docs/changelog/115142.yaml b/docs/changelog/115142.yaml deleted file mode 100644 index 2af968ae156da..0000000000000 --- a/docs/changelog/115142.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 115142 -summary: Attempt to clean up index before remote transfer -area: Recovery -type: enhancement -issues: - - 104473 diff --git a/docs/changelog/115266.yaml b/docs/changelog/115266.yaml deleted file mode 100644 index 1d7fb1368c0e8..0000000000000 --- a/docs/changelog/115266.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 115266 -summary: ES|QL CCS uses `skip_unavailable` setting for handling disconnected remote - clusters -area: ES|QL -type: enhancement -issues: [ 114531 ] diff --git a/docs/changelog/115359.yaml b/docs/changelog/115359.yaml deleted file mode 100644 index 65b3086dfc8d0..0000000000000 --- a/docs/changelog/115359.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 115359 -summary: Adding support for simulate ingest mapping adddition for indices with mappings - that do not come from templates -area: Ingest Node -type: enhancement -issues: [] diff --git a/docs/changelog/115414.yaml b/docs/changelog/115414.yaml deleted file mode 100644 index 7475b765bb30e..0000000000000 --- a/docs/changelog/115414.yaml +++ /dev/null @@ -1,9 +0,0 @@ -pr: 115414 -summary: Mitigate IOSession timeouts -area: Machine Learning -type: bug -issues: - - 114385 - - 114327 - - 114105 - - 114232 diff --git a/docs/changelog/115585.yaml b/docs/changelog/115585.yaml deleted file mode 100644 index 02eecfc3d7d2b..0000000000000 --- a/docs/changelog/115585.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 115459 -summary: Adds access to flags no_sub_matches and no_overlapping_matches to hyphenation-decompounder-tokenfilter -area: Search -type: enhancement -issues: - - 97849 diff --git a/docs/changelog/115640.yaml b/docs/changelog/115640.yaml deleted file mode 100644 index 5c4a943a9697d..0000000000000 --- a/docs/changelog/115640.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 115640 -summary: Fix NPE on plugin sync -area: Infra/CLI -type: bug -issues: - - 114818 diff --git a/docs/changelog/115655.yaml b/docs/changelog/115655.yaml deleted file mode 100644 index 7184405867657..0000000000000 --- a/docs/changelog/115655.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 115655 -summary: Better sizing `BytesRef` for Strings in Queries -area: Search -type: enhancement -issues: [] diff --git a/docs/changelog/115678.yaml b/docs/changelog/115678.yaml deleted file mode 100644 index 31240eae1ebb4..0000000000000 --- a/docs/changelog/115678.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 115678 -summary: "ESQL: extract common filter from aggs" -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/115687.yaml b/docs/changelog/115687.yaml deleted file mode 100644 index 1180b4627c635..0000000000000 --- a/docs/changelog/115687.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 115687 -summary: Add default ILM policies and switch to ILM for apm-data plugin -area: Data streams -type: feature -issues: [] diff --git a/docs/changelog/115744.yaml b/docs/changelog/115744.yaml deleted file mode 100644 index 9b8c91e59f451..0000000000000 --- a/docs/changelog/115744.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 115744 -summary: Use `SearchStats` instead of field.isAggregatable in data node planning -area: ES|QL -type: bug -issues: - - 115737 diff --git a/docs/changelog/115792.yaml b/docs/changelog/115792.yaml deleted file mode 100644 index 2945a64e3043a..0000000000000 --- a/docs/changelog/115792.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 115792 -summary: Add ES|QL `bit_length` function -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/115797.yaml b/docs/changelog/115797.yaml deleted file mode 100644 index 8adf51887c28a..0000000000000 --- a/docs/changelog/115797.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 115797 -summary: Enable `_tier` based coordinator rewrites for all indices (not just mounted - indices) -area: Search -type: enhancement -issues: [] diff --git a/docs/changelog/115807.yaml b/docs/changelog/115807.yaml deleted file mode 100644 index d17cabca4bd03..0000000000000 --- a/docs/changelog/115807.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 115807 -summary: "[Inference API] Improve chunked results error message" -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/115812.yaml b/docs/changelog/115812.yaml deleted file mode 100644 index c45c97041eb00..0000000000000 --- a/docs/changelog/115812.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 115812 -summary: "Prohibit changes to index mode, source, and sort settings during resize" -area: Logs -type: bug -issues: [] diff --git a/docs/changelog/115814.yaml b/docs/changelog/115814.yaml deleted file mode 100644 index 34f1213272d6f..0000000000000 --- a/docs/changelog/115814.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 115814 -summary: "[ES|QL] Implicit casting string literal to intervals" -area: ES|QL -type: enhancement -issues: - - 115352 diff --git a/docs/changelog/115858.yaml b/docs/changelog/115858.yaml deleted file mode 100644 index 0c0408fa656f8..0000000000000 --- a/docs/changelog/115858.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 115858 -summary: "ESQL: optimise aggregations filtered by false/null into evals" -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/115994.yaml b/docs/changelog/115994.yaml deleted file mode 100644 index ac090018c8a12..0000000000000 --- a/docs/changelog/115994.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 115994 -summary: Add logsdb telemetry -area: Logs -type: enhancement -issues: [] diff --git a/docs/changelog/116021.yaml b/docs/changelog/116021.yaml deleted file mode 100644 index 58c84b26805b2..0000000000000 --- a/docs/changelog/116021.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 116021 -summary: Fields caps does not honour ignore_unavailable -area: Search -type: bug -issues: - - 107767 diff --git a/docs/changelog/116082.yaml b/docs/changelog/116082.yaml deleted file mode 100644 index 35ca5fb1ea82e..0000000000000 --- a/docs/changelog/116082.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116082 -summary: Add support for bitwise inner-product in painless -area: Vector Search -type: enhancement -issues: [] diff --git a/docs/changelog/116128.yaml b/docs/changelog/116128.yaml deleted file mode 100644 index 7c38c0529c50d..0000000000000 --- a/docs/changelog/116128.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116128 -summary: Add num docs and size to logsdb telemetry -area: Logs -type: enhancement -issues: [] diff --git a/docs/changelog/116211.yaml b/docs/changelog/116211.yaml deleted file mode 100644 index 6f55b1b2fef34..0000000000000 --- a/docs/changelog/116211.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116211 -summary: Use underlying `ByteBuf` `refCount` for `ReleasableBytesReference` -area: Network -type: bug -issues: [] diff --git a/docs/changelog/116325.yaml b/docs/changelog/116325.yaml deleted file mode 100644 index b8cd16dc85773..0000000000000 --- a/docs/changelog/116325.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116325 -summary: Adjust analyze limit exception to be a `bad_request` -area: Analysis -type: bug -issues: [] diff --git a/docs/changelog/116346.yaml b/docs/changelog/116346.yaml deleted file mode 100644 index 1dcace88a98c0..0000000000000 --- a/docs/changelog/116346.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116346 -summary: "[ESQL] Fix Binary Comparisons on Date Nanos" -area: ES|QL -type: bug -issues: [] diff --git a/docs/changelog/116348.yaml b/docs/changelog/116348.yaml deleted file mode 100644 index 927ffc5a6121d..0000000000000 --- a/docs/changelog/116348.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116348 -summary: "ESQL: Honor skip_unavailable setting for nonmatching indices errors at planning time" -area: ES|QL -type: enhancement -issues: [ 114531 ] diff --git a/docs/changelog/116431.yaml b/docs/changelog/116431.yaml deleted file mode 100644 index 50c6baf1d01c7..0000000000000 --- a/docs/changelog/116431.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116431 -summary: Adds support for `input_type` field to Vertex inference service -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/116437.yaml b/docs/changelog/116437.yaml deleted file mode 100644 index 94c2464db9980..0000000000000 --- a/docs/changelog/116437.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116437 -summary: Ensure class resource stream is closed in `ResourceUtils` -area: Indices APIs -type: enhancement -issues: [] diff --git a/docs/changelog/116447.yaml b/docs/changelog/116447.yaml deleted file mode 100644 index 8c0cea4b54578..0000000000000 --- a/docs/changelog/116447.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116447 -summary: Adding a deprecation info API warning for data streams with old indices -area: Data streams -type: enhancement -issues: [] diff --git a/docs/changelog/116515.yaml b/docs/changelog/116515.yaml deleted file mode 100644 index 6c0d473361e52..0000000000000 --- a/docs/changelog/116515.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116515 -summary: Esql/lookup join grammar -area: ES|QL -type: feature -issues: [] diff --git a/docs/changelog/116583.yaml b/docs/changelog/116583.yaml deleted file mode 100644 index 3dc8337fe5b86..0000000000000 --- a/docs/changelog/116583.yaml +++ /dev/null @@ -1,7 +0,0 @@ -pr: 116583 -summary: Fix NPE in `EnrichLookupService` on mixed clusters with <8.14 versions -area: ES|QL -type: bug -issues: - - 116529 - - 116544 diff --git a/docs/changelog/116591.yaml b/docs/changelog/116591.yaml deleted file mode 100644 index 60ef241e197b3..0000000000000 --- a/docs/changelog/116591.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116591 -summary: "Add support for `BYTE_LENGTH` scalar function" -area: ES|QL -type: feature -issues: [] diff --git a/docs/changelog/116656.yaml b/docs/changelog/116656.yaml deleted file mode 100644 index eb5d5a1cfc201..0000000000000 --- a/docs/changelog/116656.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 116656 -summary: _validate does not honour ignore_unavailable -area: Search -type: bug -issues: - - 116594 diff --git a/docs/changelog/116664.yaml b/docs/changelog/116664.yaml deleted file mode 100644 index 36915fca39731..0000000000000 --- a/docs/changelog/116664.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 116664 -summary: Hides `hugging_face_elser` service from the `GET _inference/_services API` -area: Machine Learning -type: bug -issues: - - 116644 diff --git a/docs/changelog/116689.yaml b/docs/changelog/116689.yaml deleted file mode 100644 index 0b1d1646868aa..0000000000000 --- a/docs/changelog/116689.yaml +++ /dev/null @@ -1,10 +0,0 @@ -pr: 116689 -summary: Deprecate `_source.mode` in mappings -area: Mapping -type: deprecation -issues: [] -deprecation: - title: Deprecate `_source.mode` in mappings - area: Mapping - details: Configuring `_source.mode` in mappings is deprecated and will be removed in future versions. Use `index.mapping.source.mode` index setting instead. - impact: Use `index.mapping.source.mode` index setting instead diff --git a/docs/changelog/116809.yaml b/docs/changelog/116809.yaml deleted file mode 100644 index 61dbeb233d576..0000000000000 --- a/docs/changelog/116809.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116809 -summary: "Distinguish `LicensedFeature` by family field" -area: License -type: bug -issues: [] diff --git a/docs/changelog/116819.yaml b/docs/changelog/116819.yaml deleted file mode 100644 index afe06c583fe55..0000000000000 --- a/docs/changelog/116819.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116819 -summary: ESQL - Add match operator (:) -area: Search -type: feature -issues: [] diff --git a/docs/changelog/116931.yaml b/docs/changelog/116931.yaml deleted file mode 100644 index 8b31d236ff137..0000000000000 --- a/docs/changelog/116931.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116931 -summary: Enable built-in Inference Endpoints and default for Semantic Text -area: "Machine Learning" -type: enhancement -issues: [] diff --git a/docs/changelog/116953.yaml b/docs/changelog/116953.yaml deleted file mode 100644 index 33616510d8fd0..0000000000000 --- a/docs/changelog/116953.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 116953 -summary: Fix false positive date detection with trailing dot -area: Mapping -type: bug -issues: - - 116946 diff --git a/docs/changelog/116957.yaml b/docs/changelog/116957.yaml deleted file mode 100644 index 1020190de180d..0000000000000 --- a/docs/changelog/116957.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116957 -summary: Propagate scoring function through random sampler -area: Machine Learning -type: bug -issues: [ 110134 ] diff --git a/docs/changelog/116962.yaml b/docs/changelog/116962.yaml deleted file mode 100644 index 8f16b00e3f9fc..0000000000000 --- a/docs/changelog/116962.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 116962 -summary: "Add special case for elastic reranker in inference API" -area: Machine Learning -type: enhancement -issues: [] diff --git a/docs/changelog/116980.yaml b/docs/changelog/116980.yaml deleted file mode 100644 index 140324fd40b92..0000000000000 --- a/docs/changelog/116980.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 116980 -summary: "ESQL: Fix sorts containing `_source`" -area: ES|QL -type: bug -issues: - - 116659 diff --git a/docs/changelog/117080.yaml b/docs/changelog/117080.yaml deleted file mode 100644 index 5909f966e0fa2..0000000000000 --- a/docs/changelog/117080.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117080 -summary: Esql Enable Date Nanos (tech preview) -area: ES|QL -type: enhancement -issues: [] diff --git a/docs/changelog/117105.yaml b/docs/changelog/117105.yaml deleted file mode 100644 index de56c4d521a62..0000000000000 --- a/docs/changelog/117105.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 117105 -summary: Fix long metric deserialize & add - auto-resize needs to be set manually -area: CCS -type: bug -issues: - - 116914 diff --git a/docs/changelog/117189.yaml b/docs/changelog/117189.yaml deleted file mode 100644 index e89c2d81506d9..0000000000000 --- a/docs/changelog/117189.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117189 -summary: Fix deberta tokenizer bug caused by bug in normalizer -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/117213.yaml b/docs/changelog/117213.yaml deleted file mode 100644 index 3b4cd0cee966c..0000000000000 --- a/docs/changelog/117213.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 117213 -summary: Fix reconstituting version string from components -area: Ingest Node -type: bug -issues: - - 116950 diff --git a/docs/changelog/117271.yaml b/docs/changelog/117271.yaml deleted file mode 100644 index 1a328279b9635..0000000000000 --- a/docs/changelog/117271.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117271 -summary: Don't skip shards in coord rewrite if timestamp is an alias -area: Search -type: bug -issues: [] diff --git a/docs/changelog/117294.yaml b/docs/changelog/117294.yaml deleted file mode 100644 index f6e80690de7ff..0000000000000 --- a/docs/changelog/117294.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117294 -summary: Always Emit Inference ID in Semantic Text Mapping -area: Mapping -type: bug -issues: [] diff --git a/docs/changelog/117297.yaml b/docs/changelog/117297.yaml deleted file mode 100644 index 4a0051bbae644..0000000000000 --- a/docs/changelog/117297.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117297 -summary: Fix CCS exchange when multi cluster aliases point to same cluster -area: ES|QL -type: bug -issues: [] diff --git a/docs/changelog/117312.yaml b/docs/changelog/117312.yaml deleted file mode 100644 index 302b91388ef2b..0000000000000 --- a/docs/changelog/117312.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117312 -summary: Add missing `async_search` query parameters to rest-api-spec -area: Search -type: bug -issues: [] diff --git a/docs/changelog/117316.yaml b/docs/changelog/117316.yaml deleted file mode 100644 index 69474d68a8190..0000000000000 --- a/docs/changelog/117316.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117316 -summary: Fix validation of SORT by aggregate functions -area: ES|QL -type: bug -issues: [] diff --git a/docs/changelog/117350.yaml b/docs/changelog/117350.yaml deleted file mode 100644 index dca54f2037a87..0000000000000 --- a/docs/changelog/117350.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117350 -summary: "Improve halfbyte transposition performance, marginally improving bbq performance" -area: Vector Search -type: enhancement -issues: [] diff --git a/docs/changelog/117404.yaml b/docs/changelog/117404.yaml deleted file mode 100644 index 0bab171956ca9..0000000000000 --- a/docs/changelog/117404.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117404 -summary: Correct bit * byte and bit * float script comparisons -area: Vector Search -type: bug -issues: [] diff --git a/docs/changelog/117503.yaml b/docs/changelog/117503.yaml deleted file mode 100644 index d48741262b581..0000000000000 --- a/docs/changelog/117503.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 117503 -summary: Fix COUNT filter pushdown -area: ES|QL -type: bug -issues: - - 115522 diff --git a/docs/changelog/117551.yaml b/docs/changelog/117551.yaml deleted file mode 100644 index 081dd9203d82a..0000000000000 --- a/docs/changelog/117551.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117551 -summary: Fix stats by constant expresson with alias -area: ES|QL -type: bug -issues: [] diff --git a/docs/changelog/117575.yaml b/docs/changelog/117575.yaml deleted file mode 100644 index 781444ae97be5..0000000000000 --- a/docs/changelog/117575.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117575 -summary: Fix enrich cache size setting name -area: Ingest Node -type: bug -issues: [] diff --git a/docs/changelog/117595.yaml b/docs/changelog/117595.yaml deleted file mode 100644 index 9360c372ac97e..0000000000000 --- a/docs/changelog/117595.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117595 -summary: Fix for Deberta tokenizer when input sequence exceeds 512 tokens -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/117657.yaml b/docs/changelog/117657.yaml deleted file mode 100644 index 0a72e9dabe9e8..0000000000000 --- a/docs/changelog/117657.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117657 -summary: Ignore cancellation exceptions -area: ES|QL -type: bug -issues: [] diff --git a/docs/changelog/117762.yaml b/docs/changelog/117762.yaml deleted file mode 100644 index 123432e0f0507..0000000000000 --- a/docs/changelog/117762.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 117762 -summary: "Parse the contents of dynamic objects for [subobjects:false]" -area: Mapping -type: bug -issues: - - 117544 diff --git a/docs/changelog/117792.yaml b/docs/changelog/117792.yaml deleted file mode 100644 index 2d7ddda1ace40..0000000000000 --- a/docs/changelog/117792.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 117792 -summary: Address mapping and compute engine runtime field issues -area: Mapping -type: bug -issues: - - 117644 diff --git a/docs/changelog/117842.yaml b/docs/changelog/117842.yaml deleted file mode 100644 index 9b528a158288c..0000000000000 --- a/docs/changelog/117842.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117842 -summary: Limit size of `Literal#toString` -area: ES|QL -type: bug -issues: [] diff --git a/docs/changelog/117865.yaml b/docs/changelog/117865.yaml deleted file mode 100644 index 33dc497725f92..0000000000000 --- a/docs/changelog/117865.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117865 -summary: Fix BWC for ES|QL cluster request -area: ES|QL -type: bug -issues: [] diff --git a/docs/changelog/117914.yaml b/docs/changelog/117914.yaml deleted file mode 100644 index da58ed7bb04b7..0000000000000 --- a/docs/changelog/117914.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117914 -summary: Fix for propagating filters from compound to inner retrievers -area: Ranking -type: bug -issues: [] diff --git a/docs/changelog/117920.yaml b/docs/changelog/117920.yaml deleted file mode 100644 index 1bfddabd4462d..0000000000000 --- a/docs/changelog/117920.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 117920 -summary: Wait for the worker service to shutdown before closing task processor -area: Machine Learning -type: bug -issues: - - 117563 diff --git a/docs/changelog/117953.yaml b/docs/changelog/117953.yaml deleted file mode 100644 index 62f0218b1cdc7..0000000000000 --- a/docs/changelog/117953.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 117953 -summary: Acquire stats searcher for data stream stats -area: Data streams -type: bug -issues: [] diff --git a/docs/changelog/118354.yaml b/docs/changelog/118354.yaml deleted file mode 100644 index e2d72db121276..0000000000000 --- a/docs/changelog/118354.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 118354 -summary: Fix log message format bugs -area: Ingest Node -type: bug -issues: [] diff --git a/docs/changelog/118370.yaml b/docs/changelog/118370.yaml deleted file mode 100644 index e6a429448e493..0000000000000 --- a/docs/changelog/118370.yaml +++ /dev/null @@ -1,6 +0,0 @@ -pr: 118370 -summary: Fix concurrency issue with `ReinitializingSourceProvider` -area: Mapping -type: bug -issues: - - 118238 diff --git a/docs/changelog/118378.yaml b/docs/changelog/118378.yaml deleted file mode 100644 index d6c388b671968..0000000000000 --- a/docs/changelog/118378.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 118378 -summary: Opt into extra data stream resolution -area: ES|QL -type: bug -issues: [] From 950db572219e326d2986c744b20430dbfbd01a43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mariusz=20J=C3=B3zala?= <377355+jozala@users.noreply.github.com> Date: Fri, 13 Dec 2024 17:27:24 +0100 Subject: [PATCH 14/35] [test] Avoid running the NoImds test on AWS (#118675) Disabled the NoImds test on AWS EC2 instance where it fails because the AWS metadata are available, which is not expected by this test. --- .../ec2/DiscoveryEc2AvailabilityZoneAttributeNoImdsIT.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeNoImdsIT.java b/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeNoImdsIT.java index 602a98e17970d..73213090b6f93 100644 --- a/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeNoImdsIT.java +++ b/plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeNoImdsIT.java @@ -9,6 +9,8 @@ package org.elasticsearch.discovery.ec2; +import com.amazonaws.util.EC2MetadataUtils; + import org.elasticsearch.client.Request; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.ESRestTestCase; @@ -29,6 +31,8 @@ protected String getTestRestCluster() { } public void testAvailabilityZoneAttribute() throws IOException { + assumeTrue("test only in non-AWS environment", EC2MetadataUtils.getInstanceId() == null); + final var nodesInfoResponse = assertOKAndCreateObjectPath(client().performRequest(new Request("GET", "/_nodes/_all/_none"))); for (final var nodeId : nodesInfoResponse.evaluateMapKeys("nodes")) { assertNull(nodesInfoResponse.evaluateExact("nodes", nodeId, "attributes", "aws_availability_zone")); From 54e839b11068b9ea455befe1ce88d569b0f2c937 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Fri, 13 Dec 2024 17:42:08 +0100 Subject: [PATCH 15/35] ESQL: Fix LogicalPlanOptimizerTests testPlanSanityCheckWithBinaryPlans (#118672) --- muted-tests.yml | 3 --- .../xpack/esql/optimizer/LogicalPlanOptimizerTests.java | 3 +-- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 36dfc306b0147..e0d011c1bf239 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -317,9 +317,6 @@ tests: - class: org.elasticsearch.reservedstate.service.FileSettingsServiceTests method: testInvalidJSON issue: https://github.com/elastic/elasticsearch/issues/116521 -- class: org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizerTests - method: testPlanSanityCheckWithBinaryPlans - issue: https://github.com/elastic/elasticsearch/issues/118656 # Examples: # diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index 7e498eb6654b9..d97a8bb2bc27f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -4911,8 +4911,7 @@ public void testPlanSanityCheckWithBinaryPlans() throws Exception { """); var project = as(plan, Project.class); - var limit = as(project.child(), Limit.class); - var join = as(limit.child(), Join.class); + var join = as(project.child(), Join.class); var joinWithInvalidLeftPlan = join.replaceChildren(join.right(), join.right()); IllegalStateException e = expectThrows(IllegalStateException.class, () -> logicalOptimizer.optimize(joinWithInvalidLeftPlan)); From 6c56c32f7a872acda86232c542315f28215d24b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Slobodan=20Adamovi=C4=87?= Date: Fri, 13 Dec 2024 18:40:23 +0100 Subject: [PATCH 16/35] Grant necessary Kibana application privileges to `reporting_user` role (#118058) Previously, Kibana was authorizing (and granting application privileges) to create reports, simply based on the `reporting_user` role name. This PR makes these application privileges explicitly granted to the `reporting_user` role. --- docs/changelog/118058.yaml | 5 ++ .../authorization/built-in-roles.asciidoc | 11 ++- .../authz/store/ReservedRolesStore.java | 29 +++++-- .../authz/store/ReservedRolesStoreTests.java | 76 ++++++++++++++++++- 4 files changed, 106 insertions(+), 15 deletions(-) create mode 100644 docs/changelog/118058.yaml diff --git a/docs/changelog/118058.yaml b/docs/changelog/118058.yaml new file mode 100644 index 0000000000000..d5fad346d4d85 --- /dev/null +++ b/docs/changelog/118058.yaml @@ -0,0 +1,5 @@ +pr: 118058 +summary: Grant necessary Kibana application privileges to `reporting_user` role +area: Authorization +type: enhancement +issues: [] diff --git a/docs/reference/security/authorization/built-in-roles.asciidoc b/docs/reference/security/authorization/built-in-roles.asciidoc index d730587e7db17..13812b915dc5e 100644 --- a/docs/reference/security/authorization/built-in-roles.asciidoc +++ b/docs/reference/security/authorization/built-in-roles.asciidoc @@ -161,12 +161,11 @@ Grants the minimum privileges required to write data into the monitoring indices Grants the minimum privileges required to collect monitoring data for the {stack}. [[built-in-roles-reporting-user]] `reporting_user`:: -Grants the specific privileges required for users of {reporting} other than those -required to use {kib}. This role grants access to the reporting indices; each -user has access to only their own reports. -Reporting users should also be assigned additional roles that grant -{kibana-ref}/xpack-security-authorization.html[access to {kib}] as well as read -access to the <> that will be used to generate reports. +Grants the necessary privileges required to use {reporting} features in {kib}, +including generating and downloading reports. This role implicitly grants access +to all Kibana reporting features, with each user having access only to their own reports. +Note that reporting users should also be assigned additional roles that grant read access +to the <> that will be used to generate reports. [[built-in-roles-rollup-admin]] `rollup_admin`:: Grants `manage_rollup` cluster privileges, which enable you to manage and execute all rollup actions. diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java index fc14ec6811014..bdaf75203ee5d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java @@ -301,25 +301,40 @@ private static Map initializeReservedRoles() { "Grants access to manage all index templates and all ingest pipeline configurations." ) ), - // reporting_user doesn't have any privileges in Elasticsearch, and Kibana authorizes privileges based on this role entry( "reporting_user", new RoleDescriptor( "reporting_user", null, null, + new RoleDescriptor.ApplicationResourcePrivileges[] { + RoleDescriptor.ApplicationResourcePrivileges.builder() + .application("kibana-.kibana") + .resources("*") + .privileges( + "feature_discover.minimal_read", + "feature_discover.generate_report", + "feature_dashboard.minimal_read", + "feature_dashboard.generate_report", + "feature_dashboard.download_csv_report", + "feature_canvas.minimal_read", + "feature_canvas.generate_report", + "feature_visualize.minimal_read", + "feature_visualize.generate_report" + ) + .build() }, null, null, - null, - MetadataUtils.getDeprecatedReservedMetadata("Please use Kibana feature privileges instead"), + MetadataUtils.DEFAULT_RESERVED_METADATA, null, null, null, null, - "Grants the specific privileges required for users of X-Pack reporting other than those required to use Kibana. " - + "This role grants access to the reporting indices; each user has access to only their own reports. " - + "Reporting users should also be assigned additional roles that grant access to Kibana as well as read access " - + "to the indices that will be used to generate reports." + "Grants the necessary privileges required to use reporting features in Kibana, " + + "including generating and downloading reports. " + + "This role implicitly grants access to all Kibana reporting features, " + + "with each user having access only to their own reports. Note that reporting users should also be assigned " + + "additional roles that grant read access to the indices that will be used to generate reports." ) ), entry(KibanaSystemUser.ROLE_NAME, kibanaSystemRoleDescriptor(KibanaSystemUser.ROLE_NAME)), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java index b69b0ece89960..1b9a65d12d8d9 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java @@ -2646,12 +2646,57 @@ public void testReportingUserRole() { RoleDescriptor roleDescriptor = ReservedRolesStore.roleDescriptor("reporting_user"); assertNotNull(roleDescriptor); assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true)); - assertThat(roleDescriptor.getMetadata(), hasEntry("_deprecated", true)); + + final String applicationName = "kibana-.kibana"; + + final Set applicationPrivilegeNames = Set.of( + "feature_discover.minimal_read", + "feature_discover.generate_report", + "feature_dashboard.minimal_read", + "feature_dashboard.generate_report", + "feature_dashboard.download_csv_report", + "feature_canvas.minimal_read", + "feature_canvas.generate_report", + "feature_visualize.minimal_read", + "feature_visualize.generate_report" + ); + + final Set allowedApplicationActionPatterns = Set.of( + "login:", + "app:discover", + "app:canvas", + "app:kibana", + "ui:catalogue/canvas", + "ui:navLinks/canvas", + "ui:catalogue/discover", + "ui:navLinks/discover", + "ui:navLinks/kibana", + "saved_object:index-pattern/*", + "saved_object:search/*", + "saved_object:query/*", + "saved_object:config/*", + "saved_object:config/get", + "saved_object:config/find", + "saved_object:config-global/*", + "saved_object:telemetry/*", + "saved_object:canvas-workpad/*", + "saved_object:canvas-element/*", + "saved_object:url/*", + "ui:discover/show" + ); + + final List applicationPrivilegeDescriptors = new ArrayList<>(); + for (String appPrivilegeName : applicationPrivilegeNames) { + applicationPrivilegeDescriptors.add( + new ApplicationPrivilegeDescriptor(applicationName, appPrivilegeName, allowedApplicationActionPatterns, Map.of()) + ); + } Role reportingUserRole = Role.buildFromRoleDescriptor( roleDescriptor, new FieldPermissionsCache(Settings.EMPTY), - RESTRICTED_INDICES + RESTRICTED_INDICES, + applicationPrivilegeDescriptors ); assertThat(reportingUserRole.cluster().check(TransportClusterHealthAction.NAME, request, authentication), is(false)); assertThat(reportingUserRole.cluster().check(ClusterStateAction.NAME, request, authentication), is(false)); @@ -2723,6 +2768,33 @@ public void testReportingUserRole() { assertNoAccessAllowed(reportingUserRole, TestRestrictedIndices.SAMPLE_RESTRICTED_NAMES); assertNoAccessAllowed(reportingUserRole, XPackPlugin.ASYNC_RESULTS_INDEX + randomAlphaOfLengthBetween(0, 2)); + + applicationPrivilegeNames.forEach(appPrivilege -> { + assertThat( + reportingUserRole.application() + .grants( + ApplicationPrivilegeTests.createPrivilege( + applicationName, + appPrivilege, + allowedApplicationActionPatterns.toArray(new String[0]) + ), + "*" + ), + is(true) + ); + }); + assertThat( + reportingUserRole.application() + .grants( + ApplicationPrivilegeTests.createPrivilege( + "kibana-.*", + "feature_random.minimal_read", + allowedApplicationActionPatterns.toArray(new String[0]) + ), + "*" + ), + is(false) + ); } public void testSuperuserRole() { From c9d8a3a2b6bff2130634ec1e572cdf2ccea203e4 Mon Sep 17 00:00:00 2001 From: Joe Gallo Date: Fri, 13 Dec 2024 13:47:51 -0500 Subject: [PATCH 17/35] Add replica handling to the ILM MountSnapshotStep (#118687) --- .../xpack/core/ilm/AsyncWaitStep.java | 3 + .../xpack/core/ilm/DeleteAction.java | 6 +- .../xpack/core/ilm/DownsampleAction.java | 3 +- .../xpack/core/ilm/ForceMergeAction.java | 3 +- .../xpack/core/ilm/MountSnapshotStep.java | 29 ++-- .../xpack/core/ilm/ReadOnlyAction.java | 3 +- .../core/ilm/SearchableSnapshotAction.java | 7 +- .../xpack/core/ilm/ShrinkAction.java | 3 +- .../core/ilm/TimeseriesLifecycleType.java | 13 +- .../WaitUntilTimeSeriesEndTimePassesStep.java | 5 +- .../core/ilm/MountSnapshotStepTests.java | 161 ++++++++++++------ .../ilm/SearchableSnapshotActionTests.java | 95 ++++------- ...UntilTimeSeriesEndTimePassesStepTests.java | 9 +- 13 files changed, 182 insertions(+), 158 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AsyncWaitStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AsyncWaitStep.java index 6a72af5bce5e9..fc5e8d473b763 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AsyncWaitStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AsyncWaitStep.java @@ -8,6 +8,7 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; import org.elasticsearch.xcontent.ToXContentObject; @@ -20,6 +21,7 @@ */ public abstract class AsyncWaitStep extends Step { + @Nullable private final Client client; public AsyncWaitStep(StepKey key, StepKey nextStepKey, Client client) { @@ -27,6 +29,7 @@ public AsyncWaitStep(StepKey key, StepKey nextStepKey, Client client) { this.client = client; } + @Nullable protected Client getClient() { return client; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DeleteAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DeleteAction.java index 8712cefac5d31..6c2ab86995a6d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DeleteAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DeleteAction.java @@ -93,8 +93,7 @@ public List toSteps(Client client, String phase, Step.StepKey nextStepKey) WaitUntilTimeSeriesEndTimePassesStep waitUntilTimeSeriesEndTimeStep = new WaitUntilTimeSeriesEndTimePassesStep( waitTimeSeriesEndTimePassesKey, cleanSnapshotKey, - Instant::now, - client + Instant::now ); CleanupSnapshotStep cleanupSnapshotStep = new CleanupSnapshotStep(cleanSnapshotKey, deleteStepKey, client); DeleteStep deleteStep = new DeleteStep(deleteStepKey, nextStepKey, client); @@ -108,8 +107,7 @@ public List toSteps(Client client, String phase, Step.StepKey nextStepKey) WaitUntilTimeSeriesEndTimePassesStep waitUntilTimeSeriesEndTimeStep = new WaitUntilTimeSeriesEndTimePassesStep( waitTimeSeriesEndTimePassesKey, deleteStepKey, - Instant::now, - client + Instant::now ); DeleteStep deleteStep = new DeleteStep(deleteStepKey, nextStepKey, client); return List.of(waitForNoFollowersStep, waitUntilTimeSeriesEndTimeStep, deleteStep); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DownsampleAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DownsampleAction.java index 697f948e47832..6ce9e05e4a464 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DownsampleAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DownsampleAction.java @@ -200,8 +200,7 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { WaitUntilTimeSeriesEndTimePassesStep waitUntilTimeSeriesEndTimeStep = new WaitUntilTimeSeriesEndTimePassesStep( waitTimeSeriesEndTimePassesKey, readOnlyKey, - Instant::now, - client + Instant::now ); // Mark source index as read-only ReadOnlyStep readOnlyStep = new ReadOnlyStep(readOnlyKey, generateDownsampleIndexNameKey, client); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java index f8f4ce2bb0354..ac398bccb64e4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java @@ -162,8 +162,7 @@ public List toSteps(Client client, String phase, Step.StepKey nextStepKey) WaitUntilTimeSeriesEndTimePassesStep waitUntilTimeSeriesEndTimeStep = new WaitUntilTimeSeriesEndTimePassesStep( waitTimeSeriesEndTimePassesKey, codecChange ? closeKey : forceMergeKey, - Instant::now, - client + Instant::now ); // Indices already in this step key when upgrading need to know how to move forward but stop making the index diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStep.java index 7d045f2950e1b..82d41b91fea4f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStep.java @@ -41,6 +41,7 @@ public class MountSnapshotStep extends AsyncRetryDuringSnapshotActionStep { private final MountSearchableSnapshotRequest.Storage storageType; @Nullable private final Integer totalShardsPerNode; + private final int replicas; public MountSnapshotStep( StepKey key, @@ -48,7 +49,8 @@ public MountSnapshotStep( Client client, String restoredIndexPrefix, MountSearchableSnapshotRequest.Storage storageType, - @Nullable Integer totalShardsPerNode + @Nullable Integer totalShardsPerNode, + int replicas ) { super(key, nextStepKey, client); this.restoredIndexPrefix = restoredIndexPrefix; @@ -57,16 +59,10 @@ public MountSnapshotStep( throw new IllegalArgumentException("[" + SearchableSnapshotAction.TOTAL_SHARDS_PER_NODE.getPreferredName() + "] must be >= 1"); } this.totalShardsPerNode = totalShardsPerNode; - } - public MountSnapshotStep( - StepKey key, - StepKey nextStepKey, - Client client, - String restoredIndexPrefix, - MountSearchableSnapshotRequest.Storage storageType - ) { - this(key, nextStepKey, client, restoredIndexPrefix, storageType, null); + // this isn't directly settable by the user, so validation by assertion is sufficient + assert replicas >= 0 : "number of replicas must be gte zero, but was [" + replicas + "]"; + this.replicas = replicas; } @Override @@ -87,6 +83,10 @@ public Integer getTotalShardsPerNode() { return totalShardsPerNode; } + public int getReplicas() { + return replicas; + } + @Override void performDuringNoSnapshot(IndexMetadata indexMetadata, ClusterState currentClusterState, ActionListener listener) { String indexName = indexMetadata.getIndex().getName(); @@ -162,11 +162,13 @@ void performDuringNoSnapshot(IndexMetadata indexMetadata, ClusterState currentCl } final Settings.Builder settingsBuilder = Settings.builder(); - overrideTierPreference(this.getKey().phase()).ifPresent(override -> settingsBuilder.put(DataTier.TIER_PREFERENCE, override)); if (totalShardsPerNode != null) { settingsBuilder.put(ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), totalShardsPerNode); } + if (replicas > 0) { + settingsBuilder.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, replicas); + } final MountSearchableSnapshotRequest mountSearchableSnapshotRequest = new MountSearchableSnapshotRequest( TimeValue.MAX_VALUE, @@ -245,7 +247,7 @@ String[] ignoredIndexSettings() { @Override public int hashCode() { - return Objects.hash(super.hashCode(), restoredIndexPrefix, storageType, totalShardsPerNode); + return Objects.hash(super.hashCode(), restoredIndexPrefix, storageType, totalShardsPerNode, replicas); } @Override @@ -260,6 +262,7 @@ public boolean equals(Object obj) { return super.equals(obj) && Objects.equals(restoredIndexPrefix, other.restoredIndexPrefix) && Objects.equals(storageType, other.storageType) - && Objects.equals(totalShardsPerNode, other.totalShardsPerNode); + && Objects.equals(totalShardsPerNode, other.totalShardsPerNode) + && Objects.equals(replicas, other.replicas); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ReadOnlyAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ReadOnlyAction.java index 2b03dc77eb5b6..b36156842acf5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ReadOnlyAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ReadOnlyAction.java @@ -67,8 +67,7 @@ public List toSteps(Client client, String phase, StepKey nextStepKey) { WaitUntilTimeSeriesEndTimePassesStep waitUntilTimeSeriesEndTimeStep = new WaitUntilTimeSeriesEndTimePassesStep( waitTimeSeriesEndTimePassesKey, readOnlyKey, - Instant::now, - client + Instant::now ); ReadOnlyStep readOnlyStep = new ReadOnlyStep(readOnlyKey, nextStepKey, client); return List.of(checkNotWriteIndexStep, waitUntilTimeSeriesEndTimeStep, readOnlyStep); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java index f585575534b76..b746ee8ea7c07 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java @@ -113,6 +113,7 @@ public String getSnapshotRepository() { return snapshotRepository; } + @Nullable public Integer getTotalShardsPerNode() { return totalShardsPerNode; } @@ -230,8 +231,7 @@ public List toSteps(Client client, String phase, StepKey nextStepKey, XPac WaitUntilTimeSeriesEndTimePassesStep waitUntilTimeSeriesEndTimeStep = new WaitUntilTimeSeriesEndTimePassesStep( waitTimeSeriesEndTimePassesKey, skipGeneratingSnapshotKey, - Instant::now, - client + Instant::now ); // When generating a snapshot, we either jump to the force merge step, or we skip the @@ -318,7 +318,8 @@ public List toSteps(Client client, String phase, StepKey nextStepKey, XPac client, getRestoredIndexPrefix(mountSnapshotKey), storageType, - totalShardsPerNode + totalShardsPerNode, + 0 ); WaitForIndexColorStep waitForGreenIndexHealthStep = new WaitForIndexColorStep( waitForGreenRestoredIndexKey, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java index 70ec5da1d8a2a..f7478518613e2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java @@ -231,8 +231,7 @@ public List toSteps(Client client, String phase, Step.StepKey nextStepKey) WaitUntilTimeSeriesEndTimePassesStep waitUntilTimeSeriesEndTimeStep = new WaitUntilTimeSeriesEndTimePassesStep( waitTimeSeriesEndTimePassesKey, readOnlyKey, - Instant::now, - client + Instant::now ); ReadOnlyStep readOnlyStep = new ReadOnlyStep(readOnlyKey, checkTargetShardsCountKey, client); CheckTargetShardsCountStep checkTargetShardsCountStep = new CheckTargetShardsCountStep( diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java index 0fd280f440f39..10a4c7086a0cc 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleType.java @@ -27,7 +27,6 @@ import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; -import java.util.stream.Stream; /** * Represents the lifecycle of an index from creation to deletion. A @@ -49,7 +48,7 @@ public class TimeseriesLifecycleType implements LifecycleType { static final String DELETE_PHASE = "delete"; public static final List ORDERED_VALID_PHASES = List.of(HOT_PHASE, WARM_PHASE, COLD_PHASE, FROZEN_PHASE, DELETE_PHASE); - public static final List ORDERED_VALID_HOT_ACTIONS = Stream.of( + public static final List ORDERED_VALID_HOT_ACTIONS = List.of( SetPriorityAction.NAME, UnfollowAction.NAME, RolloverAction.NAME, @@ -58,8 +57,8 @@ public class TimeseriesLifecycleType implements LifecycleType { ShrinkAction.NAME, ForceMergeAction.NAME, SearchableSnapshotAction.NAME - ).filter(Objects::nonNull).toList(); - public static final List ORDERED_VALID_WARM_ACTIONS = Stream.of( + ); + public static final List ORDERED_VALID_WARM_ACTIONS = List.of( SetPriorityAction.NAME, UnfollowAction.NAME, ReadOnlyAction.NAME, @@ -68,8 +67,8 @@ public class TimeseriesLifecycleType implements LifecycleType { MigrateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME - ).filter(Objects::nonNull).toList(); - public static final List ORDERED_VALID_COLD_ACTIONS = Stream.of( + ); + public static final List ORDERED_VALID_COLD_ACTIONS = List.of( SetPriorityAction.NAME, UnfollowAction.NAME, ReadOnlyAction.NAME, @@ -78,7 +77,7 @@ public class TimeseriesLifecycleType implements LifecycleType { AllocateAction.NAME, MigrateAction.NAME, FreezeAction.NAME - ).filter(Objects::nonNull).toList(); + ); public static final List ORDERED_VALID_FROZEN_ACTIONS = List.of(UnfollowAction.NAME, SearchableSnapshotAction.NAME); public static final List ORDERED_VALID_DELETE_ACTIONS = List.of(WaitForSnapshotAction.NAME, DeleteAction.NAME); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStep.java index 50a7d48672c8e..3e190a26dd961 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStep.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.core.ilm; -import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.Strings; @@ -33,8 +32,8 @@ public class WaitUntilTimeSeriesEndTimePassesStep extends AsyncWaitStep { public static final String NAME = "check-ts-end-time-passed"; private final Supplier nowSupplier; - public WaitUntilTimeSeriesEndTimePassesStep(StepKey key, StepKey nextStepKey, Supplier nowSupplier, Client client) { - super(key, nextStepKey, client); + public WaitUntilTimeSeriesEndTimePassesStep(StepKey key, StepKey nextStepKey, Supplier nowSupplier) { + super(key, nextStepKey, null); this.nowSupplier = nowSupplier; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStepTests.java index 8ca7a00ab0948..7ccdb1a27326a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/MountSnapshotStepTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.metadata.LifecycleExecutionState; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.snapshots.RestoreInfo; import org.elasticsearch.test.client.NoOpClient; @@ -42,7 +43,7 @@ public MountSnapshotStep createRandomInstance() { String restoredIndexPrefix = randomAlphaOfLength(10); MountSearchableSnapshotRequest.Storage storage = randomStorageType(); Integer totalShardsPerNode = randomTotalShardsPerNode(true); - return new MountSnapshotStep(stepKey, nextStepKey, client, restoredIndexPrefix, storage, totalShardsPerNode); + return new MountSnapshotStep(stepKey, nextStepKey, client, restoredIndexPrefix, storage, totalShardsPerNode, 0); } public static MountSearchableSnapshotRequest.Storage randomStorageType() { @@ -61,7 +62,8 @@ protected MountSnapshotStep copyInstance(MountSnapshotStep instance) { instance.getClient(), instance.getRestoredIndexPrefix(), instance.getStorage(), - instance.getTotalShardsPerNode() + instance.getTotalShardsPerNode(), + instance.getReplicas() ); } @@ -72,7 +74,8 @@ public MountSnapshotStep mutateInstance(MountSnapshotStep instance) { String restoredIndexPrefix = instance.getRestoredIndexPrefix(); MountSearchableSnapshotRequest.Storage storage = instance.getStorage(); Integer totalShardsPerNode = instance.getTotalShardsPerNode(); - switch (between(0, 4)) { + int replicas = instance.getReplicas(); + switch (between(0, 5)) { case 0: key = new StepKey(key.phase(), key.action(), key.name() + randomAlphaOfLength(5)); break; @@ -94,10 +97,13 @@ public MountSnapshotStep mutateInstance(MountSnapshotStep instance) { case 4: totalShardsPerNode = totalShardsPerNode == null ? 1 : totalShardsPerNode + randomIntBetween(1, 100); break; + case 5: + replicas = replicas == 0 ? 1 : 0; // swap between 0 and 1 + break; default: throw new AssertionError("Illegal randomisation branch"); } - return new MountSnapshotStep(key, nextKey, instance.getClient(), restoredIndexPrefix, storage, totalShardsPerNode); + return new MountSnapshotStep(key, nextKey, instance.getClient(), restoredIndexPrefix, storage, totalShardsPerNode, replicas); } public void testCreateWithInvalidTotalShardsPerNode() throws Exception { @@ -111,7 +117,8 @@ public void testCreateWithInvalidTotalShardsPerNode() throws Exception { client, RESTORED_INDEX_PREFIX, randomStorageType(), - invalidTotalShardsPerNode + invalidTotalShardsPerNode, + 0 ) ); assertEquals("[total_shards_per_node] must be >= 1", exception.getMessage()); @@ -195,14 +202,18 @@ public void testPerformAction() throws Exception { indexName, RESTORED_INDEX_PREFIX, indexName, - new String[] { LifecycleSettings.LIFECYCLE_NAME } + new String[] { LifecycleSettings.LIFECYCLE_NAME }, + null, + 0 ); MountSnapshotStep step = new MountSnapshotStep( randomStepKey(), randomStepKey(), client, RESTORED_INDEX_PREFIX, - randomStorageType() + randomStorageType(), + null, + 0 ); performActionAndWait(step, indexMetadata, clusterState, null); } @@ -237,7 +248,9 @@ public void testResponseStatusHandling() throws Exception { randomStepKey(), clientPropagatingOKResponse, RESTORED_INDEX_PREFIX, - randomStorageType() + randomStorageType(), + null, + 0 ); performActionAndWait(step, indexMetadata, clusterState, null); } @@ -252,7 +265,9 @@ public void testResponseStatusHandling() throws Exception { randomStepKey(), clientPropagatingACCEPTEDResponse, RESTORED_INDEX_PREFIX, - randomStorageType() + randomStorageType(), + null, + 0 ); performActionAndWait(step, indexMetadata, clusterState, null); } @@ -289,47 +304,49 @@ public void testMountWithPartialAndRestoredPrefix() throws Exception { ); } - public void doTestMountWithoutSnapshotIndexNameInState(String prefix) throws Exception { - { - String indexNameSnippet = randomAlphaOfLength(10); - String indexName = prefix + indexNameSnippet; - String policyName = "test-ilm-policy"; - Map ilmCustom = new HashMap<>(); - String snapshotName = indexName + "-" + policyName; - ilmCustom.put("snapshot_name", snapshotName); - String repository = "repository"; - ilmCustom.put("snapshot_repository", repository); + private void doTestMountWithoutSnapshotIndexNameInState(String prefix) throws Exception { + String indexNameSnippet = randomAlphaOfLength(10); + String indexName = prefix + indexNameSnippet; + String policyName = "test-ilm-policy"; + Map ilmCustom = new HashMap<>(); + String snapshotName = indexName + "-" + policyName; + ilmCustom.put("snapshot_name", snapshotName); + String repository = "repository"; + ilmCustom.put("snapshot_repository", repository); - IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(indexName) - .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName)) - .putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, ilmCustom) - .numberOfShards(randomIntBetween(1, 5)) - .numberOfReplicas(randomIntBetween(0, 5)); - IndexMetadata indexMetadata = indexMetadataBuilder.build(); + IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(indexName) + .settings(settings(IndexVersion.current()).put(LifecycleSettings.LIFECYCLE_NAME, policyName)) + .putCustom(LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY, ilmCustom) + .numberOfShards(randomIntBetween(1, 5)) + .numberOfReplicas(randomIntBetween(0, 5)); + IndexMetadata indexMetadata = indexMetadataBuilder.build(); - ClusterState clusterState = ClusterState.builder(emptyClusterState()) - .metadata(Metadata.builder().put(indexMetadata, true).build()) - .build(); + ClusterState clusterState = ClusterState.builder(emptyClusterState()) + .metadata(Metadata.builder().put(indexMetadata, true).build()) + .build(); - try (var threadPool = createThreadPool()) { - final var client = getRestoreSnapshotRequestAssertingClient( - threadPool, - repository, - snapshotName, - indexName, - RESTORED_INDEX_PREFIX, - indexNameSnippet, - new String[] { LifecycleSettings.LIFECYCLE_NAME } - ); - MountSnapshotStep step = new MountSnapshotStep( - randomStepKey(), - randomStepKey(), - client, - RESTORED_INDEX_PREFIX, - randomStorageType() - ); - performActionAndWait(step, indexMetadata, clusterState, null); - } + try (var threadPool = createThreadPool()) { + final var client = getRestoreSnapshotRequestAssertingClient( + threadPool, + repository, + snapshotName, + indexName, + RESTORED_INDEX_PREFIX, + indexNameSnippet, + new String[] { LifecycleSettings.LIFECYCLE_NAME }, + null, + 0 + ); + MountSnapshotStep step = new MountSnapshotStep( + randomStepKey(), + randomStepKey(), + client, + RESTORED_INDEX_PREFIX, + randomStorageType(), + null, + 0 + ); + performActionAndWait(step, indexMetadata, clusterState, null); } } @@ -361,7 +378,11 @@ public void testIgnoreTotalShardsPerNodeInFrozenPhase() throws Exception { indexName, RESTORED_INDEX_PREFIX, indexName, - new String[] { LifecycleSettings.LIFECYCLE_NAME, ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING.getKey() } + new String[] { + LifecycleSettings.LIFECYCLE_NAME, + ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING.getKey() }, + null, + 0 ); MountSnapshotStep step = new MountSnapshotStep( new StepKey(TimeseriesLifecycleType.FROZEN_PHASE, randomAlphaOfLength(10), randomAlphaOfLength(10)), @@ -369,13 +390,14 @@ public void testIgnoreTotalShardsPerNodeInFrozenPhase() throws Exception { client, RESTORED_INDEX_PREFIX, randomStorageType(), - null + null, + 0 ); performActionAndWait(step, indexMetadata, clusterState, null); } } - public void testDoNotIgnoreTotalShardsPerNodeIfSet() throws Exception { + public void testDoNotIgnoreTotalShardsPerNodeAndReplicasIfSet() throws Exception { String indexName = randomAlphaOfLength(10); String policyName = "test-ilm-policy"; Map ilmCustom = new HashMap<>(); @@ -395,6 +417,9 @@ public void testDoNotIgnoreTotalShardsPerNodeIfSet() throws Exception { .metadata(Metadata.builder().put(indexMetadata, true).build()) .build(); + final Integer totalShardsPerNode = randomTotalShardsPerNode(false); + final int replicas = randomIntBetween(1, 5); + try (var threadPool = createThreadPool()) { final var client = getRestoreSnapshotRequestAssertingClient( threadPool, @@ -403,7 +428,9 @@ public void testDoNotIgnoreTotalShardsPerNodeIfSet() throws Exception { indexName, RESTORED_INDEX_PREFIX, indexName, - new String[] { LifecycleSettings.LIFECYCLE_NAME } + new String[] { LifecycleSettings.LIFECYCLE_NAME }, + totalShardsPerNode, + replicas ); MountSnapshotStep step = new MountSnapshotStep( new StepKey(TimeseriesLifecycleType.FROZEN_PHASE, randomAlphaOfLength(10), randomAlphaOfLength(10)), @@ -411,7 +438,8 @@ public void testDoNotIgnoreTotalShardsPerNodeIfSet() throws Exception { client, RESTORED_INDEX_PREFIX, randomStorageType(), - randomTotalShardsPerNode(false) + totalShardsPerNode, + replicas ); performActionAndWait(step, indexMetadata, clusterState, null); } @@ -439,7 +467,9 @@ private NoOpClient getRestoreSnapshotRequestAssertingClient( String indexName, String restoredIndexPrefix, String expectedSnapshotIndexName, - String[] expectedIgnoredIndexSettings + String[] expectedIgnoredIndexSettings, + @Nullable Integer totalShardsPerNode, + int replicas ) { return new NoOpClient(threadPool) { @Override @@ -462,6 +492,31 @@ protected void assertThat(mountSearchableSnapshotRequest.mountedIndexName(), is(restoredIndexPrefix + indexName)); assertThat(mountSearchableSnapshotRequest.snapshotIndexName(), is(expectedSnapshotIndexName)); + if (totalShardsPerNode != null) { + Integer totalShardsPerNodeSettingValue = ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING.get( + mountSearchableSnapshotRequest.indexSettings() + ); + assertThat(totalShardsPerNodeSettingValue, is(totalShardsPerNode)); + } else { + assertThat( + mountSearchableSnapshotRequest.indexSettings() + .hasValue(ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING.getKey()), + is(false) + ); + } + + if (replicas > 0) { + Integer numberOfReplicasSettingValue = IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.get( + mountSearchableSnapshotRequest.indexSettings() + ); + assertThat(numberOfReplicasSettingValue, is(replicas)); + } else { + assertThat( + mountSearchableSnapshotRequest.indexSettings().hasValue(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey()), + is(false) + ); + } + // invoke the awaiting listener with a very generic 'response', just to fulfill the contract listener.onResponse((Response) new RestoreSnapshotResponse((RestoreInfo) null)); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java index ca219fdde3d57..5304b7885f96c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java @@ -14,6 +14,8 @@ import java.io.IOException; import java.util.List; +import java.util.Objects; +import java.util.stream.Stream; import static org.elasticsearch.xpack.core.ilm.SearchableSnapshotAction.NAME; import static org.elasticsearch.xpack.core.ilm.SearchableSnapshotAction.TOTAL_SHARDS_PER_NODE; @@ -29,40 +31,23 @@ public void testToSteps() { StepKey nextStepKey = new StepKey(phase, randomAlphaOfLengthBetween(1, 5), randomAlphaOfLengthBetween(1, 5)); List steps = action.toSteps(null, phase, nextStepKey, null); - assertThat(steps.size(), is(action.isForceMergeIndex() ? 19 : 17)); - - List expectedSteps = action.isForceMergeIndex() - ? expectedStepKeysWithForceMerge(phase) - : expectedStepKeysNoForceMerge(phase); - - assertThat(steps.get(0).getKey(), is(expectedSteps.get(0))); - assertThat(steps.get(1).getKey(), is(expectedSteps.get(1))); - assertThat(steps.get(2).getKey(), is(expectedSteps.get(2))); - assertThat(steps.get(3).getKey(), is(expectedSteps.get(3))); - assertThat(steps.get(4).getKey(), is(expectedSteps.get(4))); - assertThat(steps.get(5).getKey(), is(expectedSteps.get(5))); - assertThat(steps.get(6).getKey(), is(expectedSteps.get(6))); - assertThat(steps.get(7).getKey(), is(expectedSteps.get(7))); - assertThat(steps.get(8).getKey(), is(expectedSteps.get(8))); - assertThat(steps.get(9).getKey(), is(expectedSteps.get(9))); - assertThat(steps.get(10).getKey(), is(expectedSteps.get(10))); - assertThat(steps.get(11).getKey(), is(expectedSteps.get(11))); - assertThat(steps.get(12).getKey(), is(expectedSteps.get(12))); - assertThat(steps.get(13).getKey(), is(expectedSteps.get(13))); - assertThat(steps.get(14).getKey(), is(expectedSteps.get(14))); - assertThat(steps.get(15).getKey(), is(expectedSteps.get(15))); - - if (action.isForceMergeIndex()) { - assertThat(steps.get(16).getKey(), is(expectedSteps.get(16))); - assertThat(steps.get(17).getKey(), is(expectedSteps.get(17))); - CreateSnapshotStep createSnapshotStep = (CreateSnapshotStep) steps.get(9); - assertThat(createSnapshotStep.getNextKeyOnIncomplete(), is(expectedSteps.get(8))); - validateWaitForDataTierStep(phase, steps, 10, 11); - } else { - CreateSnapshotStep createSnapshotStep = (CreateSnapshotStep) steps.get(7); - assertThat(createSnapshotStep.getNextKeyOnIncomplete(), is(expectedSteps.get(6))); - validateWaitForDataTierStep(phase, steps, 8, 9); + + List expectedSteps = expectedStepKeys(phase, action.isForceMergeIndex()); + assertThat(steps.size(), is(expectedSteps.size())); + for (int i = 0; i < expectedSteps.size(); i++) { + assertThat("steps match expectation at index " + i, steps.get(i).getKey(), is(expectedSteps.get(i))); + } + + int index = -1; + for (int i = 0; i < expectedSteps.size(); i++) { + if (expectedSteps.get(i).name().equals(CreateSnapshotStep.NAME)) { + index = i; + break; + } } + CreateSnapshotStep createSnapshotStep = (CreateSnapshotStep) steps.get(index); + assertThat(createSnapshotStep.getNextKeyOnIncomplete(), is(expectedSteps.get(index - 1))); + validateWaitForDataTierStep(phase, steps, index + 1, index + 2); } private void validateWaitForDataTierStep(String phase, List steps, int waitForDataTierStepIndex, int mountStepIndex) { @@ -108,15 +93,15 @@ public void testCreateWithInvalidTotalShardsPerNode() { assertEquals("[" + TOTAL_SHARDS_PER_NODE.getPreferredName() + "] must be >= 1", exception.getMessage()); } - private List expectedStepKeysWithForceMerge(String phase) { - return List.of( + private List expectedStepKeys(String phase, boolean forceMergeIndex) { + return Stream.of( new StepKey(phase, NAME, SearchableSnapshotAction.CONDITIONAL_SKIP_ACTION_STEP), new StepKey(phase, NAME, CheckNotDataStreamWriteIndexStep.NAME), new StepKey(phase, NAME, WaitForNoFollowersStep.NAME), new StepKey(phase, NAME, WaitUntilTimeSeriesEndTimePassesStep.NAME), new StepKey(phase, NAME, SearchableSnapshotAction.CONDITIONAL_SKIP_GENERATE_AND_CLEAN), - new StepKey(phase, NAME, ForceMergeStep.NAME), - new StepKey(phase, NAME, SegmentCountStep.NAME), + forceMergeIndex ? new StepKey(phase, NAME, ForceMergeStep.NAME) : null, + forceMergeIndex ? new StepKey(phase, NAME, SegmentCountStep.NAME) : null, new StepKey(phase, NAME, GenerateSnapshotNameStep.NAME), new StepKey(phase, NAME, CleanupSnapshotStep.NAME), new StepKey(phase, NAME, CreateSnapshotStep.NAME), @@ -129,29 +114,7 @@ private List expectedStepKeysWithForceMerge(String phase) { new StepKey(phase, NAME, ReplaceDataStreamBackingIndexStep.NAME), new StepKey(phase, NAME, DeleteStep.NAME), new StepKey(phase, NAME, SwapAliasesAndDeleteSourceIndexStep.NAME) - ); - } - - private List expectedStepKeysNoForceMerge(String phase) { - return List.of( - new StepKey(phase, NAME, SearchableSnapshotAction.CONDITIONAL_SKIP_ACTION_STEP), - new StepKey(phase, NAME, CheckNotDataStreamWriteIndexStep.NAME), - new StepKey(phase, NAME, WaitForNoFollowersStep.NAME), - new StepKey(phase, NAME, WaitUntilTimeSeriesEndTimePassesStep.NAME), - new StepKey(phase, NAME, SearchableSnapshotAction.CONDITIONAL_SKIP_GENERATE_AND_CLEAN), - new StepKey(phase, NAME, GenerateSnapshotNameStep.NAME), - new StepKey(phase, NAME, CleanupSnapshotStep.NAME), - new StepKey(phase, NAME, CreateSnapshotStep.NAME), - new StepKey(phase, NAME, WaitForDataTierStep.NAME), - new StepKey(phase, NAME, MountSnapshotStep.NAME), - new StepKey(phase, NAME, WaitForIndexColorStep.NAME), - new StepKey(phase, NAME, CopyExecutionStateStep.NAME), - new StepKey(phase, NAME, CopySettingsStep.NAME), - new StepKey(phase, NAME, SearchableSnapshotAction.CONDITIONAL_DATASTREAM_CHECK_KEY), - new StepKey(phase, NAME, ReplaceDataStreamBackingIndexStep.NAME), - new StepKey(phase, NAME, DeleteStep.NAME), - new StepKey(phase, NAME, SwapAliasesAndDeleteSourceIndexStep.NAME) - ); + ).filter(Objects::nonNull).toList(); } @Override @@ -172,8 +135,16 @@ protected Writeable.Reader instanceReader() { @Override protected SearchableSnapshotAction mutateInstance(SearchableSnapshotAction instance) { return switch (randomIntBetween(0, 2)) { - case 0 -> new SearchableSnapshotAction(randomAlphaOfLengthBetween(5, 10), instance.isForceMergeIndex()); - case 1 -> new SearchableSnapshotAction(instance.getSnapshotRepository(), instance.isForceMergeIndex() == false); + case 0 -> new SearchableSnapshotAction( + randomAlphaOfLengthBetween(5, 10), + instance.isForceMergeIndex(), + instance.getTotalShardsPerNode() + ); + case 1 -> new SearchableSnapshotAction( + instance.getSnapshotRepository(), + instance.isForceMergeIndex() == false, + instance.getTotalShardsPerNode() + ); case 2 -> new SearchableSnapshotAction( instance.getSnapshotRepository(), instance.isForceMergeIndex(), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStepTests.java index 8ca6c0016a791..15bbbe7446429 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitUntilTimeSeriesEndTimePassesStepTests.java @@ -30,7 +30,7 @@ public class WaitUntilTimeSeriesEndTimePassesStepTests extends AbstractStepTestC protected WaitUntilTimeSeriesEndTimePassesStep createRandomInstance() { Step.StepKey stepKey = randomStepKey(); Step.StepKey nextStepKey = randomStepKey(); - return new WaitUntilTimeSeriesEndTimePassesStep(stepKey, nextStepKey, Instant::now, client); + return new WaitUntilTimeSeriesEndTimePassesStep(stepKey, nextStepKey, Instant::now); } @Override @@ -42,12 +42,12 @@ protected WaitUntilTimeSeriesEndTimePassesStep mutateInstance(WaitUntilTimeSerie case 0 -> key = new Step.StepKey(key.phase(), key.action(), key.name() + randomAlphaOfLength(5)); case 1 -> nextKey = new Step.StepKey(nextKey.phase(), nextKey.action(), nextKey.name() + randomAlphaOfLength(5)); } - return new WaitUntilTimeSeriesEndTimePassesStep(key, nextKey, Instant::now, client); + return new WaitUntilTimeSeriesEndTimePassesStep(key, nextKey, Instant::now); } @Override protected WaitUntilTimeSeriesEndTimePassesStep copyInstance(WaitUntilTimeSeriesEndTimePassesStep instance) { - return new WaitUntilTimeSeriesEndTimePassesStep(instance.getKey(), instance.getNextStepKey(), Instant::now, client); + return new WaitUntilTimeSeriesEndTimePassesStep(instance.getKey(), instance.getNextStepKey(), Instant::now); } public void testEvaluateCondition() { @@ -68,8 +68,7 @@ public void testEvaluateCondition() { WaitUntilTimeSeriesEndTimePassesStep step = new WaitUntilTimeSeriesEndTimePassesStep( randomStepKey(), randomStepKey(), - () -> currentTime, - client + () -> currentTime ); { // end_time has lapsed already so condition must be met From dcadb08b57bc65ac5f989ad60e18eac9fcce42c4 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 13 Dec 2024 13:37:07 -0600 Subject: [PATCH 18/35] Unmuting XPackRestTest migrate reindex tests (#118407) These were fixed by https://github.com/elastic/elasticsearch/pull/118382 Closes #118401 Closes #118272 Closes #118273 Closes #118274 --- muted-tests.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index e0d011c1bf239..1255739e818be 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -294,18 +294,6 @@ tests: - class: org.elasticsearch.docker.test.DockerYmlTestSuiteIT method: test {p0=/11_nodes/Test cat nodes output} issue: https://github.com/elastic/elasticsearch/issues/118397 -- class: org.elasticsearch.xpack.test.rest.XPackRestIT - method: test {p0=migrate/20_reindex_status/Test get reindex status with nonexistent task id} - issue: https://github.com/elastic/elasticsearch/issues/118401 -- class: org.elasticsearch.xpack.test.rest.XPackRestIT - method: test {p0=migrate/10_reindex/Test Reindex With Nonexistent Data Stream} - issue: https://github.com/elastic/elasticsearch/issues/118274 -- class: org.elasticsearch.xpack.test.rest.XPackRestIT - method: test {p0=migrate/10_reindex/Test Reindex With Bad Data Stream Name} - issue: https://github.com/elastic/elasticsearch/issues/118272 -- class: org.elasticsearch.xpack.test.rest.XPackRestIT - method: test {p0=migrate/10_reindex/Test Reindex With Unsupported Mode} - issue: https://github.com/elastic/elasticsearch/issues/118273 - class: org.elasticsearch.xpack.security.operator.OperatorPrivilegesIT method: testEveryActionIsEitherOperatorOnlyOrNonOperator issue: https://github.com/elastic/elasticsearch/issues/118220 From 908cf9ab768c4854e51926f28b62fd2108c69aa7 Mon Sep 17 00:00:00 2001 From: Quentin Deschamps Date: Fri, 13 Dec 2024 20:54:48 +0100 Subject: [PATCH 19/35] Fix moving function linear weighted avg (#118516) Fix moving function linear weighted avg --- docs/changelog/118516.yaml | 6 ++++ modules/aggregations/build.gradle | 1 + .../test/aggregations/moving_fn.yml | 31 +++++++++++++------ .../action/search/SearchCapabilities.java | 3 ++ .../pipeline/MovingFunctions.java | 4 +-- .../MovFnWhitelistedFunctionTests.java | 2 +- 6 files changed, 34 insertions(+), 13 deletions(-) create mode 100644 docs/changelog/118516.yaml diff --git a/docs/changelog/118516.yaml b/docs/changelog/118516.yaml new file mode 100644 index 0000000000000..8a618a6d6cfd7 --- /dev/null +++ b/docs/changelog/118516.yaml @@ -0,0 +1,6 @@ +pr: 118435 +summary: Fix moving function linear weighted avg +area: Aggregations +type: bug +issues: + - 113751 diff --git a/modules/aggregations/build.gradle b/modules/aggregations/build.gradle index 94fdddf6d711a..3b5fb6ddecde9 100644 --- a/modules/aggregations/build.gradle +++ b/modules/aggregations/build.gradle @@ -48,4 +48,5 @@ tasks.named("yamlRestCompatTestTransform").configure({ task -> task.skipTest("aggregations/date_agg_per_day_of_week/Date aggregartion per day of week", "week-date behaviour has changed") task.skipTest("aggregations/time_series/Configure with no synthetic source", "temporary until backport") task.skipTest("aggregations/percentiles_hdr_metric/Negative values test", "returned exception has changed") + task.skipTest("aggregations/moving_fn/linearWeightedAvg", "math was wrong in previous versions") }) diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/moving_fn.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/moving_fn.yml index cd6feb601b1df..3abad87d57907 100644 --- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/moving_fn.yml +++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/moving_fn.yml @@ -255,6 +255,17 @@ linearWeightedAvg: - skip: features: close_to + - requires: + test_runner_features: [capabilities] + + - requires: + capabilities: + - method: POST + path: /_search + parameters: [method, path, parameters, capabilities] + capabilities: [moving_fn_right_math] + reason: "math not fixed yet" + - do: search: index: no_gaps @@ -275,11 +286,11 @@ linearWeightedAvg: - match: { hits.total.value: 6 } - length: { aggregations.@timestamp.buckets: 6 } - is_false: aggregations.@timestamp.buckets.0.d.value - - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 0.500, error: 0.0005 } } - - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 1.250, error: 0.0005 } } - - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 1.000, error: 0.0005 } } - - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 2.250, error: 0.0005 } } - - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 3.500, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 1.667, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 1.333, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 3.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 4.667, error: 0.0005 } } - do: search: @@ -301,11 +312,11 @@ linearWeightedAvg: - match: { hits.total.value: 6 } - length: { aggregations.@timestamp.buckets: 6 } - is_false: aggregations.@timestamp.buckets.0.d.value - - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 0.500, error: 0.0005 } } - - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 1.250, error: 0.0005 } } - - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 1.143, error: 0.0005 } } - - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 2.286, error: 0.0005 } } - - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 3.429, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.1.d.value: { value: 1.000, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.2.d.value: { value: 1.667, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.3.d.value: { value: 1.333, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.4.d.value: { value: 2.667, error: 0.0005 } } + - close_to: { aggregations.@timestamp.buckets.5.d.value: { value: 4.000, error: 0.0005 } } --- ewma: diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java index 86304c8c4bde2..06f8f8f3c1be6 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java @@ -42,6 +42,8 @@ private SearchCapabilities() {} private static final String RANK_VECTORS_SCRIPT_ACCESS = "rank_vectors_script_access"; /** Initial support for rank-vectors maxSim functions access. */ private static final String RANK_VECTORS_SCRIPT_MAX_SIM = "rank_vectors_script_max_sim_with_bugfix"; + /** Fixed the math in {@code moving_fn}'s {@code linearWeightedAvg}. */ + private static final String MOVING_FN_RIGHT_MATH = "moving_fn_right_math"; private static final String RANDOM_SAMPLER_WITH_SCORED_SUBAGGS = "random_sampler_with_scored_subaggs"; private static final String OPTIMIZED_SCALAR_QUANTIZATION_BBQ = "optimized_scalar_quantization_bbq"; @@ -59,6 +61,7 @@ private SearchCapabilities() {} capabilities.add(RANDOM_SAMPLER_WITH_SCORED_SUBAGGS); capabilities.add(OPTIMIZED_SCALAR_QUANTIZATION_BBQ); capabilities.add(KNN_QUANTIZED_VECTOR_RESCORE); + capabilities.add(MOVING_FN_RIGHT_MATH); if (RankVectorsFieldMapper.FEATURE_FLAG.isEnabled()) { capabilities.add(RANK_VECTORS_FIELD_MAPPER); capabilities.add(RANK_VECTORS_SCRIPT_ACCESS); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovingFunctions.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovingFunctions.java index 02e3c76e5e793..46584c171d16c 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovingFunctions.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovingFunctions.java @@ -100,7 +100,7 @@ public static double stdDev(double[] values, double avg) { */ public static double linearWeightedAvg(double[] values) { double avg = 0; - long totalWeight = 1; + long totalWeight = 0; long current = 1; for (double v : values) { @@ -110,7 +110,7 @@ public static double linearWeightedAvg(double[] values) { current += 1; } } - return totalWeight == 1 ? Double.NaN : avg / totalWeight; + return totalWeight == 0 ? Double.NaN : avg / totalWeight; } /** diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnWhitelistedFunctionTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnWhitelistedFunctionTests.java index 69173957aebab..3bc458880db0a 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnWhitelistedFunctionTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovFnWhitelistedFunctionTests.java @@ -326,7 +326,7 @@ public void testLinearMovAvg() { } double avg = 0; - long totalWeight = 1; + long totalWeight = 0; long current = 1; for (double value : window) { From b4610c8e26f292f73545b659a0bcdba7022ad1d0 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Fri, 13 Dec 2024 21:28:11 +0100 Subject: [PATCH 20/35] Remove needless sending of OriginalIndices in SearchFreeContextRequest (#117245) We don't need to use this request, the handler for freeing of scroll requests literally goes to the same transport handler and doesn't come with the list of indices. The original security need for keeping the list of indices around is long gone. --- .../action/IndicesRequestIT.java | 13 +--- .../search/AbstractSearchAsyncAction.java | 6 +- .../action/search/DfsQueryPhase.java | 6 +- .../action/search/SearchPhase.java | 6 +- .../action/search/SearchTransportService.java | 70 ++----------------- .../AbstractSearchAsyncActionTests.java | 6 +- .../action/search/MockSearchPhaseContext.java | 2 +- .../action/search/SearchAsyncActionTests.java | 15 +++- .../test/ESSingleNodeTestCase.java | 4 +- .../xpack/security/authz/RBACEngine.java | 26 +++---- 10 files changed, 44 insertions(+), 110 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java index 8bedf436e3698..f5860cedcd989 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java @@ -556,11 +556,7 @@ public void testUpdateSettings() { } public void testSearchQueryThenFetch() throws Exception { - interceptTransportActions( - SearchTransportService.QUERY_ACTION_NAME, - SearchTransportService.FETCH_ID_ACTION_NAME, - SearchTransportService.FREE_CONTEXT_ACTION_NAME - ); + interceptTransportActions(SearchTransportService.QUERY_ACTION_NAME, SearchTransportService.FETCH_ID_ACTION_NAME); String[] randomIndicesOrAliases = randomIndicesOrAliases(); for (int i = 0; i < randomIndicesOrAliases.length; i++) { @@ -580,16 +576,13 @@ public void testSearchQueryThenFetch() throws Exception { SearchTransportService.QUERY_ACTION_NAME, SearchTransportService.FETCH_ID_ACTION_NAME ); - // free context messages are not necessarily sent, but if they are, check their indices - assertIndicesSubsetOptionalRequests(Arrays.asList(searchRequest.indices()), SearchTransportService.FREE_CONTEXT_ACTION_NAME); } public void testSearchDfsQueryThenFetch() throws Exception { interceptTransportActions( SearchTransportService.DFS_ACTION_NAME, SearchTransportService.QUERY_ID_ACTION_NAME, - SearchTransportService.FETCH_ID_ACTION_NAME, - SearchTransportService.FREE_CONTEXT_ACTION_NAME + SearchTransportService.FETCH_ID_ACTION_NAME ); String[] randomIndicesOrAliases = randomIndicesOrAliases(); @@ -611,8 +604,6 @@ public void testSearchDfsQueryThenFetch() throws Exception { SearchTransportService.QUERY_ID_ACTION_NAME, SearchTransportService.FETCH_ID_ACTION_NAME ); - // free context messages are not necessarily sent, but if they are, check their indices - assertIndicesSubsetOptionalRequests(Arrays.asList(searchRequest.indices()), SearchTransportService.FREE_CONTEXT_ACTION_NAME); } private static void assertSameIndices(IndicesRequest originalRequest, String... actions) { diff --git a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java index 800193e258dba..47abfe266c524 100644 --- a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java @@ -711,7 +711,7 @@ private void raisePhaseFailure(SearchPhaseExecutionException exception) { try { SearchShardTarget searchShardTarget = entry.getSearchShardTarget(); Transport.Connection connection = getConnection(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId()); - sendReleaseSearchContext(entry.getContextId(), connection, getOriginalIndices(entry.getShardIndex())); + sendReleaseSearchContext(entry.getContextId(), connection); } catch (Exception inner) { inner.addSuppressed(exception); logger.trace("failed to release context", inner); @@ -727,10 +727,10 @@ private void raisePhaseFailure(SearchPhaseExecutionException exception) { * @see org.elasticsearch.search.fetch.FetchSearchResult#getContextId() * */ - void sendReleaseSearchContext(ShardSearchContextId contextId, Transport.Connection connection, OriginalIndices originalIndices) { + void sendReleaseSearchContext(ShardSearchContextId contextId, Transport.Connection connection) { assert isPartOfPointInTime(contextId) == false : "Must not release point in time context [" + contextId + "]"; if (connection != null) { - searchTransportService.sendFreeContext(connection, contextId, originalIndices); + searchTransportService.sendFreeContext(connection, contextId, ActionListener.noop()); } } diff --git a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java index 285dd0a22fd7e..cc8c4becea9a9 100644 --- a/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/DfsQueryPhase.java @@ -119,11 +119,7 @@ public void onFailure(Exception exception) { // the query might not have been executed at all (for example because thread pool rejected // execution) and the search context that was created in dfs phase might not be released. // release it again to be in the safe side - context.sendReleaseSearchContext( - querySearchRequest.contextId(), - connection, - context.getOriginalIndices(shardIndex) - ); + context.sendReleaseSearchContext(querySearchRequest.contextId(), connection); } } } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhase.java index 986f7210c0d1b..7d849a72abf9d 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhase.java @@ -97,11 +97,7 @@ protected static void releaseIrrelevantSearchContext(SearchPhaseResult searchPha context.getLogger().trace("trying to release search context [{}]", phaseResult.getContextId()); SearchShardTarget shardTarget = phaseResult.getSearchShardTarget(); Transport.Connection connection = context.getConnection(shardTarget.getClusterAlias(), shardTarget.getNodeId()); - context.sendReleaseSearchContext( - phaseResult.getContextId(), - connection, - context.getOriginalIndices(phaseResult.getShardIndex()) - ); + context.sendReleaseSearchContext(phaseResult.getContextId(), connection); } catch (Exception e) { context.getLogger().trace("failed to release context", e); } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java index 8444a92b24432..cfc2e1bcdaf2b 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java @@ -13,12 +13,10 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListenerResponseHandler; -import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.get.TransportGetTaskAction; import org.elasticsearch.action.support.ChannelActionListener; -import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -124,24 +122,6 @@ public SearchTransportService( this.responseWrapper = responseWrapper; } - private static final ActionListenerResponseHandler SEND_FREE_CONTEXT_LISTENER = - new ActionListenerResponseHandler<>( - ActionListener.noop(), - SearchFreeContextResponse::readFrom, - TransportResponseHandler.TRANSPORT_WORKER - ); - - public void sendFreeContext(Transport.Connection connection, final ShardSearchContextId contextId, OriginalIndices originalIndices) { - transportService.sendRequest( - connection, - FREE_CONTEXT_ACTION_NAME, - new SearchFreeContextRequest(originalIndices, contextId), - TransportRequestOptions.EMPTY, - // no need to respond if it was freed or not - SEND_FREE_CONTEXT_LISTENER - ); - } - public void sendFreeContext( Transport.Connection connection, ShardSearchContextId contextId, @@ -370,43 +350,6 @@ private static class ClearScrollContextsRequest extends TransportRequest { } } - static class SearchFreeContextRequest extends ScrollFreeContextRequest implements IndicesRequest { - private final OriginalIndices originalIndices; - - SearchFreeContextRequest(OriginalIndices originalIndices, ShardSearchContextId id) { - super(id); - this.originalIndices = originalIndices; - } - - SearchFreeContextRequest(StreamInput in) throws IOException { - super(in); - originalIndices = OriginalIndices.readOriginalIndices(in); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - OriginalIndices.writeOriginalIndices(originalIndices, out); - } - - @Override - public String[] indices() { - if (originalIndices == null) { - return null; - } - return originalIndices.indices(); - } - - @Override - public IndicesOptions indicesOptions() { - if (originalIndices == null) { - return null; - } - return originalIndices.indicesOptions(); - } - - } - public static class SearchFreeContextResponse extends TransportResponse { private static final SearchFreeContextResponse FREED = new SearchFreeContextResponse(true); @@ -456,12 +399,13 @@ public static void registerRequestHandler(TransportService transportService, Sea SearchFreeContextResponse::readFrom ); - transportService.registerRequestHandler( - FREE_CONTEXT_ACTION_NAME, - freeContextExecutor, - SearchFreeContextRequest::new, - freeContextHandler - ); + // TODO: remove this handler once the lowest compatible version stops using it + transportService.registerRequestHandler(FREE_CONTEXT_ACTION_NAME, freeContextExecutor, in -> { + var res = new ScrollFreeContextRequest(in); + // this handler exists for BwC purposes only, we don't need the original indices to free the context + OriginalIndices.readOriginalIndices(in); + return res; + }, freeContextHandler); TransportActionProxy.registerProxyAction(transportService, FREE_CONTEXT_ACTION_NAME, false, SearchFreeContextResponse::readFrom); transportService.registerRequestHandler( diff --git a/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java index 725a4583d104a..71bf2a47cfa47 100644 --- a/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/AbstractSearchAsyncActionTests.java @@ -112,11 +112,7 @@ long buildTookInMillis() { } @Override - public void sendReleaseSearchContext( - ShardSearchContextId contextId, - Transport.Connection connection, - OriginalIndices originalIndices - ) { + public void sendReleaseSearchContext(ShardSearchContextId contextId, Transport.Connection connection) { releasedContexts.add(contextId); } diff --git a/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java b/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java index 7a38858d8477a..cf65d756811ad 100644 --- a/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java +++ b/server/src/test/java/org/elasticsearch/action/search/MockSearchPhaseContext.java @@ -155,7 +155,7 @@ protected void executePhaseOnShard( } @Override - public void sendReleaseSearchContext(ShardSearchContextId contextId, Transport.Connection connection, OriginalIndices originalIndices) { + public void sendReleaseSearchContext(ShardSearchContextId contextId, Transport.Connection connection) { releasedSearchContexts.add(contextId); } diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java index b4ddd48172d01..2361beb7ad036 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java @@ -296,7 +296,11 @@ public void testFanOutAndCollect() throws InterruptedException { AtomicInteger numFreedContext = new AtomicInteger(); SearchTransportService transportService = new SearchTransportService(null, null, null) { @Override - public void sendFreeContext(Transport.Connection connection, ShardSearchContextId contextId, OriginalIndices originalIndices) { + public void sendFreeContext( + Transport.Connection connection, + ShardSearchContextId contextId, + ActionListener listener + ) { numFreedContext.incrementAndGet(); assertTrue(nodeToContextMap.containsKey(connection.getNode())); assertTrue(nodeToContextMap.get(connection.getNode()).remove(contextId)); @@ -363,7 +367,7 @@ public void run() { for (int i = 0; i < results.getNumShards(); i++) { TestSearchPhaseResult result = results.getAtomicArray().get(i); assertEquals(result.node.getId(), result.getSearchShardTarget().getNodeId()); - sendReleaseSearchContext(result.getContextId(), new MockConnection(result.node), OriginalIndices.NONE); + sendReleaseSearchContext(result.getContextId(), new MockConnection(result.node)); } responseListener.onResponse(testResponse); if (latchTriggered.compareAndSet(false, true) == false) { @@ -421,8 +425,13 @@ public void testFanOutAndFail() throws InterruptedException { ); AtomicInteger numFreedContext = new AtomicInteger(); SearchTransportService transportService = new SearchTransportService(null, null, null) { + @Override - public void sendFreeContext(Transport.Connection connection, ShardSearchContextId contextId, OriginalIndices originalIndices) { + public void sendFreeContext( + Transport.Connection connection, + ShardSearchContextId contextId, + ActionListener listener + ) { assertNotNull(contextId); numFreedContext.incrementAndGet(); assertTrue(nodeToContextMap.containsKey(connection.getNode())); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java index 459d5573d7c12..63334bd70306f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java @@ -78,7 +78,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.elasticsearch.action.search.SearchTransportService.FREE_CONTEXT_ACTION_NAME; +import static org.elasticsearch.action.search.SearchTransportService.FREE_CONTEXT_SCROLL_ACTION_NAME; import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING; import static org.elasticsearch.discovery.SettingsBasedSeedHostsProvider.DISCOVERY_SEED_HOSTS_SETTING; import static org.elasticsearch.test.NodeRoles.dataNode; @@ -482,7 +482,7 @@ protected void ensureNoInitializingShards() { */ protected void ensureAllFreeContextActionsAreConsumed() throws Exception { logger.info("--> waiting for all free_context tasks to complete within a reasonable time"); - safeGet(clusterAdmin().prepareListTasks().setActions(FREE_CONTEXT_ACTION_NAME + "*").setWaitForCompletion(true).execute()); + safeGet(clusterAdmin().prepareListTasks().setActions(FREE_CONTEXT_SCROLL_ACTION_NAME + "*").setWaitForCompletion(true).execute()); } /** diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java index fa6187798da25..2353d710059ff 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java @@ -129,6 +129,19 @@ public class RBACEngine implements AuthorizationEngine { private static final String DELETE_SUB_REQUEST_REPLICA = TransportDeleteAction.NAME + "[r]"; private static final Logger logger = LogManager.getLogger(RBACEngine.class); + + private static final Set SCROLL_RELATED_ACTIONS = Set.of( + TransportSearchScrollAction.TYPE.name(), + SearchTransportService.FETCH_ID_SCROLL_ACTION_NAME, + SearchTransportService.QUERY_FETCH_SCROLL_ACTION_NAME, + SearchTransportService.QUERY_SCROLL_ACTION_NAME, + SearchTransportService.FREE_CONTEXT_ACTION_NAME, + SearchTransportService.FREE_CONTEXT_SCROLL_ACTION_NAME, + TransportClearScrollAction.NAME, + "indices:data/read/sql/close_cursor", + SearchTransportService.CLEAR_SCROLL_CONTEXTS_ACTION_NAME + ); + private final Settings settings; private final CompositeRolesStore rolesStore; private final FieldPermissionsCache fieldPermissionsCache; @@ -320,7 +333,7 @@ public void authorizeIndexAction( // need to validate that the action is allowed and then move on listener.onResponse(role.checkIndicesAction(action) ? IndexAuthorizationResult.EMPTY : IndexAuthorizationResult.DENIED); } else if (request instanceof IndicesRequest == false) { - if (isScrollRelatedAction(action)) { + if (SCROLL_RELATED_ACTIONS.contains(action)) { // scroll is special // some APIs are indices requests that are not actually associated with indices. For example, // search scroll request, is categorized under the indices context, but doesn't hold indices names @@ -1000,17 +1013,6 @@ public int hashCode() { } } - private static boolean isScrollRelatedAction(String action) { - return action.equals(TransportSearchScrollAction.TYPE.name()) - || action.equals(SearchTransportService.FETCH_ID_SCROLL_ACTION_NAME) - || action.equals(SearchTransportService.QUERY_FETCH_SCROLL_ACTION_NAME) - || action.equals(SearchTransportService.QUERY_SCROLL_ACTION_NAME) - || action.equals(SearchTransportService.FREE_CONTEXT_SCROLL_ACTION_NAME) - || action.equals(TransportClearScrollAction.NAME) - || action.equals("indices:data/read/sql/close_cursor") - || action.equals(SearchTransportService.CLEAR_SCROLL_CONTEXTS_ACTION_NAME); - } - private static boolean isAsyncRelatedAction(String action) { return action.equals(SubmitAsyncSearchAction.NAME) || action.equals(GetAsyncSearchAction.NAME) From 0a6ce27825d15a7e8294a99ee1aca52eb05f8ae0 Mon Sep 17 00:00:00 2001 From: Parker Timmins Date: Fri, 13 Dec 2024 15:12:25 -0600 Subject: [PATCH 21/35] Add ReindexDatastreamIndexAction (#116996) Add an action to reindex a single index from a source index to a destination index. Unlike the reindex action, this action copies settings and mappings from the source index to the dest index before performing the reindex. This action is part of work to reindex data streams and will be called on each of the backing indices within a data stream. --- docs/changelog/116996.yaml | 5 + .../metadata/MetadataCreateIndexService.java | 48 ++- .../action/ReindexDatastreamIndexIT.java | 408 ++++++++++++++++++ .../xpack/migrate/MigratePlugin.java | 3 + .../action/ReindexDataStreamAction.java | 2 +- .../action/ReindexDataStreamIndexAction.java | 121 ++++++ ...ReindexDataStreamIndexTransportAction.java | 211 +++++++++ .../ReindexDatastreamIndexRequestTests.java | 29 ++ .../ReindexDatastreamIndexResponseTests.java | 29 ++ .../xpack/security/operator/Constants.java | 1 + 10 files changed, 841 insertions(+), 16 deletions(-) create mode 100644 docs/changelog/116996.yaml create mode 100644 x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexIT.java create mode 100644 x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexAction.java create mode 100644 x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java create mode 100644 x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexRequestTests.java create mode 100644 x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexResponseTests.java diff --git a/docs/changelog/116996.yaml b/docs/changelog/116996.yaml new file mode 100644 index 0000000000000..59f59355131bf --- /dev/null +++ b/docs/changelog/116996.yaml @@ -0,0 +1,5 @@ +pr: 116996 +summary: Initial work on `ReindexDatastreamIndexAction` +area: Data streams +type: enhancement +issues: [] diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 75c2c06f36c8e..a99822c9de00c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -1675,23 +1675,11 @@ static void prepareResizeIndexSettings( throw new IllegalStateException("unknown resize type is " + type); } - final Settings.Builder builder = Settings.builder(); + final Settings.Builder builder; if (copySettings) { - // copy all settings and non-copyable settings and settings that have already been set (e.g., from the request) - for (final String key : sourceMetadata.getSettings().keySet()) { - final Setting setting = indexScopedSettings.get(key); - if (setting == null) { - assert indexScopedSettings.isPrivateSetting(key) : key; - } else if (setting.getProperties().contains(Setting.Property.NotCopyableOnResize)) { - continue; - } - // do not override settings that have already been set (for example, from the request) - if (indexSettingsBuilder.keys().contains(key)) { - continue; - } - builder.copy(key, sourceMetadata.getSettings()); - } + builder = copySettingsFromSource(true, sourceMetadata.getSettings(), indexScopedSettings, indexSettingsBuilder); } else { + builder = Settings.builder(); final Predicate sourceSettingsPredicate = (s) -> (s.startsWith("index.similarity.") || s.startsWith("index.analysis.") || s.startsWith("index.sort.") @@ -1709,6 +1697,36 @@ static void prepareResizeIndexSettings( } } + public static Settings.Builder copySettingsFromSource( + boolean copyPrivateSettings, + Settings sourceSettings, + IndexScopedSettings indexScopedSettings, + Settings.Builder indexSettingsBuilder + ) { + final Settings.Builder builder = Settings.builder(); + for (final String key : sourceSettings.keySet()) { + final Setting setting = indexScopedSettings.get(key); + if (setting == null) { + assert indexScopedSettings.isPrivateSetting(key) : key; + if (copyPrivateSettings == false) { + continue; + } + } else if (setting.getProperties().contains(Setting.Property.NotCopyableOnResize)) { + continue; + } else if (setting.isPrivateIndex()) { + if (copyPrivateSettings == false) { + continue; + } + } + // do not override settings that have already been set (for example, from the request) + if (indexSettingsBuilder.keys().contains(key)) { + continue; + } + builder.copy(key, sourceSettings); + } + return builder; + } + /** * Returns a default number of routing shards based on the number of shards of the index. The default number of routing shards will * allow any index to be split at least once and at most 10 times by a factor of two. The closer the number or shards gets to 1024 diff --git a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexIT.java new file mode 100644 index 0000000000000..e492f035da866 --- /dev/null +++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexIT.java @@ -0,0 +1,408 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.migrate.action; + +import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.time.FormatNames; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.datastreams.DataStreamsPlugin; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.reindex.ReindexPlugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.transport.MockTransportService; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.migrate.MigratePlugin; + +import java.io.IOException; +import java.time.Instant; +import java.util.Collection; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.xpack.migrate.action.ReindexDataStreamAction.REINDEX_DATA_STREAM_FEATURE_FLAG; +import static org.hamcrest.Matchers.equalTo; + +public class ReindexDatastreamIndexIT extends ESIntegTestCase { + + private static final String MAPPING = """ + { + "_doc":{ + "dynamic":"strict", + "properties":{ + "foo1":{ + "type":"text" + } + } + } + } + """; + + @Override + protected Collection> nodePlugins() { + return List.of(MigratePlugin.class, ReindexPlugin.class, MockTransportService.TestPlugin.class, DataStreamsPlugin.class); + } + + public void testDestIndexDeletedIfExists() throws Exception { + assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); + + // empty source index + var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + indicesAdmin().create(new CreateIndexRequest(sourceIndex)).get(); + + // dest index with docs + var destIndex = ReindexDataStreamIndexTransportAction.generateDestIndexName(sourceIndex); + indicesAdmin().create(new CreateIndexRequest(destIndex)).actionGet(); + indexDocs(destIndex, 10); + assertHitCount(prepareSearch(destIndex).setSize(0), 10); + + // call reindex + client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)).actionGet(); + + // verify that dest still exists, but is now empty + assertTrue(indexExists(destIndex)); + assertHitCount(prepareSearch(destIndex).setSize(0), 0); + } + + public void testDestIndexNameSet() throws Exception { + assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); + + var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + indicesAdmin().create(new CreateIndexRequest(sourceIndex)).get(); + + // call reindex + var response = client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) + .actionGet(); + + var expectedDestIndexName = ReindexDataStreamIndexTransportAction.generateDestIndexName(sourceIndex); + assertEquals(expectedDestIndexName, response.getDestIndex()); + } + + public void testDestIndexContainsDocs() throws Exception { + assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); + + // source index with docs + var numDocs = randomIntBetween(1, 100); + var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + indicesAdmin().create(new CreateIndexRequest(sourceIndex)).get(); + indexDocs(sourceIndex, numDocs); + + // call reindex + var response = client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) + .actionGet(); + indicesAdmin().refresh(new RefreshRequest(response.getDestIndex())).actionGet(); + + // verify that dest contains docs + assertHitCount(prepareSearch(response.getDestIndex()).setSize(0), numDocs); + } + + public void testSetSourceToReadOnly() throws Exception { + assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); + + // empty source index + var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + indicesAdmin().create(new CreateIndexRequest(sourceIndex)).get(); + + // call reindex + client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)).actionGet(); + + // assert that write to source fails + var indexReq = new IndexRequest(sourceIndex).source(jsonBuilder().startObject().field("field", "1").endObject()); + assertThrows(ClusterBlockException.class, () -> client().index(indexReq).actionGet()); + assertHitCount(prepareSearch(sourceIndex).setSize(0), 0); + } + + public void testSettingsAddedBeforeReindex() throws Exception { + assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); + + // start with a static setting + var numShards = randomIntBetween(1, 10); + var staticSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards).build(); + var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + indicesAdmin().create(new CreateIndexRequest(sourceIndex, staticSettings)).get(); + + // update with a dynamic setting + var numReplicas = randomIntBetween(0, 10); + var dynamicSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicas).build(); + indicesAdmin().updateSettings(new UpdateSettingsRequest(dynamicSettings, sourceIndex)).actionGet(); + + // call reindex + var destIndex = client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) + .actionGet() + .getDestIndex(); + + // assert both static and dynamic settings set on dest index + var settingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(destIndex)).actionGet(); + assertEquals(numReplicas, Integer.parseInt(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_NUMBER_OF_REPLICAS))); + assertEquals(numShards, Integer.parseInt(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_NUMBER_OF_SHARDS))); + } + + public void testMappingsAddedToDestIndex() throws Exception { + assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); + + var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + String mapping = """ + { + "_doc":{ + "dynamic":"strict", + "properties":{ + "foo1":{ + "type":"text" + } + } + } + } + """; + indicesAdmin().create(new CreateIndexRequest(sourceIndex).mapping(mapping)).actionGet(); + + // call reindex + var destIndex = client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) + .actionGet() + .getDestIndex(); + + var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(sourceIndex, destIndex)).actionGet(); + Map mappings = mappingsResponse.mappings(); + var destMappings = mappings.get(destIndex).sourceAsMap(); + var sourceMappings = mappings.get(sourceIndex).sourceAsMap(); + + assertEquals(sourceMappings, destMappings); + // sanity check specific value from dest mapping + assertEquals("text", XContentMapValues.extractValue("properties.foo1.type", destMappings)); + } + + public void testReadOnlyAddedBack() { + assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); + + // Create source index with read-only and/or block-writes + var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + boolean isReadOnly = randomBoolean(); + boolean isBlockWrites = randomBoolean(); + var settings = Settings.builder() + .put(IndexMetadata.SETTING_READ_ONLY, isReadOnly) + .put(IndexMetadata.SETTING_BLOCKS_WRITE, isBlockWrites) + .build(); + indicesAdmin().create(new CreateIndexRequest(sourceIndex, settings)).actionGet(); + + // call reindex + var destIndex = client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) + .actionGet() + .getDestIndex(); + + // assert read-only settings added back to dest index + var settingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(destIndex)).actionGet(); + assertEquals(isReadOnly, Boolean.parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_READ_ONLY))); + assertEquals(isBlockWrites, Boolean.parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_BLOCKS_WRITE))); + + removeReadOnly(sourceIndex); + removeReadOnly(destIndex); + } + + public void testSettingsAndMappingsFromTemplate() throws IOException { + assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); + + var numShards = randomIntBetween(1, 10); + var numReplicas = randomIntBetween(0, 10); + + var settings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicas) + .build(); + + // Create template with settings and mappings + var template = ComposableIndexTemplate.builder() + .indexPatterns(List.of("logs-*")) + .template(new Template(settings, CompressedXContent.fromJSON(MAPPING), null)) + .build(); + var request = new TransportPutComposableIndexTemplateAction.Request("logs-template"); + request.indexTemplate(template); + client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); + + var sourceIndex = "logs-" + randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + indicesAdmin().create(new CreateIndexRequest(sourceIndex)).actionGet(); + + // call reindex + var destIndex = client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) + .actionGet() + .getDestIndex(); + + // verify settings from templates copied to dest index + { + var settingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(destIndex)).actionGet(); + assertEquals(numReplicas, Integer.parseInt(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_NUMBER_OF_REPLICAS))); + assertEquals(numShards, Integer.parseInt(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_NUMBER_OF_SHARDS))); + } + + // verify mappings from templates copied to dest index + { + var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(sourceIndex, destIndex)).actionGet(); + var destMappings = mappingsResponse.mappings().get(destIndex).sourceAsMap(); + var sourceMappings = mappingsResponse.mappings().get(sourceIndex).sourceAsMap(); + assertEquals(sourceMappings, destMappings); + // sanity check specific value from dest mapping + assertEquals("text", XContentMapValues.extractValue("properties.foo1.type", destMappings)); + } + } + + private static final String TSDB_MAPPING = """ + { + "_doc":{ + "properties": { + "@timestamp" : { + "type": "date" + }, + "metricset": { + "type": "keyword", + "time_series_dimension": true + } + } + } + }"""; + + private static final String TSDB_DOC = """ + { + "@timestamp": "$time", + "metricset": "pod", + "k8s": { + "pod": { + "name": "dog", + "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", + "ip": "10.10.55.3", + "network": { + "tx": 1434595272, + "rx": 530605511 + } + } + } + } + """; + + public void testTsdbStartEndSet() throws Exception { + assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); + + var templateSettings = Settings.builder().put("index.mode", "time_series"); + if (randomBoolean()) { + templateSettings.put("index.routing_path", "metricset"); + } + var mapping = new CompressedXContent(TSDB_MAPPING); + + // create template + var request = new TransportPutComposableIndexTemplateAction.Request("id"); + request.indexTemplate( + ComposableIndexTemplate.builder() + .indexPatterns(List.of("k8s*")) + .template(new Template(templateSettings.build(), mapping, null)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) + .build() + ); + client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); + + // index doc + Instant time = Instant.now(); + String backingIndexName; + { + var indexRequest = new IndexRequest("k8s").opType(DocWriteRequest.OpType.CREATE); + indexRequest.source(TSDB_DOC.replace("$time", formatInstant(time)), XContentType.JSON); + var indexResponse = client().index(indexRequest).actionGet(); + backingIndexName = indexResponse.getIndex(); + } + + var sourceSettings = indicesAdmin().getIndex(new GetIndexRequest().indices(backingIndexName)) + .actionGet() + .getSettings() + .get(backingIndexName); + Instant startTime = IndexSettings.TIME_SERIES_START_TIME.get(sourceSettings); + Instant endTime = IndexSettings.TIME_SERIES_END_TIME.get(sourceSettings); + + // sanity check start/end time on source + assertNotNull(startTime); + assertNotNull(endTime); + assertTrue(endTime.isAfter(startTime)); + + // force a rollover so can call reindex and delete + var rolloverRequest = new RolloverRequest("k8s", null); + var rolloverResponse = indicesAdmin().rolloverIndex(rolloverRequest).actionGet(); + rolloverResponse.getNewIndex(); + + // call reindex on the original backing index + var destIndex = client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(backingIndexName)) + .actionGet() + .getDestIndex(); + + var destSettings = indicesAdmin().getIndex(new GetIndexRequest().indices(destIndex)).actionGet().getSettings().get(destIndex); + var destStart = IndexSettings.TIME_SERIES_START_TIME.get(destSettings); + var destEnd = IndexSettings.TIME_SERIES_END_TIME.get(destSettings); + + assertEquals(startTime, destStart); + assertEquals(endTime, destEnd); + } + + // TODO more logsdb/tsdb specific tests + // TODO more data stream specific tests (how are data streams indices are different from regular indices?) + // TODO check other IndexMetadata fields that need to be fixed after the fact + // TODO what happens if don't have necessary perms for a given index? + + private static void removeReadOnly(String index) { + var settings = Settings.builder() + .put(IndexMetadata.SETTING_READ_ONLY, false) + .put(IndexMetadata.SETTING_BLOCKS_WRITE, false) + .build(); + assertAcked(indicesAdmin().updateSettings(new UpdateSettingsRequest(settings, index)).actionGet()); + } + + private static void indexDocs(String index, int numDocs) { + BulkRequest bulkRequest = new BulkRequest(); + for (int i = 0; i < numDocs; i++) { + String value = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(System.currentTimeMillis()); + bulkRequest.add( + new IndexRequest(index).opType(DocWriteRequest.OpType.CREATE) + .id(i + "") + .source(String.format(Locale.ROOT, "{\"%s\":\"%s\"}", DEFAULT_TIMESTAMP_FIELD, value), XContentType.JSON) + ); + } + BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet(); + assertThat(bulkResponse.getItems().length, equalTo(numDocs)); + indicesAdmin().refresh(new RefreshRequest(index)).actionGet(); + } + + private static String formatInstant(Instant instant) { + return DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(instant); + } + + private static String getIndexUUID(String index) { + return indicesAdmin().getIndex(new GetIndexRequest().indices(index)) + .actionGet() + .getSettings() + .get(index) + .get(IndexMetadata.SETTING_INDEX_UUID); + } +} diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigratePlugin.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigratePlugin.java index 26f8e57102a4d..f42d05727b9fd 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigratePlugin.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigratePlugin.java @@ -37,6 +37,8 @@ import org.elasticsearch.xpack.migrate.action.GetMigrationReindexStatusAction; import org.elasticsearch.xpack.migrate.action.GetMigrationReindexStatusTransportAction; import org.elasticsearch.xpack.migrate.action.ReindexDataStreamAction; +import org.elasticsearch.xpack.migrate.action.ReindexDataStreamIndexAction; +import org.elasticsearch.xpack.migrate.action.ReindexDataStreamIndexTransportAction; import org.elasticsearch.xpack.migrate.action.ReindexDataStreamTransportAction; import org.elasticsearch.xpack.migrate.rest.RestCancelReindexDataStreamAction; import org.elasticsearch.xpack.migrate.rest.RestGetMigrationReindexStatusAction; @@ -84,6 +86,7 @@ public List getRestHandlers( actions.add(new ActionHandler<>(ReindexDataStreamAction.INSTANCE, ReindexDataStreamTransportAction.class)); actions.add(new ActionHandler<>(GetMigrationReindexStatusAction.INSTANCE, GetMigrationReindexStatusTransportAction.class)); actions.add(new ActionHandler<>(CancelReindexDataStreamAction.INSTANCE, CancelReindexDataStreamTransportAction.class)); + actions.add(new ActionHandler<>(ReindexDataStreamIndexAction.INSTANCE, ReindexDataStreamIndexTransportAction.class)); } return actions; } diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamAction.java index 39d4170f6e712..b10bea9e54230 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamAction.java @@ -47,7 +47,7 @@ public class ReindexDataStreamAction extends ActionType { + + public static final String NAME = "indices:admin/data_stream/index/reindex"; + + public static final ActionType INSTANCE = new ReindexDataStreamIndexAction(); + + private ReindexDataStreamIndexAction() { + super(NAME); + } + + public static class Request extends ActionRequest implements IndicesRequest { + + private final String sourceIndex; + + public Request(String sourceIndex) { + this.sourceIndex = sourceIndex; + } + + public Request(StreamInput in) throws IOException { + super(in); + this.sourceIndex = in.readString(); + } + + public static Request readFrom(StreamInput in) throws IOException { + return new Request(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(sourceIndex); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + public String getSourceIndex() { + return sourceIndex; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Objects.equals(sourceIndex, request.sourceIndex); + } + + @Override + public int hashCode() { + return Objects.hash(sourceIndex); + } + + @Override + public String[] indices() { + return new String[] { sourceIndex }; + } + + @Override + public IndicesOptions indicesOptions() { + return IndicesOptions.strictSingleIndexNoExpandForbidClosed(); + } + } + + public static class Response extends ActionResponse { + private final String destIndex; + + public Response(String destIndex) { + this.destIndex = destIndex; + } + + public Response(StreamInput in) throws IOException { + super(in); + this.destIndex = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(destIndex); + } + + public String getDestIndex() { + return destIndex; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Response response = (Response) o; + return Objects.equals(destIndex, response.destIndex); + } + + @Override + public int hashCode() { + return Objects.hash(destIndex); + } + } +} diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java new file mode 100644 index 0000000000000..8863c45691c92 --- /dev/null +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java @@ -0,0 +1,211 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.migrate.action; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.SubscribableListener; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.ReindexAction; +import org.elasticsearch.index.reindex.ReindexRequest; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +public class ReindexDataStreamIndexTransportAction extends HandledTransportAction< + ReindexDataStreamIndexAction.Request, + ReindexDataStreamIndexAction.Response> { + + private static final Logger logger = LogManager.getLogger(ReindexDataStreamIndexTransportAction.class); + + private static final Set SETTINGS_TO_ADD_BACK = Set.of(IndexMetadata.SETTING_BLOCKS_WRITE, IndexMetadata.SETTING_READ_ONLY); + + private static final IndicesOptions IGNORE_MISSING_OPTIONS = IndicesOptions.fromOptions(true, true, false, false); + private final ClusterService clusterService; + private final Client client; + private final IndexScopedSettings indexScopedSettings; + + @Inject + public ReindexDataStreamIndexTransportAction( + TransportService transportService, + ClusterService clusterService, + ActionFilters actionFilters, + Client client, + IndexScopedSettings indexScopedSettings + ) { + super( + ReindexDataStreamIndexAction.NAME, + false, + transportService, + actionFilters, + ReindexDataStreamIndexAction.Request::new, + transportService.getThreadPool().executor(ThreadPool.Names.GENERIC) + ); + this.clusterService = clusterService; + this.client = client; + this.indexScopedSettings = indexScopedSettings; + } + + @Override + protected void doExecute( + Task task, + ReindexDataStreamIndexAction.Request request, + ActionListener listener + ) { + var sourceIndexName = request.getSourceIndex(); + var destIndexName = generateDestIndexName(sourceIndexName); + IndexMetadata sourceIndex = clusterService.state().getMetadata().index(sourceIndexName); + Settings settingsBefore = sourceIndex.getSettings(); + + var hasOldVersion = ReindexDataStreamAction.getOldIndexVersionPredicate(clusterService.state().metadata()); + if (hasOldVersion.test(sourceIndex.getIndex()) == false) { + logger.warn( + "Migrating index [{}] with version [{}] is unnecessary as its version is not before [{}]", + sourceIndexName, + sourceIndex.getCreationVersion(), + ReindexDataStreamAction.MINIMUM_WRITEABLE_VERSION_AFTER_UPGRADE + ); + } + + SubscribableListener.newForked(l -> setBlockWrites(sourceIndexName, l)) + .andThen(l -> deleteDestIfExists(destIndexName, l)) + .andThen(l -> createIndex(sourceIndex, destIndexName, l)) + .andThen(l -> reindex(sourceIndexName, destIndexName, l)) + .andThen(l -> updateSettings(settingsBefore, destIndexName, l)) + .andThenApply(ignored -> new ReindexDataStreamIndexAction.Response(destIndexName)) + .addListener(listener); + } + + private void setBlockWrites(String sourceIndexName, ActionListener listener) { + logger.debug("Setting write block on source index [{}]", sourceIndexName); + final Settings readOnlySettings = Settings.builder().put(IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.getKey(), true).build(); + var updateSettingsRequest = new UpdateSettingsRequest(readOnlySettings, sourceIndexName); + client.admin().indices().updateSettings(updateSettingsRequest, new ActionListener<>() { + @Override + public void onResponse(AcknowledgedResponse response) { + if (response.isAcknowledged()) { + listener.onResponse(null); + } else { + var errorMessage = String.format(Locale.ROOT, "Could not set read-only on source index [%s]", sourceIndexName); + listener.onFailure(new ElasticsearchException(errorMessage)); + } + } + + @Override + public void onFailure(Exception e) { + if (e instanceof ClusterBlockException || e.getCause() instanceof ClusterBlockException) { + // It's fine if read-only is already set + listener.onResponse(null); + } else { + listener.onFailure(e); + } + } + }); + } + + private void deleteDestIfExists(String destIndexName, ActionListener listener) { + logger.debug("Attempting to delete index [{}]", destIndexName); + var deleteIndexRequest = new DeleteIndexRequest(destIndexName).indicesOptions(IGNORE_MISSING_OPTIONS) + .masterNodeTimeout(TimeValue.MAX_VALUE); + var errorMessage = String.format(Locale.ROOT, "Failed to acknowledge delete of index [%s]", destIndexName); + client.admin().indices().delete(deleteIndexRequest, failIfNotAcknowledged(listener, errorMessage)); + } + + private void createIndex(IndexMetadata sourceIndex, String destIndexName, ActionListener listener) { + logger.debug("Creating destination index [{}] for source index [{}]", destIndexName, sourceIndex.getIndex().getName()); + + // Create destination with subset of source index settings that can be added before reindex + var settings = getPreSettings(sourceIndex); + + var sourceMapping = sourceIndex.mapping(); + Map mapping = sourceMapping != null ? sourceMapping.rawSourceAsMap() : Map.of(); + var createIndexRequest = new CreateIndexRequest(destIndexName).settings(settings).mapping(mapping); + + var errorMessage = String.format(Locale.ROOT, "Could not create index [%s]", destIndexName); + client.admin().indices().create(createIndexRequest, failIfNotAcknowledged(listener, errorMessage)); + } + + private void reindex(String sourceIndexName, String destIndexName, ActionListener listener) { + logger.debug("Reindex to destination index [{}] from source index [{}]", destIndexName, sourceIndexName); + var reindexRequest = new ReindexRequest(); + reindexRequest.setSourceIndices(sourceIndexName); + reindexRequest.getSearchRequest().allowPartialSearchResults(false); + reindexRequest.getSearchRequest().source().fetchSource(true); + reindexRequest.setDestIndex(destIndexName); + client.execute(ReindexAction.INSTANCE, reindexRequest, listener); + } + + private void updateSettings(Settings settingsBefore, String destIndexName, ActionListener listener) { + logger.debug("Adding settings from source index that could not be added before reindex"); + + Settings postSettings = getPostSettings(settingsBefore); + if (postSettings.isEmpty()) { + listener.onResponse(null); + return; + } + + var updateSettingsRequest = new UpdateSettingsRequest(postSettings, destIndexName); + var errorMessage = String.format(Locale.ROOT, "Could not update settings on index [%s]", destIndexName); + client.admin().indices().updateSettings(updateSettingsRequest, failIfNotAcknowledged(listener, errorMessage)); + } + + // Filter source index settings to subset of settings that can be included during reindex. + // Similar to the settings filtering done when reindexing for upgrade in Kibana + // https://github.com/elastic/kibana/blob/8a8363f02cc990732eb9cbb60cd388643a336bed/x-pack + // /plugins/upgrade_assistant/server/lib/reindexing/index_settings.ts#L155 + private Settings getPreSettings(IndexMetadata sourceIndex) { + // filter settings that will be added back later + var filtered = sourceIndex.getSettings().filter(settingName -> SETTINGS_TO_ADD_BACK.contains(settingName) == false); + + // filter private and non-copyable settings + var builder = MetadataCreateIndexService.copySettingsFromSource(false, filtered, indexScopedSettings, Settings.builder()); + return builder.build(); + } + + private Settings getPostSettings(Settings settingsBefore) { + return settingsBefore.filter(SETTINGS_TO_ADD_BACK::contains); + } + + public static String generateDestIndexName(String sourceIndex) { + return "migrated-" + sourceIndex; + } + + private static ActionListener failIfNotAcknowledged( + ActionListener listener, + String errorMessage + ) { + return listener.delegateFailureAndWrap((delegate, response) -> { + if (response.isAcknowledged()) { + delegate.onResponse(null); + } + throw new ElasticsearchException(errorMessage); + }); + } +} diff --git a/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexRequestTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexRequestTests.java new file mode 100644 index 0000000000000..a057056474ef1 --- /dev/null +++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexRequestTests.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.migrate.action; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.migrate.action.ReindexDataStreamIndexAction.Request; + +public class ReindexDatastreamIndexRequestTests extends AbstractWireSerializingTestCase { + @Override + protected Writeable.Reader instanceReader() { + return Request::new; + } + + @Override + protected Request createTestInstance() { + return new Request(randomAlphaOfLength(20)); + } + + @Override + protected Request mutateInstance(Request instance) { + return new ReindexDataStreamIndexAction.Request(randomValueOtherThan(instance.getSourceIndex(), () -> randomAlphaOfLength(20))); + } +} diff --git a/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexResponseTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexResponseTests.java new file mode 100644 index 0000000000000..752e173585f0e --- /dev/null +++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexResponseTests.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.migrate.action; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.migrate.action.ReindexDataStreamIndexAction.Response; + +public class ReindexDatastreamIndexResponseTests extends AbstractWireSerializingTestCase { + @Override + protected Writeable.Reader instanceReader() { + return Response::new; + } + + @Override + protected Response createTestInstance() { + return new Response(randomAlphaOfLength(20)); + } + + @Override + protected Response mutateInstance(Response instance) { + return new Response(randomValueOtherThan(instance.getDestIndex(), () -> randomAlphaOfLength(20))); + } +} diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index b139d1526ec20..c07e4b2c541a2 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -638,6 +638,7 @@ public class Constants { "internal:admin/indices/prevalidate_shard_path", "internal:index/metadata/migration_version/update", new FeatureFlag("reindex_data_stream").isEnabled() ? "indices:admin/migration/reindex_status" : null, + new FeatureFlag("reindex_data_stream").isEnabled() ? "indices:admin/data_stream/index/reindex" : null, new FeatureFlag("reindex_data_stream").isEnabled() ? "indices:admin/data_stream/reindex" : null, new FeatureFlag("reindex_data_stream").isEnabled() ? "indices:admin/data_stream/reindex_cancel" : null, "internal:admin/repository/verify", From e454d74de4e7eaf7054c3992632b4004e864ca7c Mon Sep 17 00:00:00 2001 From: Mark Vieira Date: Fri, 13 Dec 2024 13:13:44 -0800 Subject: [PATCH 22/35] Add optional pull request check for running with entitlements enabled (#118693) --- .../pipelines/pull-request/part-1-entitlements.yml | 11 +++++++++++ .../pipelines/pull-request/part-2-entitlements.yml | 11 +++++++++++ .../pipelines/pull-request/part-3-entitlements.yml | 11 +++++++++++ .../pipelines/pull-request/part-4-entitlements.yml | 11 +++++++++++ .../pipelines/pull-request/part-5-entitlements.yml | 11 +++++++++++ .../gradle/testclusters/ElasticsearchNode.java | 6 ++---- 6 files changed, 57 insertions(+), 4 deletions(-) create mode 100644 .buildkite/pipelines/pull-request/part-1-entitlements.yml create mode 100644 .buildkite/pipelines/pull-request/part-2-entitlements.yml create mode 100644 .buildkite/pipelines/pull-request/part-3-entitlements.yml create mode 100644 .buildkite/pipelines/pull-request/part-4-entitlements.yml create mode 100644 .buildkite/pipelines/pull-request/part-5-entitlements.yml diff --git a/.buildkite/pipelines/pull-request/part-1-entitlements.yml b/.buildkite/pipelines/pull-request/part-1-entitlements.yml new file mode 100644 index 0000000000000..abb9edf67484f --- /dev/null +++ b/.buildkite/pipelines/pull-request/part-1-entitlements.yml @@ -0,0 +1,11 @@ +config: + allow-labels: "test-entitlements" +steps: + - label: part-1-entitlements + command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.jvm.argline="-Des.entitlements.enabled=true" checkPart1 + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk diff --git a/.buildkite/pipelines/pull-request/part-2-entitlements.yml b/.buildkite/pipelines/pull-request/part-2-entitlements.yml new file mode 100644 index 0000000000000..ef889f3819c5b --- /dev/null +++ b/.buildkite/pipelines/pull-request/part-2-entitlements.yml @@ -0,0 +1,11 @@ +config: + allow-labels: "test-entitlements" +steps: + - label: part-2-entitlements + command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.jvm.argline="-Des.entitlements.enabled=true" checkPart2 + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk diff --git a/.buildkite/pipelines/pull-request/part-3-entitlements.yml b/.buildkite/pipelines/pull-request/part-3-entitlements.yml new file mode 100644 index 0000000000000..c31ae5e6a4ce3 --- /dev/null +++ b/.buildkite/pipelines/pull-request/part-3-entitlements.yml @@ -0,0 +1,11 @@ +config: + allow-labels: "test-entitlements" +steps: + - label: part-3-entitlements + command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.jvm.argline="-Des.entitlements.enabled=true" checkPart3 + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk diff --git a/.buildkite/pipelines/pull-request/part-4-entitlements.yml b/.buildkite/pipelines/pull-request/part-4-entitlements.yml new file mode 100644 index 0000000000000..5817dacbad80b --- /dev/null +++ b/.buildkite/pipelines/pull-request/part-4-entitlements.yml @@ -0,0 +1,11 @@ +config: + allow-labels: "test-entitlements" +steps: + - label: part-4-entitlements + command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.jvm.argline="-Des.entitlements.enabled=true" checkPart4 + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk diff --git a/.buildkite/pipelines/pull-request/part-5-entitlements.yml b/.buildkite/pipelines/pull-request/part-5-entitlements.yml new file mode 100644 index 0000000000000..5a92282361576 --- /dev/null +++ b/.buildkite/pipelines/pull-request/part-5-entitlements.yml @@ -0,0 +1,11 @@ +config: + allow-labels: "test-entitlements" +steps: + - label: part-5-entitlements + command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.jvm.argline="-Des.entitlements.enabled=true" checkPart5 + timeout_in_minutes: 300 + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java index 4cb67e249b0b0..2f0f85bc6e159 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java @@ -876,10 +876,8 @@ private void startElasticsearchProcess() { // Don't inherit anything from the environment for as that would lack reproducibility environment.clear(); environment.putAll(getESEnvironment()); - if (cliJvmArgs.isEmpty() == false) { - String cliJvmArgsString = String.join(" ", cliJvmArgs); - environment.put("CLI_JAVA_OPTS", cliJvmArgsString); - } + String cliJvmArgsString = String.join(" ", cliJvmArgs); + environment.put("CLI_JAVA_OPTS", cliJvmArgsString + " " + System.getProperty("tests.jvm.argline", "")); // Direct the stderr to the ES log file. This should capture any jvm problems to start. // Stdout is discarded because ES duplicates the log file to stdout when run in the foreground. From ac0ba24847a6f09ef8686659cd16ca74bc1d79d1 Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Fri, 13 Dec 2024 13:17:56 -0800 Subject: [PATCH 23/35] ESQL: Push down filter passed lookup join (#118410) Improve the planner to detect filters that can be pushed down 'through' a LOOKUP JOIN by determining the conditions scoped to the left/main side and moving them closer to the source. Relates #118305 --- docs/changelog/118410.yaml | 5 + .../core/expression/predicate/Predicates.java | 2 +- .../src/main/resources/lookup-join.csv-spec | 65 +++++ .../logical/PushDownAndCombineFilters.java | 55 ++++ .../optimizer/LogicalPlanOptimizerTests.java | 245 +++++++++++++++++- 5 files changed, 369 insertions(+), 3 deletions(-) create mode 100644 docs/changelog/118410.yaml diff --git a/docs/changelog/118410.yaml b/docs/changelog/118410.yaml new file mode 100644 index 0000000000000..ccc7f71ee2e1c --- /dev/null +++ b/docs/changelog/118410.yaml @@ -0,0 +1,5 @@ +pr: 118410 +summary: Push down filter passed lookup join +area: ES|QL +type: enhancement +issues: [] diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Predicates.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Predicates.java index e63cc1fcf25fe..32f7e181933b4 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Predicates.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Predicates.java @@ -61,7 +61,7 @@ public static Expression combineAnd(List exps) { * * using the given combiner. * - * While a bit longer, this method creates a balanced tree as oppose to a plain + * While a bit longer, this method creates a balanced tree as opposed to a plain * recursive approach which creates an unbalanced one (either to the left or right). */ private static Expression combine(List exps, BiFunction combiner) { diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index f6704d33934af..c39f4ae7b4e0c 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -418,3 +418,68 @@ count:long | type:keyword 3 | Success 1 | Disconnected ; + +// +// Filtering tests +// + +lookupWithFilterOnLeftSideField +required_capability: join_lookup_v5 + +FROM employees +| EVAL language_code = languages +| LOOKUP JOIN languages_lookup ON language_code +| SORT emp_no +| KEEP emp_no, language_code, language_name +| WHERE emp_no >= 10091 AND emp_no < 10094 +; + +emp_no:integer | language_code:integer | language_name:keyword +10091 | 3 | Spanish +10092 | 1 | English +10093 | 3 | Spanish +; + +lookupMessageWithFilterOnRightSideField-Ignore +required_capability: join_lookup_v5 + +FROM sample_data +| LOOKUP JOIN message_types_lookup ON message +| WHERE type == "Error" +| KEEP @timestamp, client_ip, event_duration, message, type +| SORT @timestamp DESC +; + +@timestamp:date | client_ip:ip | event_duration:long | message:keyword | type:keyword +2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error | Error +2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error | Error +2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error | Error +; + +lookupWithFieldAndRightSideAfterStats +required_capability: join_lookup_v5 + +FROM sample_data +| LOOKUP JOIN message_types_lookup ON message +| STATS count = count(message) BY type +| WHERE type == "Error" +; + +count:long | type:keyword +3 | Error +; + +lookupWithFieldOnJoinKey-Ignore +required_capability: join_lookup_v5 + +FROM employees +| EVAL language_code = languages +| LOOKUP JOIN languages_lookup ON language_code +| WHERE language_code > 1 AND language_name IS NOT NULL +| KEEP emp_no, language_code, language_name +; + +emp_no:integer | language_code:integer | language_name:keyword +10001 | 2 | French +10003 | 4 | German +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFilters.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFilters.java index 15e49c22a44db..9ec902e729f54 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFilters.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFilters.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.expression.predicate.Predicates; +import org.elasticsearch.xpack.esql.core.util.CollectionUtils; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.Filter; @@ -23,6 +24,8 @@ import org.elasticsearch.xpack.esql.plan.logical.Project; import org.elasticsearch.xpack.esql.plan.logical.RegexExtract; import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan; +import org.elasticsearch.xpack.esql.plan.logical.join.Join; +import org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes; import java.util.ArrayList; import java.util.List; @@ -76,11 +79,63 @@ protected LogicalPlan rule(Filter filter) { } else if (child instanceof OrderBy orderBy) { // swap the filter with its child plan = orderBy.replaceChild(filter.with(orderBy.child(), condition)); + } else if (child instanceof Join join) { + return pushDownPastJoin(filter, join); } // cannot push past a Limit, this could change the tailing result set returned return plan; } + private record ScopedFilter(List commonFilters, List leftFilters, List rightFilters) {} + + // split the filter condition in 3 parts: + // 1. filter scoped to the left + // 2. filter scoped to the right + // 3. filter that requires both sides to be evaluated + private static ScopedFilter scopeFilter(List filters, LogicalPlan left, LogicalPlan right) { + List rest = new ArrayList<>(filters); + List leftFilters = new ArrayList<>(); + List rightFilters = new ArrayList<>(); + + AttributeSet leftOutput = left.outputSet(); + AttributeSet rightOutput = right.outputSet(); + + // first remove things that are left scoped only + rest.removeIf(f -> f.references().subsetOf(leftOutput) && leftFilters.add(f)); + // followed by right scoped only + rest.removeIf(f -> f.references().subsetOf(rightOutput) && rightFilters.add(f)); + return new ScopedFilter(rest, leftFilters, rightFilters); + } + + private static LogicalPlan pushDownPastJoin(Filter filter, Join join) { + LogicalPlan plan = filter; + // pushdown only through LEFT joins + // TODO: generalize this for other join types + if (join.config().type() == JoinTypes.LEFT) { + LogicalPlan left = join.left(); + LogicalPlan right = join.right(); + + // split the filter condition in 3 parts: + // 1. filter scoped to the left + // 2. filter scoped to the right + // 3. filter that requires both sides to be evaluated + ScopedFilter scoped = scopeFilter(Predicates.splitAnd(filter.condition()), left, right); + // push the left scoped filter down to the left child, keep the rest intact + if (scoped.leftFilters.size() > 0) { + // push the filter down to the left child + left = new Filter(left.source(), left, Predicates.combineAnd(scoped.leftFilters)); + // update the join with the new left child + join = (Join) join.replaceLeft(left); + + // keep the remaining filters in place, otherwise return the new join; + Expression remainingFilter = Predicates.combineAnd(CollectionUtils.combine(scoped.commonFilters, scoped.rightFilters)); + plan = remainingFilter != null ? filter.with(join, remainingFilter) : join; + } + } + // ignore the rest of the join + return plan; + } + private static Function NO_OP = expression -> expression; private static LogicalPlan maybePushDownPastUnary( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index d97a8bb2bc27f..0a01dfc088cfb 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.optimizer; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.Build; import org.elasticsearch.common.logging.LoggerMessageFormat; import org.elasticsearch.common.lucene.BytesRefs; @@ -217,6 +218,11 @@ public static void init() { enrichResolution = new EnrichResolution(); AnalyzerTestUtils.loadEnrichPolicyResolution(enrichResolution, "languages_idx", "id", "languages_idx", "mapping-languages.json"); + var lookupMapping = loadMapping("mapping-languages.json"); + IndexResolution lookupResolution = IndexResolution.valid( + new EsIndex("language_code", lookupMapping, Map.of("language_code", IndexMode.LOOKUP)) + ); + // Most tests used data from the test index, so we load it here, and use it in the plan() function. mapping = loadMapping("mapping-basic.json"); EsIndex test = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD)); @@ -5740,7 +5746,7 @@ public void testLookupSimple() { String query = """ FROM test | RENAME languages AS int - | LOOKUP int_number_names ON int"""; + | LOOKUP_?? int_number_names ON int"""; if (Build.current().isSnapshot() == false) { var e = expectThrows(ParsingException.class, () -> analyze(query)); assertThat(e.getMessage(), containsString("line 3:3: mismatched input 'LOOKUP' expecting {")); @@ -5820,7 +5826,7 @@ public void testLookupStats() { String query = """ FROM test | RENAME languages AS int - | LOOKUP int_number_names ON int + | LOOKUP_?? int_number_names ON int | STATS MIN(emp_no) BY name"""; if (Build.current().isSnapshot() == false) { var e = expectThrows(ParsingException.class, () -> analyze(query)); @@ -5889,6 +5895,241 @@ public void testLookupStats() { ); } + // + // Lookup JOIN + // + + /** + * Filter on join keys should be pushed down + * Expects + * Project[[_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, gender{f}#9, hire_date{f}#14, job{f}#15, job.raw{f}#16, lang + * uage_code{r}#4, last_name{f}#11, long_noidx{f}#17, salary{f}#12, language_name{f}#19]] + * \_Join[LEFT,[language_code{r}#4],[language_code{r}#4],[language_code{f}#18]] + * |_EsqlProject[[_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, gender{f}#9, hire_date{f}#14, job{f}#15, job.raw{f}#16, lang + * uages{f}#10 AS language_code, last_name{f}#11, long_noidx{f}#17, salary{f}#12]] + * | \_Limit[1000[INTEGER]] + * | \_Filter[languages{f}#10 > 1[INTEGER]] + * | \_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..] + * \_EsRelation[language_code][LOOKUP][language_code{f}#18, language_name{f}#19] + */ + public void testLookupJoinPushDownFilterOnJoinKeyWithRename() { + String query = """ + FROM test + | RENAME languages AS language_code + | LOOKUP JOIN language_code ON language_code + | WHERE language_code > 1 + """; + var plan = optimizedPlan(query); + + var project = as(plan, Project.class); + var join = as(project.child(), Join.class); + assertThat(join.config().type(), equalTo(JoinTypes.LEFT)); + project = as(join.left(), Project.class); + var limit = as(project.child(), Limit.class); + assertThat(limit.limit().fold(), equalTo(1000)); + var filter = as(limit.child(), Filter.class); + // assert that the rename has been undone + var op = as(filter.condition(), GreaterThan.class); + var field = as(op.left(), FieldAttribute.class); + assertThat(field.name(), equalTo("languages")); + + var literal = as(op.right(), Literal.class); + assertThat(literal.value(), equalTo(1)); + + var leftRel = as(filter.child(), EsRelation.class); + var rightRel = as(join.right(), EsRelation.class); + } + + /** + * Filter on on left side fields (outside the join key) should be pushed down + * Expects + * Project[[_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, gender{f}#9, hire_date{f}#14, job{f}#15, job.raw{f}#16, lang + * uage_code{r}#4, last_name{f}#11, long_noidx{f}#17, salary{f}#12, language_name{f}#19]] + * \_Join[LEFT,[language_code{r}#4],[language_code{r}#4],[language_code{f}#18]] + * |_EsqlProject[[_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, gender{f}#9, hire_date{f}#14, job{f}#15, job.raw{f}#16, lang + * uages{f}#10 AS language_code, last_name{f}#11, long_noidx{f}#17, salary{f}#12]] + * | \_Limit[1000[INTEGER]] + * | \_Filter[emp_no{f}#7 > 1[INTEGER]] + * | \_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..] + * \_EsRelation[language_code][LOOKUP][language_code{f}#18, language_name{f}#19] + */ + public void testLookupJoinPushDownFilterOnLeftSideField() { + String query = """ + FROM test + | RENAME languages AS language_code + | LOOKUP JOIN language_code ON language_code + | WHERE emp_no > 1 + """; + + var plan = optimizedPlan(query); + + var project = as(plan, Project.class); + var join = as(project.child(), Join.class); + assertThat(join.config().type(), equalTo(JoinTypes.LEFT)); + project = as(join.left(), Project.class); + + var limit = as(project.child(), Limit.class); + assertThat(limit.limit().fold(), equalTo(1000)); + var filter = as(limit.child(), Filter.class); + var op = as(filter.condition(), GreaterThan.class); + var field = as(op.left(), FieldAttribute.class); + assertThat(field.name(), equalTo("emp_no")); + + var literal = as(op.right(), Literal.class); + assertThat(literal.value(), equalTo(1)); + + var leftRel = as(filter.child(), EsRelation.class); + var rightRel = as(join.right(), EsRelation.class); + } + + /** + * Filter works on the right side fields and thus cannot be pushed down + * Expects + * Project[[_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, gender{f}#9, hire_date{f}#14, job{f}#15, job.raw{f}#16, lang + * uage_code{r}#4, last_name{f}#11, long_noidx{f}#17, salary{f}#12, language_name{f}#19]] + * \_Limit[1000[INTEGER]] + * \_Filter[language_name{f}#19 == [45 6e 67 6c 69 73 68][KEYWORD]] + * \_Join[LEFT,[language_code{r}#4],[language_code{r}#4],[language_code{f}#18]] + * |_EsqlProject[[_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, gender{f}#9, hire_date{f}#14, job{f}#15, job.raw{f}#16, lang + * uages{f}#10 AS language_code, last_name{f}#11, long_noidx{f}#17, salary{f}#12]] + * | \_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..] + * \_EsRelation[language_code][LOOKUP][language_code{f}#18, language_name{f}#19] + */ + public void testLookupJoinPushDownDisabledForLookupField() { + String query = """ + FROM test + | RENAME languages AS language_code + | LOOKUP JOIN language_code ON language_code + | WHERE language_name == "English" + """; + + var plan = optimizedPlan(query); + + var project = as(plan, Project.class); + var limit = as(project.child(), Limit.class); + assertThat(limit.limit().fold(), equalTo(1000)); + + var filter = as(limit.child(), Filter.class); + var op = as(filter.condition(), Equals.class); + var field = as(op.left(), FieldAttribute.class); + assertThat(field.name(), equalTo("language_name")); + var literal = as(op.right(), Literal.class); + assertThat(literal.value(), equalTo(new BytesRef("English"))); + + var join = as(filter.child(), Join.class); + assertThat(join.config().type(), equalTo(JoinTypes.LEFT)); + project = as(join.left(), Project.class); + + var leftRel = as(project.child(), EsRelation.class); + var rightRel = as(join.right(), EsRelation.class); + } + + /** + * Split the conjunction into pushable and non pushable filters. + * Expects + * Project[[_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, gender{f}#10, hire_date{f}#15, job{f}#16, job.raw{f}#17, lan + * guage_code{r}#4, last_name{f}#12, long_noidx{f}#18, salary{f}#13, language_name{f}#20]] + * \_Limit[1000[INTEGER]] + * \_Filter[language_name{f}#20 == [45 6e 67 6c 69 73 68][KEYWORD]] + * \_Join[LEFT,[language_code{r}#4],[language_code{r}#4],[language_code{f}#19]] + * |_EsqlProject[[_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, gender{f}#10, hire_date{f}#15, job{f}#16, job.raw{f}#17, lan + * guages{f}#11 AS language_code, last_name{f}#12, long_noidx{f}#18, salary{f}#13]] + * | \_Filter[emp_no{f}#8 > 1[INTEGER]] + * | \_EsRelation[test][_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, ge..] + * \_EsRelation[language_code][LOOKUP][language_code{f}#19, language_name{f}#20] + */ + public void testLookupJoinPushDownSeparatedForConjunctionBetweenLeftAndRightField() { + String query = """ + FROM test + | RENAME languages AS language_code + | LOOKUP JOIN language_code ON language_code + | WHERE language_name == "English" AND emp_no > 1 + """; + + var plan = optimizedPlan(query); + + var project = as(plan, Project.class); + var limit = as(project.child(), Limit.class); + assertThat(limit.limit().fold(), equalTo(1000)); + // filter kept in place, working on the right side + var filter = as(limit.child(), Filter.class); + EsqlBinaryComparison op = as(filter.condition(), Equals.class); + var field = as(op.left(), FieldAttribute.class); + assertThat(field.name(), equalTo("language_name")); + var literal = as(op.right(), Literal.class); + assertThat(literal.value(), equalTo(new BytesRef("English"))); + + var join = as(filter.child(), Join.class); + assertThat(join.config().type(), equalTo(JoinTypes.LEFT)); + project = as(join.left(), Project.class); + // filter pushed down + filter = as(project.child(), Filter.class); + op = as(filter.condition(), GreaterThan.class); + field = as(op.left(), FieldAttribute.class); + assertThat(field.name(), equalTo("emp_no")); + + literal = as(op.right(), Literal.class); + assertThat(literal.value(), equalTo(1)); + + var leftRel = as(filter.child(), EsRelation.class); + var rightRel = as(join.right(), EsRelation.class); + + } + + /** + * Disjunctions however keep the filter in place, even on pushable fields + * Expects + * Project[[_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, gender{f}#10, hire_date{f}#15, job{f}#16, job.raw{f}#17, lan + * guage_code{r}#4, last_name{f}#12, long_noidx{f}#18, salary{f}#13, language_name{f}#20]] + * \_Limit[1000[INTEGER]] + * \_Filter[language_name{f}#20 == [45 6e 67 6c 69 73 68][KEYWORD] OR emp_no{f}#8 > 1[INTEGER]] + * \_Join[LEFT,[language_code{r}#4],[language_code{r}#4],[language_code{f}#19]] + * |_EsqlProject[[_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, gender{f}#10, hire_date{f}#15, job{f}#16, job.raw{f}#17, lan + * guages{f}#11 AS language_code, last_name{f}#12, long_noidx{f}#18, salary{f}#13]] + * | \_EsRelation[test][_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, ge..] + * \_EsRelation[language_code][LOOKUP][language_code{f}#19, language_name{f}#20] + */ + public void testLookupJoinPushDownDisabledForDisjunctionBetweenLeftAndRightField() { + String query = """ + FROM test + | RENAME languages AS language_code + | LOOKUP JOIN language_code ON language_code + | WHERE language_name == "English" OR emp_no > 1 + """; + + var plan = optimizedPlan(query); + + var project = as(plan, Project.class); + var limit = as(project.child(), Limit.class); + assertThat(limit.limit().fold(), equalTo(1000)); + + var filter = as(limit.child(), Filter.class); + var or = as(filter.condition(), Or.class); + EsqlBinaryComparison op = as(or.left(), Equals.class); + // OR left side + var field = as(op.left(), FieldAttribute.class); + assertThat(field.name(), equalTo("language_name")); + var literal = as(op.right(), Literal.class); + assertThat(literal.value(), equalTo(new BytesRef("English"))); + // OR right side + op = as(or.right(), GreaterThan.class); + field = as(op.left(), FieldAttribute.class); + assertThat(field.name(), equalTo("emp_no")); + literal = as(op.right(), Literal.class); + assertThat(literal.value(), equalTo(1)); + + var join = as(filter.child(), Join.class); + assertThat(join.config().type(), equalTo(JoinTypes.LEFT)); + project = as(join.left(), Project.class); + + var leftRel = as(project.child(), EsRelation.class); + var rightRel = as(join.right(), EsRelation.class); + } + + // + // + // + public void testTranslateMetricsWithoutGrouping() { assumeTrue("requires snapshot builds", Build.current().isSnapshot()); var query = "METRICS k8s max(rate(network.total_bytes_in))"; From 42792812c17646413f6346c2b4acbac35dfef44f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mariusz=20J=C3=B3zala?= <377355+jozala@users.noreply.github.com> Date: Fri, 13 Dec 2024 22:49:52 +0100 Subject: [PATCH 24/35] [ci] Add oraclelinux-9 to matrix in packaging and platform jobs (#118565) --- .buildkite/pipelines/periodic-packaging.template.yml | 1 + .buildkite/pipelines/periodic-packaging.yml | 1 + .buildkite/pipelines/periodic-platform-support.yml | 1 + .buildkite/pipelines/pull-request/packaging-tests-unix.yml | 1 + 4 files changed, 4 insertions(+) diff --git a/.buildkite/pipelines/periodic-packaging.template.yml b/.buildkite/pipelines/periodic-packaging.template.yml index b00ef51ec8f07..c42c999e08c65 100644 --- a/.buildkite/pipelines/periodic-packaging.template.yml +++ b/.buildkite/pipelines/periodic-packaging.template.yml @@ -10,6 +10,7 @@ steps: - debian-12 - opensuse-leap-15 - oraclelinux-8 + - oraclelinux-9 - sles-15 - ubuntu-2004 - ubuntu-2204 diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index 9619de3c2c98b..3c74806315922 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -11,6 +11,7 @@ steps: - debian-12 - opensuse-leap-15 - oraclelinux-8 + - oraclelinux-9 - sles-15 - ubuntu-2004 - ubuntu-2204 diff --git a/.buildkite/pipelines/periodic-platform-support.yml b/.buildkite/pipelines/periodic-platform-support.yml index 33e9a040b22cf..63d422da0f53f 100644 --- a/.buildkite/pipelines/periodic-platform-support.yml +++ b/.buildkite/pipelines/periodic-platform-support.yml @@ -10,6 +10,7 @@ steps: - debian-12 - opensuse-leap-15 - oraclelinux-8 + - oraclelinux-9 - sles-15 - ubuntu-2004 - ubuntu-2204 diff --git a/.buildkite/pipelines/pull-request/packaging-tests-unix.yml b/.buildkite/pipelines/pull-request/packaging-tests-unix.yml index 3b7973d05a338..a4ad053953509 100644 --- a/.buildkite/pipelines/pull-request/packaging-tests-unix.yml +++ b/.buildkite/pipelines/pull-request/packaging-tests-unix.yml @@ -13,6 +13,7 @@ steps: - debian-12 - opensuse-leap-15 - oraclelinux-8 + - oraclelinux-9 - sles-15 - ubuntu-2004 - ubuntu-2204 From 5b9ffefaf6d44d777f7c9a36446e7afb44783436 Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Fri, 13 Dec 2024 16:25:23 -0600 Subject: [PATCH 25/35] Avoid updating settings version in MetadataMigrateToDataStreamService when settings have not changed (#118704) If the input index already has the `index.hidden` setting set to `true`, MetadataMigrateToDataStreamService::prepareBackingIndex can incorrectly increment the settings version even if it does not change the settings. This results in an assertion failure in IndexService::updateMetadata that will take down a node if assertions are enabled. This fixes that, only incrementing the settings version if the settings actually changed. --- docs/changelog/118704.yaml | 6 ++ .../MetadataMigrateToDataStreamService.java | 9 +- ...tadataMigrateToDataStreamServiceTests.java | 82 +++++++++++++++++++ 3 files changed, 94 insertions(+), 3 deletions(-) create mode 100644 docs/changelog/118704.yaml diff --git a/docs/changelog/118704.yaml b/docs/changelog/118704.yaml new file mode 100644 index 0000000000000..c89735664f25e --- /dev/null +++ b/docs/changelog/118704.yaml @@ -0,0 +1,6 @@ +pr: 118704 +summary: Avoid updating settings version in `MetadataMigrateToDataStreamService` when + settings have not changed +area: Data streams +type: bug +issues: [] diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java index aee60c3eda57f..39acc6d3f6311 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java @@ -29,6 +29,7 @@ import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.DocumentMapper; @@ -250,9 +251,11 @@ static void prepareBackingIndex( DataStreamFailureStoreDefinition.applyFailureStoreSettings(nodeSettings, settingsUpdate); } - imb.settings(settingsUpdate.build()) - .settingsVersion(im.getSettingsVersion() + 1) - .mappingVersion(im.getMappingVersion() + 1) + Settings maybeUpdatedSettings = settingsUpdate.build(); + if (IndexSettings.same(im.getSettings(), maybeUpdatedSettings) == false) { + imb.settings(maybeUpdatedSettings).settingsVersion(im.getSettingsVersion() + 1); + } + imb.mappingVersion(im.getMappingVersion() + 1) .mappingsUpdatedVersion(IndexVersion.current()) .putMapping(new MappingMetadata(mapper)); b.put(imb); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java index 8be0f4de15500..63e92835ba8db 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java @@ -24,11 +24,13 @@ import java.io.IOException; import java.util.List; import java.util.Set; +import java.util.function.Function; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.generateMapping; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -425,6 +427,86 @@ public void testCreateDataStreamWithoutSuppliedWriteIndex() { assertThat(e.getMessage(), containsString("alias [" + dataStreamName + "] must specify a write index")); } + public void testSettingsVersion() throws IOException { + /* + * This tests that applyFailureStoreSettings updates the settings version when the settings have been modified, and does not change + * it otherwise. Incrementing the settings version when the settings have not changed can result in an assertion failing in + * IndexService::updateMetadata. + */ + String indexName = randomAlphaOfLength(30); + String dataStreamName = randomAlphaOfLength(50); + Function mapperSupplier = this::getMapperService; + boolean removeAlias = randomBoolean(); + boolean failureStore = randomBoolean(); + Settings nodeSettings = Settings.EMPTY; + + { + /* + * Here the input indexMetadata will have the index.hidden setting set to true. So we expect no change to the settings, and + * for the settings version to remain the same + */ + Metadata.Builder metadataBuilder = Metadata.builder(); + Settings indexMetadataSettings = Settings.builder() + .put(IndexMetadata.SETTING_INDEX_HIDDEN, true) + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .build(); + IndexMetadata indexMetadata = IndexMetadata.builder(indexName) + .settings(indexMetadataSettings) + .numberOfShards(1) + .numberOfReplicas(0) + .putMapping(getTestMappingWithTimestamp()) + .build(); + MetadataMigrateToDataStreamService.prepareBackingIndex( + metadataBuilder, + indexMetadata, + dataStreamName, + mapperSupplier, + removeAlias, + failureStore, + nodeSettings + ); + Metadata metadata = metadataBuilder.build(); + assertThat(indexMetadata.getSettings(), equalTo(metadata.index(indexName).getSettings())); + assertThat(metadata.index(indexName).getSettingsVersion(), equalTo(indexMetadata.getSettingsVersion())); + } + { + /* + * Here the input indexMetadata will not have the index.hidden setting set to true. So prepareBackingIndex will add that, + * meaning that the settings and settings version will change. + */ + Metadata.Builder metadataBuilder = Metadata.builder(); + Settings indexMetadataSettings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build(); + IndexMetadata indexMetadata = IndexMetadata.builder(indexName) + .settings(indexMetadataSettings) + .numberOfShards(1) + .numberOfReplicas(0) + .putMapping(getTestMappingWithTimestamp()) + .build(); + MetadataMigrateToDataStreamService.prepareBackingIndex( + metadataBuilder, + indexMetadata, + dataStreamName, + mapperSupplier, + removeAlias, + failureStore, + nodeSettings + ); + Metadata metadata = metadataBuilder.build(); + assertThat(indexMetadata.getSettings(), not(equalTo(metadata.index(indexName).getSettings()))); + assertThat(metadata.index(indexName).getSettingsVersion(), equalTo(indexMetadata.getSettingsVersion() + 1)); + } + } + + private String getTestMappingWithTimestamp() { + return """ + { + "properties": { + "@timestamp": {"type": "date"} + } + } + """; + } + private MapperService getMapperService(IndexMetadata im) { try { return createMapperService("{\"_doc\": " + im.mapping().source().toString() + "}"); From f900ae61bbc3ba9670a832206f4a6552013f67c4 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 13 Dec 2024 17:31:59 -0500 Subject: [PATCH 26/35] [CI] Automatically apply spotless changes in pull request CI (#118701) --- .buildkite/pipelines/pull-request/part-1.yml | 4 +- .../pipelines/pull-request/precommit.yml | 4 +- .buildkite/scripts/spotless.sh | 44 +++++++++++++++++++ 3 files changed, 50 insertions(+), 2 deletions(-) create mode 100755 .buildkite/scripts/spotless.sh diff --git a/.buildkite/pipelines/pull-request/part-1.yml b/.buildkite/pipelines/pull-request/part-1.yml index 3d467c6c41e43..7a09e2a162ff8 100644 --- a/.buildkite/pipelines/pull-request/part-1.yml +++ b/.buildkite/pipelines/pull-request/part-1.yml @@ -1,6 +1,8 @@ steps: - label: part-1 - command: .ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart1 + command: | + .buildkite/scripts/spotless.sh # This doesn't have to be part of part-1, it was just a convenient place to put it + .ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart1 timeout_in_minutes: 300 agents: provider: gcp diff --git a/.buildkite/pipelines/pull-request/precommit.yml b/.buildkite/pipelines/pull-request/precommit.yml index f6548dfeed9b2..1763758932581 100644 --- a/.buildkite/pipelines/pull-request/precommit.yml +++ b/.buildkite/pipelines/pull-request/precommit.yml @@ -3,7 +3,9 @@ config: skip-labels: [] steps: - label: precommit - command: .ci/scripts/run-gradle.sh -Dignore.tests.seed precommit + command: | + .buildkite/scripts/spotless.sh + .ci/scripts/run-gradle.sh -Dignore.tests.seed precommit timeout_in_minutes: 300 agents: provider: gcp diff --git a/.buildkite/scripts/spotless.sh b/.buildkite/scripts/spotless.sh new file mode 100755 index 0000000000000..b9e6094edb2c7 --- /dev/null +++ b/.buildkite/scripts/spotless.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +if [[ -z "${BUILDKITE_PULL_REQUEST:-}" ]]; then + echo "Not a pull request, skipping spotless" + exit 0 +fi + +if ! git diff --exit-code; then + echo "Changes are present before running spotless, not running" + git status + exit 0 +fi + +NEW_COMMIT_MESSAGE="[CI] Auto commit changes from spotless" +PREVIOUS_COMMIT_MESSAGE="$(git log -1 --pretty=%B)" + +echo "--- Running spotless" +.ci/scripts/run-gradle.sh -Dscan.tag.NESTED spotlessApply + +if git diff --exit-code; then + echo "No changes found after running spotless. Don't need to auto commit." + exit 0 +fi + +if [[ "$NEW_COMMIT_MESSAGE" == "$PREVIOUS_COMMIT_MESSAGE" ]]; then + echo "Changes found after running spotless" + echo "CI already attempted to commit these changes, but the file(s) seem to have changed again." + echo "Please review and fix manually." + exit 1 +fi + +git config --global user.name elasticsearchmachine +git config --global user.email 'infra-root+elasticsearchmachine@elastic.co' + +gh pr checkout "${BUILDKITE_PULL_REQUEST}" +git add -u . +git commit -m "$NEW_COMMIT_MESSAGE" +git push + +# After the git push, the new commit will trigger a new build within a few seconds and this build should get cancelled +# So, let's just sleep to give the build time to cancel itself without an error +# If it doesn't get cancelled for some reason, then exit with an error, because we don't want this build to be green (we just don't want it to generate an error either) +sleep 300 +exit 1 From b456e16c7df6d5b25aad613c79f1011f3c629f69 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 13 Dec 2024 15:23:37 -0800 Subject: [PATCH 27/35] Simplify instrumenter and tests (#118493) This commit simplifies the entitlements instrumentation service and instrumenter a bit. It especially removes some repetition in the instrumenter tests. --- .../impl/InstrumentationServiceImpl.java | 8 +- .../impl/InstrumenterImpl.java | 28 +- .../impl/InstrumentationServiceImplTests.java | 12 +- .../impl/InstrumenterTests.java | 388 +++++++++--------- .../instrumentation/impl/TestLoader.java | 20 - .../instrumentation/impl/TestMethodUtils.java | 81 ---- .../EntitlementInitialization.java | 8 +- .../InstrumentationService.java | 4 +- 8 files changed, 222 insertions(+), 327 deletions(-) delete mode 100644 libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/TestLoader.java delete mode 100644 libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/TestMethodUtils.java diff --git a/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImpl.java b/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImpl.java index 9e23d2c0412c3..9f10739486238 100644 --- a/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImpl.java +++ b/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImpl.java @@ -29,15 +29,13 @@ public class InstrumentationServiceImpl implements InstrumentationService { @Override - public Instrumenter newInstrumenter(Map checkMethods) { - return InstrumenterImpl.create(checkMethods); + public Instrumenter newInstrumenter(Class clazz, Map methods) { + return InstrumenterImpl.create(clazz, methods); } @Override - public Map lookupMethodsToInstrument(String entitlementCheckerClassName) throws ClassNotFoundException, - IOException { + public Map lookupMethods(Class checkerClass) throws IOException { var methodsToInstrument = new HashMap(); - var checkerClass = Class.forName(entitlementCheckerClassName); var classFileInfo = InstrumenterImpl.getClassFileInfo(checkerClass); ClassReader reader = new ClassReader(classFileInfo.bytecodes()); ClassVisitor visitor = new ClassVisitor(Opcodes.ASM9) { diff --git a/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterImpl.java b/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterImpl.java index 57e30c01c5c28..00efab829b2bb 100644 --- a/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterImpl.java +++ b/libs/entitlement/asm-provider/src/main/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterImpl.java @@ -58,30 +58,14 @@ public class InstrumenterImpl implements Instrumenter { this.checkMethods = checkMethods; } - static String getCheckerClassName() { - int javaVersion = Runtime.version().feature(); - final String classNamePrefix; - if (javaVersion >= 23) { - classNamePrefix = "Java23"; - } else { - classNamePrefix = ""; - } - return "org/elasticsearch/entitlement/bridge/" + classNamePrefix + "EntitlementChecker"; - } - - public static InstrumenterImpl create(Map checkMethods) { - String checkerClass = getCheckerClassName(); - String handleClass = checkerClass + "Handle"; - String getCheckerClassMethodDescriptor = Type.getMethodDescriptor(Type.getObjectType(checkerClass)); + public static InstrumenterImpl create(Class checkerClass, Map checkMethods) { + Type checkerClassType = Type.getType(checkerClass); + String handleClass = checkerClassType.getInternalName() + "Handle"; + String getCheckerClassMethodDescriptor = Type.getMethodDescriptor(checkerClassType); return new InstrumenterImpl(handleClass, getCheckerClassMethodDescriptor, "", checkMethods); } - public ClassFileInfo instrumentClassFile(Class clazz) throws IOException { - ClassFileInfo initial = getClassFileInfo(clazz); - return new ClassFileInfo(initial.fileName(), instrumentClass(Type.getInternalName(clazz), initial.bytecodes())); - } - - public static ClassFileInfo getClassFileInfo(Class clazz) throws IOException { + static ClassFileInfo getClassFileInfo(Class clazz) throws IOException { String internalName = Type.getInternalName(clazz); String fileName = "/" + internalName + ".class"; byte[] originalBytecodes; @@ -306,5 +290,5 @@ protected void pushEntitlementChecker(MethodVisitor mv) { mv.visitMethodInsn(INVOKESTATIC, handleClass, "instance", getCheckerClassMethodDescriptor, false); } - public record ClassFileInfo(String fileName, byte[] bytecodes) {} + record ClassFileInfo(String fileName, byte[] bytecodes) {} } diff --git a/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests.java b/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests.java index 9ccb72637d463..e3285cec8f883 100644 --- a/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests.java +++ b/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumentationServiceImplTests.java @@ -51,8 +51,8 @@ interface TestCheckerCtors { void check$org_example_TestTargetClass$(Class clazz, int x, String y); } - public void testInstrumentationTargetLookup() throws IOException, ClassNotFoundException { - Map checkMethods = instrumentationService.lookupMethodsToInstrument(TestChecker.class.getName()); + public void testInstrumentationTargetLookup() throws IOException { + Map checkMethods = instrumentationService.lookupMethods(TestChecker.class); assertThat(checkMethods, aMapWithSize(3)); assertThat( @@ -116,8 +116,8 @@ public void testInstrumentationTargetLookup() throws IOException, ClassNotFoundE ); } - public void testInstrumentationTargetLookupWithOverloads() throws IOException, ClassNotFoundException { - Map checkMethods = instrumentationService.lookupMethodsToInstrument(TestCheckerOverloads.class.getName()); + public void testInstrumentationTargetLookupWithOverloads() throws IOException { + Map checkMethods = instrumentationService.lookupMethods(TestCheckerOverloads.class); assertThat(checkMethods, aMapWithSize(2)); assertThat( @@ -148,8 +148,8 @@ public void testInstrumentationTargetLookupWithOverloads() throws IOException, C ); } - public void testInstrumentationTargetLookupWithCtors() throws IOException, ClassNotFoundException { - Map checkMethods = instrumentationService.lookupMethodsToInstrument(TestCheckerCtors.class.getName()); + public void testInstrumentationTargetLookupWithCtors() throws IOException { + Map checkMethods = instrumentationService.lookupMethods(TestCheckerCtors.class); assertThat(checkMethods, aMapWithSize(2)); assertThat( diff --git a/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterTests.java b/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterTests.java index 75102b0bf260d..e9af1d152dd35 100644 --- a/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterTests.java +++ b/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/InstrumenterTests.java @@ -12,31 +12,64 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.entitlement.instrumentation.CheckMethod; import org.elasticsearch.entitlement.instrumentation.MethodKey; +import org.elasticsearch.entitlement.instrumentation.impl.InstrumenterImpl.ClassFileInfo; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; import org.elasticsearch.test.ESTestCase; +import org.junit.Before; import org.objectweb.asm.Type; +import java.io.IOException; +import java.lang.reflect.AccessFlag; +import java.lang.reflect.Constructor; +import java.lang.reflect.Executable; import java.lang.reflect.InvocationTargetException; -import java.util.List; +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.HashMap; import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; import static org.elasticsearch.entitlement.instrumentation.impl.ASMUtils.bytecode2text; import static org.elasticsearch.entitlement.instrumentation.impl.InstrumenterImpl.getClassFileInfo; -import static org.elasticsearch.entitlement.instrumentation.impl.TestMethodUtils.callStaticMethod; -import static org.elasticsearch.entitlement.instrumentation.impl.TestMethodUtils.getCheckMethod; -import static org.elasticsearch.entitlement.instrumentation.impl.TestMethodUtils.methodKeyForTarget; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.startsWith; +import static org.hamcrest.Matchers.equalTo; /** - * This tests {@link InstrumenterImpl} with some ad-hoc instrumented method and checker methods, to allow us to check - * some ad-hoc test cases (e.g. overloaded methods, overloaded targets, multiple instrumentation, etc.) + * This tests {@link InstrumenterImpl} can instrument various method signatures + * (e.g. overloaded methods, overloaded targets, multiple instrumentation, etc.) */ @ESTestCase.WithoutSecurityManager public class InstrumenterTests extends ESTestCase { private static final Logger logger = LogManager.getLogger(InstrumenterTests.class); + static class TestLoader extends ClassLoader { + final byte[] testClassBytes; + final Class testClass; + + TestLoader(String testClassName, byte[] testClassBytes) { + super(InstrumenterTests.class.getClassLoader()); + this.testClassBytes = testClassBytes; + this.testClass = defineClass(testClassName, testClassBytes, 0, testClassBytes.length); + } + + Method getSameMethod(Method method) { + try { + return testClass.getMethod(method.getName(), method.getParameterTypes()); + } catch (NoSuchMethodException e) { + throw new AssertionError(e); + } + } + + Constructor getSameConstructor(Constructor ctor) { + try { + return testClass.getConstructor(ctor.getParameterTypes()); + } catch (NoSuchMethodException e) { + throw new AssertionError(e); + } + } + } + /** * Contains all the virtual methods from {@link TestClassToInstrument}, * allowing this test to call them on the dynamically loaded instrumented class. @@ -80,13 +113,15 @@ public static void anotherStaticMethod(int arg) {} public interface MockEntitlementChecker { void checkSomeStaticMethod(Class clazz, int arg); - void checkSomeStaticMethod(Class clazz, int arg, String anotherArg); + void checkSomeStaticMethodOverload(Class clazz, int arg, String anotherArg); + + void checkAnotherStaticMethod(Class clazz, int arg); void checkSomeInstanceMethod(Class clazz, Testable that, int arg, String anotherArg); void checkCtor(Class clazz); - void checkCtor(Class clazz, int arg); + void checkCtorOverload(Class clazz, int arg); } public static class TestEntitlementCheckerHolder { @@ -105,6 +140,7 @@ public static class TestEntitlementChecker implements MockEntitlementChecker { volatile boolean isActive; int checkSomeStaticMethodIntCallCount = 0; + int checkAnotherStaticMethodIntCallCount = 0; int checkSomeStaticMethodIntStringCallCount = 0; int checkSomeInstanceMethodCallCount = 0; @@ -120,28 +156,33 @@ private void throwIfActive() { @Override public void checkSomeStaticMethod(Class callerClass, int arg) { checkSomeStaticMethodIntCallCount++; - assertSame(TestMethodUtils.class, callerClass); + assertSame(InstrumenterTests.class, callerClass); assertEquals(123, arg); throwIfActive(); } @Override - public void checkSomeStaticMethod(Class callerClass, int arg, String anotherArg) { + public void checkSomeStaticMethodOverload(Class callerClass, int arg, String anotherArg) { checkSomeStaticMethodIntStringCallCount++; - assertSame(TestMethodUtils.class, callerClass); + assertSame(InstrumenterTests.class, callerClass); assertEquals(123, arg); assertEquals("abc", anotherArg); throwIfActive(); } + @Override + public void checkAnotherStaticMethod(Class callerClass, int arg) { + checkAnotherStaticMethodIntCallCount++; + assertSame(InstrumenterTests.class, callerClass); + assertEquals(123, arg); + throwIfActive(); + } + @Override public void checkSomeInstanceMethod(Class callerClass, Testable that, int arg, String anotherArg) { checkSomeInstanceMethodCallCount++; assertSame(InstrumenterTests.class, callerClass); - assertThat( - that.getClass().getName(), - startsWith("org.elasticsearch.entitlement.instrumentation.impl.InstrumenterTests$TestClassToInstrument") - ); + assertThat(that.getClass().getName(), equalTo(TestClassToInstrument.class.getName())); assertEquals(123, arg); assertEquals("def", anotherArg); throwIfActive(); @@ -155,7 +196,7 @@ public void checkCtor(Class callerClass) { } @Override - public void checkCtor(Class callerClass, int arg) { + public void checkCtorOverload(Class callerClass, int arg) { checkCtorIntCallCount++; assertSame(InstrumenterTests.class, callerClass); assertEquals(123, arg); @@ -163,206 +204,83 @@ public void checkCtor(Class callerClass, int arg) { } } - public void testClassIsInstrumented() throws Exception { - var classToInstrument = TestClassToInstrument.class; - - CheckMethod checkMethod = getCheckMethod(MockEntitlementChecker.class, "checkSomeStaticMethod", Class.class, int.class); - Map checkMethods = Map.of( - methodKeyForTarget(classToInstrument.getMethod("someStaticMethod", int.class)), - checkMethod - ); - - var instrumenter = createInstrumenter(checkMethods); - - byte[] newBytecode = instrumenter.instrumentClassFile(classToInstrument).bytecodes(); - - if (logger.isTraceEnabled()) { - logger.trace("Bytecode after instrumentation:\n{}", bytecode2text(newBytecode)); - } - - Class newClass = new TestLoader(Testable.class.getClassLoader()).defineClassFromBytes( - classToInstrument.getName() + "_NEW", - newBytecode - ); + @Before + public void resetInstance() { + TestEntitlementCheckerHolder.checkerInstance = new TestEntitlementChecker(); + } - TestEntitlementCheckerHolder.checkerInstance.isActive = false; + public void testStaticMethod() throws Exception { + Method targetMethod = TestClassToInstrument.class.getMethod("someStaticMethod", int.class); + TestLoader loader = instrumentTestClass(createInstrumenter(Map.of("checkSomeStaticMethod", targetMethod))); // Before checking is active, nothing should throw - callStaticMethod(newClass, "someStaticMethod", 123); - - TestEntitlementCheckerHolder.checkerInstance.isActive = true; - + assertStaticMethod(loader, targetMethod, 123); // After checking is activated, everything should throw - assertThrows(TestException.class, () -> callStaticMethod(newClass, "someStaticMethod", 123)); + assertStaticMethodThrows(loader, targetMethod, 123); } - public void testClassIsNotInstrumentedTwice() throws Exception { - var classToInstrument = TestClassToInstrument.class; - - CheckMethod checkMethod = getCheckMethod(MockEntitlementChecker.class, "checkSomeStaticMethod", Class.class, int.class); - Map checkMethods = Map.of( - methodKeyForTarget(classToInstrument.getMethod("someStaticMethod", int.class)), - checkMethod - ); - - var instrumenter = createInstrumenter(checkMethods); - - InstrumenterImpl.ClassFileInfo initial = getClassFileInfo(classToInstrument); - var internalClassName = Type.getInternalName(classToInstrument); - - byte[] instrumentedBytecode = instrumenter.instrumentClass(internalClassName, initial.bytecodes()); - byte[] instrumentedTwiceBytecode = instrumenter.instrumentClass(internalClassName, instrumentedBytecode); + public void testNotInstrumentedTwice() throws Exception { + Method targetMethod = TestClassToInstrument.class.getMethod("someStaticMethod", int.class); + var instrumenter = createInstrumenter(Map.of("checkSomeStaticMethod", targetMethod)); - logger.trace(() -> Strings.format("Bytecode after 1st instrumentation:\n%s", bytecode2text(instrumentedBytecode))); + var loader1 = instrumentTestClass(instrumenter); + byte[] instrumentedTwiceBytecode = instrumenter.instrumentClass(TestClassToInstrument.class.getName(), loader1.testClassBytes); logger.trace(() -> Strings.format("Bytecode after 2nd instrumentation:\n%s", bytecode2text(instrumentedTwiceBytecode))); + var loader2 = new TestLoader(TestClassToInstrument.class.getName(), instrumentedTwiceBytecode); - Class newClass = new TestLoader(Testable.class.getClassLoader()).defineClassFromBytes( - classToInstrument.getName() + "_NEW_NEW", - instrumentedTwiceBytecode - ); - - TestEntitlementCheckerHolder.checkerInstance.isActive = true; - TestEntitlementCheckerHolder.checkerInstance.checkSomeStaticMethodIntCallCount = 0; - - assertThrows(TestException.class, () -> callStaticMethod(newClass, "someStaticMethod", 123)); + assertStaticMethodThrows(loader2, targetMethod, 123); assertEquals(1, TestEntitlementCheckerHolder.checkerInstance.checkSomeStaticMethodIntCallCount); } - public void testClassAllMethodsAreInstrumentedFirstPass() throws Exception { - var classToInstrument = TestClassToInstrument.class; - - CheckMethod checkMethod = getCheckMethod(MockEntitlementChecker.class, "checkSomeStaticMethod", Class.class, int.class); - Map checkMethods = Map.of( - methodKeyForTarget(classToInstrument.getMethod("someStaticMethod", int.class)), - checkMethod, - methodKeyForTarget(classToInstrument.getMethod("anotherStaticMethod", int.class)), - checkMethod - ); - - var instrumenter = createInstrumenter(checkMethods); - - InstrumenterImpl.ClassFileInfo initial = getClassFileInfo(classToInstrument); - var internalClassName = Type.getInternalName(classToInstrument); + public void testMultipleMethods() throws Exception { + Method targetMethod1 = TestClassToInstrument.class.getMethod("someStaticMethod", int.class); + Method targetMethod2 = TestClassToInstrument.class.getMethod("anotherStaticMethod", int.class); - byte[] instrumentedBytecode = instrumenter.instrumentClass(internalClassName, initial.bytecodes()); - byte[] instrumentedTwiceBytecode = instrumenter.instrumentClass(internalClassName, instrumentedBytecode); + var instrumenter = createInstrumenter(Map.of("checkSomeStaticMethod", targetMethod1, "checkAnotherStaticMethod", targetMethod2)); + var loader = instrumentTestClass(instrumenter); - logger.trace(() -> Strings.format("Bytecode after 1st instrumentation:\n%s", bytecode2text(instrumentedBytecode))); - logger.trace(() -> Strings.format("Bytecode after 2nd instrumentation:\n%s", bytecode2text(instrumentedTwiceBytecode))); - - Class newClass = new TestLoader(Testable.class.getClassLoader()).defineClassFromBytes( - classToInstrument.getName() + "_NEW_NEW", - instrumentedTwiceBytecode - ); - - TestEntitlementCheckerHolder.checkerInstance.isActive = true; - TestEntitlementCheckerHolder.checkerInstance.checkSomeStaticMethodIntCallCount = 0; - - assertThrows(TestException.class, () -> callStaticMethod(newClass, "someStaticMethod", 123)); + assertStaticMethodThrows(loader, targetMethod1, 123); assertEquals(1, TestEntitlementCheckerHolder.checkerInstance.checkSomeStaticMethodIntCallCount); - - assertThrows(TestException.class, () -> callStaticMethod(newClass, "anotherStaticMethod", 123)); - assertEquals(2, TestEntitlementCheckerHolder.checkerInstance.checkSomeStaticMethodIntCallCount); + assertStaticMethodThrows(loader, targetMethod2, 123); + assertEquals(1, TestEntitlementCheckerHolder.checkerInstance.checkAnotherStaticMethodIntCallCount); } - public void testInstrumenterWorksWithOverloads() throws Exception { - var classToInstrument = TestClassToInstrument.class; - - Map checkMethods = Map.of( - methodKeyForTarget(classToInstrument.getMethod("someStaticMethod", int.class)), - getCheckMethod(MockEntitlementChecker.class, "checkSomeStaticMethod", Class.class, int.class), - methodKeyForTarget(classToInstrument.getMethod("someStaticMethod", int.class, String.class)), - getCheckMethod(MockEntitlementChecker.class, "checkSomeStaticMethod", Class.class, int.class, String.class) + public void testStaticMethodOverload() throws Exception { + Method targetMethod1 = TestClassToInstrument.class.getMethod("someStaticMethod", int.class); + Method targetMethod2 = TestClassToInstrument.class.getMethod("someStaticMethod", int.class, String.class); + var instrumenter = createInstrumenter( + Map.of("checkSomeStaticMethod", targetMethod1, "checkSomeStaticMethodOverload", targetMethod2) ); + var loader = instrumentTestClass(instrumenter); - var instrumenter = createInstrumenter(checkMethods); - - byte[] newBytecode = instrumenter.instrumentClassFile(classToInstrument).bytecodes(); - - if (logger.isTraceEnabled()) { - logger.trace("Bytecode after instrumentation:\n{}", bytecode2text(newBytecode)); - } - - Class newClass = new TestLoader(Testable.class.getClassLoader()).defineClassFromBytes( - classToInstrument.getName() + "_NEW", - newBytecode - ); - - TestEntitlementCheckerHolder.checkerInstance.isActive = true; - TestEntitlementCheckerHolder.checkerInstance.checkSomeStaticMethodIntCallCount = 0; - TestEntitlementCheckerHolder.checkerInstance.checkSomeStaticMethodIntStringCallCount = 0; - - // After checking is activated, everything should throw - assertThrows(TestException.class, () -> callStaticMethod(newClass, "someStaticMethod", 123)); - assertThrows(TestException.class, () -> callStaticMethod(newClass, "someStaticMethod", 123, "abc")); - + assertStaticMethodThrows(loader, targetMethod1, 123); + assertStaticMethodThrows(loader, targetMethod2, 123, "abc"); assertEquals(1, TestEntitlementCheckerHolder.checkerInstance.checkSomeStaticMethodIntCallCount); assertEquals(1, TestEntitlementCheckerHolder.checkerInstance.checkSomeStaticMethodIntStringCallCount); } - public void testInstrumenterWorksWithInstanceMethodsAndOverloads() throws Exception { - var classToInstrument = TestClassToInstrument.class; - - Map checkMethods = Map.of( - methodKeyForTarget(classToInstrument.getMethod("someMethod", int.class, String.class)), - getCheckMethod(MockEntitlementChecker.class, "checkSomeInstanceMethod", Class.class, Testable.class, int.class, String.class) - ); - - var instrumenter = createInstrumenter(checkMethods); - - byte[] newBytecode = instrumenter.instrumentClassFile(classToInstrument).bytecodes(); - - if (logger.isTraceEnabled()) { - logger.trace("Bytecode after instrumentation:\n{}", bytecode2text(newBytecode)); - } - - Class newClass = new TestLoader(Testable.class.getClassLoader()).defineClassFromBytes( - classToInstrument.getName() + "_NEW", - newBytecode - ); + public void testInstanceMethodOverload() throws Exception { + Method targetMethod = TestClassToInstrument.class.getMethod("someMethod", int.class, String.class); + var instrumenter = createInstrumenter(Map.of("checkSomeInstanceMethod", targetMethod)); + var loader = instrumentTestClass(instrumenter); TestEntitlementCheckerHolder.checkerInstance.isActive = true; - TestEntitlementCheckerHolder.checkerInstance.checkSomeInstanceMethodCallCount = 0; - - Testable testTargetClass = (Testable) (newClass.getConstructor().newInstance()); + Testable testTargetClass = (Testable) (loader.testClass.getConstructor().newInstance()); // This overload is not instrumented, so it will not throw testTargetClass.someMethod(123); - assertThrows(TestException.class, () -> testTargetClass.someMethod(123, "def")); + expectThrows(TestException.class, () -> testTargetClass.someMethod(123, "def")); assertEquals(1, TestEntitlementCheckerHolder.checkerInstance.checkSomeInstanceMethodCallCount); } - public void testInstrumenterWorksWithConstructors() throws Exception { - var classToInstrument = TestClassToInstrument.class; - - Map checkMethods = Map.of( - new MethodKey(classToInstrument.getName().replace('.', '/'), "", List.of()), - getCheckMethod(MockEntitlementChecker.class, "checkCtor", Class.class), - new MethodKey(classToInstrument.getName().replace('.', '/'), "", List.of("I")), - getCheckMethod(MockEntitlementChecker.class, "checkCtor", Class.class, int.class) - ); - - var instrumenter = createInstrumenter(checkMethods); - - byte[] newBytecode = instrumenter.instrumentClassFile(classToInstrument).bytecodes(); - - if (logger.isTraceEnabled()) { - logger.trace("Bytecode after instrumentation:\n{}", bytecode2text(newBytecode)); - } - - Class newClass = new TestLoader(Testable.class.getClassLoader()).defineClassFromBytes( - classToInstrument.getName() + "_NEW", - newBytecode - ); - - TestEntitlementCheckerHolder.checkerInstance.isActive = true; - - var ex = assertThrows(InvocationTargetException.class, () -> newClass.getConstructor().newInstance()); - assertThat(ex.getCause(), instanceOf(TestException.class)); - var ex2 = assertThrows(InvocationTargetException.class, () -> newClass.getConstructor(int.class).newInstance(123)); - assertThat(ex2.getCause(), instanceOf(TestException.class)); + public void testConstructors() throws Exception { + Constructor ctor1 = TestClassToInstrument.class.getConstructor(); + Constructor ctor2 = TestClassToInstrument.class.getConstructor(int.class); + var loader = instrumentTestClass(createInstrumenter(Map.of("checkCtor", ctor1, "checkCtorOverload", ctor2))); + assertCtorThrows(loader, ctor1); + assertCtorThrows(loader, ctor2, 123); assertEquals(1, TestEntitlementCheckerHolder.checkerInstance.checkCtorCallCount); assertEquals(1, TestEntitlementCheckerHolder.checkerInstance.checkCtorIntCallCount); } @@ -373,11 +291,107 @@ public void testInstrumenterWorksWithConstructors() throws Exception { * MethodKey and instrumentationMethod with slightly different signatures (using the common interface * Testable) which is not what would happen when it's run by the agent. */ - private InstrumenterImpl createInstrumenter(Map checkMethods) { + private static InstrumenterImpl createInstrumenter(Map methods) throws NoSuchMethodException { + Map checkMethods = new HashMap<>(); + for (var entry : methods.entrySet()) { + checkMethods.put(getMethodKey(entry.getValue()), getCheckMethod(entry.getKey(), entry.getValue())); + } String checkerClass = Type.getInternalName(InstrumenterTests.MockEntitlementChecker.class); String handleClass = Type.getInternalName(InstrumenterTests.TestEntitlementCheckerHolder.class); String getCheckerClassMethodDescriptor = Type.getMethodDescriptor(Type.getObjectType(checkerClass)); - return new InstrumenterImpl(handleClass, getCheckerClassMethodDescriptor, "_NEW", checkMethods); + return new InstrumenterImpl(handleClass, getCheckerClassMethodDescriptor, "", checkMethods); + } + + private static TestLoader instrumentTestClass(InstrumenterImpl instrumenter) throws IOException { + var clazz = TestClassToInstrument.class; + ClassFileInfo initial = getClassFileInfo(clazz); + byte[] newBytecode = instrumenter.instrumentClass(Type.getInternalName(clazz), initial.bytecodes()); + if (logger.isTraceEnabled()) { + logger.trace("Bytecode after instrumentation:\n{}", bytecode2text(newBytecode)); + } + return new TestLoader(clazz.getName(), newBytecode); + } + + private static MethodKey getMethodKey(Executable method) { + logger.info("method key: {}", method.getName()); + String methodName = method instanceof Constructor ? "" : method.getName(); + return new MethodKey( + Type.getInternalName(method.getDeclaringClass()), + methodName, + Stream.of(method.getParameterTypes()).map(Type::getType).map(Type::getInternalName).toList() + ); + } + + private static CheckMethod getCheckMethod(String methodName, Executable targetMethod) throws NoSuchMethodException { + Set flags = targetMethod.accessFlags(); + boolean isInstance = flags.contains(AccessFlag.STATIC) == false && targetMethod instanceof Method; + int extraArgs = 1; // caller class + if (isInstance) { + ++extraArgs; + } + Class[] targetParameterTypes = targetMethod.getParameterTypes(); + Class[] checkParameterTypes = new Class[targetParameterTypes.length + extraArgs]; + checkParameterTypes[0] = Class.class; + if (isInstance) { + checkParameterTypes[1] = Testable.class; + } + System.arraycopy(targetParameterTypes, 0, checkParameterTypes, extraArgs, targetParameterTypes.length); + var checkMethod = MockEntitlementChecker.class.getMethod(methodName, checkParameterTypes); + return new CheckMethod( + Type.getInternalName(MockEntitlementChecker.class), + checkMethod.getName(), + Arrays.stream(Type.getArgumentTypes(checkMethod)).map(Type::getDescriptor).toList() + ); + } + + private static void unwrapInvocationException(InvocationTargetException e) { + Throwable cause = e.getCause(); + if (cause instanceof TestException n) { + // Sometimes we're expecting this one! + throw n; + } else { + throw new AssertionError(cause); + } + } + + /** + * Calling a static method of a dynamically loaded class is significantly more cumbersome + * than calling a virtual method. + */ + static void callStaticMethod(Method method, Object... args) { + try { + method.invoke(null, args); + } catch (InvocationTargetException e) { + unwrapInvocationException(e); + } catch (IllegalAccessException e) { + throw new AssertionError(e); + } + } + + private void assertStaticMethodThrows(TestLoader loader, Method method, Object... args) { + Method testMethod = loader.getSameMethod(method); + TestEntitlementCheckerHolder.checkerInstance.isActive = true; + expectThrows(TestException.class, () -> callStaticMethod(testMethod, args)); + } + + private void assertStaticMethod(TestLoader loader, Method method, Object... args) { + Method testMethod = loader.getSameMethod(method); + TestEntitlementCheckerHolder.checkerInstance.isActive = false; + callStaticMethod(testMethod, args); + } + + private void assertCtorThrows(TestLoader loader, Constructor ctor, Object... args) { + Constructor testCtor = loader.getSameConstructor(ctor); + TestEntitlementCheckerHolder.checkerInstance.isActive = true; + expectThrows(TestException.class, () -> { + try { + testCtor.newInstance(args); + } catch (InvocationTargetException e) { + unwrapInvocationException(e); + } catch (IllegalAccessException | InstantiationException e) { + throw new AssertionError(e); + } + }); } } diff --git a/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/TestLoader.java b/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/TestLoader.java deleted file mode 100644 index 9eb8e9328ecba..0000000000000 --- a/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/TestLoader.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.entitlement.instrumentation.impl; - -class TestLoader extends ClassLoader { - TestLoader(ClassLoader parent) { - super(parent); - } - - public Class defineClassFromBytes(String name, byte[] bytes) { - return defineClass(name, bytes, 0, bytes.length); - } -} diff --git a/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/TestMethodUtils.java b/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/TestMethodUtils.java deleted file mode 100644 index de7822fea926e..0000000000000 --- a/libs/entitlement/asm-provider/src/test/java/org/elasticsearch/entitlement/instrumentation/impl/TestMethodUtils.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.entitlement.instrumentation.impl; - -import org.elasticsearch.entitlement.instrumentation.CheckMethod; -import org.elasticsearch.entitlement.instrumentation.MethodKey; -import org.objectweb.asm.Type; - -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Stream; - -class TestMethodUtils { - - /** - * @return a {@link MethodKey} suitable for looking up the given {@code targetMethod} in the entitlements trampoline - */ - static MethodKey methodKeyForTarget(Method targetMethod) { - Type actualType = Type.getMethodType(Type.getMethodDescriptor(targetMethod)); - return new MethodKey( - Type.getInternalName(targetMethod.getDeclaringClass()), - targetMethod.getName(), - Stream.of(actualType.getArgumentTypes()).map(Type::getInternalName).toList() - ); - } - - static MethodKey methodKeyForConstructor(Class classToInstrument, List params) { - return new MethodKey(classToInstrument.getName().replace('.', '/'), "", params); - } - - static CheckMethod getCheckMethod(Class clazz, String methodName, Class... parameterTypes) throws NoSuchMethodException { - var method = clazz.getMethod(methodName, parameterTypes); - return new CheckMethod( - Type.getInternalName(clazz), - method.getName(), - Arrays.stream(Type.getArgumentTypes(method)).map(Type::getDescriptor).toList() - ); - } - - /** - * Calling a static method of a dynamically loaded class is significantly more cumbersome - * than calling a virtual method. - */ - static void callStaticMethod(Class c, String methodName, int arg) throws NoSuchMethodException, IllegalAccessException { - try { - c.getMethod(methodName, int.class).invoke(null, arg); - } catch (InvocationTargetException e) { - Throwable cause = e.getCause(); - if (cause instanceof TestException n) { - // Sometimes we're expecting this one! - throw n; - } else { - throw new AssertionError(cause); - } - } - } - - static void callStaticMethod(Class c, String methodName, int arg1, String arg2) throws NoSuchMethodException, - IllegalAccessException { - try { - c.getMethod(methodName, int.class, String.class).invoke(null, arg1, arg2); - } catch (InvocationTargetException e) { - Throwable cause = e.getCause(); - if (cause instanceof TestException n) { - // Sometimes we're expecting this one! - throw n; - } else { - throw new AssertionError(cause); - } - } - } -} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java index 2956efa8eec31..9118f67cdc145 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java @@ -14,6 +14,7 @@ import org.elasticsearch.entitlement.bridge.EntitlementChecker; import org.elasticsearch.entitlement.instrumentation.CheckMethod; import org.elasticsearch.entitlement.instrumentation.InstrumentationService; +import org.elasticsearch.entitlement.instrumentation.Instrumenter; import org.elasticsearch.entitlement.instrumentation.MethodKey; import org.elasticsearch.entitlement.instrumentation.Transformer; import org.elasticsearch.entitlement.runtime.api.ElasticsearchEntitlementChecker; @@ -64,13 +65,12 @@ public static EntitlementChecker checker() { public static void initialize(Instrumentation inst) throws Exception { manager = initChecker(); - Map checkMethods = INSTRUMENTER_FACTORY.lookupMethodsToInstrument( - "org.elasticsearch.entitlement.bridge.EntitlementChecker" - ); + Map checkMethods = INSTRUMENTER_FACTORY.lookupMethods(EntitlementChecker.class); var classesToTransform = checkMethods.keySet().stream().map(MethodKey::className).collect(Collectors.toSet()); - inst.addTransformer(new Transformer(INSTRUMENTER_FACTORY.newInstrumenter(checkMethods), classesToTransform), true); + Instrumenter instrumenter = INSTRUMENTER_FACTORY.newInstrumenter(EntitlementChecker.class, checkMethods); + inst.addTransformer(new Transformer(instrumenter, classesToTransform), true); // TODO: should we limit this array somehow? var classesToRetransform = classesToTransform.stream().map(EntitlementInitialization::internalNameToClass).toArray(Class[]::new); inst.retransformClasses(classesToRetransform); diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/InstrumentationService.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/InstrumentationService.java index d0331d756d2b2..66d8ad9488cfa 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/InstrumentationService.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/instrumentation/InstrumentationService.java @@ -16,7 +16,7 @@ * The SPI service entry point for instrumentation. */ public interface InstrumentationService { - Instrumenter newInstrumenter(Map checkMethods); + Instrumenter newInstrumenter(Class clazz, Map methods); - Map lookupMethodsToInstrument(String entitlementCheckerClassName) throws ClassNotFoundException, IOException; + Map lookupMethods(Class clazz) throws IOException; } From fee6165c194c61547f54bd17c41dd0200b28eb82 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Sat, 14 Dec 2024 17:32:08 +1100 Subject: [PATCH 28/35] Mute org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizerTests org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizerTests #118721 --- muted-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 1255739e818be..09b3050359d37 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -305,6 +305,8 @@ tests: - class: org.elasticsearch.reservedstate.service.FileSettingsServiceTests method: testInvalidJSON issue: https://github.com/elastic/elasticsearch/issues/116521 +- class: org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizerTests + issue: https://github.com/elastic/elasticsearch/issues/118721 # Examples: # From d6950208508d1e961b1e6b172c2c8b148551e7ff Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sat, 14 Dec 2024 08:56:04 +0100 Subject: [PATCH 29/35] Remove lifecycle from CircuitBreakerService (#118699) CircuitBreakerService and all its implementations has no lifecycle so we don't need to extend AbstractLifecycleComponent here. Mainly motivated by wanting to have a singleton CircuitBreakerService for some optimizations in query execution, but also a worthwhile cleanup in isolation I believe. --- .../PreallocatedCircuitBreakerService.java | 2 +- .../breaker/CircuitBreakerService.java | 12 +- .../elasticsearch/node/NodeConstruction.java | 1 - ...reallocatedCircuitBreakerServiceTests.java | 138 ++++----- .../common/util/BigArraysTests.java | 69 ++--- .../HierarchyCircuitBreakerServiceTests.java | 292 +++++++++--------- .../metrics/TDigestStateReleasingTests.java | 21 +- .../aggregations/AggregatorTestCase.java | 75 +++-- .../execution/sample/CircuitBreakerTests.java | 16 +- .../sequence/CircuitBreakerTests.java | 33 +- .../DelegatingCircuitBreakerServiceTests.java | 74 ++--- 11 files changed, 340 insertions(+), 393 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/breaker/PreallocatedCircuitBreakerService.java b/server/src/main/java/org/elasticsearch/common/breaker/PreallocatedCircuitBreakerService.java index e5c9b14cf90fc..ba2382208b705 100644 --- a/server/src/main/java/org/elasticsearch/common/breaker/PreallocatedCircuitBreakerService.java +++ b/server/src/main/java/org/elasticsearch/common/breaker/PreallocatedCircuitBreakerService.java @@ -58,7 +58,7 @@ public CircuitBreakerStats stats(String name) { } @Override - protected void doClose() { + public void close() { preallocated.close(); } diff --git a/server/src/main/java/org/elasticsearch/indices/breaker/CircuitBreakerService.java b/server/src/main/java/org/elasticsearch/indices/breaker/CircuitBreakerService.java index c8d7fa0775f8f..be89dd97aab89 100644 --- a/server/src/main/java/org/elasticsearch/indices/breaker/CircuitBreakerService.java +++ b/server/src/main/java/org/elasticsearch/indices/breaker/CircuitBreakerService.java @@ -10,13 +10,12 @@ package org.elasticsearch.indices.breaker; import org.elasticsearch.common.breaker.CircuitBreaker; -import org.elasticsearch.common.component.AbstractLifecycleComponent; /** * Interface for Circuit Breaker services, which provide breakers to classes * that load field data. */ -public abstract class CircuitBreakerService extends AbstractLifecycleComponent { +public abstract class CircuitBreakerService { protected CircuitBreakerService() {} @@ -35,13 +34,4 @@ protected CircuitBreakerService() {} */ public abstract CircuitBreakerStats stats(String name); - @Override - protected void doStart() {} - - @Override - protected void doStop() {} - - @Override - protected void doClose() {} - } diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index aec8eb0c3ca67..17e56a392daff 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -1459,7 +1459,6 @@ private CircuitBreakerService createCircuitBreakerService( case "none" -> new NoneCircuitBreakerService(); default -> throw new IllegalArgumentException("Unknown circuit breaker type [" + type + "]"); }; - resourcesToClose.add(circuitBreakerService); modules.bindToInstance(CircuitBreakerService.class, circuitBreakerService); pluginBreakers.forEach(t -> { diff --git a/server/src/test/java/org/elasticsearch/common/breaker/PreallocatedCircuitBreakerServiceTests.java b/server/src/test/java/org/elasticsearch/common/breaker/PreallocatedCircuitBreakerServiceTests.java index 742c03b564307..52a30e2494974 100644 --- a/server/src/test/java/org/elasticsearch/common/breaker/PreallocatedCircuitBreakerServiceTests.java +++ b/server/src/test/java/org/elasticsearch/common/breaker/PreallocatedCircuitBreakerServiceTests.java @@ -25,100 +25,94 @@ public class PreallocatedCircuitBreakerServiceTests extends ESTestCase { public void testUseNotPreallocated() { - try (HierarchyCircuitBreakerService real = real()) { - try (PreallocatedCircuitBreakerService preallocated = preallocateRequest(real, 1024)) { - CircuitBreaker b = preallocated.getBreaker(CircuitBreaker.REQUEST); - b.addEstimateBytesAndMaybeBreak(100, "test"); - b.addWithoutBreaking(-100); - } - assertThat(real.getBreaker(CircuitBreaker.REQUEST).getUsed(), equalTo(0L)); + HierarchyCircuitBreakerService real = real(); + try (PreallocatedCircuitBreakerService preallocated = preallocateRequest(real, 1024)) { + CircuitBreaker b = preallocated.getBreaker(CircuitBreaker.REQUEST); + b.addEstimateBytesAndMaybeBreak(100, "test"); + b.addWithoutBreaking(-100); } + assertThat(real.getBreaker(CircuitBreaker.REQUEST).getUsed(), equalTo(0L)); } public void testUseLessThanPreallocated() { - try (HierarchyCircuitBreakerService real = real()) { - try (PreallocatedCircuitBreakerService preallocated = preallocateRequest(real, 1024)) { - CircuitBreaker b = preallocated.getBreaker(CircuitBreaker.REQUEST); - b.addEstimateBytesAndMaybeBreak(100, "test"); - b.addWithoutBreaking(-100); - } - assertThat(real.getBreaker(CircuitBreaker.REQUEST).getUsed(), equalTo(0L)); + HierarchyCircuitBreakerService real = real(); + try (PreallocatedCircuitBreakerService preallocated = preallocateRequest(real, 1024)) { + CircuitBreaker b = preallocated.getBreaker(CircuitBreaker.REQUEST); + b.addEstimateBytesAndMaybeBreak(100, "test"); + b.addWithoutBreaking(-100); } + assertThat(real.getBreaker(CircuitBreaker.REQUEST).getUsed(), equalTo(0L)); } public void testCloseIsIdempotent() { - try (HierarchyCircuitBreakerService real = real()) { - try (PreallocatedCircuitBreakerService preallocated = preallocateRequest(real, 1024)) { - CircuitBreaker b = preallocated.getBreaker(CircuitBreaker.REQUEST); - b.addEstimateBytesAndMaybeBreak(100, "test"); - b.addWithoutBreaking(-100); - preallocated.close(); - assertThat(real.getBreaker(CircuitBreaker.REQUEST).getUsed(), equalTo(0L)); - } // Closes again which should do nothing + HierarchyCircuitBreakerService real = real(); + try (PreallocatedCircuitBreakerService preallocated = preallocateRequest(real, 1024)) { + CircuitBreaker b = preallocated.getBreaker(CircuitBreaker.REQUEST); + b.addEstimateBytesAndMaybeBreak(100, "test"); + b.addWithoutBreaking(-100); + preallocated.close(); assertThat(real.getBreaker(CircuitBreaker.REQUEST).getUsed(), equalTo(0L)); - } + } // Closes again which should do nothing + assertThat(real.getBreaker(CircuitBreaker.REQUEST).getUsed(), equalTo(0L)); } public void testUseMoreThanPreallocated() { - try (HierarchyCircuitBreakerService real = real()) { - try (PreallocatedCircuitBreakerService preallocated = preallocateRequest(real, 1024)) { - CircuitBreaker b = preallocated.getBreaker(CircuitBreaker.REQUEST); - b.addEstimateBytesAndMaybeBreak(2048, "test"); - b.addWithoutBreaking(-2048); - } - assertThat(real.getBreaker(CircuitBreaker.REQUEST).getUsed(), equalTo(0L)); + HierarchyCircuitBreakerService real = real(); + try (PreallocatedCircuitBreakerService preallocated = preallocateRequest(real, 1024)) { + CircuitBreaker b = preallocated.getBreaker(CircuitBreaker.REQUEST); + b.addEstimateBytesAndMaybeBreak(2048, "test"); + b.addWithoutBreaking(-2048); } + assertThat(real.getBreaker(CircuitBreaker.REQUEST).getUsed(), equalTo(0L)); } public void testPreallocateMoreThanRemains() { - try (HierarchyCircuitBreakerService real = real()) { - long limit = real.getBreaker(CircuitBreaker.REQUEST).getLimit(); - Exception e = expectThrows(CircuitBreakingException.class, () -> preallocateRequest(real, limit + 1024)); - assertThat(e.getMessage(), startsWith("[request] Data too large, data for [preallocate[test]] would be [")); - } + HierarchyCircuitBreakerService real = real(); + long limit = real.getBreaker(CircuitBreaker.REQUEST).getLimit(); + Exception e = expectThrows(CircuitBreakingException.class, () -> preallocateRequest(real, limit + 1024)); + assertThat(e.getMessage(), startsWith("[request] Data too large, data for [preallocate[test]] would be [")); } public void testRandom() { - try (HierarchyCircuitBreakerService real = real()) { - CircuitBreaker realBreaker = real.getBreaker(CircuitBreaker.REQUEST); - long preallocatedBytes = randomLongBetween(1, (long) (realBreaker.getLimit() * .8)); - try (PreallocatedCircuitBreakerService preallocated = preallocateRequest(real, preallocatedBytes)) { - CircuitBreaker b = preallocated.getBreaker(CircuitBreaker.REQUEST); - boolean usedPreallocated = false; - long current = 0; - for (int i = 0; i < 10000; i++) { - if (current >= preallocatedBytes) { - usedPreallocated = true; - } - if (usedPreallocated) { - assertThat(realBreaker.getUsed(), equalTo(current)); - } else { - assertThat(realBreaker.getUsed(), equalTo(preallocatedBytes)); - } - if (current > 0 && randomBoolean()) { - long delta = randomLongBetween(-Math.min(current, realBreaker.getLimit() / 100), 0); - b.addWithoutBreaking(delta); - current += delta; - continue; - } - long delta = randomLongBetween(0, realBreaker.getLimit() / 100); - if (randomBoolean()) { - b.addWithoutBreaking(delta); - current += delta; - continue; - } - if (current + delta < realBreaker.getLimit()) { - b.addEstimateBytesAndMaybeBreak(delta, "test"); - current += delta; - continue; - } - Exception e = expectThrows(CircuitBreakingException.class, () -> b.addEstimateBytesAndMaybeBreak(delta, "test")); - assertThat(e.getMessage(), startsWith("[request] Data too large, data for [test] would be [")); + HierarchyCircuitBreakerService real = real(); + CircuitBreaker realBreaker = real.getBreaker(CircuitBreaker.REQUEST); + long preallocatedBytes = randomLongBetween(1, (long) (realBreaker.getLimit() * .8)); + try (PreallocatedCircuitBreakerService preallocated = preallocateRequest(real, preallocatedBytes)) { + CircuitBreaker b = preallocated.getBreaker(CircuitBreaker.REQUEST); + boolean usedPreallocated = false; + long current = 0; + for (int i = 0; i < 10000; i++) { + if (current >= preallocatedBytes) { + usedPreallocated = true; + } + if (usedPreallocated) { + assertThat(realBreaker.getUsed(), equalTo(current)); + } else { + assertThat(realBreaker.getUsed(), equalTo(preallocatedBytes)); + } + if (current > 0 && randomBoolean()) { + long delta = randomLongBetween(-Math.min(current, realBreaker.getLimit() / 100), 0); + b.addWithoutBreaking(delta); + current += delta; + continue; } - b.addWithoutBreaking(-current); + long delta = randomLongBetween(0, realBreaker.getLimit() / 100); + if (randomBoolean()) { + b.addWithoutBreaking(delta); + current += delta; + continue; + } + if (current + delta < realBreaker.getLimit()) { + b.addEstimateBytesAndMaybeBreak(delta, "test"); + current += delta; + continue; + } + Exception e = expectThrows(CircuitBreakingException.class, () -> b.addEstimateBytesAndMaybeBreak(delta, "test")); + assertThat(e.getMessage(), startsWith("[request] Data too large, data for [test] would be [")); } - assertThat(real.getBreaker(CircuitBreaker.REQUEST).getUsed(), equalTo(0L)); + b.addWithoutBreaking(-current); } + assertThat(real.getBreaker(CircuitBreaker.REQUEST).getUsed(), equalTo(0L)); } private HierarchyCircuitBreakerService real() { diff --git a/server/src/test/java/org/elasticsearch/common/util/BigArraysTests.java b/server/src/test/java/org/elasticsearch/common/util/BigArraysTests.java index 3ef010f760ab6..1d571c3d9b296 100644 --- a/server/src/test/java/org/elasticsearch/common/util/BigArraysTests.java +++ b/server/src/test/java/org/elasticsearch/common/util/BigArraysTests.java @@ -527,49 +527,46 @@ public void testOverSizeUsesMinPageCount() { */ public void testPreallocate() { ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + HierarchyCircuitBreakerService realBreakers = new HierarchyCircuitBreakerService( + CircuitBreakerMetrics.NOOP, + Settings.EMPTY, + List.of(), + clusterSettings + ); + BigArrays unPreAllocated = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), realBreakers); + long toPreallocate = randomLongBetween(4000, 10000); + CircuitBreaker realBreaker = realBreakers.getBreaker(CircuitBreaker.REQUEST); + assertThat(realBreaker.getUsed(), equalTo(0L)); try ( - HierarchyCircuitBreakerService realBreakers = new HierarchyCircuitBreakerService( - CircuitBreakerMetrics.NOOP, - Settings.EMPTY, - List.of(), - clusterSettings + PreallocatedCircuitBreakerService prealloctedBreakerService = new PreallocatedCircuitBreakerService( + realBreakers, + CircuitBreaker.REQUEST, + toPreallocate, + "test" ) ) { - BigArrays unPreAllocated = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), realBreakers); - long toPreallocate = randomLongBetween(4000, 10000); - CircuitBreaker realBreaker = realBreakers.getBreaker(CircuitBreaker.REQUEST); - assertThat(realBreaker.getUsed(), equalTo(0L)); - try ( - PreallocatedCircuitBreakerService prealloctedBreakerService = new PreallocatedCircuitBreakerService( - realBreakers, - CircuitBreaker.REQUEST, - toPreallocate, - "test" - ) - ) { - assertThat(realBreaker.getUsed(), equalTo(toPreallocate)); - BigArrays preallocated = unPreAllocated.withBreakerService(prealloctedBreakerService); + assertThat(realBreaker.getUsed(), equalTo(toPreallocate)); + BigArrays preallocated = unPreAllocated.withBreakerService(prealloctedBreakerService); - // We don't grab any bytes just making a new BigArrays - assertThat(realBreaker.getUsed(), equalTo(toPreallocate)); + // We don't grab any bytes just making a new BigArrays + assertThat(realBreaker.getUsed(), equalTo(toPreallocate)); - List arrays = new ArrayList<>(); - for (int i = 0; i < 30; i++) { - // We're well under the preallocation so grabbing a little array doesn't allocate anything - arrays.add(preallocated.newLongArray(1)); - assertThat(realBreaker.getUsed(), equalTo(toPreallocate)); - } - - // Allocating a large array *does* allocate some bytes - arrays.add(preallocated.newLongArray(1024)); - long expectedMin = (PageCacheRecycler.LONG_PAGE_SIZE + arrays.size()) * Long.BYTES; - assertThat(realBreaker.getUsed(), greaterThanOrEqualTo(expectedMin)); - // 64 should be enough room for each BigArray object - assertThat(realBreaker.getUsed(), lessThanOrEqualTo(expectedMin + 64 * arrays.size())); - Releasables.close(arrays); + List arrays = new ArrayList<>(); + for (int i = 0; i < 30; i++) { + // We're well under the preallocation so grabbing a little array doesn't allocate anything + arrays.add(preallocated.newLongArray(1)); + assertThat(realBreaker.getUsed(), equalTo(toPreallocate)); } - assertThat(realBreaker.getUsed(), equalTo(0L)); + + // Allocating a large array *does* allocate some bytes + arrays.add(preallocated.newLongArray(1024)); + long expectedMin = (PageCacheRecycler.LONG_PAGE_SIZE + arrays.size()) * Long.BYTES; + assertThat(realBreaker.getUsed(), greaterThanOrEqualTo(expectedMin)); + // 64 should be enough room for each BigArray object + assertThat(realBreaker.getUsed(), lessThanOrEqualTo(expectedMin + 64 * arrays.size())); + Releasables.close(arrays); } + assertThat(realBreaker.getUsed(), equalTo(0L)); } private List bigArrayCreators(final long maxSize, final boolean withBreaking) { diff --git a/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java b/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java index 0a8fbcf6d56b9..4cb6e1febd5e9 100644 --- a/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/breaker/HierarchyCircuitBreakerServiceTests.java @@ -200,39 +200,38 @@ public void testBorrowingSiblingBreakerMemory() { .put(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "150mb") .put(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "150mb") .build(); - try ( - CircuitBreakerService service = new HierarchyCircuitBreakerService( - CircuitBreakerMetrics.NOOP, - clusterSettings, - Collections.emptyList(), - new ClusterSettings(clusterSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) - ) - ) { - CircuitBreaker requestCircuitBreaker = service.getBreaker(CircuitBreaker.REQUEST); - CircuitBreaker fieldDataCircuitBreaker = service.getBreaker(CircuitBreaker.FIELDDATA); - - assertEquals(new ByteSizeValue(200, ByteSizeUnit.MB).getBytes(), service.stats().getStats(CircuitBreaker.PARENT).getLimit()); - assertEquals(new ByteSizeValue(150, ByteSizeUnit.MB).getBytes(), requestCircuitBreaker.getLimit()); - assertEquals(new ByteSizeValue(150, ByteSizeUnit.MB).getBytes(), fieldDataCircuitBreaker.getLimit()); - - fieldDataCircuitBreaker.addEstimateBytesAndMaybeBreak(new ByteSizeValue(50, ByteSizeUnit.MB).getBytes(), "should not break"); - assertEquals(new ByteSizeValue(50, ByteSizeUnit.MB).getBytes(), fieldDataCircuitBreaker.getUsed(), 0.0); - requestCircuitBreaker.addEstimateBytesAndMaybeBreak(new ByteSizeValue(50, ByteSizeUnit.MB).getBytes(), "should not break"); - assertEquals(new ByteSizeValue(50, ByteSizeUnit.MB).getBytes(), requestCircuitBreaker.getUsed(), 0.0); - requestCircuitBreaker.addEstimateBytesAndMaybeBreak(new ByteSizeValue(50, ByteSizeUnit.MB).getBytes(), "should not break"); - assertEquals(new ByteSizeValue(100, ByteSizeUnit.MB).getBytes(), requestCircuitBreaker.getUsed(), 0.0); - CircuitBreakingException exception = expectThrows( - CircuitBreakingException.class, - () -> requestCircuitBreaker.addEstimateBytesAndMaybeBreak(new ByteSizeValue(50, ByteSizeUnit.MB).getBytes(), "should break") - ); - assertThat(exception.getMessage(), containsString("[parent] Data too large, data for [should break] would be")); - assertThat(exception.getMessage(), containsString("which is larger than the limit of [209715200/200mb]")); - assertThat(exception.getMessage(), containsString("usages [")); - assertThat(exception.getMessage(), containsString("fielddata=54001664/51.5mb")); - assertThat(exception.getMessage(), containsString("inflight_requests=0/0b")); - assertThat(exception.getMessage(), containsString("request=157286400/150mb")); - assertThat(exception.getDurability(), equalTo(CircuitBreaker.Durability.TRANSIENT)); - } + + CircuitBreakerService service = new HierarchyCircuitBreakerService( + CircuitBreakerMetrics.NOOP, + clusterSettings, + Collections.emptyList(), + new ClusterSettings(clusterSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ); + + CircuitBreaker requestCircuitBreaker = service.getBreaker(CircuitBreaker.REQUEST); + CircuitBreaker fieldDataCircuitBreaker = service.getBreaker(CircuitBreaker.FIELDDATA); + + assertEquals(new ByteSizeValue(200, ByteSizeUnit.MB).getBytes(), service.stats().getStats(CircuitBreaker.PARENT).getLimit()); + assertEquals(new ByteSizeValue(150, ByteSizeUnit.MB).getBytes(), requestCircuitBreaker.getLimit()); + assertEquals(new ByteSizeValue(150, ByteSizeUnit.MB).getBytes(), fieldDataCircuitBreaker.getLimit()); + + fieldDataCircuitBreaker.addEstimateBytesAndMaybeBreak(new ByteSizeValue(50, ByteSizeUnit.MB).getBytes(), "should not break"); + assertEquals(new ByteSizeValue(50, ByteSizeUnit.MB).getBytes(), fieldDataCircuitBreaker.getUsed(), 0.0); + requestCircuitBreaker.addEstimateBytesAndMaybeBreak(new ByteSizeValue(50, ByteSizeUnit.MB).getBytes(), "should not break"); + assertEquals(new ByteSizeValue(50, ByteSizeUnit.MB).getBytes(), requestCircuitBreaker.getUsed(), 0.0); + requestCircuitBreaker.addEstimateBytesAndMaybeBreak(new ByteSizeValue(50, ByteSizeUnit.MB).getBytes(), "should not break"); + assertEquals(new ByteSizeValue(100, ByteSizeUnit.MB).getBytes(), requestCircuitBreaker.getUsed(), 0.0); + CircuitBreakingException exception = expectThrows( + CircuitBreakingException.class, + () -> requestCircuitBreaker.addEstimateBytesAndMaybeBreak(new ByteSizeValue(50, ByteSizeUnit.MB).getBytes(), "should break") + ); + assertThat(exception.getMessage(), containsString("[parent] Data too large, data for [should break] would be")); + assertThat(exception.getMessage(), containsString("which is larger than the limit of [209715200/200mb]")); + assertThat(exception.getMessage(), containsString("usages [")); + assertThat(exception.getMessage(), containsString("fielddata=54001664/51.5mb")); + assertThat(exception.getMessage(), containsString("inflight_requests=0/0b")); + assertThat(exception.getMessage(), containsString("request=157286400/150mb")); + assertThat(exception.getDurability(), equalTo(CircuitBreaker.Durability.TRANSIENT)); assertCircuitBreakerLimitWarning(); } @@ -683,42 +682,41 @@ public void testTrippedCircuitBreakerDurability() { .put(HierarchyCircuitBreakerService.REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "150mb") .put(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "150mb") .build(); - try ( - CircuitBreakerService service = new HierarchyCircuitBreakerService( - CircuitBreakerMetrics.NOOP, - clusterSettings, - Collections.emptyList(), - new ClusterSettings(clusterSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) - ) - ) { - CircuitBreaker requestCircuitBreaker = service.getBreaker(CircuitBreaker.REQUEST); - CircuitBreaker fieldDataCircuitBreaker = service.getBreaker(CircuitBreaker.FIELDDATA); - - CircuitBreaker.Durability expectedDurability; - if (randomBoolean()) { - fieldDataCircuitBreaker.addEstimateBytesAndMaybeBreak(mb(100), "should not break"); - requestCircuitBreaker.addEstimateBytesAndMaybeBreak(mb(70), "should not break"); - expectedDurability = CircuitBreaker.Durability.PERMANENT; - } else { - fieldDataCircuitBreaker.addEstimateBytesAndMaybeBreak(mb(70), "should not break"); - requestCircuitBreaker.addEstimateBytesAndMaybeBreak(mb(120), "should not break"); - expectedDurability = CircuitBreaker.Durability.TRANSIENT; - } + CircuitBreakerService service = new HierarchyCircuitBreakerService( + CircuitBreakerMetrics.NOOP, + clusterSettings, + Collections.emptyList(), + new ClusterSettings(clusterSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ); + CircuitBreaker requestCircuitBreaker = service.getBreaker(CircuitBreaker.REQUEST); + CircuitBreaker fieldDataCircuitBreaker = service.getBreaker(CircuitBreaker.FIELDDATA); + + CircuitBreaker.Durability expectedDurability; + if (randomBoolean()) { + fieldDataCircuitBreaker.addEstimateBytesAndMaybeBreak(mb(100), "should not break"); + requestCircuitBreaker.addEstimateBytesAndMaybeBreak(mb(70), "should not break"); + expectedDurability = CircuitBreaker.Durability.PERMANENT; + } else { + fieldDataCircuitBreaker.addEstimateBytesAndMaybeBreak(mb(70), "should not break"); + requestCircuitBreaker.addEstimateBytesAndMaybeBreak(mb(120), "should not break"); + expectedDurability = CircuitBreaker.Durability.TRANSIENT; + } - CircuitBreakingException exception = expectThrows( - CircuitBreakingException.class, - () -> fieldDataCircuitBreaker.addEstimateBytesAndMaybeBreak(mb(40), "should break") - ); + CircuitBreakingException exception = expectThrows( + CircuitBreakingException.class, + () -> fieldDataCircuitBreaker.addEstimateBytesAndMaybeBreak(mb(40), "should break") + ); + + assertThat(exception.getMessage(), containsString("[parent] Data too large, data for [should break] would be")); + assertThat(exception.getMessage(), containsString("which is larger than the limit of [209715200/200mb]")); + assertThat( + "Expected [" + expectedDurability + "] due to [" + exception.getMessage() + "]", + exception.getDurability(), + equalTo(expectedDurability) + ); - assertThat(exception.getMessage(), containsString("[parent] Data too large, data for [should break] would be")); - assertThat(exception.getMessage(), containsString("which is larger than the limit of [209715200/200mb]")); - assertThat( - "Expected [" + expectedDurability + "] due to [" + exception.getMessage() + "]", - exception.getDurability(), - equalTo(expectedDurability) - ); - } assertCircuitBreakerLimitWarning(); + } public void testAllocationBucketsBreaker() { @@ -727,34 +725,31 @@ public void testAllocationBucketsBreaker() { .put(HierarchyCircuitBreakerService.USE_REAL_MEMORY_USAGE_SETTING.getKey(), "false") .build(); - try ( - HierarchyCircuitBreakerService service = new HierarchyCircuitBreakerService( - CircuitBreakerMetrics.NOOP, - clusterSettings, - Collections.emptyList(), - new ClusterSettings(clusterSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) - ) - ) { + HierarchyCircuitBreakerService service = new HierarchyCircuitBreakerService( + CircuitBreakerMetrics.NOOP, + clusterSettings, + Collections.emptyList(), + new ClusterSettings(clusterSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ); - long parentLimitBytes = service.getParentLimit(); - assertEquals(new ByteSizeValue(100, ByteSizeUnit.BYTES).getBytes(), parentLimitBytes); + long parentLimitBytes = service.getParentLimit(); + assertEquals(new ByteSizeValue(100, ByteSizeUnit.BYTES).getBytes(), parentLimitBytes); - CircuitBreaker breaker = service.getBreaker(CircuitBreaker.REQUEST); - MultiBucketConsumerService.MultiBucketConsumer multiBucketConsumer = new MultiBucketConsumerService.MultiBucketConsumer( - 10000, - breaker - ); + CircuitBreaker breaker = service.getBreaker(CircuitBreaker.REQUEST); + MultiBucketConsumerService.MultiBucketConsumer multiBucketConsumer = new MultiBucketConsumerService.MultiBucketConsumer( + 10000, + breaker + ); - // make sure used bytes is greater than the total circuit breaker limit - breaker.addWithoutBreaking(200); - // make sure that we check on the following call - for (int i = 0; i < 1023; i++) { - multiBucketConsumer.accept(0); - } - CircuitBreakingException exception = expectThrows(CircuitBreakingException.class, () -> multiBucketConsumer.accept(1024)); - assertThat(exception.getMessage(), containsString("[parent] Data too large, data for [allocated_buckets] would be")); - assertThat(exception.getMessage(), containsString("which is larger than the limit of [100/100b]")); + // make sure used bytes is greater than the total circuit breaker limit + breaker.addWithoutBreaking(200); + // make sure that we check on the following call + for (int i = 0; i < 1023; i++) { + multiBucketConsumer.accept(0); } + CircuitBreakingException exception = expectThrows(CircuitBreakingException.class, () -> multiBucketConsumer.accept(1024)); + assertThat(exception.getMessage(), containsString("[parent] Data too large, data for [allocated_buckets] would be")); + assertThat(exception.getMessage(), containsString("which is larger than the limit of [100/100b]")); assertCircuitBreakerLimitWarning(); } @@ -790,21 +785,18 @@ public void testRegisterCustomCircuitBreakers_WithDuplicates() { } public void testCustomCircuitBreakers() { - try ( - CircuitBreakerService service = new HierarchyCircuitBreakerService( - CircuitBreakerMetrics.NOOP, - Settings.EMPTY, - Arrays.asList(new BreakerSettings("foo", 100, 1.2), new BreakerSettings("bar", 200, 0.1)), - new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) - ) - ) { - assertThat(service.getBreaker("foo"), is(not(nullValue()))); - assertThat(service.getBreaker("foo").getOverhead(), equalTo(1.2)); - assertThat(service.getBreaker("foo").getLimit(), equalTo(100L)); - assertThat(service.getBreaker("bar"), is(not(nullValue()))); - assertThat(service.getBreaker("bar").getOverhead(), equalTo(0.1)); - assertThat(service.getBreaker("bar").getLimit(), equalTo(200L)); - } + CircuitBreakerService service = new HierarchyCircuitBreakerService( + CircuitBreakerMetrics.NOOP, + Settings.EMPTY, + Arrays.asList(new BreakerSettings("foo", 100, 1.2), new BreakerSettings("bar", 200, 0.1)), + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ); + assertThat(service.getBreaker("foo"), is(not(nullValue()))); + assertThat(service.getBreaker("foo").getOverhead(), equalTo(1.2)); + assertThat(service.getBreaker("foo").getLimit(), equalTo(100L)); + assertThat(service.getBreaker("bar"), is(not(nullValue()))); + assertThat(service.getBreaker("bar").getOverhead(), equalTo(0.1)); + assertThat(service.getBreaker("bar").getLimit(), equalTo(200L)); } private static long mb(long size) { @@ -812,28 +804,25 @@ private static long mb(long size) { } public void testUpdatingUseRealMemory() { - try ( - HierarchyCircuitBreakerService service = new HierarchyCircuitBreakerService( - CircuitBreakerMetrics.NOOP, - Settings.EMPTY, - Collections.emptyList(), - new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) - ) - ) { - // use real memory default true - assertTrue(service.isTrackRealMemoryUsage()); - assertThat(service.getOverLimitStrategy(), instanceOf(HierarchyCircuitBreakerService.G1OverLimitStrategy.class)); - - // update use_real_memory to false - service.updateUseRealMemorySetting(false); - assertFalse(service.isTrackRealMemoryUsage()); - assertThat(service.getOverLimitStrategy(), not(instanceOf(HierarchyCircuitBreakerService.G1OverLimitStrategy.class))); - - // update use_real_memory to true - service.updateUseRealMemorySetting(true); - assertTrue(service.isTrackRealMemoryUsage()); - assertThat(service.getOverLimitStrategy(), instanceOf(HierarchyCircuitBreakerService.G1OverLimitStrategy.class)); - } + HierarchyCircuitBreakerService service = new HierarchyCircuitBreakerService( + CircuitBreakerMetrics.NOOP, + Settings.EMPTY, + Collections.emptyList(), + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ); + // use real memory default true + assertTrue(service.isTrackRealMemoryUsage()); + assertThat(service.getOverLimitStrategy(), instanceOf(HierarchyCircuitBreakerService.G1OverLimitStrategy.class)); + + // update use_real_memory to false + service.updateUseRealMemorySetting(false); + assertFalse(service.isTrackRealMemoryUsage()); + assertThat(service.getOverLimitStrategy(), not(instanceOf(HierarchyCircuitBreakerService.G1OverLimitStrategy.class))); + + // update use_real_memory to true + service.updateUseRealMemorySetting(true); + assertTrue(service.isTrackRealMemoryUsage()); + assertThat(service.getOverLimitStrategy(), instanceOf(HierarchyCircuitBreakerService.G1OverLimitStrategy.class)); } public void testApplySettingForUpdatingUseRealMemory() { @@ -842,34 +831,31 @@ public void testApplySettingForUpdatingUseRealMemory() { Settings initialSettings = Settings.builder().put(useRealMemoryUsageSetting, "true").build(); ClusterSettings clusterSettings = new ClusterSettings(initialSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); - try ( - HierarchyCircuitBreakerService service = new HierarchyCircuitBreakerService( - CircuitBreakerMetrics.NOOP, - Settings.EMPTY, - Collections.emptyList(), - clusterSettings - ) - ) { - // total.limit defaults to 95% of the JVM heap if use_real_memory is true - assertEquals( - MemorySizeValue.parseBytesSizeValueOrHeapRatio("95%", totalCircuitBreakerLimitSetting).getBytes(), - service.getParentLimit() - ); + HierarchyCircuitBreakerService service = new HierarchyCircuitBreakerService( + CircuitBreakerMetrics.NOOP, + Settings.EMPTY, + Collections.emptyList(), + clusterSettings + ); + // total.limit defaults to 95% of the JVM heap if use_real_memory is true + assertEquals( + MemorySizeValue.parseBytesSizeValueOrHeapRatio("95%", totalCircuitBreakerLimitSetting).getBytes(), + service.getParentLimit() + ); - // total.limit defaults to 70% of the JVM heap if use_real_memory set to false - clusterSettings.applySettings(Settings.builder().put(useRealMemoryUsageSetting, false).build()); - assertEquals( - MemorySizeValue.parseBytesSizeValueOrHeapRatio("70%", totalCircuitBreakerLimitSetting).getBytes(), - service.getParentLimit() - ); + // total.limit defaults to 70% of the JVM heap if use_real_memory set to false + clusterSettings.applySettings(Settings.builder().put(useRealMemoryUsageSetting, false).build()); + assertEquals( + MemorySizeValue.parseBytesSizeValueOrHeapRatio("70%", totalCircuitBreakerLimitSetting).getBytes(), + service.getParentLimit() + ); - // total.limit defaults to 95% of the JVM heap if use_real_memory set to true - clusterSettings.applySettings(Settings.builder().put(useRealMemoryUsageSetting, true).build()); - assertEquals( - MemorySizeValue.parseBytesSizeValueOrHeapRatio("95%", totalCircuitBreakerLimitSetting).getBytes(), - service.getParentLimit() - ); - } + // total.limit defaults to 95% of the JVM heap if use_real_memory set to true + clusterSettings.applySettings(Settings.builder().put(useRealMemoryUsageSetting, true).build()); + assertEquals( + MemorySizeValue.parseBytesSizeValueOrHeapRatio("95%", totalCircuitBreakerLimitSetting).getBytes(), + service.getParentLimit() + ); } public void testSizeBelowMinimumWarning() { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestStateReleasingTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestStateReleasingTests.java index c03578b0017bb..b9f8037efb9ca 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestStateReleasingTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestStateReleasingTests.java @@ -102,19 +102,18 @@ public void testReadWithData() throws IOException { */ public void testCircuitBreakerTrip(CheckedFunction tDigestStateFactory) throws E { - try (CrankyCircuitBreakerService circuitBreakerService = new CrankyCircuitBreakerService()) { - CircuitBreaker breaker = circuitBreakerService.getBreaker("test"); + CrankyCircuitBreakerService circuitBreakerService = new CrankyCircuitBreakerService(); + CircuitBreaker breaker = circuitBreakerService.getBreaker("test"); - try (TDigestState state = tDigestStateFactory.apply(breaker)) { - // Add some data to make it trip. It won't work in all digest types - for (int i = 0; i < 10; i++) { - state.add(randomDoubleBetween(-Double.MAX_VALUE, Double.MAX_VALUE, true)); - } - } catch (CircuitBreakingException e) { - // Expected - } finally { - assertThat("unreleased bytes", breaker.getUsed(), equalTo(0L)); + try (TDigestState state = tDigestStateFactory.apply(breaker)) { + // Add some data to make it trip. It won't work in all digest types + for (int i = 0; i < 10; i++) { + state.add(randomDoubleBetween(-Double.MAX_VALUE, Double.MAX_VALUE, true)); } + } catch (CircuitBreakingException e) { + // Expected + } finally { + assertThat("unreleased bytes", breaker.getUsed(), equalTo(0L)); } } } diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index d491e4bb52fa2..f057d35d6e7a9 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -671,54 +671,49 @@ private A searchAndReduce( Releasables.close(context); } } - - try { - if (aggTestConfig.incrementalReduce() && internalAggs.size() > 1) { - // sometimes do an incremental reduce - int toReduceSize = internalAggs.size(); - Collections.shuffle(internalAggs, random()); - int r = randomIntBetween(1, toReduceSize); - List toReduce = internalAggs.subList(0, r); - AggregationReduceContext reduceContext = new AggregationReduceContext.ForPartial( - bigArraysForReduction, - getMockScriptService(), - () -> false, - builder, - b -> {} - ); - internalAggs = new ArrayList<>(internalAggs.subList(r, toReduceSize)); - internalAggs.add(InternalAggregations.topLevelReduce(toReduce, reduceContext)); - for (InternalAggregations internalAggregation : internalAggs) { - assertRoundTrip(internalAggregation.copyResults()); - } - } - - // now do the final reduce - MultiBucketConsumer reduceBucketConsumer = new MultiBucketConsumer( - maxBucket, - new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST) - ); - AggregationReduceContext reduceContext = new AggregationReduceContext.ForFinal( + if (aggTestConfig.incrementalReduce() && internalAggs.size() > 1) { + // sometimes do an incremental reduce + int toReduceSize = internalAggs.size(); + Collections.shuffle(internalAggs, random()); + int r = randomIntBetween(1, toReduceSize); + List toReduce = internalAggs.subList(0, r); + AggregationReduceContext reduceContext = new AggregationReduceContext.ForPartial( bigArraysForReduction, getMockScriptService(), () -> false, builder, - reduceBucketConsumer + b -> {} ); + internalAggs = new ArrayList<>(internalAggs.subList(r, toReduceSize)); + internalAggs.add(InternalAggregations.topLevelReduce(toReduce, reduceContext)); + for (InternalAggregations internalAggregation : internalAggs) { + assertRoundTrip(internalAggregation.copyResults()); + } + } - @SuppressWarnings("unchecked") - A internalAgg = (A) doInternalAggregationsReduce(internalAggs, reduceContext); - assertRoundTrip(internalAgg); + // now do the final reduce + MultiBucketConsumer reduceBucketConsumer = new MultiBucketConsumer( + maxBucket, + new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST) + ); + AggregationReduceContext reduceContext = new AggregationReduceContext.ForFinal( + bigArraysForReduction, + getMockScriptService(), + () -> false, + builder, + reduceBucketConsumer + ); - doAssertReducedMultiBucketConsumer(internalAgg, reduceBucketConsumer); - assertRoundTrip(internalAgg); - if (aggTestConfig.builder instanceof ValuesSourceAggregationBuilder.MetricsAggregationBuilder) { - verifyMetricNames((ValuesSourceAggregationBuilder.MetricsAggregationBuilder) aggTestConfig.builder, internalAgg); - } - return internalAgg; - } finally { - Releasables.close(breakerService); + @SuppressWarnings("unchecked") + A internalAgg = (A) doInternalAggregationsReduce(internalAggs, reduceContext); + assertRoundTrip(internalAgg); + + doAssertReducedMultiBucketConsumer(internalAgg, reduceBucketConsumer); + assertRoundTrip(internalAgg); + if (aggTestConfig.builder instanceof ValuesSourceAggregationBuilder.MetricsAggregationBuilder) { + verifyMetricNames((ValuesSourceAggregationBuilder.MetricsAggregationBuilder) aggTestConfig.builder, internalAgg); } + return internalAgg; } private InternalAggregation doReduce(List aggregators, AggregationReduceContext reduceContext) { diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sample/CircuitBreakerTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sample/CircuitBreakerTests.java index 9cd6549b4be2c..dc132659417ff 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sample/CircuitBreakerTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sample/CircuitBreakerTests.java @@ -101,15 +101,13 @@ public void testMemoryCleared() { } private void testMemoryCleared(boolean fail) { - try ( - CircuitBreakerService service = new HierarchyCircuitBreakerService( - CircuitBreakerMetrics.NOOP, - Settings.EMPTY, - Collections.singletonList(EqlTestUtils.circuitBreakerSettings(Settings.EMPTY)), - new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) - ); - var threadPool = createThreadPool() - ) { + CircuitBreakerService service = new HierarchyCircuitBreakerService( + CircuitBreakerMetrics.NOOP, + Settings.EMPTY, + Collections.singletonList(EqlTestUtils.circuitBreakerSettings(Settings.EMPTY)), + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ); + try (var threadPool = createThreadPool()) { final var esClient = new ESMockClient(threadPool, service.getBreaker(CIRCUIT_BREAKER_NAME)); CircuitBreaker eqlCircuitBreaker = service.getBreaker(CIRCUIT_BREAKER_NAME); IndexResolver indexResolver = new IndexResolver(esClient, "cluster", DefaultDataTypeRegistry.INSTANCE, () -> emptySet()); diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java index ecf5ef61ac49a..fe1fca45364e3 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/sequence/CircuitBreakerTests.java @@ -209,15 +209,13 @@ private void assertMemoryCleared( TriFunction esClientSupplier ) { final int searchRequestsExpectedCount = 2; - try ( - CircuitBreakerService service = new HierarchyCircuitBreakerService( - CircuitBreakerMetrics.NOOP, - Settings.EMPTY, - breakerSettings(), - new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) - ); - var threadPool = createThreadPool() - ) { + CircuitBreakerService service = new HierarchyCircuitBreakerService( + CircuitBreakerMetrics.NOOP, + Settings.EMPTY, + breakerSettings(), + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ); + try (var threadPool = createThreadPool()) { final var esClient = esClientSupplier.apply(threadPool, service.getBreaker(CIRCUIT_BREAKER_NAME), searchRequestsExpectedCount); CircuitBreaker eqlCircuitBreaker = service.getBreaker(CIRCUIT_BREAKER_NAME); QueryClient eqlClient = buildQueryClient(esClient, eqlCircuitBreaker); @@ -248,16 +246,13 @@ public void testEqlCBCleanedUp_on_ParentCBBreak() { Settings settings = Settings.builder() .put(HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "0%") .build(); - - try ( - CircuitBreakerService service = new HierarchyCircuitBreakerService( - CircuitBreakerMetrics.NOOP, - settings, - breakerSettings(), - new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) - ); - var threadPool = createThreadPool() - ) { + CircuitBreakerService service = new HierarchyCircuitBreakerService( + CircuitBreakerMetrics.NOOP, + settings, + breakerSettings(), + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ); + try (var threadPool = createThreadPool()) { final var esClient = new SuccessfulESMockClient( threadPool, service.getBreaker(CIRCUIT_BREAKER_NAME), diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/DelegatingCircuitBreakerServiceTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/DelegatingCircuitBreakerServiceTests.java index aa1a82116387e..a452ff49b7a74 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/DelegatingCircuitBreakerServiceTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/aggs/frequentitemsets/mr/DelegatingCircuitBreakerServiceTests.java @@ -26,49 +26,43 @@ public class DelegatingCircuitBreakerServiceTests extends ESTestCase { * the purpose of the test is not hitting the `IllegalStateException("already closed")` in * PreallocatedCircuitBreaker#addEstimateBytesAndMaybeBreak in {@link PreallocatedCircuitBreakerService} */ - public void testThreadedExecution() { + public void testThreadedExecution() throws InterruptedException { + HierarchyCircuitBreakerService topBreaker = new HierarchyCircuitBreakerService( + CircuitBreakerMetrics.NOOP, + Settings.builder() + .put(REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "10mb") + // Disable the real memory checking because it causes other tests to interfere with this one. + .put(USE_REAL_MEMORY_USAGE_SETTING.getKey(), false) + .build(), + List.of(), + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) + ); + PreallocatedCircuitBreakerService preallocated = new PreallocatedCircuitBreakerService( + topBreaker, + CircuitBreaker.REQUEST, + 10_000, + "test" + ); - try ( - HierarchyCircuitBreakerService topBreaker = new HierarchyCircuitBreakerService( - CircuitBreakerMetrics.NOOP, - Settings.builder() - .put(REQUEST_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "10mb") - // Disable the real memory checking because it causes other tests to interfere with this one. - .put(USE_REAL_MEMORY_USAGE_SETTING.getKey(), false) - .build(), - List.of(), - new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS) - ) - ) { - PreallocatedCircuitBreakerService preallocated = new PreallocatedCircuitBreakerService( - topBreaker, - CircuitBreaker.REQUEST, - 10_000, - "test" - ); + CircuitBreaker breaker = preallocated.getBreaker(CircuitBreaker.REQUEST); - CircuitBreaker breaker = preallocated.getBreaker(CircuitBreaker.REQUEST); + DelegatingCircuitBreakerService delegatingCircuitBreakerService = new DelegatingCircuitBreakerService(breaker, (bytes -> { + breaker.addEstimateBytesAndMaybeBreak(bytes, "test"); + })); - DelegatingCircuitBreakerService delegatingCircuitBreakerService = new DelegatingCircuitBreakerService(breaker, (bytes -> { - breaker.addEstimateBytesAndMaybeBreak(bytes, "test"); - })); + Thread consumerThread = new Thread(() -> { + for (int i = 0; i < 100; ++i) { + delegatingCircuitBreakerService.getBreaker("ignored").addEstimateBytesAndMaybeBreak(i % 2 == 0 ? 10 : -10, "ignored"); + } + }); - Thread consumerThread = new Thread(() -> { - for (int i = 0; i < 100; ++i) { - delegatingCircuitBreakerService.getBreaker("ignored").addEstimateBytesAndMaybeBreak(i % 2 == 0 ? 10 : -10, "ignored"); - } - }); - - final Thread producerThread = new Thread(() -> { - delegatingCircuitBreakerService.disconnect(); - preallocated.close(); - }); - consumerThread.start(); - producerThread.start(); - consumerThread.join(); - producerThread.join(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } + final Thread producerThread = new Thread(() -> { + delegatingCircuitBreakerService.disconnect(); + preallocated.close(); + }); + consumerThread.start(); + producerThread.start(); + consumerThread.join(); + producerThread.join(); } } From ce2a7dee866ad2050ee88512c99f33cc8fb5971a Mon Sep 17 00:00:00 2001 From: Chris Hegarty <62058229+ChrisHegarty@users.noreply.github.com> Date: Sat, 14 Dec 2024 21:40:13 +0000 Subject: [PATCH 30/35] Add Lucene 9.12.1 index version constant (#118557) This commit adds the IndexVersion constant for Lucene 9.12.1. Since main is already on Lucene 10, this change trivially just ensures that the constant version value in main is the same as that in the 8.x branch. --- server/src/main/java/org/elasticsearch/index/IndexVersions.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 0aaae2104576a..fd321f6256194 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -133,6 +133,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion TIME_BASED_K_ORDERED_DOC_ID_BACKPORT = def(8_520_00_0, Version.LUCENE_9_12_0); public static final IndexVersion V8_DEPRECATE_SOURCE_MODE_MAPPER = def(8_521_00_0, Version.LUCENE_9_12_0); public static final IndexVersion USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BACKPORT = def(8_522_00_0, Version.LUCENE_9_12_0); + public static final IndexVersion UPGRADE_TO_LUCENE_9_12_1 = def(8_523_00_0, parseUnchecked("9.12.1")); public static final IndexVersion UPGRADE_TO_LUCENE_10_0_0 = def(9_000_00_0, Version.LUCENE_10_0_0); public static final IndexVersion LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT = def(9_001_00_0, Version.LUCENE_10_0_0); public static final IndexVersion TIME_BASED_K_ORDERED_DOC_ID = def(9_002_00_0, Version.LUCENE_10_0_0); From ba52a225dd9d04e8adfde7998fe4698151cbb7b9 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Sun, 15 Dec 2024 10:49:25 +0100 Subject: [PATCH 31/35] [Build] Make checkPart4 configuration cache compatible (#117250) This makes checkPart4 tasks in general configuration cache compliant. We do not enable it by default in our pipelines yet. That will eventually be done in separate PRs. To track any regressions we cover the checkPart4 task in the weekly gradle cache validation build job --- .../gradle-configuration-cache-validation.sh | 6 ++-- .../gradle/internal/BwcSetupExtension.java | 31 ++++++++++++------- .../plugin/sql/qa/jdbc/security/build.gradle | 11 ++++--- .../sql/qa/server/security/build.gradle | 25 +++++++++++++-- 4 files changed, 50 insertions(+), 23 deletions(-) diff --git a/.buildkite/scripts/gradle-configuration-cache-validation.sh b/.buildkite/scripts/gradle-configuration-cache-validation.sh index 55a4b18a1e887..2257b4cfb17a5 100755 --- a/.buildkite/scripts/gradle-configuration-cache-validation.sh +++ b/.buildkite/scripts/gradle-configuration-cache-validation.sh @@ -3,16 +3,16 @@ set -euo pipefail # This is a workaround for https://github.com/gradle/gradle/issues/28159 -.ci/scripts/run-gradle.sh --no-daemon precommit +.ci/scripts/run-gradle.sh --no-daemon $@ -.ci/scripts/run-gradle.sh --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 +.ci/scripts/run-gradle.sh --configuration-cache -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 $@ # Create a temporary file tmpOutputFile=$(mktemp) trap "rm $tmpOutputFile" EXIT echo "2nd run" -.ci/scripts/run-gradle.sh --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 | tee $tmpOutputFile +.ci/scripts/run-gradle.sh --configuration-cache -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 $@ | tee $tmpOutputFile # Check if the command was successful if grep -q "Configuration cache entry reused." $tmpOutputFile; then diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java index 5992a40275b46..6050e5c9a60d4 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java @@ -16,6 +16,7 @@ import org.gradle.api.Action; import org.gradle.api.GradleException; import org.gradle.api.Project; +import org.gradle.api.Task; import org.gradle.api.logging.LogLevel; import org.gradle.api.model.ObjectFactory; import org.gradle.api.provider.Property; @@ -103,10 +104,16 @@ private static TaskProvider createRunBwcGradleTask( loggedExec.dependsOn("checkoutBwcBranch"); loggedExec.getWorkingDir().set(checkoutDir.get()); - loggedExec.getNonTrackedEnvironment().put("JAVA_HOME", providerFactory.of(JavaHomeValueSource.class, spec -> { - spec.getParameters().getVersion().set(unreleasedVersionInfo.map(it -> it.version())); - spec.getParameters().getCheckoutDir().set(checkoutDir); - }).flatMap(s -> getJavaHome(objectFactory, toolChainService, Integer.parseInt(s)))); + loggedExec.doFirst(new Action() { + @Override + public void execute(Task task) { + Provider minimumCompilerVersionValueSource = providerFactory.of(JavaHomeValueSource.class, spec -> { + spec.getParameters().getVersion().set(unreleasedVersionInfo.map(it -> it.version())); + spec.getParameters().getCheckoutDir().set(checkoutDir); + }).flatMap(s -> getJavaHome(objectFactory, toolChainService, Integer.parseInt(s))); + loggedExec.getNonTrackedEnvironment().put("JAVA_HOME", minimumCompilerVersionValueSource.get()); + } + }); if (isCi && OS.current() != OS.WINDOWS) { // TODO: Disabled for now until we can figure out why files are getting corrupted @@ -169,14 +176,6 @@ private static Provider getJavaHome(ObjectFactory objectFactory, JavaToo .map(launcher -> launcher.getMetadata().getInstallationPath().getAsFile().getAbsolutePath()); } - private static String readFromFile(File file) { - try { - return FileUtils.readFileToString(file).trim(); - } catch (IOException ioException) { - throw new GradleException("Cannot read java properties file.", ioException); - } - } - public abstract static class JavaHomeValueSource implements ValueSource { private String minimumCompilerVersionPath(Version bwcVersion) { @@ -185,6 +184,14 @@ private String minimumCompilerVersionPath(Version bwcVersion) { : "buildSrc/" + MINIMUM_COMPILER_VERSION_PATH; } + private static String readFromFile(File file) { + try { + return FileUtils.readFileToString(file).trim(); + } catch (IOException ioException) { + throw new GradleException("Cannot read java properties file.", ioException); + } + } + @Override public String obtain() { return readFromFile( diff --git a/x-pack/plugin/sql/qa/jdbc/security/build.gradle b/x-pack/plugin/sql/qa/jdbc/security/build.gradle index 82510285cb996..b40a8aaaa5580 100644 --- a/x-pack/plugin/sql/qa/jdbc/security/build.gradle +++ b/x-pack/plugin/sql/qa/jdbc/security/build.gradle @@ -50,6 +50,7 @@ subprojects { tasks.withType(RestIntegTestTask).configureEach { + def taskName = name dependsOn copyTestClasses classpath += configurations.testArtifacts testClassesDirs = project.files(testArtifactsDir) @@ -58,15 +59,15 @@ subprojects { project.gradle.sharedServices, TestClustersPlugin.REGISTRY_SERVICE_NAME ) - project.getProviders().of(TestClusterValueSource.class) { + + def clusterInfo = project.getProviders().of(TestClusterValueSource.class) { it.parameters.path.set(clusterPath) - it.parameters.clusterName.set("javaRestTest") + it.parameters.clusterName.set(taskName) it.parameters.service = serviceProvider } - nonInputProperties.systemProperty 'tests.audit.logfile', - "${-> testClusters.javaRestTest.singleNode().getAuditLog()}" + nonInputProperties.systemProperty 'tests.audit.logfile', clusterInfo.map { it.auditLogs.get(0) } nonInputProperties.systemProperty 'tests.audit.yesterday.logfile', - "${-> testClusters.javaRestTest.singleNode().getAuditLog().getParentFile()}/javaRestTest_audit-${new Date().format('yyyy-MM-dd')}.json" + clusterInfo.map { it.auditLogs.get(0).getParentFile().toString() + "/javaRestTest_audit-${new Date().format('yyyy-MM-dd')}-1.json" } } } diff --git a/x-pack/plugin/sql/qa/server/security/build.gradle b/x-pack/plugin/sql/qa/server/security/build.gradle index 2d9f7b563d073..ef81ebda5cb89 100644 --- a/x-pack/plugin/sql/qa/server/security/build.gradle +++ b/x-pack/plugin/sql/qa/server/security/build.gradle @@ -1,3 +1,9 @@ +import org.elasticsearch.gradle.internal.test.RestIntegTestTask +import org.elasticsearch.gradle.testclusters.TestClusterValueSource +import org.elasticsearch.gradle.testclusters.TestClustersPlugin +import org.elasticsearch.gradle.testclusters.TestClustersRegistry +import org.elasticsearch.gradle.util.GradleUtils + apply plugin: 'elasticsearch.internal-test-artifact' dependencies { @@ -16,6 +22,8 @@ subprojects { // Use tests from the root security qa project in subprojects configurations.create('testArtifacts').transitive(false) + def clusterPath = getPath() + dependencies { javaRestTestImplementation project(":x-pack:plugin:core") javaRestTestImplementation(testArtifact(project(xpackModule('core')))) @@ -50,12 +58,23 @@ subprojects { tasks.named("javaRestTest").configure { dependsOn copyTestClasses + + Provider serviceProvider = GradleUtils.getBuildService( + project.gradle.sharedServices, + TestClustersPlugin.REGISTRY_SERVICE_NAME + ) + + def clusterInfo = project.getProviders().of(TestClusterValueSource.class) { + it.parameters.path.set(clusterPath) + it.parameters.clusterName.set("javaRestTest") + it.parameters.service = serviceProvider + } + testClassesDirs += project.files(testArtifactsDir) classpath += configurations.testArtifacts - nonInputProperties.systemProperty 'tests.audit.logfile', - "${-> testClusters.javaRestTest.singleNode().getAuditLog()}" + nonInputProperties.systemProperty 'tests.audit.logfile', clusterInfo.map { it.auditLogs.get(0) } nonInputProperties.systemProperty 'tests.audit.yesterday.logfile', - "${-> testClusters.javaRestTest.singleNode().getAuditLog().getParentFile()}/javaRestTest_audit-${new Date().format('yyyy-MM-dd')}-1.json.gz" + clusterInfo.map { it.auditLogs.get(0).getParentFile().toString() + "/javaRestTest_audit-${new Date().format('yyyy-MM-dd')}-1.json.gz"} } } From c1569b2e0909d6cd38d26515bf94403c8eac1dd1 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Sun, 15 Dec 2024 18:39:46 +0100 Subject: [PATCH 32/35] [CI] Allow explicitly override of worker count in run gradle script (#118734) This makes running experiments on ci way easier without changing default behaviour --- .ci/scripts/run-gradle.sh | 56 +++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/.ci/scripts/run-gradle.sh b/.ci/scripts/run-gradle.sh index af5db8a6b4063..b091c4a7c7d89 100755 --- a/.ci/scripts/run-gradle.sh +++ b/.ci/scripts/run-gradle.sh @@ -5,35 +5,39 @@ mkdir -p ~/.gradle/init.d && cp -v $WORKSPACE/.ci/init.gradle ~/.gradle/init.d MAX_WORKERS=4 -# Don't run this stuff on Windows -if ! uname -a | grep -q MING; then - # drop page cache and kernel slab objects on linux - [[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches +if [ -z "$MAX_WORKER_ENV" ]; then + # Don't run this stuff on Windows + if ! uname -a | grep -q MING; then + # drop page cache and kernel slab objects on linux + [[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches - if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then - MAX_WORKERS=16 - elif [ -f /proc/cpuinfo ]; then - MAX_WORKERS=`grep '^cpu\scores' /proc/cpuinfo | uniq | sed 's/\s\+//g' | cut -d':' -f 2` - else - if [[ "$OSTYPE" == "darwin"* ]]; then - MAX_WORKERS=`sysctl -n hw.physicalcpu | sed 's/\s\+//g'` - else - echo "Unsupported OS Type: $OSTYPE" - exit 1 - fi - fi - if pwd | grep -v -q ^/dev/shm ; then - echo "Not running on a ramdisk, reducing number of workers" - MAX_WORKERS=$(($MAX_WORKERS*2/3)) - fi + if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then + MAX_WORKERS=16 + elif [ -f /proc/cpuinfo ]; then + MAX_WORKERS=`grep '^cpu\scores' /proc/cpuinfo | uniq | sed 's/\s\+//g' | cut -d':' -f 2` + else + if [[ "$OSTYPE" == "darwin"* ]]; then + MAX_WORKERS=`sysctl -n hw.physicalcpu | sed 's/\s\+//g'` + else + echo "Unsupported OS Type: $OSTYPE" + exit 1 + fi + fi + if pwd | grep -v -q ^/dev/shm ; then + echo "Not running on a ramdisk, reducing number of workers" + MAX_WORKERS=$(($MAX_WORKERS*2/3)) + fi - # Export glibc version as environment variable since some BWC tests are incompatible with later versions - export GLIBC_VERSION=$(ldd --version | grep '^ldd' | sed 's/.* \([1-9]\.[0-9]*\).*/\1/') -fi + # Export glibc version as environment variable since some BWC tests are incompatible with later versions + export GLIBC_VERSION=$(ldd --version | grep '^ldd' | sed 's/.* \([1-9]\.[0-9]*\).*/\1/') + fi -# Running on 2-core machines without ramdisk can make this value be 0 -if [[ "$MAX_WORKERS" == "0" ]]; then - MAX_WORKERS=1 + # Running on 2-core machines without ramdisk can make this value be 0 + if [[ "$MAX_WORKERS" == "0" ]]; then + MAX_WORKERS=1 + fi +else + MAX_WORKERS=$MAX_WORKER_ENV fi set -e From 55ae512f186d0772c33c728a48004e929378cbe0 Mon Sep 17 00:00:00 2001 From: Tim Vernum Date: Mon, 16 Dec 2024 19:39:49 +1100 Subject: [PATCH 33/35] More realistic test scenario in Authc service test (#118519) This changes `AuthenticationServiceTests.testInvalidToken` to configure the security (tokens) index correctly. Previously `defensiveCopy()` would return `null` which would cause `getTokenDocById` to throw an exception, which `decodeToken` would catch and ignore. But that is not a realistic scenario, and is testing by side-effect. --- .../authc/AuthenticationServiceTests.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java index 7b66a95609b05..5eb9fb9b41a22 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java @@ -124,6 +124,7 @@ import java.util.function.Consumer; import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM; +import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; import static org.elasticsearch.test.ActionListenerUtils.anyActionListener; import static org.elasticsearch.test.SecurityTestsUtils.assertAuthenticationException; import static org.elasticsearch.test.TestMatchers.throwableWithMessage; @@ -1955,6 +1956,37 @@ public void testInvalidToken() throws Exception { final User user = new User("_username", "r1"); when(firstRealm.token(threadContext)).thenReturn(token); when(firstRealm.supports(token)).thenReturn(true); + + when(securityIndex.defensiveCopy()).thenReturn(securityIndex); + // An invalid token might decode to something that looks like a UUID + // Randomise it being invalid because the index doesn't exist, or the document doesn't exist + if (randomBoolean()) { + when(securityIndex.isAvailable(any())).thenReturn(false); + when(securityIndex.getUnavailableReason(any())).thenReturn(new ElasticsearchException(getTestName())); + } else { + when(securityIndex.isAvailable(any())).thenReturn(true); + doAnswer(inv -> { + final GetRequest request = inv.getArgument(0); + final ActionListener listener = inv.getArgument(1); + listener.onResponse( + new GetResponse( + new GetResult( + request.index(), + request.id(), + UNASSIGNED_SEQ_NO, + UNASSIGNED_PRIMARY_TERM, + 0, + false, + null, + Map.of(), + Map.of() + ) + ) + ); + return null; + }).when(client).get(any(GetRequest.class), any()); + } + mockAuthenticate(firstRealm, token, user); final int numBytes = randomIntBetween(TokenService.MINIMUM_BYTES, TokenService.MINIMUM_BYTES + 32); final byte[] randomBytes = new byte[numBytes]; From 09ce855d83d41225cb6e988745f2933c034577db Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Mon, 16 Dec 2024 09:08:06 +0000 Subject: [PATCH 34/35] Remove some 7.7 and 7.8 transport version checks (#118563) --- .../metadata/ComponentTemplateMetadata.java | 4 +-- .../ComposableIndexTemplateMetadata.java | 4 +-- .../cluster/metadata/DataStreamMetadata.java | 4 +-- .../elasticsearch/script/ScriptException.java | 15 ++++------ .../elasticsearch/search/DocValueFormat.java | 14 ---------- .../tasks/TaskCancellationService.java | 10 ++----- .../transport/ProxyConnectionStrategy.java | 11 ++------ .../action/OriginalIndicesTests.java | 11 +------- .../ClusterSearchShardsRequestTests.java | 8 +----- .../indices/close/CloseIndexRequestTests.java | 17 ++--------- .../index/mapper/ParametrizedMapperTests.java | 22 --------------- .../boxplot/BoxplotAggregationBuilder.java | 2 +- .../TopMetricsAggregationBuilder.java | 2 +- .../ttest/TTestAggregationBuilder.java | 2 +- .../search/CancellingAggregationBuilder.java | 2 +- .../autoscaling/AutoscalingMetadata.java | 4 +-- .../action/AnalyticsStatsAction.java | 28 ++----------------- .../xpack/core/indexing/IndexerJobStats.java | 14 +++------- .../ClassificationConfigUpdate.java | 2 +- .../trainedmodel/RegressionConfigUpdate.java | 2 +- ...DeleteSamlServiceProviderRequestTests.java | 2 +- .../PutSamlServiceProviderRequestTests.java | 2 +- .../sp/SamlServiceProviderDocumentTests.java | 2 +- .../support/SamlAuthenticationStateTests.java | 2 +- 24 files changed, 38 insertions(+), 148 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java index 1151a99a24403..2aedfcef272ed 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java @@ -89,7 +89,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.V_7_7_0; + return TransportVersions.ZERO; } @Override @@ -166,7 +166,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.V_7_7_0; + return TransportVersions.ZERO; } } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java index e798b0f6add4f..f34375d9c8a6d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java @@ -94,7 +94,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.V_7_7_0; + return TransportVersions.ZERO; } @Override @@ -167,7 +167,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.V_7_7_0; + return TransportVersions.ZERO; } } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java index 3f5e7a2e0c4aa..e49b991c21887 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java @@ -217,7 +217,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.V_7_7_0; + return TransportVersions.ZERO; } @Override @@ -311,7 +311,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.V_7_7_0; + return TransportVersions.ZERO; } } } diff --git a/server/src/main/java/org/elasticsearch/script/ScriptException.java b/server/src/main/java/org/elasticsearch/script/ScriptException.java index 72349e408210e..ab8d804b82ce8 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptException.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptException.java @@ -10,7 +10,6 @@ package org.elasticsearch.script; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.TransportVersions; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -79,7 +78,7 @@ public ScriptException(StreamInput in) throws IOException { scriptStack = Arrays.asList(in.readStringArray()); script = in.readString(); lang = in.readString(); - if (in.getTransportVersion().onOrAfter(TransportVersions.V_7_7_0) && in.readBoolean()) { + if (in.readBoolean()) { pos = new Position(in); } else { pos = null; @@ -92,13 +91,11 @@ protected void writeTo(StreamOutput out, Writer nestedExceptionsWrite out.writeStringCollection(scriptStack); out.writeString(script); out.writeString(lang); - if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_7_0)) { - if (pos == null) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - pos.writeTo(out); - } + if (pos == null) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + pos.writeTo(out); } } diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index f8d161ef1f5e5..dca3da9419380 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -271,13 +271,6 @@ private DateTime(StreamInput in) throws IOException { this.formatter = DateFormatter.forPattern(formatterPattern).withZone(this.timeZone).withLocale(locale); this.parser = formatter.toDateMathParser(); this.resolution = DateFieldMapper.Resolution.ofOrdinal(in.readVInt()); - if (in.getTransportVersion().between(TransportVersions.V_7_7_0, TransportVersions.V_8_0_0)) { - /* when deserialising from 7.7+ nodes expect a flag indicating if a pattern is of joda style - This is only used to support joda style indices in 7.x, in 8 we no longer support this. - All indices in 8 should use java style pattern. Hence we can ignore this flag. - */ - in.readBoolean(); - } this.formatSortValues = in.readBoolean(); } @@ -302,13 +295,6 @@ public void writeTo(StreamOutput out) throws IOException { } out.writeString(timeZone.getId()); out.writeVInt(resolution.ordinal()); - if (out.getTransportVersion().between(TransportVersions.V_7_7_0, TransportVersions.V_8_0_0)) { - /* when serializing to 7.7+ send out a flag indicating if a pattern is of joda style - This is only used to support joda style indices in 7.x, in 8 we no longer support this. - All indices in 8 should use java style pattern. Hence this flag is always false. - */ - out.writeBoolean(false); - } out.writeBoolean(formatSortValues); } diff --git a/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java b/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java index d457607d0bc7f..32a71bbe8a26a 100644 --- a/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java +++ b/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java @@ -333,11 +333,7 @@ private BanParentTaskRequest(StreamInput in) throws IOException { parentTaskId = TaskId.readFromStream(in); ban = in.readBoolean(); reason = ban ? in.readString() : null; - if (in.getTransportVersion().onOrAfter(TransportVersions.V_7_8_0)) { - waitForCompletion = in.readBoolean(); - } else { - waitForCompletion = false; - } + waitForCompletion = in.readBoolean(); } @Override @@ -348,9 +344,7 @@ public void writeTo(StreamOutput out) throws IOException { if (ban) { out.writeString(reason); } - if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_8_0)) { - out.writeBoolean(waitForCompletion); - } + out.writeBoolean(waitForCompletion); } } diff --git a/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java b/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java index 4b3678d75af7c..d5047a61e4606 100644 --- a/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java +++ b/server/src/main/java/org/elasticsearch/transport/ProxyConnectionStrategy.java @@ -9,7 +9,6 @@ package org.elasticsearch.transport; -import org.elasticsearch.TransportVersions; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.cluster.ClusterName; @@ -355,11 +354,7 @@ public ProxyModeInfo(String address, String serverName, int maxSocketConnections private ProxyModeInfo(StreamInput input) throws IOException { address = input.readString(); - if (input.getTransportVersion().onOrAfter(TransportVersions.V_7_7_0)) { - serverName = input.readString(); - } else { - serverName = null; - } + serverName = input.readString(); maxSocketConnections = input.readVInt(); numSocketsConnected = input.readVInt(); } @@ -376,9 +371,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(address); - if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_7_0)) { - out.writeString(serverName); - } + out.writeString(serverName); out.writeVInt(maxSocketConnections); out.writeVInt(numSocketsConnected); } diff --git a/server/src/test/java/org/elasticsearch/action/OriginalIndicesTests.java b/server/src/test/java/org/elasticsearch/action/OriginalIndicesTests.java index 4a2fb698468b4..1db1b62facc9d 100644 --- a/server/src/test/java/org/elasticsearch/action/OriginalIndicesTests.java +++ b/server/src/test/java/org/elasticsearch/action/OriginalIndicesTests.java @@ -9,7 +9,6 @@ package org.elasticsearch.action; -import org.elasticsearch.TransportVersions; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; @@ -43,15 +42,7 @@ public void testOriginalIndicesSerialization() throws IOException { OriginalIndices originalIndices2 = OriginalIndices.readOriginalIndices(in); assertThat(originalIndices2.indices(), equalTo(originalIndices.indices())); - // indices options are not equivalent when sent to an older version and re-read due - // to the addition of hidden indices as expand to hidden indices is always true when - // read from a prior version - if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_7_0) - || originalIndices.indicesOptions().expandWildcardsHidden()) { - assertThat(originalIndices2.indicesOptions(), equalTo(originalIndices.indicesOptions())); - } else if (originalIndices.indicesOptions().expandWildcardsHidden()) { - assertThat(originalIndices2.indicesOptions(), equalTo(originalIndices.indicesOptions())); - } + assertThat(originalIndices2.indicesOptions(), equalTo(originalIndices.indicesOptions())); } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequestTests.java index 7bf2cc600dd3b..d4a3d29ea68f2 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/shards/ClusterSearchShardsRequestTests.java @@ -10,7 +10,6 @@ package org.elasticsearch.action.admin.cluster.shards; import org.elasticsearch.TransportVersion; -import org.elasticsearch.TransportVersions; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; @@ -54,12 +53,7 @@ public void testSerialization() throws Exception { in.setTransportVersion(version); ClusterSearchShardsRequest deserialized = new ClusterSearchShardsRequest(in); assertArrayEquals(request.indices(), deserialized.indices()); - // indices options are not equivalent when sent to an older version and re-read due - // to the addition of hidden indices as expand to hidden indices is always true when - // read from a prior version - if (version.onOrAfter(TransportVersions.V_7_7_0) || request.indicesOptions().expandWildcardsHidden()) { - assertEquals(request.indicesOptions(), deserialized.indicesOptions()); - } + assertEquals(request.indicesOptions(), deserialized.indicesOptions()); assertEquals(request.routing(), deserialized.routing()); assertEquals(request.preference(), deserialized.preference()); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java index dc267c7c1d80b..e45e940334a9e 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/close/CloseIndexRequestTests.java @@ -56,14 +56,7 @@ public void testBwcSerialization() throws Exception { assertEquals(request.ackTimeout(), in.readTimeValue()); assertArrayEquals(request.indices(), in.readStringArray()); final IndicesOptions indicesOptions = IndicesOptions.readIndicesOptions(in); - // indices options are not equivalent when sent to an older version and re-read due - // to the addition of hidden indices as expand to hidden indices is always true when - // read from a prior version - // TODO update version on backport! - if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_7_0) - || request.indicesOptions().expandWildcardsHidden()) { - assertEquals(request.indicesOptions(), indicesOptions); - } + assertEquals(request.indicesOptions(), indicesOptions); assertEquals(request.waitForActiveShards(), ActiveShardCount.readFrom(in)); } } @@ -92,13 +85,7 @@ public void testBwcSerialization() throws Exception { assertEquals(sample.masterNodeTimeout(), deserializedRequest.masterNodeTimeout()); assertEquals(sample.ackTimeout(), deserializedRequest.ackTimeout()); assertArrayEquals(sample.indices(), deserializedRequest.indices()); - // indices options are not equivalent when sent to an older version and re-read due - // to the addition of hidden indices as expand to hidden indices is always true when - // read from a prior version - // TODO change version on backport - if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_7_0) || sample.indicesOptions().expandWildcardsHidden()) { - assertEquals(sample.indicesOptions(), deserializedRequest.indicesOptions()); - } + assertEquals(sample.indicesOptions(), deserializedRequest.indicesOptions()); assertEquals(sample.waitForActiveShards(), deserializedRequest.waitForActiveShards()); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java index 2681f70743f56..a161bcbc5d6d2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java @@ -11,16 +11,13 @@ import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.elasticsearch.TransportVersion; -import org.elasticsearch.TransportVersions; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; @@ -578,25 +575,6 @@ public void testAnalyzers() { ); } - @UpdateForV9(owner = UpdateForV9.Owner.SEARCH_FOUNDATIONS) - @AwaitsFix(bugUrl = "this is testing legacy functionality so can likely be removed in 9.0") - public void testDeprecatedParameters() { - // 'index' is declared explicitly, 'store' is not, but is one of the previously always-accepted params - String mapping = """ - {"type":"test_mapper","index":false,"store":true,"required":"value"}"""; - TestMapper mapper = fromMapping(mapping, IndexVersions.V_7_8_0, TransportVersions.V_7_8_0); - assertWarnings("Parameter [store] has no effect on type [test_mapper] and will be removed in future"); - assertFalse(mapper.index); - assertEquals(""" - {"field":{"type":"test_mapper","index":false,"required":"value"}}""", Strings.toString(mapper)); - - MapperParsingException e = expectThrows( - MapperParsingException.class, - () -> fromMapping(mapping, IndexVersions.V_8_0_0, TransportVersions.V_8_0_0) - ); - assertEquals("unknown parameter [store] on mapper [field] of type [test_mapper]", e.getMessage()); - } - public void testLinkedAnalyzers() throws IOException { String mapping = """ {"type":"test_mapper","analyzer":"_standard","required":"value"}"""; diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregationBuilder.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregationBuilder.java index 61917220f10d1..37e05898dfc48 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregationBuilder.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/boxplot/BoxplotAggregationBuilder.java @@ -194,6 +194,6 @@ public Optional> getOutputFieldNames() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.V_7_7_0; + return TransportVersions.ZERO; } } diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregationBuilder.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregationBuilder.java index e7e06946a5289..93fd71b55332a 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregationBuilder.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/topmetrics/TopMetricsAggregationBuilder.java @@ -235,6 +235,6 @@ public Optional> getOutputFieldNames() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.V_7_7_0; + return TransportVersions.ZERO; } } diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregationBuilder.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregationBuilder.java index 7137302459c0b..bbe497718b62a 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregationBuilder.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/ttest/TTestAggregationBuilder.java @@ -182,6 +182,6 @@ public String getType() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.V_7_8_0; + return TransportVersions.ZERO; } } diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/CancellingAggregationBuilder.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/CancellingAggregationBuilder.java index 90358014e1ee4..c87057a7cd2e1 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/CancellingAggregationBuilder.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/CancellingAggregationBuilder.java @@ -107,6 +107,6 @@ public BucketCardinality bucketCardinality() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.V_7_7_0; + return TransportVersions.ZERO; } } diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingMetadata.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingMetadata.java index 38c654f94fff3..b8911d24e1a28 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingMetadata.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingMetadata.java @@ -114,7 +114,7 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.V_7_8_0; + return TransportVersions.ZERO; } @Override @@ -173,7 +173,7 @@ static Diff readFrom(final StreamInput in) throws IOE @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.V_7_8_0; + return TransportVersions.ZERO; } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/action/AnalyticsStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/action/AnalyticsStatsAction.java index 1420b1f86f12e..1792ba7aa4637 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/action/AnalyticsStatsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/analytics/action/AnalyticsStatsAction.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.core.analytics.action; -import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.nodes.BaseNodeResponse; @@ -138,36 +137,13 @@ public NodeResponse(DiscoveryNode node, EnumCounters counters) { public NodeResponse(StreamInput in) throws IOException { super(in); - if (in.getTransportVersion().onOrAfter(TransportVersions.V_7_8_0)) { - counters = new EnumCounters<>(in, Item.class); - } else { - counters = new EnumCounters<>(Item.class); - if (in.getTransportVersion().onOrAfter(TransportVersions.V_7_7_0)) { - counters.inc(Item.BOXPLOT, in.readVLong()); - } - counters.inc(Item.CUMULATIVE_CARDINALITY, in.readZLong()); - if (in.getTransportVersion().onOrAfter(TransportVersions.V_7_7_0)) { - counters.inc(Item.STRING_STATS, in.readVLong()); - counters.inc(Item.TOP_METRICS, in.readVLong()); - } - } + counters = new EnumCounters<>(in, Item.class); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_8_0)) { - counters.writeTo(out); - } else { - if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_7_0)) { - out.writeVLong(counters.get(Item.BOXPLOT)); - } - out.writeZLong(counters.get(Item.CUMULATIVE_CARDINALITY)); - if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_7_0)) { - out.writeVLong(counters.get(Item.STRING_STATS)); - out.writeVLong(counters.get(Item.TOP_METRICS)); - } - } + counters.writeTo(out); } public EnumCounters getStats() { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/IndexerJobStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/IndexerJobStats.java index a13cdf1966811..9f9755b6c9874 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/IndexerJobStats.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexing/IndexerJobStats.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.core.indexing; -import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -83,11 +82,8 @@ public IndexerJobStats(StreamInput in) throws IOException { this.searchTotal = in.readVLong(); this.indexFailures = in.readVLong(); this.searchFailures = in.readVLong(); - - if (in.getTransportVersion().onOrAfter(TransportVersions.V_7_7_0)) { - this.processingTime = in.readVLong(); - this.processingTotal = in.readVLong(); - } + this.processingTime = in.readVLong(); + this.processingTotal = in.readVLong(); } public long getNumPages() { @@ -205,10 +201,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(searchTotal); out.writeVLong(indexFailures); out.writeVLong(searchFailures); - if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_7_0)) { - out.writeVLong(processingTime); - out.writeVLong(processingTotal); - } + out.writeVLong(processingTime); + out.writeVLong(processingTotal); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfigUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfigUpdate.java index de4004792af7c..1cf1c0ddeb570 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfigUpdate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/ClassificationConfigUpdate.java @@ -210,7 +210,7 @@ public boolean isSupported(InferenceConfig inferenceConfig) { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.V_7_8_0; + return TransportVersions.ZERO; } public static class Builder implements InferenceConfigUpdate.Builder { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfigUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfigUpdate.java index dc1a7bdeef104..ceebe3105eb63 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfigUpdate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/trainedmodel/RegressionConfigUpdate.java @@ -114,7 +114,7 @@ public String getName() { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.V_7_8_0; + return TransportVersions.ZERO; } @Override diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderRequestTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderRequestTests.java index 4beda3cc18792..110605b6a6de5 100644 --- a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderRequestTests.java +++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/DeleteSamlServiceProviderRequestTests.java @@ -29,7 +29,7 @@ public void testSerialization() throws IOException { ); final TransportVersion version = TransportVersionUtils.randomVersionBetween( random(), - TransportVersions.V_7_7_0, + TransportVersions.V_8_0_0, TransportVersion.current() ); final DeleteSamlServiceProviderRequest read = copyWriteable( diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderRequestTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderRequestTests.java index 60675b9355973..c9b1008ae795a 100644 --- a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderRequestTests.java +++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/action/PutSamlServiceProviderRequestTests.java @@ -87,7 +87,7 @@ public void testSerialization() throws IOException { final PutSamlServiceProviderRequest request = new PutSamlServiceProviderRequest(doc, RefreshPolicy.NONE); final TransportVersion version = TransportVersionUtils.randomVersionBetween( random(), - TransportVersions.V_7_7_0, + TransportVersions.V_8_0_0, TransportVersion.current() ); final PutSamlServiceProviderRequest read = copyWriteable( diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderDocumentTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderDocumentTests.java index 233702d7ddd9a..b6c20cebe595f 100644 --- a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderDocumentTests.java +++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/sp/SamlServiceProviderDocumentTests.java @@ -162,7 +162,7 @@ private SamlServiceProviderDocument assertXContentRoundTrip(SamlServiceProviderD private SamlServiceProviderDocument assertSerializationRoundTrip(SamlServiceProviderDocument doc) throws IOException { final TransportVersion version = TransportVersionUtils.randomVersionBetween( random(), - TransportVersions.V_7_7_0, + TransportVersions.V_8_0_0, TransportVersion.current() ); final SamlServiceProviderDocument read = copyWriteable( diff --git a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/support/SamlAuthenticationStateTests.java b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/support/SamlAuthenticationStateTests.java index 934126e7f0fa0..61018fe77d47e 100644 --- a/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/support/SamlAuthenticationStateTests.java +++ b/x-pack/plugin/identity-provider/src/test/java/org/elasticsearch/xpack/idp/saml/support/SamlAuthenticationStateTests.java @@ -90,7 +90,7 @@ private SamlAuthenticationState assertXContentRoundTrip(SamlAuthenticationState private SamlAuthenticationState assertSerializationRoundTrip(SamlAuthenticationState state) throws IOException { final TransportVersion version = TransportVersionUtils.randomVersionBetween( random(), - TransportVersions.V_7_7_0, + TransportVersions.V_8_0_0, TransportVersion.current() ); final SamlAuthenticationState read = copyWriteable( From 1fb677f461018d4ecdd61d6cc284a02f911224d0 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Mon, 16 Dec 2024 09:17:17 +0000 Subject: [PATCH 35/35] Unmute test UpdateResponseTests.testToAndFromXContent (#118449) Test is no longer failing... --- muted-tests.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 09b3050359d37..33f5434d6a963 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -79,9 +79,6 @@ tests: - class: org.elasticsearch.xpack.restart.MLModelDeploymentFullClusterRestartIT method: testDeploymentSurvivesRestart {cluster=UPGRADED} issue: https://github.com/elastic/elasticsearch/issues/115528 -- class: org.elasticsearch.action.update.UpdateResponseTests - method: testToAndFromXContent - issue: https://github.com/elastic/elasticsearch/issues/115689 - class: org.elasticsearch.xpack.shutdown.NodeShutdownIT method: testStalledShardMigrationProperlyDetected issue: https://github.com/elastic/elasticsearch/issues/115697