() {
@Override
public void execute(Task task) {
if (expectedOutputFile.exists() == false) {
- Path relativeOutputPath = project.getRootDir().toPath().relativize(expectedOutputFile.toPath());
+ Path relativeOutputPath = rootDir.toPath().relativize(expectedOutputFile.toPath());
final String message = "Building %s didn't generate expected artifact [%s]. The working branch may be "
+ "out-of-date - try merging in the latest upstream changes to the branch.";
throw new InvalidUserDataException(message.formatted(bwcVersion.get(), relativeOutputPath));
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java
index 16c7bf6d32862..f92789f701049 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java
@@ -23,16 +23,17 @@
import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin;
import org.elasticsearch.gradle.util.GradleUtils;
import org.gradle.api.GradleException;
-import org.gradle.api.NamedDomainObjectContainer;
import org.gradle.api.Plugin;
import org.gradle.api.Project;
import org.gradle.api.artifacts.Dependency;
+import org.gradle.api.artifacts.dsl.DependencyHandler;
import org.gradle.api.provider.Provider;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import java.util.function.Function;
-import static org.elasticsearch.gradle.util.GradleUtils.projectDependency;
-
/**
* An internal elasticsearch build plugin that registers additional
* distribution resolution strategies to the 'elasticsearch.download-distribution' plugin
@@ -64,18 +65,18 @@ public void apply(Project project) {
*
* BWC versions are resolved as project to projects under `:distribution:bwc`.
*/
- private void registerInternalDistributionResolutions(NamedDomainObjectContainer resolutions) {
- resolutions.register("localBuild", distributionResolution -> distributionResolution.setResolver((project, distribution) -> {
+ private void registerInternalDistributionResolutions(List resolutions) {
+ resolutions.add(new DistributionResolution("local-build", (project, distribution) -> {
if (isCurrentVersion(distribution)) {
// non-external project, so depend on local build
return new ProjectBasedDistributionDependency(
- config -> projectDependency(project, distributionProjectPath(distribution), config)
+ config -> projectDependency(project.getDependencies(), distributionProjectPath(distribution), config)
);
}
return null;
}));
- resolutions.register("bwc", distributionResolution -> distributionResolution.setResolver((project, distribution) -> {
+ resolutions.add(new DistributionResolution("bwc", (project, distribution) -> {
BwcVersions.UnreleasedVersionInfo unreleasedInfo = BuildParams.getBwcVersions()
.unreleasedInfo(Version.fromString(distribution.getVersion()));
if (unreleasedInfo != null) {
@@ -89,7 +90,7 @@ private void registerInternalDistributionResolutions(NamedDomainObjectContainer<
}
String projectConfig = getProjectConfig(distribution, unreleasedInfo);
return new ProjectBasedDistributionDependency(
- (config) -> projectDependency(project, unreleasedInfo.gradleProjectPath(), projectConfig)
+ (config) -> projectDependency(project.getDependencies(), unreleasedInfo.gradleProjectPath(), projectConfig)
);
}
return null;
@@ -116,6 +117,13 @@ private static String getProjectConfig(ElasticsearchDistribution distribution, B
}
}
+ private static Dependency projectDependency(DependencyHandler dependencyHandler, String projectPath, String projectConfig) {
+ Map depConfig = new HashMap<>();
+ depConfig.put("path", projectPath);
+ depConfig.put("configuration", projectConfig);
+ return dependencyHandler.project(depConfig);
+ }
+
private static String distributionProjectPath(ElasticsearchDistribution distribution) {
String projectPath = ":distribution";
if (distribution.getType() == ElasticsearchDistributionTypes.INTEG_TEST_ZIP) {
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/NoticeTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/NoticeTask.java
index 1e2506908d108..751ac92512dad 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/NoticeTask.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/NoticeTask.java
@@ -16,11 +16,11 @@
import org.gradle.api.file.ProjectLayout;
import org.gradle.api.file.SourceDirectorySet;
import org.gradle.api.internal.file.FileOperations;
-import org.gradle.api.model.ObjectFactory;
import org.gradle.api.provider.ListProperty;
import org.gradle.api.tasks.CacheableTask;
import org.gradle.api.tasks.InputFile;
import org.gradle.api.tasks.InputFiles;
+import org.gradle.api.tasks.Internal;
import org.gradle.api.tasks.Optional;
import org.gradle.api.tasks.OutputFile;
import org.gradle.api.tasks.PathSensitive;
@@ -43,7 +43,7 @@
* A task to create a notice file which includes dependencies' notices.
*/
@CacheableTask
-public class NoticeTask extends DefaultTask {
+public abstract class NoticeTask extends DefaultTask {
@InputFile
@PathSensitive(PathSensitivity.RELATIVE)
@@ -57,19 +57,17 @@ public class NoticeTask extends DefaultTask {
/**
* Directories to include notices from
*/
- private final ListProperty licensesDirs;
+ @Internal
+ abstract ListProperty getLicenseDirs();
private final FileOperations fileOperations;
- private ObjectFactory objectFactory;
@Inject
- public NoticeTask(BuildLayout buildLayout, ProjectLayout projectLayout, FileOperations fileOperations, ObjectFactory objectFactory) {
- this.objectFactory = objectFactory;
+ public NoticeTask(BuildLayout buildLayout, ProjectLayout projectLayout, FileOperations fileOperations) {
this.fileOperations = fileOperations;
setDescription("Create a notice file from dependencies");
// Default licenses directory is ${projectDir}/licenses (if it exists)
- licensesDirs = objectFactory.listProperty(File.class);
- licensesDirs.add(projectLayout.getProjectDirectory().dir("licenses").getAsFile());
+ getLicenseDirs().add(projectLayout.getProjectDirectory().dir("licenses").getAsFile());
inputFile = new File(buildLayout.getRootDirectory(), "NOTICE.txt");
outputFile = projectLayout.getBuildDirectory().dir("notices/" + getName()).get().file("NOTICE.txt").getAsFile();
}
@@ -78,7 +76,7 @@ public NoticeTask(BuildLayout buildLayout, ProjectLayout projectLayout, FileOper
* Add notices from the specified directory.
*/
public void licensesDir(File licensesDir) {
- licensesDirs.add(licensesDir);
+ getLicenseDirs().add(licensesDir);
}
public void source(Object source) {
@@ -185,7 +183,7 @@ public FileCollection getNoticeFiles() {
}
private List existingLicenseDirs() {
- return licensesDirs.get().stream().filter(d -> d.exists()).collect(Collectors.toList());
+ return getLicenseDirs().get().stream().filter(d -> d.exists()).collect(Collectors.toList());
}
@InputFiles
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java
index b32c566363e88..cafa02941d77c 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java
@@ -28,22 +28,14 @@ private static ListMultimap, String> createLegacyRestTestBasePluginUsag
ListMultimap, String> map = ArrayListMultimap.create(1, 200);
map.put(LegacyRestTestBasePlugin.class, ":docs");
map.put(LegacyRestTestBasePlugin.class, ":distribution:docker");
- map.put(LegacyRestTestBasePlugin.class, ":modules:analysis-common");
- map.put(LegacyRestTestBasePlugin.class, ":modules:ingest-attachment");
- map.put(LegacyRestTestBasePlugin.class, ":modules:ingest-common");
- map.put(LegacyRestTestBasePlugin.class, ":modules:ingest-user-agent");
- map.put(LegacyRestTestBasePlugin.class, ":modules:kibana");
map.put(LegacyRestTestBasePlugin.class, ":modules:lang-expression");
map.put(LegacyRestTestBasePlugin.class, ":modules:lang-mustache");
- map.put(LegacyRestTestBasePlugin.class, ":modules:lang-painless");
map.put(LegacyRestTestBasePlugin.class, ":modules:mapper-extras");
map.put(LegacyRestTestBasePlugin.class, ":modules:parent-join");
map.put(LegacyRestTestBasePlugin.class, ":modules:percolator");
map.put(LegacyRestTestBasePlugin.class, ":modules:rank-eval");
map.put(LegacyRestTestBasePlugin.class, ":modules:reindex");
- map.put(LegacyRestTestBasePlugin.class, ":modules:repository-s3");
map.put(LegacyRestTestBasePlugin.class, ":modules:repository-url");
- map.put(LegacyRestTestBasePlugin.class, ":modules:runtime-fields-common");
map.put(LegacyRestTestBasePlugin.class, ":modules:transport-netty4");
map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-icu");
map.put(LegacyRestTestBasePlugin.class, ":plugins:analysis-kuromoji");
@@ -74,14 +66,13 @@ private static ListMultimap, String> createLegacyRestTestBasePluginUsag
map.put(LegacyRestTestBasePlugin.class, ":qa:system-indices");
map.put(LegacyRestTestBasePlugin.class, ":qa:unconfigured-node-name");
map.put(LegacyRestTestBasePlugin.class, ":qa:verify-version-constants");
+ map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-apm-integration");
map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-delayed-aggs");
map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-die-with-dignity");
- map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-apm-integration");
map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-error-query");
map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-latency-simulating-directory");
map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-seek-tracking-directory");
map.put(LegacyRestTestBasePlugin.class, ":test:yaml-rest-runner");
- map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin");
map.put(LegacyRestTestBasePlugin.class, ":distribution:archives:integ-test-zip");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:core");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ent-search");
@@ -92,17 +83,13 @@ private static ListMultimap, String> createLegacyRestTestBasePluginUsag
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:mapper-version");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:vector-tile");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:wildcard");
- map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:kerberos-tests");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:mixed-tier-cluster");
- map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:password-protected-keystore");
- map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:reindex-tests-with-security");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:repository-old-versions");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:rolling-upgrade");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:rolling-upgrade-basic");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:rolling-upgrade-multi-cluster");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:runtime-fields:core-with-mapped");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:runtime-fields:core-with-search");
- map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:saml-idp-tests");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:security-example-spi-extension");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:security-setup-password-tests");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:smoke-test-plugins");
@@ -115,12 +102,10 @@ private static ListMultimap, String> createLegacyRestTestBasePluginUsag
map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:multi-cluster-search-security:legacy-with-basic-license");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:multi-cluster-search-security:legacy-with-full-license");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:multi-cluster-search-security:legacy-with-restricted-trust");
- map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:runtime-fields:with-security");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:third-party:jira");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:third-party:pagerduty");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:qa:third-party:slack");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:async-search:qa:rest");
- map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:async-search:qa:security");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:autoscaling:qa:rest");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ccr:qa:downgrade-to-basic-license");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ccr:qa:multi-cluster");
@@ -130,7 +115,6 @@ private static ListMultimap, String> createLegacyRestTestBasePluginUsag
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ccr:qa:security");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:deprecation:qa:early-deprecation-rest");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:deprecation:qa:rest");
- map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:downsample:qa:rest");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:downsample:qa:with-security");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:enrich:qa:rest");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:enrich:qa:rest-with-advanced-security");
@@ -139,55 +123,40 @@ private static ListMultimap, String> createLegacyRestTestBasePluginUsag
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:eql:qa:ccs-rolling-upgrade");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:eql:qa:correctness");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:eql:qa:mixed-node");
- map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:eql:qa:multi-cluster-with-security");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:esql:qa:security");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:esql:qa:server:multi-node");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:esql:qa:server:single-node");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:fleet:qa:rest");
- map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:graph:qa:with-security");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:identity-provider:qa:idp-rest-tests");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ilm:qa:multi-cluster");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ilm:qa:multi-node");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ilm:qa:rest");
- map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ilm:qa:with-security");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ml:qa:basic-multi-node");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ml:qa:disabled");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ml:qa:ml-with-security");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ml:qa:multi-cluster-tests-with-security");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ml:qa:native-multi-node-tests");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:ml:qa:single-node-tests");
- map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:repositories-metering-api:qa:s3");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:searchable-snapshots:qa:hdfs");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:searchable-snapshots:qa:minio");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:searchable-snapshots:qa:rest");
- map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:searchable-snapshots:qa:s3");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:searchable-snapshots:qa:url");
- map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:security:qa:operator-privileges-tests");
- map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:security:qa:profile");
- map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:security:qa:security-disabled");
- map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:security:qa:smoke-test-all-realms");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:security:qa:tls-basic");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:shutdown:qa:multi-node");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:shutdown:qa:rolling-upgrade");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:slm:qa:multi-node");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:slm:qa:rest");
- map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:slm:qa:with-security");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-based-recoveries:qa:fs");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-based-recoveries:qa:license-enforcing");
- map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-based-recoveries:qa:s3");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-repo-test-kit:qa:hdfs");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-repo-test-kit:qa:minio");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-repo-test-kit:qa:rest");
- map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:snapshot-repo-test-kit:qa:s3");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:jdbc:multi-node");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:jdbc:no-sql");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:jdbc:single-node");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:jdbc:security:with-ssl");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:jdbc:security:without-ssl");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:mixed-node");
- map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:server:multi-cluster-with-security");
- map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:server:multi-node");
- map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:server:single-node");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:server:security:with-ssl");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:sql:qa:server:security:without-ssl");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:stack:qa:rest");
@@ -197,8 +166,8 @@ private static ListMultimap, String> createLegacyRestTestBasePluginUsag
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:transform:qa:single-node-tests");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:vector-tile:qa:multi-cluster");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:watcher:qa:rest");
- map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:watcher:qa:with-monitoring");
map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:watcher:qa:with-security");
+ map.put(LegacyRestTestBasePlugin.class, ":x-pack:plugin:esql:qa:server:mixed-cluster");
return map;
}
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckForbiddenApisTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckForbiddenApisTask.java
index bb0b8dcf04437..d69a355a3595d 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckForbiddenApisTask.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/CheckForbiddenApisTask.java
@@ -10,6 +10,7 @@
import de.thetaphi.forbiddenapis.Checker;
import de.thetaphi.forbiddenapis.Constants;
+import de.thetaphi.forbiddenapis.ForbiddenApiException;
import de.thetaphi.forbiddenapis.Logger;
import de.thetaphi.forbiddenapis.ParseException;
import groovy.lang.Closure;
@@ -43,6 +44,7 @@
import org.gradle.api.tasks.PathSensitivity;
import org.gradle.api.tasks.SkipWhenEmpty;
import org.gradle.api.tasks.TaskAction;
+import org.gradle.api.tasks.VerificationException;
import org.gradle.api.tasks.VerificationTask;
import org.gradle.api.tasks.util.PatternFilterable;
import org.gradle.api.tasks.util.PatternSet;
@@ -469,6 +471,8 @@ public void execute() {
}
checker.run();
writeMarker(getParameters().getSuccessMarker().getAsFile().get());
+ } catch (ForbiddenApiException e) {
+ throw new VerificationException("Forbidden API verification failed", e);
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java
index a25ad34a241d4..d2ba86bb99cf2 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java
@@ -218,7 +218,8 @@ public void runThirdPartyAudit() throws IOException {
if (bogousExcludesCount != 0 && bogousExcludesCount == missingClassExcludes.size() + violationsExcludes.size()) {
logForbiddenAPIsOutput(forbiddenApisOutput);
throw new IllegalStateException(
- "All excluded classes seem to have no issues. This is sometimes an indication that the check silently failed"
+ "All excluded classes seem to have no issues. This is sometimes an indication that the check silently failed "
+ + "or that exclusions are configured unnecessarily"
);
}
assertNoPointlessExclusions("are not missing", missingClassExcludes, missingClasses);
@@ -261,10 +262,6 @@ private void logForbiddenAPIsOutput(String forbiddenApisOutput) {
getLogger().error("Forbidden APIs output:\n{}==end of forbidden APIs==", forbiddenApisOutput);
}
- private void throwNotConfiguredCorrectlyException() {
- throw new IllegalArgumentException("Audit of third party dependencies is not configured correctly");
- }
-
/**
* Ideally we would do unpacking already via artifact transform and keep unpacked jars across builds.
* At the moment transform target folder is not configurable and forbidden CLI only takes one common
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java
index bcbe1740630ce..42d3a770dbbcc 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/DistroTestPlugin.java
@@ -101,7 +101,7 @@ public void apply(Project project) {
addDistributionSysprop(t, DISTRIBUTION_SYSPROP, distribution::getFilepath);
addDistributionSysprop(t, EXAMPLE_PLUGIN_SYSPROP, () -> examplePlugin.getSingleFile().toString());
t.exclude("**/PackageUpgradeTests.class");
- }, distribution.getArchiveDependencies(), examplePlugin.getDependencies());
+ }, distribution, examplePlugin.getDependencies());
if (distribution.getPlatform() == Platform.WINDOWS) {
windowsTestTasks.add(destructiveTask);
@@ -235,6 +235,7 @@ private static ElasticsearchDistribution createDistro(
d.setBundledJdk(bundledJdk);
}
d.setVersion(version);
+ d.setPreferArchive(true);
});
// Allow us to gracefully omit building Docker distributions if Docker is not available on the system.
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/HistoricalFeaturesMetadataPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/HistoricalFeaturesMetadataPlugin.java
new file mode 100644
index 0000000000000..bd9df6d3903ca
--- /dev/null
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/HistoricalFeaturesMetadataPlugin.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.test;
+
+import org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.type.ArtifactTypeDefinition;
+import org.gradle.api.tasks.SourceSet;
+import org.gradle.api.tasks.SourceSetContainer;
+import org.gradle.api.tasks.TaskProvider;
+
+import java.util.Map;
+
+/**
+ * Extracts historical feature metadata into a machine-readable format for use in backward compatibility testing.
+ */
+public class HistoricalFeaturesMetadataPlugin implements Plugin {
+ public static final String HISTORICAL_FEATURES_JSON = "historical-features.json";
+ public static final String FEATURES_METADATA_TYPE = "features-metadata-json";
+ public static final String FEATURES_METADATA_CONFIGURATION = "featuresMetadata";
+
+ @Override
+ public void apply(Project project) {
+ Configuration featureMetadataExtractorConfig = project.getConfigurations().create("featuresMetadataExtractor", c -> {
+ // Don't bother adding this dependency if the project doesn't exist which simplifies testing
+ if (project.findProject(":test:metadata-extractor") != null) {
+ c.defaultDependencies(d -> d.add(project.getDependencies().project(Map.of("path", ":test:metadata-extractor"))));
+ }
+ });
+
+ SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
+ SourceSet mainSourceSet = sourceSets.getByName(SourceSet.MAIN_SOURCE_SET_NAME);
+
+ TaskProvider generateTask = project.getTasks()
+ .register("generateHistoricalFeaturesMetadata", HistoricalFeaturesMetadataTask.class, task -> {
+ task.setClasspath(
+ featureMetadataExtractorConfig.plus(mainSourceSet.getRuntimeClasspath())
+ .plus(project.getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME))
+ );
+ task.getOutputFile().convention(project.getLayout().getBuildDirectory().file(HISTORICAL_FEATURES_JSON));
+ });
+
+ Configuration featuresMetadataArtifactConfig = project.getConfigurations().create(FEATURES_METADATA_CONFIGURATION, c -> {
+ c.setCanBeResolved(false);
+ c.setCanBeConsumed(true);
+ c.attributes(a -> { a.attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, FEATURES_METADATA_TYPE); });
+ });
+
+ project.getArtifacts().add(featuresMetadataArtifactConfig.getName(), generateTask);
+ }
+}
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/HistoricalFeaturesMetadataTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/HistoricalFeaturesMetadataTask.java
new file mode 100644
index 0000000000000..0891225d1e1ef
--- /dev/null
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/HistoricalFeaturesMetadataTask.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.test;
+
+import org.elasticsearch.gradle.LoggedExec;
+import org.gradle.api.DefaultTask;
+import org.gradle.api.file.ConfigurableFileCollection;
+import org.gradle.api.file.FileCollection;
+import org.gradle.api.file.RegularFileProperty;
+import org.gradle.api.tasks.CacheableTask;
+import org.gradle.api.tasks.Classpath;
+import org.gradle.api.tasks.OutputFile;
+import org.gradle.api.tasks.TaskAction;
+import org.gradle.process.ExecOperations;
+import org.gradle.workers.WorkAction;
+import org.gradle.workers.WorkParameters;
+import org.gradle.workers.WorkerExecutor;
+
+import javax.inject.Inject;
+
+@CacheableTask
+public abstract class HistoricalFeaturesMetadataTask extends DefaultTask {
+ private FileCollection classpath;
+
+ @OutputFile
+ public abstract RegularFileProperty getOutputFile();
+
+ @Classpath
+ public FileCollection getClasspath() {
+ return classpath;
+ }
+
+ public void setClasspath(FileCollection classpath) {
+ this.classpath = classpath;
+ }
+
+ @Inject
+ public abstract WorkerExecutor getWorkerExecutor();
+
+ @TaskAction
+ public void execute() {
+ getWorkerExecutor().noIsolation().submit(HistoricalFeaturesMetadataWorkAction.class, params -> {
+ params.getClasspath().setFrom(getClasspath());
+ params.getOutputFile().set(getOutputFile());
+ });
+ }
+
+ public interface HistoricalFeaturesWorkParameters extends WorkParameters {
+ ConfigurableFileCollection getClasspath();
+
+ RegularFileProperty getOutputFile();
+ }
+
+ public abstract static class HistoricalFeaturesMetadataWorkAction implements WorkAction {
+ private final ExecOperations execOperations;
+
+ @Inject
+ public HistoricalFeaturesMetadataWorkAction(ExecOperations execOperations) {
+ this.execOperations = execOperations;
+ }
+
+ @Override
+ public void execute() {
+ LoggedExec.javaexec(execOperations, spec -> {
+ spec.getMainClass().set("org.elasticsearch.extractor.features.HistoricalFeaturesMetadataExtractor");
+ spec.classpath(getParameters().getClasspath());
+ spec.args(getParameters().getOutputFile().get().getAsFile().getAbsolutePath());
+ });
+ }
+ }
+}
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/LegacyRestTestBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/LegacyRestTestBasePlugin.java
index eacc5da6220ab..b29efbfab069f 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/LegacyRestTestBasePlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/LegacyRestTestBasePlugin.java
@@ -22,9 +22,12 @@
import org.gradle.api.NamedDomainObjectContainer;
import org.gradle.api.Plugin;
import org.gradle.api.Project;
+import org.gradle.api.Task;
import org.gradle.api.plugins.JavaBasePlugin;
import org.gradle.api.provider.Provider;
import org.gradle.api.provider.ProviderFactory;
+import org.gradle.api.specs.NotSpec;
+import org.gradle.api.specs.Spec;
import org.gradle.api.tasks.Sync;
import org.gradle.api.tasks.bundling.Zip;
@@ -47,6 +50,7 @@ public class LegacyRestTestBasePlugin implements Plugin {
private static final String TESTS_CLUSTER_REMOTE_ACCESS = "tests.cluster.remote_access";
private ProviderFactory providerFactory;
+ private Project project;
@Inject
public LegacyRestTestBasePlugin(ProviderFactory providerFactory) {
@@ -55,6 +59,7 @@ public LegacyRestTestBasePlugin(ProviderFactory providerFactory) {
@Override
public void apply(Project project) {
+ this.project = project;
Provider serviceProvider = project.getGradle()
.getSharedServices()
.registerIfAbsent("restrictedBuildAPI", RestrictedBuildApiService.class, spec -> {
@@ -118,9 +123,26 @@ public void apply(Project project) {
t.getClusters().forEach(c -> c.plugin(bundle));
}
});
+ configureCacheability(t);
});
}
+ private void configureCacheability(StandaloneRestIntegTestTask testTask) {
+ Spec taskSpec = task -> testTask.getClusters().stream().anyMatch(ElasticsearchCluster::isShared);
+ testTask.getOutputs()
+ .doNotCacheIf(
+ "Caching disabled for this task since it uses a cluster shared by other tasks",
+ /*
+ * Look for any other tasks which use the same cluster as this task. Since tests often have side effects for the cluster
+ * they execute against, this state can cause issues when trying to cache tests results of tasks that share a cluster. To
+ * avoid any undesired behavior we simply disable the cache if we detect that this task uses a cluster shared between
+ * multiple tasks.
+ */
+ taskSpec
+ );
+ testTask.getOutputs().upToDateWhen(new NotSpec(taskSpec));
+ }
+
private String systemProperty(String propName) {
return providerFactory.systemProperty(propName).getOrNull();
}
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestWithSslPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestWithSslPlugin.java
index 524f3dfedf95f..833c7ad546a4a 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestWithSslPlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/TestWithSslPlugin.java
@@ -85,7 +85,7 @@ public void apply(Project project) {
NamedDomainObjectContainer clusters = (NamedDomainObjectContainer) project
.getExtensions()
.getByName(TestClustersPlugin.EXTENSION_NAME);
- clusters.all(c -> {
+ clusters.configureEach(c -> {
if (BuildParams.isInFipsJvm()) {
c.setting("xpack.security.transport.ssl.key", "test-node.key");
c.keystore("xpack.security.transport.ssl.secure_key_passphrase", "test-node-key-password");
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java
index c602a50c2adb8..a7e72b55f9117 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java
@@ -21,6 +21,7 @@
import org.elasticsearch.gradle.internal.ElasticsearchTestBasePlugin;
import org.elasticsearch.gradle.internal.InternalDistributionDownloadPlugin;
import org.elasticsearch.gradle.internal.info.BuildParams;
+import org.elasticsearch.gradle.internal.test.HistoricalFeaturesMetadataPlugin;
import org.elasticsearch.gradle.plugin.BasePluginBuildPlugin;
import org.elasticsearch.gradle.plugin.PluginBuildPlugin;
import org.elasticsearch.gradle.plugin.PluginPropertiesExtension;
@@ -35,9 +36,12 @@
import org.gradle.api.Task;
import org.gradle.api.artifacts.Configuration;
import org.gradle.api.artifacts.Dependency;
+import org.gradle.api.artifacts.DependencySet;
import org.gradle.api.artifacts.ProjectDependency;
import org.gradle.api.artifacts.type.ArtifactTypeDefinition;
import org.gradle.api.attributes.Attribute;
+import org.gradle.api.file.ConfigurableFileCollection;
+import org.gradle.api.file.FileCollection;
import org.gradle.api.file.FileTree;
import org.gradle.api.provider.ProviderFactory;
import org.gradle.api.tasks.ClasspathNormalizer;
@@ -72,6 +76,9 @@ public class RestTestBasePlugin implements Plugin {
private static final String PLUGINS_CONFIGURATION = "clusterPlugins";
private static final String EXTRACTED_PLUGINS_CONFIGURATION = "extractedPlugins";
private static final Attribute CONFIGURATION_ATTRIBUTE = Attribute.of("test-cluster-artifacts", String.class);
+ private static final String FEATURES_METADATA_CONFIGURATION = "featuresMetadataDeps";
+ private static final String DEFAULT_DISTRO_FEATURES_METADATA_CONFIGURATION = "defaultDistrofeaturesMetadataDeps";
+ private static final String TESTS_FEATURES_METADATA_PATH = "tests.features.metadata.path";
private final ProviderFactory providerFactory;
@@ -105,6 +112,36 @@ public void apply(Project project) {
extractedPluginsConfiguration.extendsFrom(pluginsConfiguration);
configureArtifactTransforms(project);
+ // Create configuration for aggregating historical feature metadata
+ FileCollection featureMetadataConfig = project.getConfigurations().create(FEATURES_METADATA_CONFIGURATION, c -> {
+ c.setCanBeConsumed(false);
+ c.setCanBeResolved(true);
+ c.attributes(
+ a -> a.attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, HistoricalFeaturesMetadataPlugin.FEATURES_METADATA_TYPE)
+ );
+ c.defaultDependencies(d -> d.add(project.getDependencies().project(Map.of("path", ":server"))));
+ c.withDependencies(dependencies -> {
+ // We can't just use Configuration#extendsFrom() here as we'd inherit the wrong project configuration
+ copyDependencies(project, dependencies, modulesConfiguration);
+ copyDependencies(project, dependencies, pluginsConfiguration);
+ });
+ });
+
+ FileCollection defaultDistroFeatureMetadataConfig = project.getConfigurations()
+ .create(DEFAULT_DISTRO_FEATURES_METADATA_CONFIGURATION, c -> {
+ c.setCanBeConsumed(false);
+ c.setCanBeResolved(true);
+ c.attributes(
+ a -> a.attribute(
+ ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE,
+ HistoricalFeaturesMetadataPlugin.FEATURES_METADATA_TYPE
+ )
+ );
+ c.defaultDependencies(
+ d -> d.add(project.getDependencies().project(Map.of("path", ":distribution", "configuration", "featuresMetadata")))
+ );
+ });
+
// For plugin and module projects, register the current project plugin bundle as a dependency
project.getPluginManager().withPlugin("elasticsearch.esplugin", plugin -> {
if (GradleUtils.isModuleProject(project.getPath())) {
@@ -122,6 +159,10 @@ public void apply(Project project) {
task.dependsOn(integTestDistro, modulesConfiguration);
registerDistributionInputs(task, integTestDistro);
+ // Pass feature metadata on to tests
+ task.getInputs().files(featureMetadataConfig).withPathSensitivity(PathSensitivity.NONE);
+ nonInputSystemProperties.systemProperty(TESTS_FEATURES_METADATA_PATH, () -> featureMetadataConfig.getAsPath());
+
// Enable parallel execution for these tests since each test gets its own cluster
task.setMaxParallelForks(task.getProject().getGradle().getStartParameter().getMaxWorkerCount() / 2);
nonInputSystemProperties.systemProperty(TESTS_MAX_PARALLEL_FORKS_SYSPROP, () -> String.valueOf(task.getMaxParallelForks()));
@@ -134,16 +175,20 @@ public void apply(Project project) {
task.systemProperty("tests.system_call_filter", "false");
// Register plugins and modules as task inputs and pass paths as system properties to tests
- nonInputSystemProperties.systemProperty(TESTS_CLUSTER_MODULES_PATH_SYSPROP, modulesConfiguration::getAsPath);
- registerConfigurationInputs(task, modulesConfiguration);
- nonInputSystemProperties.systemProperty(TESTS_CLUSTER_PLUGINS_PATH_SYSPROP, pluginsConfiguration::getAsPath);
- registerConfigurationInputs(task, extractedPluginsConfiguration);
+ var modulePath = project.getObjects().fileCollection().from(modulesConfiguration);
+ nonInputSystemProperties.systemProperty(TESTS_CLUSTER_MODULES_PATH_SYSPROP, modulePath::getAsPath);
+ registerConfigurationInputs(task, modulesConfiguration.getName(), modulePath);
+ var pluginPath = project.getObjects().fileCollection().from(pluginsConfiguration);
+ nonInputSystemProperties.systemProperty(TESTS_CLUSTER_PLUGINS_PATH_SYSPROP, pluginPath::getAsPath);
+ registerConfigurationInputs(
+ task,
+ extractedPluginsConfiguration.getName(),
+ project.getObjects().fileCollection().from(extractedPluginsConfiguration)
+ );
// Wire up integ-test distribution by default for all test tasks
- nonInputSystemProperties.systemProperty(
- INTEG_TEST_DISTRIBUTION_SYSPROP,
- () -> integTestDistro.getExtracted().getSingleFile().getPath()
- );
+ FileCollection extracted = integTestDistro.getExtracted();
+ nonInputSystemProperties.systemProperty(INTEG_TEST_DISTRIBUTION_SYSPROP, () -> extracted.getSingleFile().getPath());
nonInputSystemProperties.systemProperty(TESTS_RUNTIME_JAVA_SYSPROP, BuildParams.getRuntimeJavaHome());
// Add `usesDefaultDistribution()` extension method to test tasks to indicate they require the default distro
@@ -157,6 +202,11 @@ public Void call(Object... args) {
DEFAULT_DISTRIBUTION_SYSPROP,
providerFactory.provider(() -> defaultDistro.getExtracted().getSingleFile().getPath())
);
+
+ // If we are using the default distribution we need to register all module feature metadata
+ task.getInputs().files(defaultDistroFeatureMetadataConfig).withPathSensitivity(PathSensitivity.NONE);
+ nonInputSystemProperties.systemProperty(TESTS_FEATURES_METADATA_PATH, defaultDistroFeatureMetadataConfig::getAsPath);
+
return null;
}
});
@@ -192,6 +242,14 @@ public Void call(Object... args) {
});
}
+ private void copyDependencies(Project project, DependencySet dependencies, Configuration configuration) {
+ configuration.getDependencies()
+ .stream()
+ .filter(d -> d instanceof ProjectDependency)
+ .map(d -> project.getDependencies().project(Map.of("path", ((ProjectDependency) d).getDependencyProject().getPath())))
+ .forEach(dependencies::add);
+ }
+
private ElasticsearchDistribution createDistribution(Project project, String name, String version) {
return createDistribution(project, name, version, null);
}
@@ -216,15 +274,15 @@ private FileTree getDistributionFiles(ElasticsearchDistribution distribution, Ac
return distribution.getExtracted().getAsFileTree().matching(patternFilter);
}
- private void registerConfigurationInputs(Task task, Configuration configuration) {
+ private void registerConfigurationInputs(Task task, String configurationName, ConfigurableFileCollection configuration) {
task.getInputs()
.files(providerFactory.provider(() -> configuration.getAsFileTree().filter(f -> f.getName().endsWith(".jar") == false)))
- .withPropertyName(configuration.getName() + "-files")
+ .withPropertyName(configurationName + "-files")
.withPathSensitivity(PathSensitivity.RELATIVE);
task.getInputs()
.files(providerFactory.provider(() -> configuration.getAsFileTree().filter(f -> f.getName().endsWith(".jar"))))
- .withPropertyName(configuration.getName() + "-classpath")
+ .withPropertyName(configurationName + "-classpath")
.withNormalizer(ClasspathNormalizer.class);
}
diff --git a/build-tools-internal/src/main/resources/fips_java.policy b/build-tools-internal/src/main/resources/fips_java.policy
index 4ef62e03c2546..bbfc1caf7593a 100644
--- a/build-tools-internal/src/main/resources/fips_java.policy
+++ b/build-tools-internal/src/main/resources/fips_java.policy
@@ -1,6 +1,10 @@
grant {
permission java.security.SecurityPermission "putProviderProperty.BCFIPS";
permission java.security.SecurityPermission "putProviderProperty.BCJSSE";
+ permission java.security.SecurityPermission "getProperty.keystore.type.compat";
+ permission java.security.SecurityPermission "getProperty.jdk.tls.disabledAlgorithms";
+ permission java.security.SecurityPermission "getProperty.jdk.certpath.disabledAlgorithms";
+ permission java.security.SecurityPermission "getProperty.jdk.tls.server.defaultDHEParameters";
permission java.lang.RuntimePermission "getProtectionDomain";
permission java.util.PropertyPermission "java.runtime.name", "read";
permission org.bouncycastle.crypto.CryptoServicesPermission "tlsAlgorithmsEnabled";
diff --git a/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt b/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt
index 34f39bbc4ca54..48c888acd35e2 100644
--- a/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt
+++ b/build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt
@@ -158,6 +158,8 @@ org.elasticsearch.cluster.ClusterState#compatibilityVersions()
@defaultMessage ClusterFeatures#nodeFeatures is for internal use only. Use FeatureService#clusterHasFeature to determine if a feature is present on the cluster.
org.elasticsearch.cluster.ClusterFeatures#nodeFeatures()
+@defaultMessage ClusterFeatures#allNodeFeatures is for internal use only. Use FeatureService#clusterHasFeature to determine if a feature is present on the cluster.
+org.elasticsearch.cluster.ClusterFeatures#allNodeFeatures()
@defaultMessage ClusterFeatures#clusterHasFeature is for internal use only. Use FeatureService#clusterHasFeature to determine if a feature is present on the cluster.
org.elasticsearch.cluster.ClusterFeatures#clusterHasFeature(org.elasticsearch.features.NodeFeature)
diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties
index dc43523b747b3..98d3ad1eff10b 100644
--- a/build-tools-internal/version.properties
+++ b/build-tools-internal/version.properties
@@ -20,10 +20,9 @@ google_oauth_client = 1.34.1
antlr4 = 4.11.1
# when updating this version, you need to ensure compatibility with:
-# - modules/ingest-attachment (transitive dependency, check the upstream POM)
# - distribution/tools/plugin-cli
# - x-pack/plugin/security
-bouncycastle=1.64
+bouncycastle=1.76
# used by security and idp (need to be in sync due to cross-dependency in testing)
opensaml = 4.3.0
@@ -42,6 +41,12 @@ junit5 = 5.7.1
hamcrest = 2.1
mocksocket = 1.2
+# test container dependencies
+testcontainer = 1.19.2
+dockerJava = 3.3.4
+ductTape = 1.0.8
+commonsCompress = 1.24.0
+
# benchmark dependencies
jmh = 1.26
diff --git a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy
index 6b662b8165034..22efa8d08d3e7 100644
--- a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy
+++ b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy
@@ -34,7 +34,7 @@ class TestClustersPluginFuncTest extends AbstractGradleFuncTest {
id 'elasticsearch.testclusters'
}
- class SomeClusterAwareTask extends DefaultTask implements TestClustersAware {
+ abstract class SomeClusterAwareTask extends DefaultTask implements TestClustersAware {
private Collection clusters = new HashSet<>();
@@ -166,7 +166,7 @@ class TestClustersPluginFuncTest extends AbstractGradleFuncTest {
then:
result.output.contains("Task ':myTask' is not up-to-date because:\n" +
- " Input property 'clusters.myCluster\$0.nodes.\$0.$propertyName'")
+ " Input property 'clusters.myCluster\$0.$propertyName'")
result.output.contains("elasticsearch-keystore script executed!")
assertEsOutputContains("myCluster", "Starting Elasticsearch process")
assertEsOutputContains("myCluster", "Stopping node")
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java
index d08dc469e5ba5..fb8416b24d052 100644
--- a/build-tools/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java
@@ -11,6 +11,7 @@
import org.elasticsearch.gradle.distribution.ElasticsearchDistributionTypes;
import org.elasticsearch.gradle.transform.SymbolicLinkPreservingUntarTransform;
import org.elasticsearch.gradle.transform.UnzipTransform;
+import org.gradle.api.Action;
import org.gradle.api.NamedDomainObjectContainer;
import org.gradle.api.Plugin;
import org.gradle.api.Project;
@@ -22,7 +23,8 @@
import org.gradle.api.provider.Property;
import org.gradle.api.provider.Provider;
-import java.util.Comparator;
+import java.util.ArrayList;
+import java.util.List;
import javax.inject.Inject;
@@ -42,9 +44,10 @@ public class DistributionDownloadPlugin implements Plugin {
private static final String DOWNLOAD_REPO_NAME = "elasticsearch-downloads";
private static final String SNAPSHOT_REPO_NAME = "elasticsearch-snapshots";
public static final String DISTRO_EXTRACTED_CONFIG_PREFIX = "es_distro_extracted_";
+ public static final String DISTRO_CONFIG_PREFIX = "es_distro_file_";
private NamedDomainObjectContainer distributionsContainer;
- private NamedDomainObjectContainer distributionsResolutionStrategiesContainer;
+ private List distributionsResolutionStrategies;
private Property dockerAvailability;
@@ -76,8 +79,9 @@ public void apply(Project project) {
}
private void setupDistributionContainer(Project project, Property dockerAvailable) {
+
distributionsContainer = project.container(ElasticsearchDistribution.class, name -> {
- Configuration fileConfiguration = project.getConfigurations().create("es_distro_file_" + name);
+ Configuration fileConfiguration = project.getConfigurations().create(DISTRO_CONFIG_PREFIX + name);
Configuration extractedConfiguration = project.getConfigurations().create(DISTRO_EXTRACTED_CONFIG_PREFIX + name);
extractedConfiguration.getAttributes()
.attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE);
@@ -85,21 +89,17 @@ private void setupDistributionContainer(Project project, Property docke
name,
project.getObjects(),
dockerAvailability,
- fileConfiguration,
- extractedConfiguration,
- (dist) -> finalizeDistributionDependencies(project, dist)
+ project.getObjects().fileCollection().from(fileConfiguration),
+ project.getObjects().fileCollection().from(extractedConfiguration),
+ new FinalizeDistributionAction(distributionsResolutionStrategies, project)
);
});
project.getExtensions().add(CONTAINER_NAME, distributionsContainer);
}
private void setupResolutionsContainer(Project project) {
- distributionsResolutionStrategiesContainer = project.container(DistributionResolution.class);
- // We want this ordered in the same resolution strategies are added
- distributionsResolutionStrategiesContainer.whenObjectAdded(
- resolveDependencyNotation -> resolveDependencyNotation.setPriority(distributionsResolutionStrategiesContainer.size())
- );
- project.getExtensions().add(RESOLUTION_CONTAINER_NAME, distributionsResolutionStrategiesContainer);
+ distributionsResolutionStrategies = new ArrayList<>();
+ project.getExtensions().add(RESOLUTION_CONTAINER_NAME, distributionsResolutionStrategies);
}
@SuppressWarnings("unchecked")
@@ -108,30 +108,8 @@ public static NamedDomainObjectContainer getContainer
}
@SuppressWarnings("unchecked")
- public static NamedDomainObjectContainer getRegistrationsContainer(Project project) {
- return (NamedDomainObjectContainer) project.getExtensions().getByName(RESOLUTION_CONTAINER_NAME);
- }
-
- private void finalizeDistributionDependencies(Project project, ElasticsearchDistribution distribution) {
- DependencyHandler dependencies = project.getDependencies();
- // for the distribution as a file, just depend on the artifact directly
- DistributionDependency distributionDependency = resolveDependencyNotation(project, distribution);
- dependencies.add(distribution.configuration.getName(), distributionDependency.getDefaultNotation());
- // no extraction needed for rpm, deb or docker
- if (distribution.getType().shouldExtract()) {
- // The extracted configuration depends on the artifact directly but has
- // an artifact transform registered to resolve it as an unpacked folder.
- dependencies.add(distribution.getExtracted().getName(), distributionDependency.getExtractedNotation());
- }
- }
-
- private DistributionDependency resolveDependencyNotation(Project p, ElasticsearchDistribution distribution) {
- return distributionsResolutionStrategiesContainer.stream()
- .sorted(Comparator.comparingInt(DistributionResolution::getPriority))
- .map(r -> r.getResolver().resolve(p, distribution))
- .filter(d -> d != null)
- .findFirst()
- .orElseGet(() -> DistributionDependency.of(dependencyNotation(distribution)));
+ public static List getRegistrationsContainer(Project project) {
+ return (List) project.getExtensions().getByName(RESOLUTION_CONTAINER_NAME);
}
private static void addIvyRepo(Project project, String name, String url, String group) {
@@ -155,22 +133,53 @@ private static void setupDownloadServiceRepo(Project project) {
addIvyRepo(project, SNAPSHOT_REPO_NAME, "https://snapshots-no-kpi.elastic.co", FAKE_SNAPSHOT_IVY_GROUP);
}
- /**
- * Returns a dependency object representing the given distribution.
- *
- * The returned object is suitable to be passed to {@link DependencyHandler}.
- * The concrete type of the object will be a set of maven coordinates as a {@link String}.
- * Maven coordinates point to either the integ-test-zip coordinates on maven central, or a set of artificial
- * coordinates that resolve to the Elastic download service through an ivy repository.
- */
- private String dependencyNotation(ElasticsearchDistribution distribution) {
- if (distribution.getType() == ElasticsearchDistributionTypes.INTEG_TEST_ZIP) {
- return "org.elasticsearch.distribution.integ-test-zip:elasticsearch:" + distribution.getVersion() + "@zip";
+ private record FinalizeDistributionAction(List resolutionList, Project project)
+ implements
+ Action {
+ @Override
+
+ public void execute(ElasticsearchDistribution distro) {
+ finalizeDistributionDependencies(project, distro);
+ }
+
+ private void finalizeDistributionDependencies(Project project, ElasticsearchDistribution distribution) {
+ // for the distribution as a file, just depend on the artifact directly
+ DistributionDependency distributionDependency = resolveDependencyNotation(project, distribution);
+ project.getDependencies().add(DISTRO_CONFIG_PREFIX + distribution.getName(), distributionDependency.getDefaultNotation());
+ // no extraction needed for rpm, deb or docker
+ if (distribution.getType().shouldExtract()) {
+ // The extracted configuration depends on the artifact directly but has
+ // an artifact transform registered to resolve it as an unpacked folder.
+ project.getDependencies()
+ .add(DISTRO_EXTRACTED_CONFIG_PREFIX + distribution.getName(), distributionDependency.getExtractedNotation());
+ }
+ }
+
+ private DistributionDependency resolveDependencyNotation(Project project, ElasticsearchDistribution distro) {
+ return resolutionList.stream()
+ .map(r -> r.getResolver().resolve(project, distro))
+ .filter(d -> d != null)
+ .findFirst()
+ .orElseGet(() -> DistributionDependency.of(dependencyNotation(distro)));
+ }
+
+ /**
+ * Returns a dependency object representing the given distribution.
+ *
+ * The returned object is suitable to be passed to {@link DependencyHandler}.
+ * The concrete type of the object will be a set of maven coordinates as a {@link String}.
+ * Maven coordinates point to either the integ-test-zip coordinates on maven central, or a set of artificial
+ * coordinates that resolve to the Elastic download service through an ivy repository.
+ */
+ private String dependencyNotation(ElasticsearchDistribution distribution) {
+ if (distribution.getType() == ElasticsearchDistributionTypes.INTEG_TEST_ZIP) {
+ return "org.elasticsearch.distribution.integ-test-zip:elasticsearch:" + distribution.getVersion() + "@zip";
+ }
+ Version distroVersion = Version.fromString(distribution.getVersion());
+ String extension = distribution.getType().getExtension(distribution.getPlatform());
+ String classifier = distribution.getType().getClassifier(distribution.getPlatform(), distroVersion);
+ String group = distribution.getVersion().endsWith("-SNAPSHOT") ? FAKE_SNAPSHOT_IVY_GROUP : FAKE_IVY_GROUP;
+ return group + ":elasticsearch" + ":" + distribution.getVersion() + classifier + "@" + extension;
}
- Version distroVersion = Version.fromString(distribution.getVersion());
- String extension = distribution.getType().getExtension(distribution.getPlatform());
- String classifier = distribution.getType().getClassifier(distribution.getPlatform(), distroVersion);
- String group = distribution.getVersion().endsWith("-SNAPSHOT") ? FAKE_SNAPSHOT_IVY_GROUP : FAKE_IVY_GROUP;
- return group + ":elasticsearch" + ":" + distribution.getVersion() + classifier + "@" + extension;
}
}
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/DistributionResolution.java b/build-tools/src/main/java/org/elasticsearch/gradle/DistributionResolution.java
index 3b82c9f6975a0..0d8177dea5cb6 100644
--- a/build-tools/src/main/java/org/elasticsearch/gradle/DistributionResolution.java
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/DistributionResolution.java
@@ -12,9 +12,14 @@
public class DistributionResolution {
private Resolver resolver;
- private String name;
+ private final String name;
private int priority;
+ public DistributionResolution(String name, Resolver resolver) {
+ this(name);
+ this.resolver = resolver;
+ }
+
public DistributionResolution(String name) {
this.name = name;
}
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java b/build-tools/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java
index 5350b6698cb30..fab6926008d6c 100644
--- a/build-tools/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java
@@ -11,7 +11,8 @@
import org.elasticsearch.gradle.distribution.ElasticsearchDistributionTypes;
import org.gradle.api.Action;
import org.gradle.api.Buildable;
-import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.file.ConfigurableFileCollection;
+import org.gradle.api.file.FileCollection;
import org.gradle.api.model.ObjectFactory;
import org.gradle.api.provider.Property;
import org.gradle.api.tasks.TaskDependency;
@@ -44,7 +45,7 @@ public String toString() {
private final String name;
private final Property dockerAvailability;
// pkg private so plugin can configure
- final Configuration configuration;
+ final FileCollection configuration;
private final Property architecture;
private final Property version;
@@ -52,16 +53,17 @@ public String toString() {
private final Property platform;
private final Property bundledJdk;
private final Property failIfUnavailable;
- private final Configuration extracted;
- private Action distributionFinalizer;
+ private final Property preferArchive;
+ private final ConfigurableFileCollection extracted;
+ private transient Action distributionFinalizer;
private boolean frozen = false;
ElasticsearchDistribution(
String name,
ObjectFactory objectFactory,
Property dockerAvailability,
- Configuration fileConfiguration,
- Configuration extractedConfiguration,
+ ConfigurableFileCollection fileConfiguration,
+ ConfigurableFileCollection extractedConfiguration,
Action distributionFinalizer
) {
this.name = name;
@@ -74,6 +76,7 @@ public String toString() {
this.platform = objectFactory.property(Platform.class);
this.bundledJdk = objectFactory.property(Boolean.class);
this.failIfUnavailable = objectFactory.property(Boolean.class).convention(true);
+ this.preferArchive = objectFactory.property(Boolean.class).convention(false);
this.extracted = extractedConfiguration;
this.distributionFinalizer = distributionFinalizer;
}
@@ -140,6 +143,14 @@ public void setFailIfUnavailable(boolean failIfUnavailable) {
this.failIfUnavailable.set(failIfUnavailable);
}
+ public boolean getPreferArchive() {
+ return preferArchive.get();
+ }
+
+ public void setPreferArchive(boolean preferArchive) {
+ this.preferArchive.set(preferArchive);
+ }
+
public void setArchitecture(Architecture architecture) {
this.architecture.set(architecture);
}
@@ -172,7 +183,7 @@ public String getFilepath() {
return configuration.getSingleFile().toString();
}
- public Configuration getExtracted() {
+ public ConfigurableFileCollection getExtracted() {
if (getType().shouldExtract() == false) {
throw new UnsupportedOperationException(
"distribution type [" + getType().getName() + "] for " + "elasticsearch distribution [" + name + "] cannot be extracted"
@@ -187,7 +198,9 @@ public TaskDependency getBuildDependencies() {
return task -> Collections.emptySet();
} else {
maybeFreeze();
- return getType().shouldExtract() ? extracted.getBuildDependencies() : configuration.getBuildDependencies();
+ return getType().shouldExtract() && (preferArchive.get() == false)
+ ? extracted.getBuildDependencies()
+ : configuration.getBuildDependencies();
}
}
@@ -252,13 +265,4 @@ void finalizeValues() {
type.finalizeValue();
bundledJdk.finalizeValue();
}
-
- public TaskDependency getArchiveDependencies() {
- if (skippingDockerDistributionBuild()) {
- return task -> Collections.emptySet();
- } else {
- maybeFreeze();
- return configuration.getBuildDependencies();
- }
- }
}
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/DefaultTestClustersTask.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/DefaultTestClustersTask.java
index 5c98ab3bf4364..e80d2ed64cabd 100644
--- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/DefaultTestClustersTask.java
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/DefaultTestClustersTask.java
@@ -12,7 +12,7 @@
import java.util.Collection;
import java.util.HashSet;
-public class DefaultTestClustersTask extends DefaultTask implements TestClustersAware {
+public abstract class DefaultTestClustersTask extends DefaultTask implements TestClustersAware {
private Collection clusters = new HashSet<>();
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java
index f6705bdb62faa..bf539efaf3c30 100644
--- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java
@@ -14,7 +14,11 @@
import org.gradle.api.Named;
import org.gradle.api.NamedDomainObjectContainer;
import org.gradle.api.Project;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.Dependency;
+import org.gradle.api.artifacts.type.ArtifactTypeDefinition;
import org.gradle.api.file.ArchiveOperations;
+import org.gradle.api.file.ConfigurableFileCollection;
import org.gradle.api.file.FileCollection;
import org.gradle.api.file.FileSystemOperations;
import org.gradle.api.file.RegularFile;
@@ -22,10 +26,15 @@
import org.gradle.api.logging.Logger;
import org.gradle.api.logging.Logging;
import org.gradle.api.provider.Provider;
+import org.gradle.api.tasks.Classpath;
+import org.gradle.api.tasks.InputFiles;
import org.gradle.api.tasks.Internal;
import org.gradle.api.tasks.Nested;
+import org.gradle.api.tasks.PathSensitive;
+import org.gradle.api.tasks.PathSensitivity;
import org.gradle.api.tasks.Sync;
import org.gradle.api.tasks.TaskProvider;
+import org.gradle.api.tasks.bundling.AbstractArchiveTask;
import org.gradle.api.tasks.bundling.Zip;
import org.gradle.process.ExecOperations;
@@ -35,6 +44,8 @@
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.security.GeneralSecurityException;
+import java.util.Collection;
+import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -46,6 +57,9 @@
import java.util.function.Supplier;
import java.util.stream.Collectors;
+import static org.elasticsearch.gradle.plugin.BasePluginBuildPlugin.EXPLODED_BUNDLE_CONFIG;
+import static org.elasticsearch.gradle.testclusters.TestClustersPlugin.BUNDLE_ATTRIBUTE;
+
public class ElasticsearchCluster implements TestClusterConfiguration, Named {
private static final Logger LOGGER = Logging.getLogger(ElasticsearchNode.class);
@@ -59,7 +73,7 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named {
private final FileOperations fileOperations;
private final File workingDirBase;
private final LinkedHashMap> waitConditions = new LinkedHashMap<>();
- private final Project project;
+ private final transient Project project;
private final Provider reaper;
private final FileSystemOperations fileSystemOperations;
private final ArchiveOperations archiveOperations;
@@ -68,6 +82,10 @@ public class ElasticsearchCluster implements TestClusterConfiguration, Named {
private final Function isReleasedVersion;
private int nodeIndex = 0;
+ private final ConfigurableFileCollection pluginAndModuleConfiguration;
+
+ private boolean shared = false;
+
public ElasticsearchCluster(
String path,
String clusterName,
@@ -93,6 +111,7 @@ public ElasticsearchCluster(
this.runtimeJava = runtimeJava;
this.isReleasedVersion = isReleasedVersion;
this.nodes = project.container(ElasticsearchNode.class);
+ this.pluginAndModuleConfiguration = project.getObjects().fileCollection();
this.nodes.add(
new ElasticsearchNode(
safeName(clusterName),
@@ -113,6 +132,29 @@ public ElasticsearchCluster(
addWaitForClusterHealth();
}
+ /**
+ * this cluster si marked as shared across TestClusterAware tasks
+ * */
+ @Internal
+ public boolean isShared() {
+ return shared;
+ }
+
+ protected void setShared(boolean shared) {
+ this.shared = shared;
+ }
+
+ @Classpath
+ public FileCollection getInstalledClasspath() {
+ return pluginAndModuleConfiguration.getAsFileTree().filter(f -> f.getName().endsWith(".jar"));
+ }
+
+ @InputFiles
+ @PathSensitive(PathSensitivity.RELATIVE)
+ public FileCollection getInstalledFiles() {
+ return pluginAndModuleConfiguration.getAsFileTree().filter(f -> f.getName().endsWith(".jar") == false);
+ }
+
public void setNumberOfNodes(int numberOfNodes) {
checkFrozen();
@@ -195,34 +237,70 @@ public void setTestDistribution(TestDistribution distribution) {
nodes.all(each -> each.setTestDistribution(distribution));
}
- @Override
- public void plugin(Provider plugin) {
- nodes.all(each -> each.plugin(plugin));
+ private void registerExtractedConfig(Provider pluginProvider) {
+ Dependency pluginDependency = this.project.getDependencies().create(project.files(pluginProvider));
+ Configuration extractedConfig = project.getConfigurations().detachedConfiguration(pluginDependency);
+ extractedConfig.getAttributes().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE);
+ extractedConfig.getAttributes().attribute(BUNDLE_ATTRIBUTE, true);
+ pluginAndModuleConfiguration.from(extractedConfig);
}
@Override
+ public void plugin(String pluginProjectPath) {
+ plugin(maybeCreatePluginOrModuleDependency(pluginProjectPath, "zip"));
+ }
+
public void plugin(TaskProvider plugin) {
- nodes.all(each -> each.plugin(plugin));
+ plugin(plugin.flatMap(AbstractArchiveTask::getArchiveFile));
}
@Override
- public void plugin(String pluginProjectPath) {
- nodes.all(each -> each.plugin(pluginProjectPath));
+ public void plugin(Provider plugin) {
+ registerExtractedConfig(plugin);
+ nodes.all(each -> each.plugin(plugin));
}
@Override
public void module(Provider module) {
+ registerExtractedConfig(module);
nodes.all(each -> each.module(module));
}
- @Override
public void module(TaskProvider module) {
- nodes.all(each -> each.module(module));
+ module(project.getLayout().file(module.map(Sync::getDestinationDir)));
}
@Override
public void module(String moduleProjectPath) {
- nodes.all(each -> each.module(moduleProjectPath));
+ module(maybeCreatePluginOrModuleDependency(moduleProjectPath, EXPLODED_BUNDLE_CONFIG));
+ }
+
+ private final Map pluginAndModuleConfigurations = new HashMap<>();
+
+ // package protected so only TestClustersAware can access
+ @Internal
+ Collection getPluginAndModuleConfigurations() {
+ return pluginAndModuleConfigurations.values();
+ }
+
+ // creates a configuration to depend on the given plugin project, then wraps that configuration
+ // to grab the zip as a file provider
+ private Provider maybeCreatePluginOrModuleDependency(String path, String consumingConfiguration) {
+ var configuration = pluginAndModuleConfigurations.computeIfAbsent(path, key -> {
+ var bundleDependency = this.project.getDependencies().project(Map.of("path", path, "configuration", consumingConfiguration));
+ return project.getConfigurations().detachedConfiguration(bundleDependency);
+ });
+
+ Provider fileProvider = configuration.getElements()
+ .map(
+ s -> s.stream()
+ .findFirst()
+ .orElseThrow(
+ () -> new IllegalStateException(consumingConfiguration + " configuration of project " + path + " had no files")
+ )
+ .getAsFile()
+ );
+ return project.getLayout().file(fileProvider);
}
@Override
@@ -579,4 +657,5 @@ public int hashCode() {
public String toString() {
return "cluster{" + path + ":" + clusterName + "}";
}
+
}
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
index f0ab67fe51a34..ce4fd7502f417 100644
--- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
@@ -21,18 +21,12 @@
import org.elasticsearch.gradle.Version;
import org.elasticsearch.gradle.VersionProperties;
import org.elasticsearch.gradle.distribution.ElasticsearchDistributionTypes;
-import org.elasticsearch.gradle.transform.UnzipTransform;
import org.elasticsearch.gradle.util.Pair;
import org.gradle.api.Action;
import org.gradle.api.Named;
import org.gradle.api.NamedDomainObjectContainer;
import org.gradle.api.Project;
-import org.gradle.api.artifacts.Configuration;
-import org.gradle.api.artifacts.Dependency;
-import org.gradle.api.artifacts.type.ArtifactTypeDefinition;
-import org.gradle.api.attributes.Attribute;
import org.gradle.api.file.ArchiveOperations;
-import org.gradle.api.file.ConfigurableFileCollection;
import org.gradle.api.file.FileCollection;
import org.gradle.api.file.FileSystemOperations;
import org.gradle.api.file.FileTree;
@@ -52,7 +46,6 @@
import org.gradle.api.tasks.PathSensitivity;
import org.gradle.api.tasks.Sync;
import org.gradle.api.tasks.TaskProvider;
-import org.gradle.api.tasks.bundling.AbstractArchiveTask;
import org.gradle.api.tasks.bundling.Zip;
import org.gradle.api.tasks.util.PatternFilterable;
import org.gradle.process.ExecOperations;
@@ -75,7 +68,6 @@
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -98,7 +90,6 @@
import static java.util.Objects.requireNonNull;
import static java.util.Optional.ofNullable;
-import static org.elasticsearch.gradle.plugin.BasePluginBuildPlugin.EXPLODED_BUNDLE_CONFIG;
public class ElasticsearchNode implements TestClusterConfiguration {
@@ -130,7 +121,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
private final String path;
private final String name;
- private final Project project;
+ transient private final Project project;
private final Provider reaperServiceProvider;
private final FileSystemOperations fileSystemOperations;
private final ArchiveOperations archiveOperations;
@@ -140,8 +131,6 @@ public class ElasticsearchNode implements TestClusterConfiguration {
private final Path workingDir;
private final LinkedHashMap> waitConditions = new LinkedHashMap<>();
- private final Map pluginAndModuleConfigurations = new HashMap<>();
- private final ConfigurableFileCollection pluginAndModuleConfiguration;
private final List> plugins = new ArrayList<>();
private final List> modules = new ArrayList<>();
private final LazyPropertyMap settings = new LazyPropertyMap<>("Settings", this);
@@ -151,6 +140,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
private final LazyPropertyMap systemProperties = new LazyPropertyMap<>("System properties", this);
private final LazyPropertyMap environment = new LazyPropertyMap<>("Environment", this);
private final LazyPropertyList jvmArgs = new LazyPropertyList<>("JVM arguments", this);
+ private final LazyPropertyList cliJvmArgs = new LazyPropertyList<>("CLI JVM arguments", this);
private final LazyPropertyMap extraConfigFiles = new LazyPropertyMap<>("Extra config files", this, FileEntry::new);
private final LazyPropertyList extraJarConfigurations = new LazyPropertyList<>("Extra jar files", this);
private final List> credentials = new ArrayList<>();
@@ -171,12 +161,10 @@ public class ElasticsearchNode implements TestClusterConfiguration {
private final Provider runtimeJava;
private final Function isReleasedVersion;
private final List distributions = new ArrayList<>();
- private final Attribute bundleAttribute = Attribute.of("bundle", Boolean.class);
-
private int currentDistro = 0;
private TestDistribution testDistribution;
private volatile Process esProcess;
- private Function nameCustomization = Function.identity();
+ private Function nameCustomization = s -> s;
private boolean isWorkingDirConfigured = false;
private String httpPort = "0";
private String transportPort = "0";
@@ -223,10 +211,8 @@ public class ElasticsearchNode implements TestClusterConfiguration {
waitConditions.put("ports files", this::checkPortsFilesExistWithDelay);
defaultConfig.put("cluster.name", clusterName);
- pluginAndModuleConfiguration = project.getObjects().fileCollection();
setTestDistribution(TestDistribution.INTEG_TEST);
setVersion(VersionProperties.getElasticsearch());
- configureArtifactTransforms();
}
@Input
@@ -302,84 +288,34 @@ private void setDistributionType(ElasticsearchDistribution distribution, TestDis
}
}
- // package protected so only TestClustersAware can access
- @Internal
- Collection getPluginAndModuleConfigurations() {
- return pluginAndModuleConfigurations.values();
- }
-
- // creates a configuration to depend on the given plugin project, then wraps that configuration
- // to grab the zip as a file provider
- private Provider maybeCreatePluginOrModuleDependency(String path, String consumingConfiguration) {
- var configuration = pluginAndModuleConfigurations.computeIfAbsent(path, key -> {
- var bundleDependency = this.project.getDependencies().project(Map.of("path", path, "configuration", consumingConfiguration));
- return project.getConfigurations().detachedConfiguration(bundleDependency);
- });
-
- Provider fileProvider = configuration.getElements()
- .map(
- s -> s.stream()
- .findFirst()
- .orElseThrow(
- () -> new IllegalStateException(consumingConfiguration + " configuration of project " + path + " had no files")
- )
- .getAsFile()
- );
- return project.getLayout().file(fileProvider);
- }
-
@Override
public void plugin(Provider plugin) {
checkFrozen();
- registerExtractedConfig(plugin);
this.plugins.add(plugin.map(RegularFile::getAsFile));
}
@Override
public void plugin(String pluginProjectPath) {
- plugin(maybeCreatePluginOrModuleDependency(pluginProjectPath, "zip"));
+ throw new UnsupportedOperationException("Not Supported API");
}
public void plugin(TaskProvider plugin) {
- plugin(plugin.flatMap(AbstractArchiveTask::getArchiveFile));
+ throw new UnsupportedOperationException("Not Supported API");
}
@Override
public void module(Provider module) {
checkFrozen();
- registerExtractedConfig(module);
this.modules.add(module.map(RegularFile::getAsFile));
}
public void module(TaskProvider module) {
- module(project.getLayout().file(module.map(Sync::getDestinationDir)));
- }
-
- private void registerExtractedConfig(Provider pluginProvider) {
- Dependency pluginDependency = this.project.getDependencies().create(project.files(pluginProvider));
- Configuration extractedConfig = project.getConfigurations().detachedConfiguration(pluginDependency);
- extractedConfig.getAttributes().attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE);
- extractedConfig.getAttributes().attribute(bundleAttribute, true);
- pluginAndModuleConfiguration.from(extractedConfig);
- }
-
- private void configureArtifactTransforms() {
- project.getDependencies().getAttributesSchema().attribute(bundleAttribute);
- project.getDependencies().getArtifactTypes().maybeCreate(ArtifactTypeDefinition.ZIP_TYPE);
- project.getDependencies().registerTransform(UnzipTransform.class, transformSpec -> {
- transformSpec.getFrom()
- .attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.ZIP_TYPE)
- .attribute(bundleAttribute, true);
- transformSpec.getTo()
- .attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE)
- .attribute(bundleAttribute, true);
- transformSpec.getParameters().setAsFiletreeOutput(true);
- });
+ throw new IllegalStateException("Not Supported API");
}
@Override
public void module(String moduleProjectPath) {
- module(maybeCreatePluginOrModuleDependency(moduleProjectPath, EXPLODED_BUNDLE_CONFIG));
+ throw new IllegalStateException("Not Supported API");
}
@Override
@@ -471,6 +407,10 @@ public void jvmArgs(String... values) {
jvmArgs.addAll(Arrays.asList(values));
}
+ public void cliJvmArgs(String... values) {
+ cliJvmArgs.addAll(Arrays.asList(values));
+ }
+
@Internal
public Path getConfigDir() {
return configFile.getParent();
@@ -932,6 +872,10 @@ private void startElasticsearchProcess() {
// Don't inherit anything from the environment for as that would lack reproducibility
environment.clear();
environment.putAll(getESEnvironment());
+ if (cliJvmArgs.isEmpty() == false) {
+ String cliJvmArgsString = String.join(" ", cliJvmArgs);
+ environment.put("CLI_JAVA_OPTS", cliJvmArgsString);
+ }
// Direct the stderr to the ES log file. This should capture any jvm problems to start.
// Stdout is discarded because ES duplicates the log file to stdout when run in the foreground.
@@ -1551,17 +1495,6 @@ private Path getExtractedDistributionDir() {
return distributions.get(currentDistro).getExtracted().getSingleFile().toPath();
}
- @Classpath
- public FileCollection getInstalledClasspath() {
- return pluginAndModuleConfiguration.getAsFileTree().filter(f -> f.getName().endsWith(".jar"));
- }
-
- @InputFiles
- @PathSensitive(PathSensitivity.RELATIVE)
- public FileCollection getInstalledFiles() {
- return pluginAndModuleConfiguration.getAsFileTree().filter(f -> f.getName().endsWith(".jar") == false);
- }
-
@Classpath
public List getDistributionClasspath() {
return getDistributionFiles(filter -> filter.include("**/*.jar"));
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/MockApmServer.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/MockApmServer.java
index 7ec74ee19d1bb..7c1d4b6015d2e 100644
--- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/MockApmServer.java
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/MockApmServer.java
@@ -25,10 +25,10 @@
* This is a server which just accepts lines of JSON code and if the JSON
* is valid and the root node is "transaction", then adds that JSON object
* to a transaction list which is accessible externally to the class.
- *
+ *
* The Elastic agent sends lines of JSON code, and so this mock server
* can be used as a basic APM server for testing.
- *
+ *
* The HTTP server used is the JDK embedded com.sun.net.httpserver
*/
public class MockApmServer {
@@ -54,12 +54,16 @@ public static void main(String[] args) throws IOException, InterruptedException
/**
* Start the Mock APM server. Just returns empty JSON structures for every incoming message
+ *
* @return - the port the Mock APM server started on
* @throws IOException
*/
public synchronized int start() throws IOException {
if (instance != null) {
- throw new IOException("MockApmServer: Ooops, you can't start this instance more than once");
+ String hostname = instance.getAddress().getHostName();
+ int port = instance.getAddress().getPort();
+ logger.lifecycle("MockApmServer is already running. Reusing on address:port " + hostname + ":" + port);
+ return port;
}
InetSocketAddress addr = new InetSocketAddress("0.0.0.0", port);
HttpServer server = HttpServer.create(addr, 10);
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java
index 953c0447ec71b..477842a201bb9 100644
--- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java
@@ -40,6 +40,7 @@ public abstract class RunTask extends DefaultTestClustersTask {
private static final String transportCertificate = "private-cert2.p12";
private Boolean debug = false;
+ private Boolean cliDebug = false;
private Boolean apmServerEnabled = false;
private Boolean preserveData = false;
@@ -62,11 +63,21 @@ public void setDebug(boolean enabled) {
this.debug = enabled;
}
+ @Option(option = "debug-cli-jvm", description = "Enable debugging configuration, to allow attaching a debugger to the cli launcher.")
+ public void setCliDebug(boolean enabled) {
+ this.cliDebug = enabled;
+ }
+
@Input
public Boolean getDebug() {
return debug;
}
+ @Input
+ public Boolean getCliDebug() {
+ return cliDebug;
+ }
+
@Input
public Boolean getApmServerEnabled() {
return apmServerEnabled;
@@ -194,7 +205,9 @@ public void beforeStart() {
} catch (IOException e) {
logger.warn("Unable to start APM server", e);
}
-
+ } else {
+ // metrics are enabled by default, if the --with-apm-server was not used we should disable it
+ node.setting("telemetry.metrics.enabled", "false");
}
}
@@ -202,6 +215,9 @@ public void beforeStart() {
if (debug) {
enableDebug();
}
+ if (cliDebug) {
+ enableCliDebug();
+ }
}
@TaskAction
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/StandaloneRestIntegTestTask.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/StandaloneRestIntegTestTask.java
index 2bd8219dc48e5..5e6b33aa980f0 100644
--- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/StandaloneRestIntegTestTask.java
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/StandaloneRestIntegTestTask.java
@@ -8,11 +8,9 @@
package org.elasticsearch.gradle.testclusters;
import org.elasticsearch.gradle.FileSystemOperationsAware;
-import org.gradle.api.Task;
+import org.gradle.api.provider.ProviderFactory;
import org.gradle.api.services.internal.BuildServiceProvider;
import org.gradle.api.services.internal.BuildServiceRegistryInternal;
-import org.gradle.api.specs.NotSpec;
-import org.gradle.api.specs.Spec;
import org.gradle.api.tasks.CacheableTask;
import org.gradle.api.tasks.Internal;
import org.gradle.api.tasks.Nested;
@@ -28,6 +26,8 @@
import java.util.HashSet;
import java.util.List;
+import javax.inject.Inject;
+
import static org.elasticsearch.gradle.testclusters.TestClustersPlugin.THROTTLE_SERVICE_NAME;
/**
@@ -42,23 +42,6 @@ public abstract class StandaloneRestIntegTestTask extends Test implements TestCl
private boolean debugServer = false;
public StandaloneRestIntegTestTask() {
- Spec taskSpec = t -> getProject().getTasks()
- .withType(StandaloneRestIntegTestTask.class)
- .stream()
- .filter(task -> task != this)
- .anyMatch(task -> Collections.disjoint(task.getClusters(), getClusters()) == false);
- this.getOutputs()
- .doNotCacheIf(
- "Caching disabled for this task since it uses a cluster shared by other tasks",
- /*
- * Look for any other tasks which use the same cluster as this task. Since tests often have side effects for the cluster
- * they execute against, this state can cause issues when trying to cache tests results of tasks that share a cluster. To
- * avoid any undesired behavior we simply disable the cache if we detect that this task uses a cluster shared between
- * multiple tasks.
- */
- taskSpec
- );
- this.getOutputs().upToDateWhen(new NotSpec(taskSpec));
this.getOutputs()
.doNotCacheIf(
"Caching disabled for this task since it is configured to preserve data directory",
@@ -79,16 +62,22 @@ public Collection getClusters() {
return clusters;
}
+ @Override
+ @Inject
+ public abstract ProviderFactory getProviderFactory();
+
@Override
@Internal
public List getSharedResources() {
+ // Since we need to have the buildservice registered for configuration cache compatibility,
+ // we already get one lock for throttle service
List locks = new ArrayList<>(super.getSharedResources());
BuildServiceRegistryInternal serviceRegistry = getServices().get(BuildServiceRegistryInternal.class);
BuildServiceProvider serviceProvider = serviceRegistry.consume(THROTTLE_SERVICE_NAME, TestClustersThrottle.class);
SharedResource resource = serviceRegistry.forService(serviceProvider);
int nodeCount = clusters.stream().mapToInt(cluster -> cluster.getNodes().size()).sum();
if (nodeCount > 0) {
- for (int i = 0; i < Math.min(nodeCount, resource.getMaxUsages()); i++) {
+ for (int i = 0; i < Math.min(nodeCount, resource.getMaxUsages() - 1); i++) {
locks.add(resource.getResourceLock());
}
}
@@ -101,6 +90,7 @@ public WorkResult delete(Object... objects) {
@Override
public void beforeStart() {
+ TestClustersAware.super.beforeStart();
if (debugServer) {
enableDebug();
}
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersAware.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersAware.java
index 9537162b5d109..3fef77688c48d 100644
--- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersAware.java
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersAware.java
@@ -8,18 +8,27 @@
package org.elasticsearch.gradle.testclusters;
import org.gradle.api.Task;
-import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.provider.Property;
import org.gradle.api.provider.Provider;
+import org.gradle.api.services.ServiceReference;
import org.gradle.api.tasks.Nested;
import java.util.Collection;
-import java.util.concurrent.Callable;
+
+import static org.elasticsearch.gradle.testclusters.TestClustersPlugin.REGISTRY_SERVICE_NAME;
+import static org.elasticsearch.gradle.testclusters.TestClustersPlugin.TEST_CLUSTER_TASKS_SERVICE;
public interface TestClustersAware extends Task {
@Nested
Collection getClusters();
+ @ServiceReference(REGISTRY_SERVICE_NAME)
+ Property getRegistery();
+
+ @ServiceReference(TEST_CLUSTER_TASKS_SERVICE)
+ Property getTasksService();
+
default void useCluster(ElasticsearchCluster cluster) {
if (cluster.getPath().equals(getProject().getPath()) == false) {
throw new TestClustersException("Task " + getPath() + " can't use test cluster from" + " another project " + cluster);
@@ -27,7 +36,7 @@ default void useCluster(ElasticsearchCluster cluster) {
cluster.getNodes()
.all(node -> node.getDistributions().forEach(distro -> dependsOn(getProject().provider(() -> distro.maybeFreeze()))));
- cluster.getNodes().all(node -> dependsOn((Callable>) node::getPluginAndModuleConfigurations));
+ dependsOn(cluster.getPluginAndModuleConfigurations());
getClusters().add(cluster);
}
@@ -35,7 +44,9 @@ default void useCluster(Provider cluster) {
useCluster(cluster.get());
}
- default void beforeStart() {}
+ default void beforeStart() {
+ getTasksService().get().register(this);
+ }
default void enableDebug() {
int debugPort = 5007;
@@ -47,4 +58,19 @@ default void enableDebug() {
}
}
}
+
+ default void enableCliDebug() {
+ int cliDebugPort = 5107;
+ for (ElasticsearchCluster cluster : getClusters()) {
+ for (ElasticsearchNode node : cluster.getNodes()) {
+ getLogger().lifecycle(
+ "Running cli launcher in debug mode, {} expecting running debug server on port {}",
+ node,
+ cliDebugPort
+ );
+ node.cliJvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=" + cliDebugPort);
+ cliDebugPort += 1;
+ }
+ }
+ }
}
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java
index 72a462c3cd8c9..d4ae65d43893a 100644
--- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java
@@ -11,11 +11,14 @@
import org.elasticsearch.gradle.ReaperPlugin;
import org.elasticsearch.gradle.ReaperService;
import org.elasticsearch.gradle.Version;
+import org.elasticsearch.gradle.transform.UnzipTransform;
import org.elasticsearch.gradle.util.GradleUtils;
import org.gradle.api.NamedDomainObjectContainer;
import org.gradle.api.Plugin;
import org.gradle.api.Project;
import org.gradle.api.Task;
+import org.gradle.api.artifacts.type.ArtifactTypeDefinition;
+import org.gradle.api.attributes.Attribute;
import org.gradle.api.file.ArchiveOperations;
import org.gradle.api.file.FileSystemOperations;
import org.gradle.api.internal.file.FileOperations;
@@ -45,12 +48,15 @@
public class TestClustersPlugin implements Plugin {
+ public static final Attribute BUNDLE_ATTRIBUTE = Attribute.of("bundle", Boolean.class);
+
public static final String EXTENSION_NAME = "testClusters";
public static final String THROTTLE_SERVICE_NAME = "testClustersThrottle";
private static final String LIST_TASK_NAME = "listTestClusters";
- private static final String REGISTRY_SERVICE_NAME = "testClustersRegistry";
+ public static final String REGISTRY_SERVICE_NAME = "testClustersRegistry";
private static final Logger logger = Logging.getLogger(TestClustersPlugin.class);
+ public static final String TEST_CLUSTER_TASKS_SERVICE = "testClusterTasksService";
private final ProviderFactory providerFactory;
private Provider runtimeJavaProvider;
private Function isReleasedVersion = v -> true;
@@ -109,7 +115,7 @@ public void apply(Project project) {
project.getGradle().getSharedServices().registerIfAbsent(REGISTRY_SERVICE_NAME, TestClustersRegistry.class, noop());
// register throttle so we only run at most max-workers/2 nodes concurrently
- project.getGradle()
+ Provider testClustersThrottleProvider = project.getGradle()
.getSharedServices()
.registerIfAbsent(
THROTTLE_SERVICE_NAME,
@@ -117,8 +123,23 @@ public void apply(Project project) {
spec -> spec.getMaxParallelUsages().set(Math.max(1, project.getGradle().getStartParameter().getMaxWorkerCount() / 2))
);
- // register cluster hooks
+ project.getTasks().withType(TestClustersAware.class).configureEach(task -> { task.usesService(testClustersThrottleProvider); });
project.getRootProject().getPluginManager().apply(TestClustersHookPlugin.class);
+ configureArtifactTransforms(project);
+ }
+
+ private void configureArtifactTransforms(Project project) {
+ project.getDependencies().getAttributesSchema().attribute(BUNDLE_ATTRIBUTE);
+ project.getDependencies().getArtifactTypes().maybeCreate(ArtifactTypeDefinition.ZIP_TYPE);
+ project.getDependencies().registerTransform(UnzipTransform.class, transformSpec -> {
+ transformSpec.getFrom()
+ .attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.ZIP_TYPE)
+ .attribute(BUNDLE_ATTRIBUTE, true);
+ transformSpec.getTo()
+ .attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ArtifactTypeDefinition.DIRECTORY_TYPE)
+ .attribute(BUNDLE_ATTRIBUTE, true);
+ transformSpec.getParameters().setAsFiletreeOutput(true);
+ });
}
private NamedDomainObjectContainer createTestClustersContainerExtension(
@@ -156,13 +177,13 @@ private void createListClustersTask(Project project, NamedDomainObjectContainer<
(Task t) -> container.forEach(cluster -> logger.lifecycle(" * {}: {}", cluster.getName(), cluster.getNumberOfNodes()))
);
});
-
}
static abstract class TestClustersHookPlugin implements Plugin {
@Inject
public abstract BuildEventsListenerRegistry getEventsListenerRegistry();
+ @SuppressWarnings("checkstyle:RedundantModifier")
@Inject
public TestClustersHookPlugin() {}
@@ -177,10 +198,9 @@ public void apply(Project project) {
Provider testClusterTasksService = project.getGradle()
.getSharedServices()
- .registerIfAbsent("testClusterTasksService", TaskEventsService.class, spec -> {});
+ .registerIfAbsent(TEST_CLUSTER_TASKS_SERVICE, TaskEventsService.class, spec -> {});
TestClustersRegistry registry = registryProvider.get();
-
// When we know what tasks will run, we claim the clusters of those task to differentiate between clusters
// that are defined in the build script and the ones that will actually be used in this invocation of gradle
// we use this information to determine when the last task that required the cluster executed so that we can
@@ -219,10 +239,9 @@ private void configureStartClustersHook(
.filter(task -> task instanceof TestClustersAware)
.map(task -> (TestClustersAware) task)
.forEach(awareTask -> {
- testClusterTasksService.get().register(awareTask.getPath(), awareTask);
awareTask.doFirst(task -> {
awareTask.beforeStart();
- awareTask.getClusters().forEach(registry::maybeStartCluster);
+ awareTask.getClusters().forEach(awareTask.getRegistery().get()::maybeStartCluster);
});
});
});
@@ -234,12 +253,12 @@ static public abstract class TaskEventsService implements BuildService tasksMap = new HashMap<>();
private TestClustersRegistry registryProvider;
- public void register(String path, TestClustersAware task) {
- tasksMap.put(path, task);
+ public void register(TestClustersAware task) {
+ tasksMap.put(task.getPath(), task);
}
public void registry(TestClustersRegistry registry) {
- registryProvider = registry;
+ this.registryProvider = registry;
}
@Override
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersRegistry.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersRegistry.java
index b46e86ca84bdd..1d6efdabcd59f 100644
--- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersRegistry.java
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersRegistry.java
@@ -22,11 +22,16 @@ public abstract class TestClustersRegistry implements BuildService claimsInventory = new HashMap<>();
+
private final Set runningClusters = new HashSet<>();
public void claimCluster(ElasticsearchCluster cluster) {
cluster.freeze();
- claimsInventory.put(cluster, claimsInventory.getOrDefault(cluster, 0) + 1);
+ int claim = claimsInventory.getOrDefault(cluster, 0) + 1;
+ claimsInventory.put(cluster, claim);
+ if (claim > 1) {
+ cluster.setShared(true);
+ }
}
public void maybeStartCluster(ElasticsearchCluster cluster) {
@@ -63,7 +68,6 @@ public void stopCluster(ElasticsearchCluster cluster, boolean taskFailed) {
} else {
int currentClaims = claimsInventory.getOrDefault(cluster, 0) - 1;
claimsInventory.put(cluster, currentClaims);
-
if (currentClaims <= 0 && runningClusters.contains(cluster)) {
cluster.stop(false);
runningClusters.remove(cluster);
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersThrottle.java b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersThrottle.java
index 6ba3261691cb9..4dfd950e0aaf1 100644
--- a/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersThrottle.java
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersThrottle.java
@@ -10,4 +10,6 @@
import org.gradle.api.services.BuildService;
import org.gradle.api.services.BuildServiceParameters;
-public abstract class TestClustersThrottle implements BuildService {}
+public abstract class TestClustersThrottle implements BuildService {
+
+}
diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/util/GradleUtils.java b/build-tools/src/main/java/org/elasticsearch/gradle/util/GradleUtils.java
index ce69c4ec476f9..00e5834b0f826 100644
--- a/build-tools/src/main/java/org/elasticsearch/gradle/util/GradleUtils.java
+++ b/build-tools/src/main/java/org/elasticsearch/gradle/util/GradleUtils.java
@@ -13,7 +13,6 @@
import org.gradle.api.Task;
import org.gradle.api.UnknownTaskException;
import org.gradle.api.artifacts.Configuration;
-import org.gradle.api.artifacts.Dependency;
import org.gradle.api.artifacts.ModuleDependency;
import org.gradle.api.artifacts.ProjectDependency;
import org.gradle.api.plugins.JavaBasePlugin;
@@ -34,7 +33,6 @@
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
@@ -183,16 +181,6 @@ public static void extendSourceSet(Project project, String parentSourceSetName,
}
}
- public static Dependency projectDependency(Project project, String projectPath, String projectConfig) {
- if (project.findProject(projectPath) == null) {
- throw new GradleException("no project [" + projectPath + "], project names: " + project.getRootProject().getAllprojects());
- }
- Map depConfig = new HashMap<>();
- depConfig.put("path", projectPath);
- depConfig.put("configuration", projectConfig);
- return project.getDependencies().project(depConfig);
- }
-
/**
* To calculate the project path from a task path without relying on Task#getProject() which is discouraged during
* task execution time.
diff --git a/build.gradle b/build.gradle
index d05c2bf53f660..c0b613beefea4 100644
--- a/build.gradle
+++ b/build.gradle
@@ -29,8 +29,8 @@ plugins {
id 'lifecycle-base'
id 'elasticsearch.docker-support'
id 'elasticsearch.global-build-info'
- id 'elasticsearch.build-scan'
id 'elasticsearch.build-complete'
+ id 'elasticsearch.build-scan'
id 'elasticsearch.jdk-download'
id 'elasticsearch.internal-distribution-download'
id 'elasticsearch.runtime-jdk-provision'
@@ -161,8 +161,10 @@ tasks.register("verifyVersions") {
String versionMapping = backportConfig.get("branchLabelMapping").fields().find { it.value.textValue() == 'main' }.key
String expectedMapping = "^v${versions.elasticsearch.replaceAll('-SNAPSHOT', '')}\$"
if (versionMapping != expectedMapping) {
- throw new GradleException("Backport label mapping for branch 'main' is '${versionMapping}' but should be " +
- "'${expectedMapping}'. Update .backportrc.json.")
+ throw new GradleException(
+ "Backport label mapping for branch 'main' is '${versionMapping}' but should be " +
+ "'${expectedMapping}'. Update .backportrc.json."
+ )
}
}
}
@@ -211,9 +213,9 @@ allprojects {
project.ext {
// for ide hacks...
isEclipse = providers.systemProperty("eclipse.launcher").isPresent() || // Detects gradle launched from Eclipse's IDE
- providers.systemProperty("eclipse.application").isPresent() || // Detects gradle launched from the Eclipse compiler server
- gradle.startParameter.taskNames.contains('eclipse') || // Detects gradle launched from the command line to do eclipse stuff
- gradle.startParameter.taskNames.contains('cleanEclipse')
+ providers.systemProperty("eclipse.application").isPresent() || // Detects gradle launched from the Eclipse compiler server
+ gradle.startParameter.taskNames.contains('eclipse') || // Detects gradle launched from the command line to do eclipse stuff
+ gradle.startParameter.taskNames.contains('cleanEclipse')
}
ext.bwc_tests_enabled = bwc_tests_enabled
@@ -229,10 +231,10 @@ allprojects {
eclipse.classpath.file.whenMerged { classpath ->
if (false == forbiddenApisTest.bundledSignatures.contains('jdk-non-portable')) {
classpath.entries
- .findAll { it.kind == "con" && it.toString().contains("org.eclipse.jdt.launching.JRE_CONTAINER") }
- .each {
- it.accessRules.add(new AccessRule("accessible", "com/sun/net/httpserver/*"))
- }
+ .findAll { it.kind == "con" && it.toString().contains("org.eclipse.jdt.launching.JRE_CONTAINER") }
+ .each {
+ it.accessRules.add(new AccessRule("accessible", "com/sun/net/httpserver/*"))
+ }
}
}
}
@@ -248,6 +250,8 @@ allprojects {
plugins.withId('lifecycle-base') {
if (project.path.startsWith(":x-pack:")) {
if (project.path.contains("security") || project.path.contains(":ml")) {
+ tasks.register('checkPart4') { dependsOn 'check' }
+ } else if (project.path == ":x-pack:plugin" || project.path.contains("ql") || project.path.contains("smoke-test")) {
tasks.register('checkPart3') { dependsOn 'check' }
} else {
tasks.register('checkPart2') { dependsOn 'check' }
@@ -256,7 +260,7 @@ allprojects {
tasks.register('checkPart1') { dependsOn 'check' }
}
- tasks.register('functionalTests') { dependsOn 'check'}
+ tasks.register('functionalTests') { dependsOn 'check' }
}
/*
@@ -281,7 +285,7 @@ allprojects {
// :test:framework:test cannot run before and after :server:test
return
}
- tasks.matching { it.name.equals('integTest')}.configureEach {integTestTask ->
+ tasks.matching { it.name.equals('integTest') }.configureEach { integTestTask ->
integTestTask.mustRunAfter tasks.matching { it.name.equals("test") }
}
@@ -290,7 +294,7 @@ allprojects {
Project upstreamProject = dep.dependencyProject
if (project.path != upstreamProject?.path) {
for (String taskName : ['test', 'integTest']) {
- project.tasks.matching { it.name == taskName }.configureEach {task ->
+ project.tasks.matching { it.name == taskName }.configureEach { task ->
task.shouldRunAfter(upstreamProject.tasks.matching { upStreamTask -> upStreamTask.name == taskName })
}
}
diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle
index 743f64b3b28d3..bcbc73f643298 100644
--- a/client/rest-high-level/build.gradle
+++ b/client/rest-high-level/build.gradle
@@ -61,11 +61,6 @@ tasks.named('forbiddenApisMain').configure {
signaturesFiles += files('src/main/resources/forbidden/rest-high-level-signatures.txt')
}
-tasks.named('splitPackagesAudit').configure {
- // the client package should be owned by the client, but server has some classes there too
- ignoreClasses 'org.elasticsearch.client.*'
-}
-
// we don't have tests now, as HLRC is in the process of being removed
tasks.named("test").configure {enabled = false }
diff --git a/client/rest-high-level/roles.yml b/client/rest-high-level/roles.yml
deleted file mode 100644
index d3d0630f43058..0000000000000
--- a/client/rest-high-level/roles.yml
+++ /dev/null
@@ -1,12 +0,0 @@
-admin:
- cluster:
- - all
- indices:
- - names: '*'
- privileges:
- - all
- run_as: [ '*' ]
- applications:
- - application: '*'
- privileges: [ '*' ]
- resources: [ '*' ]
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
deleted file mode 100644
index fca1e5d29efaf..0000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
+++ /dev/null
@@ -1,554 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.client;
-
-import org.apache.http.HttpEntity;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.entity.ContentType;
-import org.apache.http.nio.entity.NByteArrayEntity;
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.action.DocWriteRequest;
-import org.elasticsearch.action.bulk.BulkRequest;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchScrollRequest;
-import org.elasticsearch.action.support.ActiveShardCount;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
-import org.elasticsearch.action.update.UpdateRequest;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.lucene.uid.Versions;
-import org.elasticsearch.common.xcontent.XContentHelper;
-import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.SuppressForbidden;
-import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.index.VersionType;
-import org.elasticsearch.index.seqno.SequenceNumbers;
-import org.elasticsearch.rest.action.search.RestSearchAction;
-import org.elasticsearch.xcontent.DeprecationHandler;
-import org.elasticsearch.xcontent.NamedXContentRegistry;
-import org.elasticsearch.xcontent.ToXContent;
-import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xcontent.XContentType;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.charset.Charset;
-import java.util.HashMap;
-import java.util.Locale;
-import java.util.Map;
-import java.util.StringJoiner;
-
-final class RequestConverters {
- static final XContentType REQUEST_BODY_CONTENT_TYPE = XContentType.JSON;
-
- private RequestConverters() {
- // Contains only status utility methods
- }
-
- static Request bulk(BulkRequest bulkRequest) throws IOException {
- Request request = new Request(HttpPost.METHOD_NAME, "/_bulk");
-
- Params parameters = new Params();
- parameters.withTimeout(bulkRequest.timeout());
- parameters.withRefreshPolicy(bulkRequest.getRefreshPolicy());
- parameters.withPipeline(bulkRequest.pipeline());
- parameters.withRouting(bulkRequest.routing());
- // Bulk API only supports newline delimited JSON or Smile. Before executing
- // the bulk, we need to check that all requests have the same content-type
- // and this content-type is supported by the Bulk API.
- XContentType bulkContentType = null;
- for (int i = 0; i < bulkRequest.numberOfActions(); i++) {
- DocWriteRequest action = bulkRequest.requests().get(i);
-
- DocWriteRequest.OpType opType = action.opType();
- if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
- bulkContentType = enforceSameContentType((IndexRequest) action, bulkContentType);
-
- } else if (opType == DocWriteRequest.OpType.UPDATE) {
- UpdateRequest updateRequest = (UpdateRequest) action;
- if (updateRequest.doc() != null) {
- bulkContentType = enforceSameContentType(updateRequest.doc(), bulkContentType);
- }
- if (updateRequest.upsertRequest() != null) {
- bulkContentType = enforceSameContentType(updateRequest.upsertRequest(), bulkContentType);
- }
- }
- }
-
- if (bulkContentType == null) {
- bulkContentType = XContentType.JSON;
- }
-
- final byte separator = bulkContentType.xContent().streamSeparator();
- final ContentType requestContentType = createContentType(bulkContentType);
-
- ByteArrayOutputStream content = new ByteArrayOutputStream();
- for (DocWriteRequest action : bulkRequest.requests()) {
- DocWriteRequest.OpType opType = action.opType();
-
- try (XContentBuilder metadata = XContentBuilder.builder(bulkContentType.xContent())) {
- metadata.startObject();
- {
- metadata.startObject(opType.getLowercase());
- if (Strings.hasLength(action.index())) {
- metadata.field("_index", action.index());
- }
- if (Strings.hasLength(action.id())) {
- metadata.field("_id", action.id());
- }
- if (Strings.hasLength(action.routing())) {
- metadata.field("routing", action.routing());
- }
- if (action.version() != Versions.MATCH_ANY) {
- metadata.field("version", action.version());
- }
-
- VersionType versionType = action.versionType();
- if (versionType != VersionType.INTERNAL) {
- if (versionType == VersionType.EXTERNAL) {
- metadata.field("version_type", "external");
- } else if (versionType == VersionType.EXTERNAL_GTE) {
- metadata.field("version_type", "external_gte");
- }
- }
-
- if (action.ifSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO) {
- metadata.field("if_seq_no", action.ifSeqNo());
- metadata.field("if_primary_term", action.ifPrimaryTerm());
- }
-
- if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
- IndexRequest indexRequest = (IndexRequest) action;
- if (Strings.hasLength(indexRequest.getPipeline())) {
- metadata.field("pipeline", indexRequest.getPipeline());
- }
- } else if (opType == DocWriteRequest.OpType.UPDATE) {
- UpdateRequest updateRequest = (UpdateRequest) action;
- if (updateRequest.retryOnConflict() > 0) {
- metadata.field("retry_on_conflict", updateRequest.retryOnConflict());
- }
- if (updateRequest.fetchSource() != null) {
- metadata.field("_source", updateRequest.fetchSource());
- }
- }
- metadata.endObject();
- }
- metadata.endObject();
-
- BytesRef metadataSource = BytesReference.bytes(metadata).toBytesRef();
- content.write(metadataSource.bytes, metadataSource.offset, metadataSource.length);
- content.write(separator);
- }
-
- BytesRef source = null;
- if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
- IndexRequest indexRequest = (IndexRequest) action;
- BytesReference indexSource = indexRequest.source();
- XContentType indexXContentType = indexRequest.getContentType();
-
- try (
- XContentParser parser = XContentHelper.createParser(
- /*
- * EMPTY and THROW are fine here because we just call
- * copyCurrentStructure which doesn't touch the
- * registry or deprecation.
- */
- NamedXContentRegistry.EMPTY,
- DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
- indexSource,
- indexXContentType
- )
- ) {
- try (XContentBuilder builder = XContentBuilder.builder(bulkContentType.xContent())) {
- builder.copyCurrentStructure(parser);
- source = BytesReference.bytes(builder).toBytesRef();
- }
- }
- } else if (opType == DocWriteRequest.OpType.UPDATE) {
- source = XContentHelper.toXContent((UpdateRequest) action, bulkContentType, false).toBytesRef();
- }
-
- if (source != null) {
- content.write(source.bytes, source.offset, source.length);
- content.write(separator);
- }
- }
- request.addParameters(parameters.asMap());
- request.setEntity(new NByteArrayEntity(content.toByteArray(), 0, content.size(), requestContentType));
- return request;
- }
-
- static Request index(IndexRequest indexRequest) {
- String method = Strings.hasLength(indexRequest.id()) ? HttpPut.METHOD_NAME : HttpPost.METHOD_NAME;
-
- String endpoint;
- if (indexRequest.opType() == DocWriteRequest.OpType.CREATE) {
- endpoint = endpoint(indexRequest.index(), "_create", indexRequest.id());
- } else {
- endpoint = endpoint(indexRequest.index(), indexRequest.id());
- }
-
- Request request = new Request(method, endpoint);
-
- Params parameters = new Params();
- parameters.withRouting(indexRequest.routing());
- parameters.withTimeout(indexRequest.timeout());
- parameters.withVersion(indexRequest.version());
- parameters.withVersionType(indexRequest.versionType());
- parameters.withIfSeqNo(indexRequest.ifSeqNo());
- parameters.withIfPrimaryTerm(indexRequest.ifPrimaryTerm());
- parameters.withPipeline(indexRequest.getPipeline());
- parameters.withRefreshPolicy(indexRequest.getRefreshPolicy());
- parameters.withWaitForActiveShards(indexRequest.waitForActiveShards());
- parameters.withRequireAlias(indexRequest.isRequireAlias());
-
- BytesRef source = indexRequest.source().toBytesRef();
- ContentType contentType = createContentType(indexRequest.getContentType());
- request.addParameters(parameters.asMap());
- request.setEntity(new NByteArrayEntity(source.bytes, source.offset, source.length, contentType));
- return request;
- }
-
- /**
- * Convert a {@linkplain SearchRequest} into a {@linkplain Request}.
- * @param searchRequest the request to convert
- * @param searchEndpoint the name of the search endpoint. {@literal _search}
- * for standard searches and {@literal _rollup_search} for rollup
- * searches.
- */
- static Request search(SearchRequest searchRequest, String searchEndpoint) throws IOException {
- Request request = new Request(HttpPost.METHOD_NAME, endpoint(searchRequest.indices(), searchEndpoint));
-
- Params params = new Params();
- addSearchRequestParams(params, searchRequest);
-
- if (searchRequest.source() != null) {
- request.setEntity(createEntity(searchRequest.source(), REQUEST_BODY_CONTENT_TYPE));
- }
- request.addParameters(params.asMap());
- return request;
- }
-
- static void addSearchRequestParams(Params params, SearchRequest searchRequest) {
- params.putParam(RestSearchAction.TYPED_KEYS_PARAM, "true");
- params.withRouting(searchRequest.routing());
- params.withPreference(searchRequest.preference());
- if (SearchRequest.DEFAULT_INDICES_OPTIONS.equals(searchRequest.indicesOptions()) == false) {
- params.withIndicesOptions(searchRequest.indicesOptions());
- }
- params.withSearchType(searchRequest.searchType().name().toLowerCase(Locale.ROOT));
- if (searchRequest.isCcsMinimizeRoundtrips() != SearchRequest.defaultCcsMinimizeRoundtrips(searchRequest)) {
- params.putParam("ccs_minimize_roundtrips", Boolean.toString(searchRequest.isCcsMinimizeRoundtrips()));
- }
- if (searchRequest.getPreFilterShardSize() != null) {
- params.putParam("pre_filter_shard_size", Integer.toString(searchRequest.getPreFilterShardSize()));
- }
- params.withMaxConcurrentShardRequests(searchRequest.getMaxConcurrentShardRequests());
- if (searchRequest.requestCache() != null) {
- params.withRequestCache(searchRequest.requestCache());
- }
- if (searchRequest.allowPartialSearchResults() != null) {
- params.withAllowPartialResults(searchRequest.allowPartialSearchResults());
- }
- params.withBatchedReduceSize(searchRequest.getBatchedReduceSize());
- if (searchRequest.scroll() != null) {
- params.putParam("scroll", searchRequest.scroll().keepAlive());
- }
- }
-
- static Request searchScroll(SearchScrollRequest searchScrollRequest) throws IOException {
- Request request = new Request(HttpPost.METHOD_NAME, "/_search/scroll");
- request.setEntity(createEntity(searchScrollRequest, REQUEST_BODY_CONTENT_TYPE));
- return request;
- }
-
- static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType) throws IOException {
- return createEntity(toXContent, xContentType, ToXContent.EMPTY_PARAMS);
- }
-
- static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType, ToXContent.Params toXContentParams)
- throws IOException {
- BytesRef source = XContentHelper.toXContent(toXContent, xContentType, toXContentParams, false).toBytesRef();
- return new NByteArrayEntity(source.bytes, source.offset, source.length, createContentType(xContentType));
- }
-
- @Deprecated
- static String endpoint(String index, String type, String id) {
- return new EndpointBuilder().addPathPart(index, type, id).build();
- }
-
- static String endpoint(String index, String id) {
- return new EndpointBuilder().addPathPart(index, "_doc", id).build();
- }
-
- @Deprecated
- static String endpoint(String index, String type, String id, String endpoint) {
- return new EndpointBuilder().addPathPart(index, type, id).addPathPartAsIs(endpoint).build();
- }
-
- static String endpoint(String[] indices, String endpoint) {
- return new EndpointBuilder().addCommaSeparatedPathParts(indices).addPathPartAsIs(endpoint).build();
- }
-
- @Deprecated
- static String endpoint(String[] indices, String[] types, String endpoint) {
- return new EndpointBuilder().addCommaSeparatedPathParts(indices)
- .addCommaSeparatedPathParts(types)
- .addPathPartAsIs(endpoint)
- .build();
- }
-
- @Deprecated
- static String endpoint(String[] indices, String endpoint, String type) {
- return new EndpointBuilder().addCommaSeparatedPathParts(indices).addPathPartAsIs(endpoint).addPathPart(type).build();
- }
-
- /**
- * Returns a {@link ContentType} from a given {@link XContentType}.
- *
- * @param xContentType the {@link XContentType}
- * @return the {@link ContentType}
- */
- @SuppressForbidden(reason = "Only allowed place to convert a XContentType to a ContentType")
- public static ContentType createContentType(final XContentType xContentType) {
- return ContentType.create(xContentType.mediaTypeWithoutParameters(), (Charset) null);
- }
-
- /**
- * Utility class to help with common parameter names and patterns. Wraps
- * a {@link Request} and adds the parameters to it directly.
- */
- static class Params {
- private final Map parameters = new HashMap<>();
-
- Params() {}
-
- Params putParam(String name, String value) {
- if (Strings.hasLength(value)) {
- parameters.put(name, value);
- }
- return this;
- }
-
- Params putParam(String key, TimeValue value) {
- if (value != null) {
- return putParam(key, value.getStringRep());
- }
- return this;
- }
-
- Map asMap() {
- return parameters;
- }
-
- Params withPipeline(String pipeline) {
- return putParam("pipeline", pipeline);
- }
-
- Params withPreference(String preference) {
- return putParam("preference", preference);
- }
-
- Params withSearchType(String searchType) {
- return putParam("search_type", searchType);
- }
-
- Params withMaxConcurrentShardRequests(int maxConcurrentShardRequests) {
- return putParam("max_concurrent_shard_requests", Integer.toString(maxConcurrentShardRequests));
- }
-
- Params withBatchedReduceSize(int batchedReduceSize) {
- return putParam("batched_reduce_size", Integer.toString(batchedReduceSize));
- }
-
- Params withRequestCache(boolean requestCache) {
- return putParam("request_cache", Boolean.toString(requestCache));
- }
-
- Params withAllowPartialResults(boolean allowPartialSearchResults) {
- return putParam("allow_partial_search_results", Boolean.toString(allowPartialSearchResults));
- }
-
- Params withRefreshPolicy(RefreshPolicy refreshPolicy) {
- if (refreshPolicy != RefreshPolicy.NONE) {
- return putParam("refresh", refreshPolicy.getValue());
- }
- return this;
- }
-
- Params withRouting(String routing) {
- return putParam("routing", routing);
- }
-
- Params withTimeout(TimeValue timeout) {
- return putParam("timeout", timeout);
- }
-
- Params withVersion(long version) {
- if (version != Versions.MATCH_ANY) {
- return putParam("version", Long.toString(version));
- }
- return this;
- }
-
- Params withVersionType(VersionType versionType) {
- if (versionType != VersionType.INTERNAL) {
- return putParam("version_type", versionType.name().toLowerCase(Locale.ROOT));
- }
- return this;
- }
-
- Params withIfSeqNo(long ifSeqNo) {
- if (ifSeqNo != SequenceNumbers.UNASSIGNED_SEQ_NO) {
- return putParam("if_seq_no", Long.toString(ifSeqNo));
- }
- return this;
- }
-
- Params withIfPrimaryTerm(long ifPrimaryTerm) {
- if (ifPrimaryTerm != SequenceNumbers.UNASSIGNED_PRIMARY_TERM) {
- return putParam("if_primary_term", Long.toString(ifPrimaryTerm));
- }
- return this;
- }
-
- Params withWaitForActiveShards(ActiveShardCount activeShardCount) {
- return withWaitForActiveShards(activeShardCount, ActiveShardCount.DEFAULT);
- }
-
- Params withWaitForActiveShards(ActiveShardCount activeShardCount, ActiveShardCount defaultActiveShardCount) {
- if (activeShardCount != null && activeShardCount != defaultActiveShardCount) {
- return putParam("wait_for_active_shards", activeShardCount.toString().toLowerCase(Locale.ROOT));
- }
- return this;
- }
-
- Params withRequireAlias(boolean requireAlias) {
- if (requireAlias) {
- return putParam("require_alias", Boolean.toString(requireAlias));
- }
- return this;
- }
-
- Params withIndicesOptions(IndicesOptions indicesOptions) {
- if (indicesOptions != null) {
- withIgnoreUnavailable(indicesOptions.ignoreUnavailable());
- putParam("allow_no_indices", Boolean.toString(indicesOptions.allowNoIndices()));
- String expandWildcards;
- if (indicesOptions.expandWildcardExpressions() == false) {
- expandWildcards = "none";
- } else {
- StringJoiner joiner = new StringJoiner(",");
- if (indicesOptions.expandWildcardsOpen()) {
- joiner.add("open");
- }
- if (indicesOptions.expandWildcardsClosed()) {
- joiner.add("closed");
- }
- expandWildcards = joiner.toString();
- }
- putParam("expand_wildcards", expandWildcards);
- putParam("ignore_throttled", Boolean.toString(indicesOptions.ignoreThrottled()));
- }
- return this;
- }
-
- Params withIgnoreUnavailable(boolean ignoreUnavailable) {
- // Always explicitly place the ignore_unavailable value.
- putParam("ignore_unavailable", Boolean.toString(ignoreUnavailable));
- return this;
- }
- }
-
- /**
- * Ensure that the {@link IndexRequest}'s content type is supported by the Bulk API and that it conforms
- * to the current {@link BulkRequest}'s content type (if it's known at the time of this method get called).
- *
- * @return the {@link IndexRequest}'s content type
- */
- static XContentType enforceSameContentType(IndexRequest indexRequest, @Nullable XContentType xContentType) {
- XContentType requestContentType = indexRequest.getContentType();
- if (requestContentType.canonical() != XContentType.JSON && requestContentType.canonical() != XContentType.SMILE) {
- throw new IllegalArgumentException(
- "Unsupported content-type found for request with content-type ["
- + requestContentType
- + "], only JSON and SMILE are supported"
- );
- }
- if (xContentType == null) {
- return requestContentType;
- }
- if (requestContentType.canonical() != xContentType.canonical()) {
- throw new IllegalArgumentException(
- "Mismatching content-type found for request with content-type ["
- + requestContentType
- + "], previous requests have content-type ["
- + xContentType
- + "]"
- );
- }
- return xContentType;
- }
-
- /**
- * Utility class to build request's endpoint given its parts as strings
- */
- static class EndpointBuilder {
-
- private final StringJoiner joiner = new StringJoiner("/", "/", "");
-
- EndpointBuilder addPathPart(String... parts) {
- for (String part : parts) {
- if (Strings.hasLength(part)) {
- joiner.add(encodePart(part));
- }
- }
- return this;
- }
-
- EndpointBuilder addCommaSeparatedPathParts(String[] parts) {
- addPathPart(String.join(",", parts));
- return this;
- }
-
- EndpointBuilder addPathPartAsIs(String... parts) {
- for (String part : parts) {
- if (Strings.hasLength(part)) {
- joiner.add(part);
- }
- }
- return this;
- }
-
- String build() {
- return joiner.toString();
- }
-
- private static String encodePart(String pathPart) {
- try {
- // encode each part (e.g. index, type and id) separately before merging them into the path
- // we prepend "/" to the path part to make this path absolute, otherwise there can be issues with
- // paths that start with `-` or contain `:`
- // the authority must be an empty string and not null, else paths that being with slashes could have them
- // misinterpreted as part of the authority.
- URI uri = new URI(null, "", "/" + pathPart, null, null);
- // manually encode any slash that each part may contain
- return uri.getRawPath().substring(1).replaceAll("/", "%2F");
- } catch (URISyntaxException e) {
- throw new IllegalArgumentException("Path part [" + pathPart + "] couldn't be encoded", e);
- }
- }
- }
-}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
deleted file mode 100644
index b0998957910a2..0000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java
+++ /dev/null
@@ -1,1042 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.client;
-
-import org.apache.http.Header;
-import org.apache.http.HttpEntity;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.ElasticsearchStatusException;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ActionRequest;
-import org.elasticsearch.action.ActionRequestValidationException;
-import org.elasticsearch.action.bulk.BulkRequest;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.index.IndexResponse;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.search.SearchScrollRequest;
-import org.elasticsearch.action.support.PlainActionFuture;
-import org.elasticsearch.aggregations.bucket.adjacency.AdjacencyMatrixAggregationBuilder;
-import org.elasticsearch.aggregations.bucket.adjacency.ParsedAdjacencyMatrix;
-import org.elasticsearch.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder;
-import org.elasticsearch.aggregations.bucket.histogram.ParsedAutoDateHistogram;
-import org.elasticsearch.aggregations.bucket.timeseries.ParsedTimeSeries;
-import org.elasticsearch.aggregations.bucket.timeseries.TimeSeriesAggregationBuilder;
-import org.elasticsearch.aggregations.pipeline.DerivativePipelineAggregationBuilder;
-import org.elasticsearch.client.analytics.ParsedStringStats;
-import org.elasticsearch.client.analytics.ParsedTopMetrics;
-import org.elasticsearch.client.analytics.StringStatsAggregationBuilder;
-import org.elasticsearch.client.analytics.TopMetricsAggregationBuilder;
-import org.elasticsearch.client.core.MainResponse;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.util.concurrent.FutureUtils;
-import org.elasticsearch.common.util.concurrent.ListenableFuture;
-import org.elasticsearch.core.CheckedConsumer;
-import org.elasticsearch.core.CheckedFunction;
-import org.elasticsearch.plugins.spi.NamedXContentProvider;
-import org.elasticsearch.rest.RestResponse;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.search.aggregations.Aggregation;
-import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.composite.ParsedComposite;
-import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.filter.FiltersAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilter;
-import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilters;
-import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedGeoHashGrid;
-import org.elasticsearch.search.aggregations.bucket.geogrid.ParsedGeoTileGrid;
-import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.global.ParsedGlobal;
-import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.histogram.ParsedDateHistogram;
-import org.elasticsearch.search.aggregations.bucket.histogram.ParsedHistogram;
-import org.elasticsearch.search.aggregations.bucket.histogram.ParsedVariableWidthHistogram;
-import org.elasticsearch.search.aggregations.bucket.histogram.VariableWidthHistogramAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.missing.MissingAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.missing.ParsedMissing;
-import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.nested.ParsedNested;
-import org.elasticsearch.search.aggregations.bucket.nested.ParsedReverseNested;
-import org.elasticsearch.search.aggregations.bucket.nested.ReverseNestedAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.range.DateRangeAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.range.GeoDistanceAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.range.IpRangeAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.range.ParsedBinaryRange;
-import org.elasticsearch.search.aggregations.bucket.range.ParsedDateRange;
-import org.elasticsearch.search.aggregations.bucket.range.ParsedGeoDistance;
-import org.elasticsearch.search.aggregations.bucket.range.ParsedRange;
-import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder;
-import org.elasticsearch.search.aggregations.bucket.sampler.InternalSampler;
-import org.elasticsearch.search.aggregations.bucket.sampler.ParsedSampler;
-import org.elasticsearch.search.aggregations.bucket.terms.DoubleTerms;
-import org.elasticsearch.search.aggregations.bucket.terms.LongRareTerms;
-import org.elasticsearch.search.aggregations.bucket.terms.LongTerms;
-import org.elasticsearch.search.aggregations.bucket.terms.ParsedDoubleTerms;
-import org.elasticsearch.search.aggregations.bucket.terms.ParsedLongRareTerms;
-import org.elasticsearch.search.aggregations.bucket.terms.ParsedLongTerms;
-import org.elasticsearch.search.aggregations.bucket.terms.ParsedSignificantLongTerms;
-import org.elasticsearch.search.aggregations.bucket.terms.ParsedSignificantStringTerms;
-import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringRareTerms;
-import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms;
-import org.elasticsearch.search.aggregations.bucket.terms.SignificantLongTerms;
-import org.elasticsearch.search.aggregations.bucket.terms.SignificantStringTerms;
-import org.elasticsearch.search.aggregations.bucket.terms.StringRareTerms;
-import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
-import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder;
-import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder;
-import org.elasticsearch.search.aggregations.metrics.ExtendedStatsAggregationBuilder;
-import org.elasticsearch.search.aggregations.metrics.GeoBoundsAggregationBuilder;
-import org.elasticsearch.search.aggregations.metrics.GeoCentroidAggregationBuilder;
-import org.elasticsearch.search.aggregations.metrics.InternalHDRPercentileRanks;
-import org.elasticsearch.search.aggregations.metrics.InternalHDRPercentiles;
-import org.elasticsearch.search.aggregations.metrics.InternalTDigestPercentileRanks;
-import org.elasticsearch.search.aggregations.metrics.InternalTDigestPercentiles;
-import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder;
-import org.elasticsearch.search.aggregations.metrics.MedianAbsoluteDeviationAggregationBuilder;
-import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder;
-import org.elasticsearch.search.aggregations.metrics.ParsedAvg;
-import org.elasticsearch.search.aggregations.metrics.ParsedCardinality;
-import org.elasticsearch.search.aggregations.metrics.ParsedExtendedStats;
-import org.elasticsearch.search.aggregations.metrics.ParsedGeoBounds;
-import org.elasticsearch.search.aggregations.metrics.ParsedGeoCentroid;
-import org.elasticsearch.search.aggregations.metrics.ParsedHDRPercentileRanks;
-import org.elasticsearch.search.aggregations.metrics.ParsedHDRPercentiles;
-import org.elasticsearch.search.aggregations.metrics.ParsedMax;
-import org.elasticsearch.search.aggregations.metrics.ParsedMedianAbsoluteDeviation;
-import org.elasticsearch.search.aggregations.metrics.ParsedMin;
-import org.elasticsearch.search.aggregations.metrics.ParsedScriptedMetric;
-import org.elasticsearch.search.aggregations.metrics.ParsedStats;
-import org.elasticsearch.search.aggregations.metrics.ParsedSum;
-import org.elasticsearch.search.aggregations.metrics.ParsedTDigestPercentileRanks;
-import org.elasticsearch.search.aggregations.metrics.ParsedTDigestPercentiles;
-import org.elasticsearch.search.aggregations.metrics.ParsedTopHits;
-import org.elasticsearch.search.aggregations.metrics.ParsedValueCount;
-import org.elasticsearch.search.aggregations.metrics.ParsedWeightedAvg;
-import org.elasticsearch.search.aggregations.metrics.ScriptedMetricAggregationBuilder;
-import org.elasticsearch.search.aggregations.metrics.StatsAggregationBuilder;
-import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder;
-import org.elasticsearch.search.aggregations.metrics.TopHitsAggregationBuilder;
-import org.elasticsearch.search.aggregations.metrics.ValueCountAggregationBuilder;
-import org.elasticsearch.search.aggregations.metrics.WeightedAvgAggregationBuilder;
-import org.elasticsearch.search.aggregations.pipeline.ExtendedStatsBucketPipelineAggregationBuilder;
-import org.elasticsearch.search.aggregations.pipeline.InternalBucketMetricValue;
-import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValue;
-import org.elasticsearch.search.aggregations.pipeline.ParsedBucketMetricValue;
-import org.elasticsearch.search.aggregations.pipeline.ParsedDerivative;
-import org.elasticsearch.search.aggregations.pipeline.ParsedExtendedStatsBucket;
-import org.elasticsearch.search.aggregations.pipeline.ParsedPercentilesBucket;
-import org.elasticsearch.search.aggregations.pipeline.ParsedSimpleValue;
-import org.elasticsearch.search.aggregations.pipeline.ParsedStatsBucket;
-import org.elasticsearch.search.aggregations.pipeline.PercentilesBucketPipelineAggregationBuilder;
-import org.elasticsearch.search.aggregations.pipeline.StatsBucketPipelineAggregationBuilder;
-import org.elasticsearch.search.suggest.Suggest;
-import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
-import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestion;
-import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder;
-import org.elasticsearch.search.suggest.term.TermSuggestion;
-import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
-import org.elasticsearch.xcontent.ContextParser;
-import org.elasticsearch.xcontent.DeprecationHandler;
-import org.elasticsearch.xcontent.NamedXContentRegistry;
-import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xcontent.XContentParserConfiguration;
-import org.elasticsearch.xcontent.XContentType;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.ServiceLoader;
-import java.util.Set;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static java.util.Collections.emptySet;
-import static java.util.stream.Collectors.toList;
-
-/**
- * High level REST client that wraps an instance of the low level {@link RestClient} and allows to build requests and read responses. The
- * {@link RestClient} instance is internally built based on the provided {@link RestClientBuilder} and it gets closed automatically when
- * closing the {@link RestHighLevelClient} instance that wraps it.
- *
- *
- * In case an already existing instance of a low-level REST client needs to be provided, this class can be subclassed and the
- * {@link #RestHighLevelClient(RestClient, CheckedConsumer, List)} constructor can be used.
- *
- *
- * This class can also be sub-classed to expose additional client methods that make use of endpoints added to Elasticsearch through plugins,
- * or to add support for custom response sections, again added to Elasticsearch through plugins.
- *
- *
- * The majority of the methods in this class come in two flavors, a blocking and an asynchronous version (e.g.
- * {@link #search(SearchRequest, RequestOptions)} and {@link #searchAsync(SearchRequest, RequestOptions, ActionListener)}, where the later
- * takes an implementation of an {@link ActionListener} as an argument that needs to implement methods that handle successful responses and
- * failure scenarios. Most of the blocking calls can throw an {@link IOException} or an unchecked {@link ElasticsearchException} in the
- * following cases:
- *
- *
- * - an {@link IOException} is usually thrown in case of failing to parse the REST response in the high-level REST client, the request
- * times out or similar cases where there is no response coming back from the Elasticsearch server
- * - an {@link ElasticsearchException} is usually thrown in case where the server returns a 4xx or 5xx error code. The high-level client
- * then tries to parse the response body error details into a generic ElasticsearchException and suppresses the original
- * {@link ResponseException}
- *
- *
- * @deprecated The High Level Rest Client is deprecated in favor of the
- *
- * Elasticsearch Java API Client
- */
-@Deprecated(since = "7.16.0", forRemoval = true)
-@SuppressWarnings("removal")
-public class RestHighLevelClient implements Closeable {
-
- private static final Logger logger = LogManager.getLogger(RestHighLevelClient.class);
- /**
- * Environment variable determining whether to send the 7.x compatibility header
- */
- public static final String API_VERSIONING_ENV_VARIABLE = "ELASTIC_CLIENT_APIVERSIONING";
-
- // To be called using performClientRequest and performClientRequestAsync to ensure version compatibility check
- private final RestClient client;
- private final XContentParserConfiguration parserConfig;
- private final CheckedConsumer doClose;
- private final boolean useAPICompatibility;
-
- /** Do not access directly but through getVersionValidationFuture() */
- private volatile ListenableFuture> versionValidationFuture;
-
- /**
- * Creates a {@link RestHighLevelClient} given the low level {@link RestClientBuilder} that allows to build the
- * {@link RestClient} to be used to perform requests.
- */
- public RestHighLevelClient(RestClientBuilder restClientBuilder) {
- this(restClientBuilder.build(), RestClient::close, Collections.emptyList());
- }
-
- /**
- * Creates a {@link RestHighLevelClient} given the low level {@link RestClient} that it should use to perform requests and
- * a list of entries that allow to parse custom response sections added to Elasticsearch through plugins.
- * This constructor can be called by subclasses in case an externally created low-level REST client needs to be provided.
- * The consumer argument allows to control what needs to be done when the {@link #close()} method is called.
- * Also subclasses can provide parsers for custom response sections added to Elasticsearch through plugins.
- */
- protected RestHighLevelClient(
- RestClient restClient,
- CheckedConsumer doClose,
- List namedXContentEntries
- ) {
- this(restClient, doClose, namedXContentEntries, null);
- }
-
- /**
- * Creates a {@link RestHighLevelClient} given the low level {@link RestClient} that it should use to perform requests and
- * a list of entries that allow to parse custom response sections added to Elasticsearch through plugins.
- * This constructor can be called by subclasses in case an externally created low-level REST client needs to be provided.
- * The consumer argument allows to control what needs to be done when the {@link #close()} method is called.
- * Also subclasses can provide parsers for custom response sections added to Elasticsearch through plugins.
- */
- private RestHighLevelClient(
- RestClient restClient,
- CheckedConsumer doClose,
- List namedXContentEntries,
- Boolean useAPICompatibility
- ) {
- this.client = Objects.requireNonNull(restClient, "restClient must not be null");
- this.doClose = Objects.requireNonNull(doClose, "doClose consumer must not be null");
- NamedXContentRegistry registry = new NamedXContentRegistry(
- Stream.of(getDefaultNamedXContents().stream(), getProvidedNamedXContents().stream(), namedXContentEntries.stream())
- .flatMap(Function.identity())
- .collect(toList())
- );
- /*
- * Ignores deprecation warnings. This is appropriate because it is only
- * used to parse responses from Elasticsearch. Any deprecation warnings
- * emitted there just mean that you are talking to an old version of
- * Elasticsearch. There isn't anything you can do about the deprecation.
- */
- this.parserConfig = XContentParserConfiguration.EMPTY.withRegistry(registry)
- .withDeprecationHandler(DeprecationHandler.IGNORE_DEPRECATIONS);
- if (useAPICompatibility == null && "true".equals(System.getenv(API_VERSIONING_ENV_VARIABLE))) {
- this.useAPICompatibility = true;
- } else {
- this.useAPICompatibility = Boolean.TRUE.equals(useAPICompatibility);
- }
- }
-
- /**
- * Returns the low-level client that the current high-level client instance is using to perform requests
- */
- public final RestClient getLowLevelClient() {
- return client;
- }
-
- public final XContentParserConfiguration getParserConfig() {
- return parserConfig;
- }
-
- @Override
- public final void close() throws IOException {
- doClose.accept(client);
- }
-
- /**
- * Asynchronously executes a bulk request using the Bulk API.
- * See Bulk API on elastic.co
- * @param bulkRequest the request
- * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
- * @param listener the listener to be notified upon request completion
- * @return cancellable that may be used to cancel the request
- */
- public final Cancellable bulkAsync(BulkRequest bulkRequest, RequestOptions options, ActionListener listener) {
- return performRequestAsyncAndParseEntity(
- bulkRequest,
- RequestConverters::bulk,
- options,
- BulkResponse::fromXContent,
- listener,
- emptySet()
- );
- }
-
- /**
- * Index a document using the Index API.
- * See Index API on elastic.co
- * @param indexRequest the request
- * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
- * @return the response
- */
- public final IndexResponse index(IndexRequest indexRequest, RequestOptions options) throws IOException {
- return performRequestAndParseEntity(indexRequest, RequestConverters::index, options, IndexResponse::fromXContent, emptySet());
- }
-
- /**
- * Executes a search request using the Search API.
- * See Search API on elastic.co
- * @param searchRequest the request
- * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
- * @return the response
- */
- public final SearchResponse search(SearchRequest searchRequest, RequestOptions options) throws IOException {
- return performRequestAndParseEntity(
- searchRequest,
- r -> RequestConverters.search(r, "_search"),
- options,
- SearchResponse::fromXContent,
- emptySet()
- );
- }
-
- /**
- * Asynchronously executes a search using the Search API.
- * See Search API on elastic.co
- * @param searchRequest the request
- * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
- * @param listener the listener to be notified upon request completion
- * @return cancellable that may be used to cancel the request
- */
- public final Cancellable searchAsync(SearchRequest searchRequest, RequestOptions options, ActionListener listener) {
- return performRequestAsyncAndParseEntity(
- searchRequest,
- r -> RequestConverters.search(r, "_search"),
- options,
- SearchResponse::fromXContent,
- listener,
- emptySet()
- );
- }
-
- /**
- * Executes a search using the Search Scroll API.
- * See Search
- * Scroll API on elastic.co
- * @param searchScrollRequest the request
- * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
- * @return the response
- */
- public final SearchResponse scroll(SearchScrollRequest searchScrollRequest, RequestOptions options) throws IOException {
- return performRequestAndParseEntity(
- searchScrollRequest,
- RequestConverters::searchScroll,
- options,
- SearchResponse::fromXContent,
- emptySet()
- );
- }
-
- /**
- * @deprecated If creating a new HLRC ReST API call, consider creating new actions instead of reusing server actions. The Validation
- * layer has been added to the ReST client, and requests should extend {@link Validatable} instead of {@link ActionRequest}.
- */
- @Deprecated
- private Resp performRequestAndParseEntity(
- Req request,
- CheckedFunction requestConverter,
- RequestOptions options,
- CheckedFunction entityParser,
- Set ignores
- ) throws IOException {
- return performRequest(request, requestConverter, options, response -> parseEntity(response.getEntity(), entityParser), ignores);
- }
-
- /**
- * @deprecated If creating a new HLRC ReST API call, consider creating new actions instead of reusing server actions. The Validation
- * layer has been added to the ReST client, and requests should extend {@link Validatable} instead of {@link ActionRequest}.
- */
- @Deprecated
- private Resp performRequest(
- Req request,
- CheckedFunction requestConverter,
- RequestOptions options,
- CheckedFunction responseConverter,
- Set ignores
- ) throws IOException {
- ActionRequestValidationException validationException = request.validate();
- if (validationException != null && validationException.validationErrors().isEmpty() == false) {
- throw validationException;
- }
- return internalPerformRequest(request, requestConverter, options, responseConverter, ignores);
- }
-
- /**
- * Provides common functionality for performing a request.
- */
- private Resp internalPerformRequest(
- Req request,
- CheckedFunction requestConverter,
- RequestOptions options,
- CheckedFunction responseConverter,
- Set ignores
- ) throws IOException {
- Request req = requestConverter.apply(request);
- req.setOptions(options);
- Response response;
- try {
- response = performClientRequest(req);
- } catch (ResponseException e) {
- if (ignores.contains(e.getResponse().getStatusLine().getStatusCode())) {
- try {
- return responseConverter.apply(e.getResponse());
- } catch (Exception innerException) {
- // the exception is ignored as we now try to parse the response as an error.
- // this covers cases like get where 404 can either be a valid document not found response,
- // or an error for which parsing is completely different. We try to consider the 404 response as a valid one
- // first. If parsing of the response breaks, we fall back to parsing it as an error.
- throw parseResponseException(e);
- }
- }
- throw parseResponseException(e);
- }
-
- try {
- return responseConverter.apply(response);
- } catch (Exception e) {
- throw new IOException("Unable to parse response body for " + response, e);
- }
- }
-
- /**
- * @deprecated If creating a new HLRC ReST API call, consider creating new actions instead of reusing server actions. The Validation
- * layer has been added to the ReST client, and requests should extend {@link Validatable} instead of {@link ActionRequest}.
- * @return Cancellable instance that may be used to cancel the request
- */
- @Deprecated
- private Cancellable performRequestAsyncAndParseEntity(
- Req request,
- CheckedFunction requestConverter,
- RequestOptions options,
- CheckedFunction entityParser,
- ActionListener listener,
- Set ignores
- ) {
- return performRequestAsync(
- request,
- requestConverter,
- options,
- response -> parseEntity(response.getEntity(), entityParser),
- listener,
- ignores
- );
- }
-
- /**
- * @deprecated If creating a new HLRC ReST API call, consider creating new actions instead of reusing server actions. The Validation
- * layer has been added to the ReST client, and requests should extend {@link Validatable} instead of {@link ActionRequest}.
- * @return Cancellable instance that may be used to cancel the request
- */
- @Deprecated
- private Cancellable performRequestAsync(
- Req request,
- CheckedFunction requestConverter,
- RequestOptions options,
- CheckedFunction responseConverter,
- ActionListener listener,
- Set ignores
- ) {
- ActionRequestValidationException validationException = request.validate();
- if (validationException != null && validationException.validationErrors().isEmpty() == false) {
- listener.onFailure(validationException);
- return Cancellable.NO_OP;
- }
- return internalPerformRequestAsync(request, requestConverter, options, responseConverter, listener, ignores);
- }
-
- /**
- * Provides common functionality for asynchronously performing a request.
- * @return Cancellable instance that may be used to cancel the request
- */
- private Cancellable internalPerformRequestAsync(
- Req request,
- CheckedFunction requestConverter,
- RequestOptions options,
- CheckedFunction responseConverter,
- ActionListener listener,
- Set ignores
- ) {
- Request req;
- try {
- req = requestConverter.apply(request);
- } catch (Exception e) {
- listener.onFailure(e);
- return Cancellable.NO_OP;
- }
- req.setOptions(options);
-
- ResponseListener responseListener = wrapResponseListener(responseConverter, listener, ignores);
- return performClientRequestAsync(req, responseListener);
- }
-
- private ResponseListener wrapResponseListener(
- CheckedFunction responseConverter,
- ActionListener actionListener,
- Set ignores
- ) {
- return new ResponseListener() {
- @Override
- public void onSuccess(Response response) {
- try {
- actionListener.onResponse(responseConverter.apply(response));
- } catch (Exception e) {
- IOException ioe = new IOException("Unable to parse response body for " + response, e);
- onFailure(ioe);
- }
- }
-
- @Override
- public void onFailure(Exception exception) {
- if (exception instanceof ResponseException responseException) {
- Response response = responseException.getResponse();
- if (ignores.contains(response.getStatusLine().getStatusCode())) {
- try {
- actionListener.onResponse(responseConverter.apply(response));
- } catch (Exception innerException) {
- // the exception is ignored as we now try to parse the response as an error.
- // this covers cases like get where 404 can either be a valid document not found response,
- // or an error for which parsing is completely different. We try to consider the 404 response as a valid one
- // first. If parsing of the response breaks, we fall back to parsing it as an error.
- actionListener.onFailure(parseResponseException(responseException));
- }
- } else {
- actionListener.onFailure(parseResponseException(responseException));
- }
- } else {
- actionListener.onFailure(exception);
- }
- }
- };
- }
-
- /**
- * Converts a {@link ResponseException} obtained from the low level REST client into an {@link ElasticsearchException}.
- * If a response body was returned, tries to parse it as an error returned from Elasticsearch.
- * If no response body was returned or anything goes wrong while parsing the error, returns a new {@link ElasticsearchStatusException}
- * that wraps the original {@link ResponseException}. The potential exception obtained while parsing is added to the returned
- * exception as a suppressed exception. This method is guaranteed to not throw any exception eventually thrown while parsing.
- */
- private ElasticsearchStatusException parseResponseException(ResponseException responseException) {
- Response response = responseException.getResponse();
- HttpEntity entity = response.getEntity();
- ElasticsearchStatusException elasticsearchException;
- RestStatus restStatus = RestStatus.fromCode(response.getStatusLine().getStatusCode());
-
- if (entity == null) {
- elasticsearchException = new ElasticsearchStatusException(responseException.getMessage(), restStatus, responseException);
- } else {
- try {
- elasticsearchException = parseEntity(entity, RestResponse::errorFromXContent);
- elasticsearchException.addSuppressed(responseException);
- } catch (Exception e) {
- elasticsearchException = new ElasticsearchStatusException("Unable to parse response body", restStatus, responseException);
- elasticsearchException.addSuppressed(e);
- }
- }
- return elasticsearchException;
- }
-
- private Resp parseEntity(final HttpEntity entity, final CheckedFunction entityParser)
- throws IOException {
- if (entity == null) {
- throw new IllegalStateException("Response body expected but not returned");
- }
- if (entity.getContentType() == null) {
- throw new IllegalStateException("Elasticsearch didn't return the [Content-Type] header, unable to parse response body");
- }
- XContentType xContentType = XContentType.fromMediaType(entity.getContentType().getValue());
- if (xContentType == null) {
- throw new IllegalStateException("Unsupported Content-Type: " + entity.getContentType().getValue());
- }
- try (XContentParser parser = xContentType.xContent().createParser(parserConfig, entity.getContent())) {
- return entityParser.apply(parser);
- }
- }
-
- private enum EntityType {
- JSON() {
- @Override
- public String header() {
- return "application/json";
- }
-
- @Override
- public String compatibleHeader() {
- return "application/vnd.elasticsearch+json; compatible-with=7";
- }
- },
- NDJSON() {
- @Override
- public String header() {
- return "application/x-ndjson";
- }
-
- @Override
- public String compatibleHeader() {
- return "application/vnd.elasticsearch+x-ndjson; compatible-with=7";
- }
- },
- STAR() {
- @Override
- public String header() {
- return "application/*";
- }
-
- @Override
- public String compatibleHeader() {
- return "application/vnd.elasticsearch+json; compatible-with=7";
- }
- },
- YAML() {
- @Override
- public String header() {
- return "application/yaml";
- }
-
- @Override
- public String compatibleHeader() {
- return "application/vnd.elasticsearch+yaml; compatible-with=7";
- }
- },
- SMILE() {
- @Override
- public String header() {
- return "application/smile";
- }
-
- @Override
- public String compatibleHeader() {
- return "application/vnd.elasticsearch+smile; compatible-with=7";
- }
- },
- CBOR() {
- @Override
- public String header() {
- return "application/cbor";
- }
-
- @Override
- public String compatibleHeader() {
- return "application/vnd.elasticsearch+cbor; compatible-with=7";
- }
- };
-
- public abstract String header();
-
- public abstract String compatibleHeader();
-
- @Override
- public String toString() {
- return header();
- }
- }
-
- private Cancellable performClientRequestAsync(Request request, ResponseListener listener) {
- // Add compatibility request headers if compatibility mode has been enabled
- if (this.useAPICompatibility) {
- modifyRequestForCompatibility(request);
- }
-
- ListenableFuture> versionCheck = getVersionValidationFuture();
-
- // Create a future that tracks cancellation of this method's result and forwards cancellation to the actual LLRC request.
- CompletableFuture cancellationForwarder = new CompletableFuture<>();
- Cancellable result = new Cancellable() {
- @Override
- public void cancel() {
- // Raise the flag by completing the future
- FutureUtils.cancel(cancellationForwarder);
- }
-
- @Override
- void runIfNotCancelled(Runnable runnable) {
- if (cancellationForwarder.isCancelled()) {
- throw newCancellationException();
- }
- runnable.run();
- }
- };
-
- // Send the request after we have done the version compatibility check. Note that if it has already happened, the listener will
- // be called immediately on the same thread with no asynchronous scheduling overhead.
- versionCheck.addListener(new ActionListener<>() {
- @Override
- public void onResponse(Optional validation) {
- if (validation.isPresent() == false) {
- // Send the request and propagate cancellation
- Cancellable call = client.performRequestAsync(request, listener);
- cancellationForwarder.whenComplete((r, t) ->
- // Forward cancellation to the actual request (no need to check parameters as the
- // only way for cancellationForwarder to be completed is by being cancelled).
- call.cancel());
- } else {
- // Version validation wasn't successful, fail the request with the validation result.
- listener.onFailure(new ElasticsearchException(validation.get()));
- }
- }
-
- @Override
- public void onFailure(Exception e) {
- // Propagate validation request failure. This will be transient since `getVersionValidationFuture` clears the validation
- // future if the request fails, leading to retries at the next HLRC request (see comments below).
- listener.onFailure(e);
- }
- });
-
- return result;
- }
-
- /**
- * Go through all the request's existing headers, looking for {@code headerName} headers and if they exist,
- * changing them to use version compatibility. If no request headers are changed, modify the entity type header if appropriate
- */
- private boolean addCompatibilityFor(RequestOptions.Builder newOptions, Header entityHeader, String headerName) {
- // Modify any existing "Content-Type" headers on the request to use the version compatibility, if available
- boolean contentTypeModified = false;
- for (Header header : new ArrayList<>(newOptions.getHeaders())) {
- if (headerName.equalsIgnoreCase(header.getName()) == false) {
- continue;
- }
- contentTypeModified = contentTypeModified || modifyHeader(newOptions, header, headerName);
- }
-
- // If there were no request-specific headers, modify the request entity's header to be compatible
- if (entityHeader != null && contentTypeModified == false) {
- contentTypeModified = modifyHeader(newOptions, entityHeader, headerName);
- }
-
- return contentTypeModified;
- }
-
- /**
- * Modify the given header to be version compatible, if necessary.
- * Returns true if a modification was made, false otherwise.
- */
- private boolean modifyHeader(RequestOptions.Builder newOptions, Header header, String headerName) {
- for (EntityType type : EntityType.values()) {
- final String headerValue = header.getValue();
- if (headerValue.startsWith(type.header())) {
- String newHeaderValue = headerValue.replace(type.header(), type.compatibleHeader());
- newOptions.removeHeader(header.getName());
- newOptions.addHeader(headerName, newHeaderValue);
- return true;
- }
- }
- return false;
- }
-
- /**
- * Make all necessary changes to support API compatibility for the given request. This includes
- * modifying the "Content-Type" and "Accept" headers if present, or modifying the header based
- * on the request's entity type.
- */
- private void modifyRequestForCompatibility(Request request) {
- final Header entityHeader = request.getEntity() == null ? null : request.getEntity().getContentType();
- final RequestOptions.Builder newOptions = request.getOptions().toBuilder();
-
- addCompatibilityFor(newOptions, entityHeader, "Content-Type");
- if (request.getOptions().containsHeader("Accept")) {
- addCompatibilityFor(newOptions, entityHeader, "Accept");
- } else {
- // There is no entity, and no existing accept header, but we still need one
- // with compatibility, so use the compatible JSON (default output) format
- newOptions.addHeader("Accept", EntityType.JSON.compatibleHeader());
- }
- request.setOptions(newOptions);
- }
-
- private Response performClientRequest(Request request) throws IOException {
- // Add compatibility request headers if compatibility mode has been enabled
- if (this.useAPICompatibility) {
- modifyRequestForCompatibility(request);
- }
-
- Optional versionValidation;
- try {
- final var future = new PlainActionFuture>();
- getVersionValidationFuture().addListener(future);
- versionValidation = future.get();
- } catch (InterruptedException | ExecutionException e) {
- // Unlikely to happen
- throw new ElasticsearchException(e);
- }
-
- if (versionValidation.isPresent() == false) {
- return client.performRequest(request);
- } else {
- throw new ElasticsearchException(versionValidation.get());
- }
- }
-
- /**
- * Returns a future that asynchronously validates the Elasticsearch product version. Its result is an optional string: if empty then
- * validation was successful, if present it contains the validation error. API requests should be chained to this future and check
- * the validation result before going further.
- *
- * This future is a memoization of the first successful request to the "/" endpoint and the subsequent compatibility check
- * ({@see #versionValidationFuture}). Further client requests reuse its result.
- *
- * If the version check request fails (e.g. network error), {@link #versionValidationFuture} is cleared so that a new validation
- * request is sent at the next HLRC request. This allows retries to happen while avoiding a busy retry loop (LLRC retries on the node
- * pool still happen).
- */
- private ListenableFuture> getVersionValidationFuture() {
- ListenableFuture> currentFuture = this.versionValidationFuture;
- if (currentFuture != null) {
- return currentFuture;
- } else {
- synchronized (this) {
- // Re-check in synchronized block
- currentFuture = this.versionValidationFuture;
- if (currentFuture != null) {
- return currentFuture;
- }
- ListenableFuture> future = new ListenableFuture<>();
- this.versionValidationFuture = future;
-
- // Asynchronously call the info endpoint and complete the future with the version validation result.
- Request req = new Request("GET", "/");
- // These status codes are nominal in the context of product version verification
- req.addParameter("ignore", "401,403");
- client.performRequestAsync(req, new ResponseListener() {
- @Override
- public void onSuccess(Response response) {
- Optional validation;
- try {
- validation = getVersionValidation(response);
- } catch (Exception e) {
- logger.error("Failed to parse info response", e);
- validation = Optional.of(
- "Failed to parse info response. Check logs for detailed information - " + e.getMessage()
- );
- }
- future.onResponse(validation);
- }
-
- @Override
- public void onFailure(Exception exception) {
-
- // Fail the requests (this one and the ones waiting for it) and clear the future
- // so that we retry the next time the client executes a request.
- versionValidationFuture = null;
- future.onFailure(exception);
- }
- });
-
- return future;
- }
- }
- }
-
- /**
- * Validates that the response info() is a compatible Elasticsearch version.
- *
- * @return an optional string. If empty, version is compatible. Otherwise, it's the message to return to the application.
- */
- private Optional getVersionValidation(Response response) throws IOException {
- // Let requests go through if the client doesn't have permissions for the info endpoint.
- int statusCode = response.getStatusLine().getStatusCode();
- if (statusCode == 401 || statusCode == 403) {
- return Optional.empty();
- }
-
- MainResponse mainResponse;
- try {
- mainResponse = parseEntity(response.getEntity(), MainResponse::fromXContent);
- } catch (ResponseException e) {
- throw parseResponseException(e);
- }
-
- String version = mainResponse.getVersion().getNumber();
- if (Strings.hasLength(version) == false) {
- return Optional.of("Missing version.number in info response");
- }
-
- String[] parts = version.split("\\.");
- if (parts.length < 2) {
- return Optional.of("Wrong version.number format in info response");
- }
-
- int major = Integer.parseInt(parts[0]);
- int minor = Integer.parseInt(parts[1]);
-
- if (major < 6) {
- return Optional.of("Elasticsearch version 6 or more is required");
- }
-
- if (major == 6 || (major == 7 && minor < 14)) {
- if ("You Know, for Search".equalsIgnoreCase(mainResponse.getTagline()) == false) {
- return Optional.of("Invalid or missing tagline [" + mainResponse.getTagline() + "]");
- }
-
- return Optional.empty();
- }
-
- String header = response.getHeader("X-Elastic-Product");
- if (header == null) {
- return Optional.of(
- "Missing [X-Elastic-Product] header. Please check that you are connecting to an Elasticsearch "
- + "instance, and that any networking filters are preserving that header."
- );
- }
-
- if ("Elasticsearch".equals(header) == false) {
- return Optional.of("Invalid value [" + header + "] for [X-Elastic-Product] header.");
- }
-
- return Optional.empty();
- }
-
- private static List getDefaultNamedXContents() {
- Map> map = new HashMap<>();
- map.put(CardinalityAggregationBuilder.NAME, (p, c) -> ParsedCardinality.fromXContent(p, (String) c));
- map.put(InternalHDRPercentiles.NAME, (p, c) -> ParsedHDRPercentiles.fromXContent(p, (String) c));
- map.put(InternalHDRPercentileRanks.NAME, (p, c) -> ParsedHDRPercentileRanks.fromXContent(p, (String) c));
- map.put(InternalTDigestPercentiles.NAME, (p, c) -> ParsedTDigestPercentiles.fromXContent(p, (String) c));
- map.put(InternalTDigestPercentileRanks.NAME, (p, c) -> ParsedTDigestPercentileRanks.fromXContent(p, (String) c));
- map.put(PercentilesBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedPercentilesBucket.fromXContent(p, (String) c));
- map.put(MedianAbsoluteDeviationAggregationBuilder.NAME, (p, c) -> ParsedMedianAbsoluteDeviation.fromXContent(p, (String) c));
- map.put(MinAggregationBuilder.NAME, (p, c) -> ParsedMin.fromXContent(p, (String) c));
- map.put(MaxAggregationBuilder.NAME, (p, c) -> ParsedMax.fromXContent(p, (String) c));
- map.put(SumAggregationBuilder.NAME, (p, c) -> ParsedSum.fromXContent(p, (String) c));
- map.put(AvgAggregationBuilder.NAME, (p, c) -> ParsedAvg.fromXContent(p, (String) c));
- map.put(WeightedAvgAggregationBuilder.NAME, (p, c) -> ParsedWeightedAvg.fromXContent(p, (String) c));
- map.put(ValueCountAggregationBuilder.NAME, (p, c) -> ParsedValueCount.fromXContent(p, (String) c));
- map.put(InternalSimpleValue.NAME, (p, c) -> ParsedSimpleValue.fromXContent(p, (String) c));
- map.put(DerivativePipelineAggregationBuilder.NAME, (p, c) -> ParsedDerivative.fromXContent(p, (String) c));
- map.put(InternalBucketMetricValue.NAME, (p, c) -> ParsedBucketMetricValue.fromXContent(p, (String) c));
- map.put(StatsAggregationBuilder.NAME, (p, c) -> ParsedStats.fromXContent(p, (String) c));
- map.put(StatsBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedStatsBucket.fromXContent(p, (String) c));
- map.put(ExtendedStatsAggregationBuilder.NAME, (p, c) -> ParsedExtendedStats.fromXContent(p, (String) c));
- map.put(ExtendedStatsBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedExtendedStatsBucket.fromXContent(p, (String) c));
- map.put(GeoBoundsAggregationBuilder.NAME, (p, c) -> ParsedGeoBounds.fromXContent(p, (String) c));
- map.put(GeoCentroidAggregationBuilder.NAME, (p, c) -> ParsedGeoCentroid.fromXContent(p, (String) c));
- map.put(HistogramAggregationBuilder.NAME, (p, c) -> ParsedHistogram.fromXContent(p, (String) c));
- map.put(DateHistogramAggregationBuilder.NAME, (p, c) -> ParsedDateHistogram.fromXContent(p, (String) c));
- map.put(AutoDateHistogramAggregationBuilder.NAME, (p, c) -> ParsedAutoDateHistogram.fromXContent(p, (String) c));
- map.put(VariableWidthHistogramAggregationBuilder.NAME, (p, c) -> ParsedVariableWidthHistogram.fromXContent(p, (String) c));
- map.put(StringTerms.NAME, (p, c) -> ParsedStringTerms.fromXContent(p, (String) c));
- map.put(LongTerms.NAME, (p, c) -> ParsedLongTerms.fromXContent(p, (String) c));
- map.put(DoubleTerms.NAME, (p, c) -> ParsedDoubleTerms.fromXContent(p, (String) c));
- map.put(LongRareTerms.NAME, (p, c) -> ParsedLongRareTerms.fromXContent(p, (String) c));
- map.put(StringRareTerms.NAME, (p, c) -> ParsedStringRareTerms.fromXContent(p, (String) c));
- map.put(MissingAggregationBuilder.NAME, (p, c) -> ParsedMissing.fromXContent(p, (String) c));
- map.put(NestedAggregationBuilder.NAME, (p, c) -> ParsedNested.fromXContent(p, (String) c));
- map.put(ReverseNestedAggregationBuilder.NAME, (p, c) -> ParsedReverseNested.fromXContent(p, (String) c));
- map.put(GlobalAggregationBuilder.NAME, (p, c) -> ParsedGlobal.fromXContent(p, (String) c));
- map.put(FilterAggregationBuilder.NAME, (p, c) -> ParsedFilter.fromXContent(p, (String) c));
- map.put(InternalSampler.PARSER_NAME, (p, c) -> ParsedSampler.fromXContent(p, (String) c));
- map.put(GeoHashGridAggregationBuilder.NAME, (p, c) -> ParsedGeoHashGrid.fromXContent(p, (String) c));
- map.put(GeoTileGridAggregationBuilder.NAME, (p, c) -> ParsedGeoTileGrid.fromXContent(p, (String) c));
- map.put(RangeAggregationBuilder.NAME, (p, c) -> ParsedRange.fromXContent(p, (String) c));
- map.put(DateRangeAggregationBuilder.NAME, (p, c) -> ParsedDateRange.fromXContent(p, (String) c));
- map.put(GeoDistanceAggregationBuilder.NAME, (p, c) -> ParsedGeoDistance.fromXContent(p, (String) c));
- map.put(FiltersAggregationBuilder.NAME, (p, c) -> ParsedFilters.fromXContent(p, (String) c));
- map.put(AdjacencyMatrixAggregationBuilder.NAME, (p, c) -> ParsedAdjacencyMatrix.fromXContent(p, (String) c));
- map.put(SignificantLongTerms.NAME, (p, c) -> ParsedSignificantLongTerms.fromXContent(p, (String) c));
- map.put(SignificantStringTerms.NAME, (p, c) -> ParsedSignificantStringTerms.fromXContent(p, (String) c));
- map.put(ScriptedMetricAggregationBuilder.NAME, (p, c) -> ParsedScriptedMetric.fromXContent(p, (String) c));
- map.put(IpRangeAggregationBuilder.NAME, (p, c) -> ParsedBinaryRange.fromXContent(p, (String) c));
- map.put(TopHitsAggregationBuilder.NAME, (p, c) -> ParsedTopHits.fromXContent(p, (String) c));
- map.put(CompositeAggregationBuilder.NAME, (p, c) -> ParsedComposite.fromXContent(p, (String) c));
- map.put(StringStatsAggregationBuilder.NAME, (p, c) -> ParsedStringStats.PARSER.parse(p, (String) c));
- map.put(TopMetricsAggregationBuilder.NAME, (p, c) -> ParsedTopMetrics.PARSER.parse(p, (String) c));
- map.put(TimeSeriesAggregationBuilder.NAME, (p, c) -> ParsedTimeSeries.fromXContent(p, (String) (c)));
- List entries = map.entrySet()
- .stream()
- .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue()))
- .collect(Collectors.toList());
- entries.add(
- new NamedXContentRegistry.Entry(
- Suggest.Suggestion.class,
- new ParseField(TermSuggestionBuilder.SUGGESTION_NAME),
- (parser, context) -> TermSuggestion.fromXContent(parser, (String) context)
- )
- );
- entries.add(
- new NamedXContentRegistry.Entry(
- Suggest.Suggestion.class,
- new ParseField(PhraseSuggestionBuilder.SUGGESTION_NAME),
- (parser, context) -> PhraseSuggestion.fromXContent(parser, (String) context)
- )
- );
- entries.add(
- new NamedXContentRegistry.Entry(
- Suggest.Suggestion.class,
- new ParseField(CompletionSuggestionBuilder.SUGGESTION_NAME),
- (parser, context) -> CompletionSuggestion.fromXContent(parser, (String) context)
- )
- );
- return entries;
- }
-
- /**
- * Loads and returns the {@link NamedXContentRegistry.Entry} parsers provided by plugins.
- */
- private static List getProvidedNamedXContents() {
- List entries = new ArrayList<>();
- for (NamedXContentProvider service : ServiceLoader.load(NamedXContentProvider.class)) {
- entries.addAll(service.getNamedXContentParsers());
- }
- return entries;
- }
-}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/Validatable.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/Validatable.java
deleted file mode 100644
index b7635f7054299..0000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/Validatable.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.client;
-
-import java.util.Optional;
-
-/**
- * Defines a validation layer for Requests.
- */
-public interface Validatable {
-
- Validatable EMPTY = new Validatable() {
- };
-
- /**
- * Perform validation. This method does not have to be overridden in the event that no validation needs to be done,
- * or the validation was done during object construction time. A {@link ValidationException} that is not null is
- * assumed to contain validation errors and will be thrown.
- *
- * @return An {@link Optional} {@link ValidationException} that contains a list of validation errors.
- */
- default Optional validate() {
- return Optional.empty();
- }
-}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ValidationException.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ValidationException.java
deleted file mode 100644
index d5701c5723096..0000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ValidationException.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-package org.elasticsearch.client;
-
-import org.elasticsearch.core.Nullable;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * Encapsulates an accumulation of validation errors
- */
-public class ValidationException extends IllegalArgumentException {
-
- /**
- * Creates {@link ValidationException} instance initialized with given error messages.
- * @param error the errors to add
- * @return {@link ValidationException} instance
- */
- public static ValidationException withError(String... error) {
- return withErrors(Arrays.asList(error));
- }
-
- /**
- * Creates {@link ValidationException} instance initialized with given error messages.
- * @param errors the list of errors to add
- * @return {@link ValidationException} instance
- */
- public static ValidationException withErrors(List errors) {
- ValidationException e = new ValidationException();
- for (String error : errors) {
- e.addValidationError(error);
- }
- return e;
- }
-
- private final List validationErrors = new ArrayList<>();
-
- /**
- * Add a new validation error to the accumulating validation errors
- * @param error the error to add
- */
- public void addValidationError(final String error) {
- validationErrors.add(error);
- }
-
- /**
- * Adds validation errors from an existing {@link ValidationException} to
- * the accumulating validation errors
- * @param exception the {@link ValidationException} to add errors from
- */
- public final void addValidationErrors(final @Nullable ValidationException exception) {
- if (exception != null) {
- for (String error : exception.validationErrors()) {
- addValidationError(error);
- }
- }
- }
-
- /**
- * Returns the validation errors accumulated
- */
- public final List validationErrors() {
- return validationErrors;
- }
-
- @Override
- public final String getMessage() {
- StringBuilder sb = new StringBuilder();
- sb.append("Validation Failed: ");
- int index = 0;
- for (String error : validationErrors) {
- sb.append(++index).append(": ").append(error).append(";");
- }
- return sb.toString();
- }
-}
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/MainResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/MainResponse.java
deleted file mode 100644
index bf7b1a665e098..0000000000000
--- a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/MainResponse.java
+++ /dev/null
@@ -1,220 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0 and the Server Side Public License, v 1; you may not use this file except
- * in compliance with, at your election, the Elastic License 2.0 or the Server
- * Side Public License, v 1.
- */
-
-package org.elasticsearch.client.core;
-
-import org.elasticsearch.xcontent.ConstructingObjectParser;
-import org.elasticsearch.xcontent.ParseField;
-import org.elasticsearch.xcontent.XContentParser;
-
-import java.util.Objects;
-
-public class MainResponse {
-
- private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(
- MainResponse.class.getName(),
- true,
- args -> {
- return new MainResponse((String) args[0], (Version) args[1], (String) args[2], (String) args[3], (String) args[4]);
- }
- );
-
- static {
- PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("name"));
- PARSER.declareObject(ConstructingObjectParser.constructorArg(), Version.PARSER, new ParseField("version"));
- PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("cluster_name"));
- PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("cluster_uuid"));
- PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("tagline"));
-
- }
-
- private final String nodeName;
- private final Version version;
- private final String clusterName;
- private final String clusterUuid;
- private final String tagline;
-
- public MainResponse(String nodeName, Version version, String clusterName, String clusterUuid, String tagline) {
- this.nodeName = nodeName;
- this.version = version;
- this.clusterName = clusterName;
- this.clusterUuid = clusterUuid;
- this.tagline = tagline;
- }
-
- public String getNodeName() {
- return nodeName;
- }
-
- public Version getVersion() {
- return version;
- }
-
- public String getClusterName() {
- return clusterName;
- }
-
- public String getClusterUuid() {
- return clusterUuid;
- }
-
- public String getTagline() {
- return tagline;
- }
-
- public static MainResponse fromXContent(XContentParser parser) {
- return PARSER.apply(parser, null);
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
- MainResponse that = (MainResponse) o;
- return nodeName.equals(that.nodeName)
- && version.equals(that.version)
- && clusterName.equals(that.clusterName)
- && clusterUuid.equals(that.clusterUuid)
- && tagline.equals(that.tagline);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(nodeName, version, clusterName, clusterUuid, tagline);
- }
-
- public static class Version {
- private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(
- Version.class.getName(),
- true,
- args -> {
- return new Version(
- (String) args[0],
- (String) args[1],
- (String) args[2],
- (String) args[3],
- (String) args[4],
- (Boolean) args[5],
- (String) args[6],
- (String) args[7],
- (String) args[8]
- );
- }
- );
-
- static {
- PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("number"));
- PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField("build_flavor"));
- PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField("build_type"));
- PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("build_hash"));
- PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("build_date"));
- PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), new ParseField("build_snapshot"));
- PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("lucene_version"));
- PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("minimum_wire_compatibility_version"));
- PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("minimum_index_compatibility_version"));
- }
- private final String number;
- private final String buildFlavor;
- private final String buildType;
- private final String buildHash;
- private final String buildDate;
- private final boolean isSnapshot;
- private final String luceneVersion;
- private final String minimumWireCompatibilityVersion;
- private final String minimumIndexCompatibilityVersion;
-
- public Version(
- String number,
- String buildFlavor,
- String buildType,
- String buildHash,
- String buildDate,
- boolean isSnapshot,
- String luceneVersion,
- String minimumWireCompatibilityVersion,
- String minimumIndexCompatibilityVersion
- ) {
- this.number = number;
- this.buildFlavor = buildFlavor;
- this.buildType = buildType;
- this.buildHash = buildHash;
- this.buildDate = buildDate;
- this.isSnapshot = isSnapshot;
- this.luceneVersion = luceneVersion;
- this.minimumWireCompatibilityVersion = minimumWireCompatibilityVersion;
- this.minimumIndexCompatibilityVersion = minimumIndexCompatibilityVersion;
- }
-
- public String getNumber() {
- return number;
- }
-
- public String getBuildFlavor() {
- return buildFlavor;
- }
-
- public String getBuildType() {
- return buildType;
- }
-
- public String getBuildHash() {
- return buildHash;
- }
-
- public String getBuildDate() {
- return buildDate;
- }
-
- public boolean isSnapshot() {
- return isSnapshot;
- }
-
- public String getLuceneVersion() {
- return luceneVersion;
- }
-
- public String getMinimumWireCompatibilityVersion() {
- return minimumWireCompatibilityVersion;
- }
-
- public String getMinimumIndexCompatibilityVersion() {
- return minimumIndexCompatibilityVersion;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
- Version version = (Version) o;
- return isSnapshot == version.isSnapshot
- && number.equals(version.number)
- && Objects.equals(buildFlavor, version.buildFlavor)
- && Objects.equals(buildType, version.buildType)
- && buildHash.equals(version.buildHash)
- && buildDate.equals(version.buildDate)
- && luceneVersion.equals(version.luceneVersion)
- && minimumWireCompatibilityVersion.equals(version.minimumWireCompatibilityVersion)
- && minimumIndexCompatibilityVersion.equals(version.minimumIndexCompatibilityVersion);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(
- number,
- buildFlavor,
- buildType,
- buildHash,
- buildDate,
- isSnapshot,
- luceneVersion,
- minimumWireCompatibilityVersion,
- minimumIndexCompatibilityVersion
- );
- }
- }
-}
diff --git a/client/rest-high-level/testnode.crt b/client/rest-high-level/testnode.crt
deleted file mode 100644
index 08c160bcea5ff..0000000000000
--- a/client/rest-high-level/testnode.crt
+++ /dev/null
@@ -1,23 +0,0 @@
------BEGIN CERTIFICATE-----
-MIID0zCCArugAwIBAgIJALi5bDfjMszLMA0GCSqGSIb3DQEBCwUAMEgxDDAKBgNV
-BAoTA29yZzEWMBQGA1UECxMNZWxhc3RpY3NlYXJjaDEgMB4GA1UEAxMXRWxhc3Rp
-Y3NlYXJjaCBUZXN0IE5vZGUwHhcNMTUwOTIzMTg1MjU3WhcNMTkwOTIyMTg1MjU3
-WjBIMQwwCgYDVQQKEwNvcmcxFjAUBgNVBAsTDWVsYXN0aWNzZWFyY2gxIDAeBgNV
-BAMTF0VsYXN0aWNzZWFyY2ggVGVzdCBOb2RlMIIBIjANBgkqhkiG9w0BAQEFAAOC
-AQ8AMIIBCgKCAQEA3rGZ1QbsW0+MuyrSLmMfDFKtLBkIFW8V0gRuurFg1PUKKNR1
-Mq2tMVwjjYETAU/UY0iKZOzjgvYPKhDTYBTte/WHR1ZK4CYVv7TQX/gtFQG/ge/c
-7u0sLch9p7fbd+/HZiLS/rBEZDIohvgUvzvnA8+OIYnw4kuxKo/5iboAIS41klMg
-/lATm8V71LMY68inht71/ZkQoAHKgcR9z4yNYvQ1WqKG8DG8KROXltll3sTrKbl5
-zJhn660es/1ZnR6nvwt6xnSTl/mNHMjkfv1bs4rJ/py3qPxicdoSIn/KyojUcgHV
-F38fuAy2CQTdjVG5fWj9iz+mQvLm3+qsIYQdFwIDAQABo4G/MIG8MAkGA1UdEwQC
-MAAwHQYDVR0OBBYEFEMMWLWQi/g83PzlHYqAVnty5L7HMIGPBgNVHREEgYcwgYSC
-CWxvY2FsaG9zdIIVbG9jYWxob3N0LmxvY2FsZG9tYWluggpsb2NhbGhvc3Q0ghds
-b2NhbGhvc3Q0LmxvY2FsZG9tYWluNIIKbG9jYWxob3N0NoIXbG9jYWxob3N0Ni5s
-b2NhbGRvbWFpbjaHBH8AAAGHEAAAAAAAAAAAAAAAAAAAAAEwDQYJKoZIhvcNAQEL
-BQADggEBAMjGGXT8Nt1tbl2GkiKtmiuGE2Ej66YuZ37WSJViaRNDVHLlg87TCcHe
-k2rdO+6sFqQbbzEfwQ05T7xGmVu7tm54HwKMRugoQ3wct0bQC5wEWYN+oMDvSyO6
-M28mZwWb4VtR2IRyWP+ve5DHwTM9mxWa6rBlGzsQqH6YkJpZojzqk/mQTug+Y8aE
-mVoqRIPMHq9ob+S9qd5lp09+MtYpwPfTPx/NN+xMEooXWW/ARfpGhWPkg/FuCu4z
-1tFmCqHgNcWirzMm3dQpF78muE9ng6OB2MXQwL4VgnVkxmlZNHbkR2v/t8MyZJxC
-y4g6cTMM3S/UMt5/+aIB2JAuMKyuD+A=
------END CERTIFICATE-----
diff --git a/client/rest-high-level/testnode.jks b/client/rest-high-level/testnode.jks
deleted file mode 100644
index ebe6146124e8f..0000000000000
Binary files a/client/rest-high-level/testnode.jks and /dev/null differ
diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java
index 7154a2be5bbd8..ed087bef0ac76 100644
--- a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java
+++ b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java
@@ -87,6 +87,7 @@
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Collections.singletonList;
+import static org.elasticsearch.client.RestClient.IGNORE_RESPONSE_CODES_PARAM;
/**
* Client that connects to an Elasticsearch cluster through HTTP.
@@ -106,6 +107,9 @@
* Requests can be traced by enabling trace logging for "tracer". The trace logger outputs requests and responses in curl format.
*/
public class RestClient implements Closeable {
+
+ public static final String IGNORE_RESPONSE_CODES_PARAM = "ignore";
+
private static final Log logger = LogFactory.getLog(RestClient.class);
private final CloseableHttpAsyncClient client;
@@ -780,8 +784,8 @@ private class InternalRequest {
this.request = request;
Map params = new HashMap<>(request.getParameters());
params.putAll(request.getOptions().getParameters());
- // ignore is a special parameter supported by the clients, shouldn't be sent to es
- String ignoreString = params.remove("ignore");
+ // IGNORE_RESPONSE_CODES_PARAM is a special parameter supported by the clients, shouldn't be sent to es
+ String ignoreString = params.remove(IGNORE_RESPONSE_CODES_PARAM);
this.ignoreErrorCodes = getIgnoreErrorCodes(ignoreString, request.getMethod());
URI uri = buildUri(pathPrefix, request.getEndpoint(), params);
this.httpRequest = createHttpRequest(request.getMethod(), uri, request.getEntity(), compressionEnabled);
diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
index a1c4d3fab076a..10d24242ae620 100644
--- a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
+++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java
@@ -275,6 +275,7 @@ public void testErrorStatusCodes() throws Exception {
try {
Request request = new Request(method, "/" + errorStatusCode);
if (false == ignoreParam.isEmpty()) {
+ // literal "ignore" rather than IGNORE_RESPONSE_CODES_PARAM since this is something on which callers might rely
request.addParameter("ignore", ignoreParam);
}
Response response = restClient.performRequest(request);
@@ -568,6 +569,7 @@ private HttpUriRequest performRandomRequest(String method) throws Exception {
if (randomBoolean()) {
ignore += "," + randomFrom(RestClientTestUtil.getAllErrorStatusCodes());
}
+ // literal "ignore" rather than IGNORE_RESPONSE_CODES_PARAM since this is something on which callers might rely
request.addParameter("ignore", ignore);
}
URI uri = uriBuilder.build();
diff --git a/distribution/build.gradle b/distribution/build.gradle
index 90af1472deb2e..e45f1d09625d6 100644
--- a/distribution/build.gradle
+++ b/distribution/build.gradle
@@ -14,6 +14,7 @@ import org.elasticsearch.gradle.internal.ConcatFilesTask
import org.elasticsearch.gradle.internal.DependenciesInfoPlugin
import org.elasticsearch.gradle.internal.NoticeTask
import org.elasticsearch.gradle.internal.info.BuildParams
+import org.elasticsearch.gradle.internal.test.HistoricalFeaturesMetadataPlugin
import java.nio.file.Files
import java.nio.file.Path
@@ -30,6 +31,15 @@ configurations {
attribute(Category.CATEGORY_ATTRIBUTE, project.getObjects().named(Category.class, Category.DOCUMENTATION))
}
}
+ featuresMetadata {
+ attributes {
+ attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, HistoricalFeaturesMetadataPlugin.FEATURES_METADATA_TYPE)
+ }
+ }
+}
+
+dependencies {
+ featuresMetadata project(':server')
}
def thisProj = project
@@ -196,6 +206,7 @@ project.rootProject.subprojects.findAll { it.parent.path == ':modules' }.each {
}
distro.copyModule(processDefaultOutputsTaskProvider, module)
+ dependencies.add('featuresMetadata', module)
if (module.name.startsWith('transport-') || (BuildParams.snapshotBuild == false && module.name == 'apm')) {
distro.copyModule(processIntegTestOutputsTaskProvider, module)
}
@@ -214,6 +225,7 @@ xpack.subprojects.findAll { it.parent == xpack }.each { Project xpackModule ->
}
}
distro.copyModule(processDefaultOutputsTaskProvider, xpackModule)
+ dependencies.add('featuresMetadata', xpackModule)
if (xpackModule.name.equals('core') || xpackModule.name.equals('security')) {
distro.copyModule(processIntegTestOutputsTaskProvider, xpackModule)
}
diff --git a/distribution/tools/java-version-checker/build.gradle b/distribution/tools/java-version-checker/build.gradle
index 39f9bbf536dda..0a47d0652e465 100644
--- a/distribution/tools/java-version-checker/build.gradle
+++ b/distribution/tools/java-version-checker/build.gradle
@@ -8,15 +8,17 @@ tasks.named(sourceSets.unsupportedJdkVersionEntrypoint.compileJavaTaskName).conf
targetCompatibility = JavaVersion.VERSION_1_8
}
+
tasks.named("jar") {
manifest {
attributes("Multi-Release": "true")
}
+ FileCollection mainOutput = sourceSets.main.output;
from(sourceSets.unsupportedJdkVersionEntrypoint.output)
eachFile { details ->
if (details.path.equals("org/elasticsearch/tools/java_version_checker/JavaVersionChecker.class") &&
- sourceSets.main.output.asFileTree.contains(details.file)) {
+ mainOutput.asFileTree.contains(details.file)) {
details.relativePath = details.relativePath.prepend("META-INF/versions/17")
}
}
diff --git a/distribution/tools/plugin-cli/build.gradle b/distribution/tools/plugin-cli/build.gradle
index 25b5883166ccc..e0d1dd983c0de 100644
--- a/distribution/tools/plugin-cli/build.gradle
+++ b/distribution/tools/plugin-cli/build.gradle
@@ -24,8 +24,8 @@ dependencies {
implementation 'org.ow2.asm:asm:9.5'
implementation 'org.ow2.asm:asm-tree:9.5'
- api "org.bouncycastle:bcpg-fips:1.0.4"
- api "org.bouncycastle:bc-fips:1.0.2"
+ api "org.bouncycastle:bcpg-fips:1.0.7.1"
+ api "org.bouncycastle:bc-fips:1.0.2.4"
testImplementation project(":test:framework")
testImplementation "com.google.jimfs:jimfs:${versions.jimfs}"
testRuntimeOnly "com.google.guava:guava:${versions.jimfs_guava}"
diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java
index e9adf9882b6db..c088e89338e74 100644
--- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java
+++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java
@@ -88,7 +88,6 @@
import java.util.Arrays;
import java.util.Date;
import java.util.HashSet;
-import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -1591,15 +1590,6 @@ public void testGetSemanticVersion() {
assertThat(InstallPluginAction.getSemanticVersion("foo-1.2.3"), nullValue());
}
- private Map> namedComponentsMap() {
- Map> result = new LinkedHashMap<>();
- Map extensibles = new LinkedHashMap<>();
- extensibles.put("a_component", "p.A");
- extensibles.put("b_component", "p.B");
- result.put("org.elasticsearch.plugins.cli.test_model.ExtensibleInterface", extensibles);
- return result;
- }
-
private static String namedComponentsJSON() {
return """
{
diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/RemovePluginActionTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/RemovePluginActionTests.java
index 7d900155488b2..73e89fc948029 100644
--- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/RemovePluginActionTests.java
+++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/RemovePluginActionTests.java
@@ -140,10 +140,6 @@ public void testRemoveMultiple() throws Exception {
assertRemoveCleaned(env);
}
- private static Version minimumCompatibleVersion(Version v) {
- return Version.fromString((v.major - 1) + ".0.0");
- }
-
public void testBin() throws Exception {
createPlugin("fake");
Path binDir = env.binFile().resolve("fake");
diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java
index b6cd680cb5816..9dcd630f52631 100644
--- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java
+++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java
@@ -145,7 +145,7 @@ static List apmJvmOptions(Settings settings, @Nullable SecureSettings se
// Configures a log file to write to. Don't disable writing to a log file,
// as the agent will then require extra Security Manager permissions when
// it tries to do something else, and it's just painful.
- propertiesMap.put("log_file", logsDir.resolve("apm-agent.log").toString());
+ propertiesMap.put("log_file", logsDir.resolve("apm-agent.json").toString());
// No point doing anything if we don't have a destination for the trace data, and it can't be configured dynamically
if (propertiesMap.containsKey("server_url") == false && propertiesMap.containsKey("server_urls") == false) {
diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java
index 25c61c41638d1..ea2df72fb2c0b 100644
--- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java
+++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java
@@ -107,6 +107,14 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce
// we are running in the foreground, so wait for the server to exit
int exitCode = server.waitFor();
+ onExit(exitCode);
+ }
+
+ /**
+ * A post-exit hook to perform additional processing before the command terminates
+ * @param exitCode the server process exit code
+ */
+ protected void onExit(int exitCode) throws UserException {
if (exitCode != ExitCodes.OK) {
throw new UserException(exitCode, "Elasticsearch exited unexpectedly");
}
diff --git a/docs/build.gradle b/docs/build.gradle
index 33e6cc6080a95..da3d83378e894 100644
--- a/docs/build.gradle
+++ b/docs/build.gradle
@@ -72,6 +72,9 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach {
setting 'xpack.license.self_generated.type', 'trial'
setting 'indices.lifecycle.history_index_enabled', 'false'
keystorePassword 'keystore-password'
+ if (BuildParams.isSnapshotBuild() == false) {
+ requiresFeature 'es.failure_store_feature_flag_enabled', new Version(8, 12, 0)
+ }
}
// debug ccr test failures:
diff --git a/docs/changelog/100408.yaml b/docs/changelog/100408.yaml
new file mode 100644
index 0000000000000..275c3b4a0de48
--- /dev/null
+++ b/docs/changelog/100408.yaml
@@ -0,0 +1,5 @@
+pr: 100408
+summary: "ESQL: Make blocks ref counted"
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/100570.yaml b/docs/changelog/100570.yaml
new file mode 100644
index 0000000000000..b68a905b0e046
--- /dev/null
+++ b/docs/changelog/100570.yaml
@@ -0,0 +1,5 @@
+pr: 100570
+summary: Added metric for cache eviction of entries with non zero frequency
+area: Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/100921.yaml b/docs/changelog/100921.yaml
new file mode 100644
index 0000000000000..e6e2caa93d465
--- /dev/null
+++ b/docs/changelog/100921.yaml
@@ -0,0 +1,5 @@
+pr: 100921
+summary: "Add support for Serbian Language Analyzer"
+area: Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/100938.yaml b/docs/changelog/100938.yaml
new file mode 100644
index 0000000000000..b21f6955c992e
--- /dev/null
+++ b/docs/changelog/100938.yaml
@@ -0,0 +1,5 @@
+pr: 100938
+summary: "Set includeShardsStats = false in NodesStatsRequest where the caller does not use shards-level statistics"
+area: Stats
+type: enhancement
+issues: []
diff --git a/docs/changelog/101333.yaml b/docs/changelog/101333.yaml
new file mode 100644
index 0000000000000..4452687b995d3
--- /dev/null
+++ b/docs/changelog/101333.yaml
@@ -0,0 +1,29 @@
+pr: 101333
+summary: Fixed JWT principal from claims
+area: Authorization
+type: breaking
+issues: []
+breaking:
+ title: Fixed JWT principal from claims
+ area: Authorization
+ details: "This changes the format of a JWT's principal before the JWT is actually\
+ \ validated by any JWT realm. The JWT's principal is a convenient way to refer\
+ \ to a JWT that has not yet been verified by a JWT realm. The JWT's principal\
+ \ is printed in the audit and regular logs (notably for auditing authn failures)\
+ \ as well as the smart realm chain reordering optimization. The JWT principal\
+ \ is NOT required to be identical to the JWT-authenticated user's principal, but\
+ \ in general, they should be similar. Previously, the JWT's principal was built\
+ \ by individual realms in the same way the realms built the authenticated user's\
+ \ principal. This had the advantage that, in simpler JWT realms configurations\
+ \ (e.g. a single JWT realm in the chain), the JWT principal and the authenticated\
+ \ user's principal are very similar. However the drawback is that, in general,\
+ \ the JWT principal and the user principal can be very different (i.e. in the\
+ \ case where one JWT realm builds the JWT principal and a different one builds\
+ \ the user principal). Another downside is that the (unauthenticated) JWT principal\
+ \ depended on realm ordering, which makes identifying the JWT from its principal\
+ \ dependent on the ES authn realm configuration. This PR implements a consistent\
+ \ fixed logic to build the JWT principal, which only depends on the JWT's claims\
+ \ and no ES configuration."
+ impact: "Users will observe changed format and values for the `user.name` attribute\
+ \ of `authentication_failed` audit log events, in the JWT (failed) authn case."
+ notable: false
diff --git a/docs/changelog/101390.yaml b/docs/changelog/101390.yaml
new file mode 100644
index 0000000000000..23bdef6e39dfe
--- /dev/null
+++ b/docs/changelog/101390.yaml
@@ -0,0 +1,5 @@
+pr: 101390
+summary: Enable inter-segment concurrency for terms aggs
+area: Aggregations
+type: enhancement
+issues: []
diff --git a/docs/changelog/101409.yaml b/docs/changelog/101409.yaml
new file mode 100644
index 0000000000000..82e7f339fdd89
--- /dev/null
+++ b/docs/changelog/101409.yaml
@@ -0,0 +1,5 @@
+pr: 101409
+summary: Adding a simulate ingest api
+area: Ingest Node
+type: feature
+issues: []
diff --git a/docs/changelog/101423.yaml b/docs/changelog/101423.yaml
new file mode 100644
index 0000000000000..a5497d444797f
--- /dev/null
+++ b/docs/changelog/101423.yaml
@@ -0,0 +1,5 @@
+pr: 101423
+summary: Export circuit breaker trip count as a counter metric
+area: Aggregations
+type: enhancement
+issues: []
diff --git a/docs/changelog/101476.yaml b/docs/changelog/101476.yaml
deleted file mode 100644
index ee4cd9b1e4b1a..0000000000000
--- a/docs/changelog/101476.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 101476
-summary: Mark legacy stack templates as deprecated
-area: Indices APIs
-type: enhancement
-issues: []
diff --git a/docs/changelog/101577.yaml b/docs/changelog/101577.yaml
new file mode 100644
index 0000000000000..e485fd3811cb6
--- /dev/null
+++ b/docs/changelog/101577.yaml
@@ -0,0 +1,5 @@
+pr: 101577
+summary: Add metrics to the shared blob cache
+area: Search
+type: enhancement
+issues: []
diff --git a/docs/changelog/101609.yaml b/docs/changelog/101609.yaml
new file mode 100644
index 0000000000000..27993574743d2
--- /dev/null
+++ b/docs/changelog/101609.yaml
@@ -0,0 +1,9 @@
+pr: 101609
+summary: >
+ Add a node feature join barrier. This prevents nodes from joining clusters that do not have
+ all the features already present in the cluster. This ensures that once a features is supported
+ by all the nodes in a cluster, that feature will never then not be supported in the future.
+ This is the corresponding functionality for the version join barrier, but for features
+area: "Cluster Coordination"
+type: feature
+issues: []
diff --git a/docs/changelog/101682.yaml b/docs/changelog/101682.yaml
new file mode 100644
index 0000000000000..e512006057581
--- /dev/null
+++ b/docs/changelog/101682.yaml
@@ -0,0 +1,5 @@
+pr: 101682
+summary: "Add manage_enrich cluster privilege to kibana_system role"
+area: Authentication
+type: enhancement
+issues: []
diff --git a/docs/changelog/101705.yaml b/docs/changelog/101705.yaml
deleted file mode 100644
index baa7e69d48d88..0000000000000
--- a/docs/changelog/101705.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-pr: 101705
-summary: Respect regional AWS STS endpoints
-area: Snapshot/Restore
-type: bug
-issues:
- - 89175
diff --git a/docs/changelog/101723.yaml b/docs/changelog/101723.yaml
new file mode 100644
index 0000000000000..146d164805f00
--- /dev/null
+++ b/docs/changelog/101723.yaml
@@ -0,0 +1,6 @@
+pr: 101723
+summary: Allowing non-dynamic index settings to be updated by automatically unassigning
+ shards
+area: Indices APIs
+type: enhancement
+issues: []
diff --git a/docs/changelog/101799.yaml b/docs/changelog/101799.yaml
deleted file mode 100644
index a3ef5fb190177..0000000000000
--- a/docs/changelog/101799.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-pr: 101799
-summary: Fix memory leak from JWT cache (and fix the usage of the JWT auth cache)
-area: Authentication
-type: bug
-issues: []
diff --git a/docs/changelog/101826.yaml b/docs/changelog/101826.yaml
new file mode 100644
index 0000000000000..87f3f8df1b0c2
--- /dev/null
+++ b/docs/changelog/101826.yaml
@@ -0,0 +1,6 @@
+pr: 101826
+summary: Support keyed histograms
+area: Aggregations
+type: enhancement
+issues:
+ - 100242
diff --git a/docs/changelog/101845.yaml b/docs/changelog/101845.yaml
new file mode 100644
index 0000000000000..0dd95bdabca57
--- /dev/null
+++ b/docs/changelog/101845.yaml
@@ -0,0 +1,5 @@
+pr: 101845
+summary: Introduce new endpoint to expose data stream lifecycle stats
+area: Data streams
+type: enhancement
+issues: []
diff --git a/docs/changelog/101859.yaml b/docs/changelog/101859.yaml
new file mode 100644
index 0000000000000..54f3fb12810ca
--- /dev/null
+++ b/docs/changelog/101859.yaml
@@ -0,0 +1,6 @@
+pr: 101859
+summary: Cover head/tail commands edge cases and data types coverage
+area: EQL
+type: bug
+issues:
+ - 101724
diff --git a/docs/changelog/101904.yaml b/docs/changelog/101904.yaml
new file mode 100644
index 0000000000000..cad422cc52e15
--- /dev/null
+++ b/docs/changelog/101904.yaml
@@ -0,0 +1,5 @@
+pr: 101904
+summary: Allow granting API keys with JWT as the access_token
+area: Security
+type: feature
+issues: []
diff --git a/docs/changelog/101915.yaml b/docs/changelog/101915.yaml
new file mode 100644
index 0000000000000..aed7ca62021a5
--- /dev/null
+++ b/docs/changelog/101915.yaml
@@ -0,0 +1,5 @@
+pr: 101915
+summary: Add inference counts by model to the machine learning usage stats
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/101979.yaml b/docs/changelog/101979.yaml
new file mode 100644
index 0000000000000..ad119df24d36f
--- /dev/null
+++ b/docs/changelog/101979.yaml
@@ -0,0 +1,5 @@
+pr: 101979
+summary: Calculate CO2 and emmission and costs
+area: Application
+type: enhancement
+issues: []
diff --git a/docs/changelog/101989.yaml b/docs/changelog/101989.yaml
new file mode 100644
index 0000000000000..d294d194bd4e8
--- /dev/null
+++ b/docs/changelog/101989.yaml
@@ -0,0 +1,5 @@
+pr: 101989
+summary: Add message field to `HealthPeriodicLogger` and `S3RequestRetryStats`
+area: Health
+type: enhancement
+issues: []
diff --git a/docs/changelog/102020.yaml b/docs/changelog/102020.yaml
new file mode 100644
index 0000000000000..7c74e9676d342
--- /dev/null
+++ b/docs/changelog/102020.yaml
@@ -0,0 +1,5 @@
+pr: 102020
+summary: Retrieve stacktrace events from a custom index
+area: Application
+type: enhancement
+issues: []
diff --git a/docs/changelog/102048.yaml b/docs/changelog/102048.yaml
new file mode 100644
index 0000000000000..54bc1d9eae52e
--- /dev/null
+++ b/docs/changelog/102048.yaml
@@ -0,0 +1,5 @@
+pr: 102048
+summary: "Repo analysis: verify empty register"
+area: Snapshot/Restore
+type: enhancement
+issues: []
diff --git a/docs/changelog/102051.yaml b/docs/changelog/102051.yaml
new file mode 100644
index 0000000000000..c3ca4a546928f
--- /dev/null
+++ b/docs/changelog/102051.yaml
@@ -0,0 +1,5 @@
+pr: 102051
+summary: "Repo analysis: allow configuration of register ops"
+area: Snapshot/Restore
+type: enhancement
+issues: []
diff --git a/docs/changelog/102056.yaml b/docs/changelog/102056.yaml
new file mode 100644
index 0000000000000..455f66ba90b03
--- /dev/null
+++ b/docs/changelog/102056.yaml
@@ -0,0 +1,5 @@
+pr: 102056
+summary: Use `BulkRequest` to store Application Privileges
+area: Authorization
+type: enhancement
+issues: []
diff --git a/docs/changelog/102057.yaml b/docs/changelog/102057.yaml
new file mode 100644
index 0000000000000..d5b664ba14c29
--- /dev/null
+++ b/docs/changelog/102057.yaml
@@ -0,0 +1,6 @@
+pr: 102057
+summary: Simplify `BlobStoreRepository` idle check
+area: Snapshot/Restore
+type: bug
+issues:
+ - 101948
diff --git a/docs/changelog/102065.yaml b/docs/changelog/102065.yaml
new file mode 100644
index 0000000000000..1a9a219df4502
--- /dev/null
+++ b/docs/changelog/102065.yaml
@@ -0,0 +1,5 @@
+pr: 102065
+summary: Add more desired balance stats
+area: Allocation
+type: enhancement
+issues: []
diff --git a/docs/changelog/102075.yaml b/docs/changelog/102075.yaml
new file mode 100644
index 0000000000000..54daae04169db
--- /dev/null
+++ b/docs/changelog/102075.yaml
@@ -0,0 +1,5 @@
+pr: 102075
+summary: Accept a single or multiple inputs to `_inference`
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/102089.yaml b/docs/changelog/102089.yaml
new file mode 100644
index 0000000000000..9f33c0648d09f
--- /dev/null
+++ b/docs/changelog/102089.yaml
@@ -0,0 +1,5 @@
+pr: 102089
+summary: Add prefix strings option to trained models
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/102114.yaml b/docs/changelog/102114.yaml
new file mode 100644
index 0000000000000..a08389da0351b
--- /dev/null
+++ b/docs/changelog/102114.yaml
@@ -0,0 +1,6 @@
+pr: 102114
+summary: Fix double-completion in `SecurityUsageTransportAction`
+area: Security
+type: bug
+issues:
+ - 102111
diff --git a/docs/changelog/102138.yaml b/docs/changelog/102138.yaml
new file mode 100644
index 0000000000000..3819e3201150e
--- /dev/null
+++ b/docs/changelog/102138.yaml
@@ -0,0 +1,5 @@
+pr: 102138
+summary: Skip shards that don't match the source query during checkpointing
+area: Transform
+type: enhancement
+issues: []
diff --git a/docs/changelog/102140.yaml b/docs/changelog/102140.yaml
new file mode 100644
index 0000000000000..0f086649b9710
--- /dev/null
+++ b/docs/changelog/102140.yaml
@@ -0,0 +1,6 @@
+pr: 102140
+summary: Collect data tiers usage stats more efficiently
+area: ILM+SLM
+type: bug
+issues:
+ - 100230
\ No newline at end of file
diff --git a/docs/changelog/102151.yaml b/docs/changelog/102151.yaml
new file mode 100644
index 0000000000000..652ae555af97d
--- /dev/null
+++ b/docs/changelog/102151.yaml
@@ -0,0 +1,5 @@
+pr: 102151
+summary: Default `run_ml_inference` should be true
+area: Application
+type: bug
+issues: []
diff --git a/docs/changelog/102165.yaml b/docs/changelog/102165.yaml
new file mode 100644
index 0000000000000..e1c4c76f1f6ff
--- /dev/null
+++ b/docs/changelog/102165.yaml
@@ -0,0 +1,6 @@
+pr: 102165
+summary: Fix planning of duplicate aggs
+area: ES|QL
+type: bug
+issues:
+ - 102083
diff --git a/docs/changelog/102172.yaml b/docs/changelog/102172.yaml
new file mode 100644
index 0000000000000..485c2c4327e11
--- /dev/null
+++ b/docs/changelog/102172.yaml
@@ -0,0 +1,5 @@
+pr: 102172
+summary: Adjust Histogram's bucket accounting to be iteratively
+area: Aggregations
+type: bug
+issues: []
diff --git a/docs/changelog/102183.yaml b/docs/changelog/102183.yaml
new file mode 100644
index 0000000000000..3daa1418ba5d0
--- /dev/null
+++ b/docs/changelog/102183.yaml
@@ -0,0 +1,13 @@
+pr: 102183
+summary: "[ES|QL] pow function always returns double"
+area: ES|QL
+type: "breaking"
+issues:
+ - 99055
+breaking:
+ title: "[ES|QL] pow function always returns double"
+ area: REST API
+ details: "In ES|QL, the pow function no longer returns the type of its inputs, instead\
+ \ always returning a double."
+ impact: low. Most queries should continue to function with the change.
+ notable: false
diff --git a/docs/changelog/102184.yaml b/docs/changelog/102184.yaml
new file mode 100644
index 0000000000000..ba4d045b6b0aa
--- /dev/null
+++ b/docs/changelog/102184.yaml
@@ -0,0 +1,5 @@
+pr: 102184
+summary: Track ESQL enrich memory
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/102188.yaml b/docs/changelog/102188.yaml
new file mode 100644
index 0000000000000..595a8395fab5c
--- /dev/null
+++ b/docs/changelog/102188.yaml
@@ -0,0 +1,5 @@
+pr: 102188
+summary: Track blocks in `AsyncOperator`
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/102190.yaml b/docs/changelog/102190.yaml
new file mode 100644
index 0000000000000..cd04e041fca5e
--- /dev/null
+++ b/docs/changelog/102190.yaml
@@ -0,0 +1,5 @@
+pr: 102190
+summary: Track pages in ESQL enrich request/response
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/102192.yaml b/docs/changelog/102192.yaml
new file mode 100644
index 0000000000000..531aa943c9e36
--- /dev/null
+++ b/docs/changelog/102192.yaml
@@ -0,0 +1,5 @@
+pr: 102192
+summary: "ESQL: Load more than one field at once"
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/102193.yaml b/docs/changelog/102193.yaml
new file mode 100644
index 0000000000000..4d64493602ff2
--- /dev/null
+++ b/docs/changelog/102193.yaml
@@ -0,0 +1,5 @@
+pr: 102193
+summary: Fix cache invalidation on privilege modification
+area: Authorization
+type: bug
+issues: []
diff --git a/docs/changelog/102208.yaml b/docs/changelog/102208.yaml
new file mode 100644
index 0000000000000..b566a85753d82
--- /dev/null
+++ b/docs/changelog/102208.yaml
@@ -0,0 +1,5 @@
+pr: 102208
+summary: Add static node settings to set default values for max merged segment sizes
+area: Engine
+type: enhancement
+issues: []
diff --git a/docs/changelog/102220.yaml b/docs/changelog/102220.yaml
new file mode 100644
index 0000000000000..d24dab1f91b31
--- /dev/null
+++ b/docs/changelog/102220.yaml
@@ -0,0 +1,5 @@
+pr: 102220
+summary: Upgrade xmlsec to 2.3.4
+area: Security
+type: enhancement
+issues: []
diff --git a/docs/changelog/102230.yaml b/docs/changelog/102230.yaml
new file mode 100644
index 0000000000000..20e8d8d1f10a6
--- /dev/null
+++ b/docs/changelog/102230.yaml
@@ -0,0 +1,6 @@
+pr: 102230
+summary: Set region for the STS client via privileged calls in AWS SDK
+area: Snapshot/Restore
+type: bug
+issues:
+ - 102173
diff --git a/docs/changelog/102240.yaml b/docs/changelog/102240.yaml
new file mode 100644
index 0000000000000..5df0046ee92fc
--- /dev/null
+++ b/docs/changelog/102240.yaml
@@ -0,0 +1,5 @@
+pr: 102240
+summary: Exclude stack traces from transform audit messages and health
+area: Transform
+type: bug
+issues: []
diff --git a/docs/changelog/102244.yaml b/docs/changelog/102244.yaml
new file mode 100644
index 0000000000000..3b160e033b57e
--- /dev/null
+++ b/docs/changelog/102244.yaml
@@ -0,0 +1,5 @@
+pr: 102244
+summary: Expose reconciliation metrics via APM
+area: Allocation
+type: enhancement
+issues: []
diff --git a/docs/changelog/102245.yaml b/docs/changelog/102245.yaml
new file mode 100644
index 0000000000000..387540d96290c
--- /dev/null
+++ b/docs/changelog/102245.yaml
@@ -0,0 +1,5 @@
+pr: 102245
+summary: Add non-green indicator names to `HealthPeriodicLogger` message
+area: Health
+type: enhancement
+issues: []
diff --git a/docs/changelog/102250.yaml b/docs/changelog/102250.yaml
new file mode 100644
index 0000000000000..755341d9a3a64
--- /dev/null
+++ b/docs/changelog/102250.yaml
@@ -0,0 +1,6 @@
+pr: 102250
+summary: "[ILM] Fix downsample to skip already downsampled indices"
+area: ILM+SLM
+type: bug
+issues:
+ - 102249
diff --git a/docs/changelog/102259.yaml b/docs/changelog/102259.yaml
new file mode 100644
index 0000000000000..3d8a1c6381f6d
--- /dev/null
+++ b/docs/changelog/102259.yaml
@@ -0,0 +1,5 @@
+pr: 102259
+summary: "[Usage API] Count all the data streams that have lifecycle"
+area: Data streams
+type: bug
+issues: []
diff --git a/docs/changelog/102273.yaml b/docs/changelog/102273.yaml
new file mode 100644
index 0000000000000..78ecc8b2d2734
--- /dev/null
+++ b/docs/changelog/102273.yaml
@@ -0,0 +1,5 @@
+pr: 102273
+summary: Improve analyzer reload log message
+area: Mapping
+type: enhancement
+issues: []
diff --git a/docs/changelog/102281.yaml b/docs/changelog/102281.yaml
new file mode 100644
index 0000000000000..ac6c17591e013
--- /dev/null
+++ b/docs/changelog/102281.yaml
@@ -0,0 +1,5 @@
+pr: 102281
+summary: Improve failure handling in `ContinuousComputation`
+area: Allocation
+type: bug
+issues: []
diff --git a/docs/changelog/102282.yaml b/docs/changelog/102282.yaml
new file mode 100644
index 0000000000000..4860d70f99ccc
--- /dev/null
+++ b/docs/changelog/102282.yaml
@@ -0,0 +1,6 @@
+pr: 102282
+summary: "ES|QL: Fix drop of renamed grouping"
+area: ES|QL
+type: bug
+issues:
+ - 102121
diff --git a/docs/changelog/102292.yaml b/docs/changelog/102292.yaml
new file mode 100644
index 0000000000000..953c3ffdf6150
--- /dev/null
+++ b/docs/changelog/102292.yaml
@@ -0,0 +1,5 @@
+pr: 102292
+summary: Consider duplicate stacktraces in custom index
+area: Application
+type: enhancement
+issues: []
diff --git a/docs/changelog/102311.yaml b/docs/changelog/102311.yaml
new file mode 100644
index 0000000000000..bb1769527fdd4
--- /dev/null
+++ b/docs/changelog/102311.yaml
@@ -0,0 +1,5 @@
+pr: 102311
+summary: Upgrade reactor netty http version
+area: Snapshot/Restore
+type: upgrade
+issues: []
diff --git a/docs/changelog/102317.yaml b/docs/changelog/102317.yaml
new file mode 100644
index 0000000000000..89b2ae5432101
--- /dev/null
+++ b/docs/changelog/102317.yaml
@@ -0,0 +1,6 @@
+pr: 102317
+summary: "ESQL: Fix single value query"
+area: ES|QL
+type: bug
+issues:
+ - 102298
diff --git a/docs/changelog/102350.yaml b/docs/changelog/102350.yaml
new file mode 100644
index 0000000000000..00a311c5d99f8
--- /dev/null
+++ b/docs/changelog/102350.yaml
@@ -0,0 +1,6 @@
+pr: 102350
+summary: "ESQL: Fix rare bug with empty string"
+area: ES|QL
+type: bug
+issues:
+ - 101969
diff --git a/docs/changelog/102379.yaml b/docs/changelog/102379.yaml
new file mode 100644
index 0000000000000..0773b137779a5
--- /dev/null
+++ b/docs/changelog/102379.yaml
@@ -0,0 +1,6 @@
+pr: 102379
+summary: Pass source query to `_field_caps` (as `index_filter`) when deducing destination index mappings for better
+ performance
+area: Transform
+type: enhancement
+issues: []
diff --git a/docs/changelog/102388.yaml b/docs/changelog/102388.yaml
new file mode 100644
index 0000000000000..3e65e46949bda
--- /dev/null
+++ b/docs/changelog/102388.yaml
@@ -0,0 +1,6 @@
+pr: 102388
+summary: Add support for `index_filter` to open pit
+area: Search
+type: enhancement
+issues:
+ - 99740
diff --git a/docs/changelog/102391.yaml b/docs/changelog/102391.yaml
new file mode 100644
index 0000000000000..5fcbb9e6d2858
--- /dev/null
+++ b/docs/changelog/102391.yaml
@@ -0,0 +1,5 @@
+pr: 102391
+summary: "ESQL: Support the `_source` metadata field"
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/102396.yaml b/docs/changelog/102396.yaml
new file mode 100644
index 0000000000000..9ea53ca5b6840
--- /dev/null
+++ b/docs/changelog/102396.yaml
@@ -0,0 +1,5 @@
+pr: 102396
+summary: Add more logging to the real memory circuit breaker and lower minimum interval
+area: "Infra/Circuit Breakers"
+type: bug
+issues: []
diff --git a/docs/changelog/102399.yaml b/docs/changelog/102399.yaml
new file mode 100644
index 0000000000000..7a4e1ff7ddab6
--- /dev/null
+++ b/docs/changelog/102399.yaml
@@ -0,0 +1,6 @@
+pr: 102399
+summary: "ES|QL: Fix layout management for Project"
+area: ES|QL
+type: bug
+issues:
+ - 102120
diff --git a/docs/changelog/102434.yaml b/docs/changelog/102434.yaml
new file mode 100644
index 0000000000000..ab6aa886c13b1
--- /dev/null
+++ b/docs/changelog/102434.yaml
@@ -0,0 +1,5 @@
+pr: 102434
+summary: "ESQL: Short circuit loading empty doc values"
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/102447.yaml b/docs/changelog/102447.yaml
new file mode 100644
index 0000000000000..76823153670bd
--- /dev/null
+++ b/docs/changelog/102447.yaml
@@ -0,0 +1,6 @@
+pr: 102447
+summary: Pass transform source query as `index_filter` to `open_point_in_time` request
+area: Transform
+type: enhancement
+issues:
+ - 101049
diff --git a/docs/changelog/102456.yaml b/docs/changelog/102456.yaml
new file mode 100644
index 0000000000000..6ef3b8f16f53c
--- /dev/null
+++ b/docs/changelog/102456.yaml
@@ -0,0 +1,6 @@
+pr: 102456
+summary: Switch logs data streams to search all fields by default
+area: Data streams
+type: enhancement
+issues:
+ - 99872
diff --git a/docs/changelog/102461.yaml b/docs/changelog/102461.yaml
new file mode 100644
index 0000000000000..c0c07554ed21f
--- /dev/null
+++ b/docs/changelog/102461.yaml
@@ -0,0 +1,5 @@
+pr: 102461
+summary: Enable concurrency for scripted metric agg
+area: Aggregations
+type: enhancement
+issues: []
diff --git a/docs/changelog/102462.yaml b/docs/changelog/102462.yaml
new file mode 100644
index 0000000000000..d44ccc4cbbc5c
--- /dev/null
+++ b/docs/changelog/102462.yaml
@@ -0,0 +1,5 @@
+pr: 102462
+summary: Check the real memory circuit breaker when building global ordinals
+area: Aggregations
+type: enhancement
+issues: []
diff --git a/docs/changelog/102467.yaml b/docs/changelog/102467.yaml
new file mode 100644
index 0000000000000..580788a5aa2f9
--- /dev/null
+++ b/docs/changelog/102467.yaml
@@ -0,0 +1,6 @@
+pr: 102467
+summary: Fix dense_vector cluster stats indexed_vector_dim_min/max values
+area: Mapping
+type: bug
+issues:
+ - 102416
diff --git a/docs/changelog/102472.yaml b/docs/changelog/102472.yaml
new file mode 100644
index 0000000000000..b0f5bfc714643
--- /dev/null
+++ b/docs/changelog/102472.yaml
@@ -0,0 +1,5 @@
+pr: 102472
+summary: Expose the `invalidation` field in Get/Query `ApiKey` APIs
+area: Security
+type: enhancement
+issues: [ ]
diff --git a/docs/changelog/102476.yaml b/docs/changelog/102476.yaml
new file mode 100644
index 0000000000000..a53a20ecfec20
--- /dev/null
+++ b/docs/changelog/102476.yaml
@@ -0,0 +1,5 @@
+pr: 102476
+summary: Unwrap `ExecutionException` when loading from cache in `AbstractIndexOrdinalsFieldData`
+area: Aggregations
+type: bug
+issues: []
diff --git a/docs/changelog/102490.yaml b/docs/changelog/102490.yaml
new file mode 100644
index 0000000000000..8ff554ab0f0fe
--- /dev/null
+++ b/docs/changelog/102490.yaml
@@ -0,0 +1,6 @@
+pr: 102490
+summary: "ESQL: Load text field from parent keyword field"
+area: ES|QL
+type: enhancement
+issues:
+ - 102473
diff --git a/docs/changelog/102492.yaml b/docs/changelog/102492.yaml
new file mode 100644
index 0000000000000..943d82873e0b6
--- /dev/null
+++ b/docs/changelog/102492.yaml
@@ -0,0 +1,5 @@
+pr: 102492
+summary: Ensure datafeed previews with no start or end time don't search the cold or frozen tiers
+area: Machine Learning
+type: bug
+issues: []
diff --git a/docs/changelog/102495.yaml b/docs/changelog/102495.yaml
new file mode 100644
index 0000000000000..77ae42f7eebcb
--- /dev/null
+++ b/docs/changelog/102495.yaml
@@ -0,0 +1,6 @@
+pr: 102495
+summary: "Add support for configuring proxy scheme in S3 client settings and EC2 discovery plugin"
+area: Distributed
+type: enhancement
+issues:
+ - 101873
diff --git a/docs/changelog/102510.yaml b/docs/changelog/102510.yaml
new file mode 100644
index 0000000000000..2b654b5c85929
--- /dev/null
+++ b/docs/changelog/102510.yaml
@@ -0,0 +1,7 @@
+pr: 102510
+summary: "ESQL: Make fieldcaps calls lighter"
+area: ES|QL
+type: enhancement
+issues:
+ - 101763
+ - 102393
diff --git a/docs/changelog/102511.yaml b/docs/changelog/102511.yaml
new file mode 100644
index 0000000000000..cf80ca03e197f
--- /dev/null
+++ b/docs/changelog/102511.yaml
@@ -0,0 +1,5 @@
+pr: 102511
+summary: Trigger parent circuit breaker when building scorers in filters aggregation
+area: Aggregations
+type: bug
+issues: []
diff --git a/docs/changelog/102512.yaml b/docs/changelog/102512.yaml
new file mode 100644
index 0000000000000..d4bc765ecaf5f
--- /dev/null
+++ b/docs/changelog/102512.yaml
@@ -0,0 +1,6 @@
+pr: 102512
+summary: Implement exponential backoff for transform state persistence retrying
+area: Transform
+type: enhancement
+issues:
+ - 102528
diff --git a/docs/changelog/102562.yaml b/docs/changelog/102562.yaml
new file mode 100644
index 0000000000000..a4b4f5a095118
--- /dev/null
+++ b/docs/changelog/102562.yaml
@@ -0,0 +1,5 @@
+pr: 102562
+summary: Track blocks of intermediate state of aggs
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/102570.yaml b/docs/changelog/102570.yaml
new file mode 100644
index 0000000000000..2d3f878dbbb27
--- /dev/null
+++ b/docs/changelog/102570.yaml
@@ -0,0 +1,5 @@
+pr: 102570
+summary: Added `beat.stats.libbeat.pipeline.queue.max_events`
+area: Monitoring
+type: enhancement
+issues: []
diff --git a/docs/changelog/102571.yaml b/docs/changelog/102571.yaml
new file mode 100644
index 0000000000000..25272408161db
--- /dev/null
+++ b/docs/changelog/102571.yaml
@@ -0,0 +1,5 @@
+pr: 102571
+summary: Allow executing multiple periodic flushes while they are being made durable
+area: Store
+type: enhancement
+issues: []
diff --git a/docs/changelog/102580.yaml b/docs/changelog/102580.yaml
new file mode 100644
index 0000000000000..50d315efd7071
--- /dev/null
+++ b/docs/changelog/102580.yaml
@@ -0,0 +1,6 @@
+pr: 102580
+summary: Fix DISSECT with empty patterns
+area: ES|QL
+type: bug
+issues:
+ - 102577
diff --git a/docs/changelog/102598.yaml b/docs/changelog/102598.yaml
new file mode 100644
index 0000000000000..c32519acdf6d1
--- /dev/null
+++ b/docs/changelog/102598.yaml
@@ -0,0 +1,5 @@
+pr: 102598
+summary: Add apm api for asynchronous counters (always increasing)
+area: Infra/Core
+type: enhancement
+issues: []
diff --git a/docs/changelog/102599.yaml b/docs/changelog/102599.yaml
new file mode 100644
index 0000000000000..74e3d89421463
--- /dev/null
+++ b/docs/changelog/102599.yaml
@@ -0,0 +1,5 @@
+pr: 102599
+summary: "Recreate the Elasticsearch private temporary directory if it doesn't exist when an ML job is opened"
+area: Machine Learning
+type: bug
+issues: []
diff --git a/docs/changelog/102602.yaml b/docs/changelog/102602.yaml
new file mode 100644
index 0000000000000..dd01eaa98b214
--- /dev/null
+++ b/docs/changelog/102602.yaml
@@ -0,0 +1,5 @@
+pr: 102602
+summary: Consider search context missing exceptions as recoverable
+area: Transform
+type: bug
+issues: []
diff --git a/docs/changelog/102612.yaml b/docs/changelog/102612.yaml
new file mode 100644
index 0000000000000..60808ae72801a
--- /dev/null
+++ b/docs/changelog/102612.yaml
@@ -0,0 +1,5 @@
+pr: 102612
+summary: Track blocks when hashing single multi-valued field
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/102636.yaml b/docs/changelog/102636.yaml
new file mode 100644
index 0000000000000..8b32e0568b0fb
--- /dev/null
+++ b/docs/changelog/102636.yaml
@@ -0,0 +1,5 @@
+pr: 102636
+summary: Revert non-semantic `NodeInfo`
+area: Infra/Core
+type: regression
+issues: []
diff --git a/docs/changelog/102637.yaml b/docs/changelog/102637.yaml
new file mode 100644
index 0000000000000..4d5d689934bd6
--- /dev/null
+++ b/docs/changelog/102637.yaml
@@ -0,0 +1,5 @@
+pr: 102637
+summary: Improve stability of spike and dip detection for the change point aggregation
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/102644.yaml b/docs/changelog/102644.yaml
new file mode 100644
index 0000000000000..17c5cbebed7cc
--- /dev/null
+++ b/docs/changelog/102644.yaml
@@ -0,0 +1,5 @@
+pr: 102644
+summary: Disable parallelism for composite agg against high cardinality fields
+area: Aggregations
+type: enhancement
+issues: []
diff --git a/docs/changelog/102673.yaml b/docs/changelog/102673.yaml
new file mode 100644
index 0000000000000..16546edb3cf3c
--- /dev/null
+++ b/docs/changelog/102673.yaml
@@ -0,0 +1,5 @@
+pr: 102673
+summary: "ESQL: Share constant null Blocks"
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/102680.yaml b/docs/changelog/102680.yaml
new file mode 100644
index 0000000000000..8b32c5029ea2a
--- /dev/null
+++ b/docs/changelog/102680.yaml
@@ -0,0 +1,5 @@
+pr: 102680
+summary: Make `api_key.delete.interval` a dynamic setting
+area: Security
+type: enhancement
+issues: []
diff --git a/docs/changelog/102682.yaml b/docs/changelog/102682.yaml
new file mode 100644
index 0000000000000..190ff3df5a7f6
--- /dev/null
+++ b/docs/changelog/102682.yaml
@@ -0,0 +1,5 @@
+pr: 102682
+summary: Introduce fielddata cache ttl
+area: Aggregations
+type: enhancement
+issues: []
diff --git a/docs/changelog/102710.yaml b/docs/changelog/102710.yaml
new file mode 100644
index 0000000000000..ee805c70180a0
--- /dev/null
+++ b/docs/changelog/102710.yaml
@@ -0,0 +1,5 @@
+pr: 102710
+summary: Enable concurrency for multi terms agg
+area: Aggregations
+type: enhancement
+issues: []
diff --git a/docs/changelog/102715.yaml b/docs/changelog/102715.yaml
new file mode 100644
index 0000000000000..7311db66ce151
--- /dev/null
+++ b/docs/changelog/102715.yaml
@@ -0,0 +1,6 @@
+pr: 102715
+summary: Fix leaking blocks in TopN
+area: ES|QL
+type: bug
+issues:
+ - 102646
diff --git a/docs/changelog/102716.yaml b/docs/changelog/102716.yaml
new file mode 100644
index 0000000000000..39317fdb38415
--- /dev/null
+++ b/docs/changelog/102716.yaml
@@ -0,0 +1,5 @@
+pr: 102716
+summary: Fix leaking blocks in `BlockUtils`
+area: ES|QL
+type: bug
+issues: []
diff --git a/docs/changelog/102735.yaml b/docs/changelog/102735.yaml
new file mode 100644
index 0000000000000..4726e08d1f314
--- /dev/null
+++ b/docs/changelog/102735.yaml
@@ -0,0 +1,5 @@
+pr: 102735
+summary: "[Profiling] Report in status API if docs exist"
+area: Application
+type: enhancement
+issues: []
diff --git a/docs/changelog/99134.yaml b/docs/changelog/99134.yaml
new file mode 100644
index 0000000000000..10156b9b30066
--- /dev/null
+++ b/docs/changelog/99134.yaml
@@ -0,0 +1,5 @@
+pr: 99134
+summary: Add ability to create a data stream failure store
+area: Data streams
+type: feature
+issues: []
diff --git a/docs/plugins/discovery-ec2.asciidoc b/docs/plugins/discovery-ec2.asciidoc
index 3947ed71ea9ae..44acba4752aaa 100644
--- a/docs/plugins/discovery-ec2.asciidoc
+++ b/docs/plugins/discovery-ec2.asciidoc
@@ -97,6 +97,11 @@ The available settings for the EC2 discovery plugin are as follows.
this setting determines the port to use to connect to the proxy. Defaults to
`80`.
+`discovery.ec2.proxy.scheme`::
+
+ The scheme to use when connecting to the EC2 service endpoint through proxy specified
+ in `discovery.ec2.proxy.host`. Valid values are `http` or `https`. Defaults to `http`.
+
`discovery.ec2.proxy.username` ({ref}/secure-settings.html[Secure], {ref}/secure-settings.html#reloadable-secure-settings[reloadable])::
When the address of an HTTP proxy is given in `discovery.ec2.proxy.host`,
diff --git a/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc b/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc
index 2bedcd4698b42..d7d837b2f8364 100644
--- a/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc
+++ b/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc
@@ -127,8 +127,7 @@ init_script:: Executed prior to any collection of documents. Allows the ag
+
In the above example, the `init_script` creates an array `transactions` in the `state` object.
-map_script:: Executed once per document collected. This is a required script. If no combine_script is specified, the resulting state
- needs to be stored in the `state` object.
+map_script:: Executed once per document collected. This is a required script.
+
In the above example, the `map_script` checks the value of the type field. If the value is 'sale' the value of the amount field
is added to the transactions array. If the value of the type field is not 'sale' the negated value of the amount field is added
diff --git a/docs/reference/aggregations/pipeline/movfn-aggregation.asciidoc b/docs/reference/aggregations/pipeline/movfn-aggregation.asciidoc
index b5f1315531916..44a00b9f5b99e 100644
--- a/docs/reference/aggregations/pipeline/movfn-aggregation.asciidoc
+++ b/docs/reference/aggregations/pipeline/movfn-aggregation.asciidoc
@@ -68,7 +68,7 @@ POST /_search
--------------------------------------------------
// TEST[setup:sales]
-<1> A `date_histogram` named "my_date_histo" is constructed on the "timestamp" field, with one-day intervals
+<1> A `date_histogram` named "my_date_histo" is constructed on the "timestamp" field, with one-month intervals
<2> A `sum` metric is used to calculate the sum of a field. This could be any numeric metric (sum, min, max, etc)
<3> Finally, we specify a `moving_fn` aggregation which uses "the_sum" metric as its input.
diff --git a/docs/reference/analysis/analyzers/lang-analyzer.asciidoc b/docs/reference/analysis/analyzers/lang-analyzer.asciidoc
index 45cb725492f07..5273537389e3d 100644
--- a/docs/reference/analysis/analyzers/lang-analyzer.asciidoc
+++ b/docs/reference/analysis/analyzers/lang-analyzer.asciidoc
@@ -36,6 +36,7 @@ following types are supported:
<>,
<>,
<>,
+<>,
<>,
<>,
<>,
@@ -64,8 +65,8 @@ The following analyzers support setting custom `stem_exclusion` list:
`arabic`, `armenian`, `basque`, `bengali`, `bulgarian`, `catalan`, `czech`,
`dutch`, `english`, `finnish`, `french`, `galician`,
`german`, `hindi`, `hungarian`, `indonesian`, `irish`, `italian`, `latvian`,
-`lithuanian`, `norwegian`, `portuguese`, `romanian`, `russian`, `sorani`,
-`spanish`, `swedish`, `turkish`.
+`lithuanian`, `norwegian`, `portuguese`, `romanian`, `russian`, `serbian`,
+`sorani`, `spanish`, `swedish`, `turkish`.
==== Reimplementing language analyzers
@@ -1588,6 +1589,55 @@ PUT /russian_example
<2> This filter should be removed unless there are words which should
be excluded from stemming.
+[[serbian-analyzer]]
+===== `serbian` analyzer
+
+The `serbian` analyzer could be reimplemented as a `custom` analyzer as follows:
+
+[source,console]
+----------------------------------------------------
+PUT /serbian_example
+{
+ "settings": {
+ "analysis": {
+ "filter": {
+ "serbian_stop": {
+ "type": "stop",
+ "stopwords": "_serbian_" <1>
+ },
+ "serbian_keywords": {
+ "type": "keyword_marker",
+ "keywords": ["пример"] <2>
+ },
+ "serbian_stemmer": {
+ "type": "stemmer",
+ "language": "serbian"
+ }
+ },
+ "analyzer": {
+ "rebuilt_serbian": {
+ "tokenizer": "standard",
+ "filter": [
+ "lowercase",
+ "serbian_stop",
+ "serbian_keywords",
+ "serbian_stemmer",
+ "serbian_normalization"
+ ]
+ }
+ }
+ }
+ }
+}
+----------------------------------------------------
+// TEST[s/"serbian_keywords",//]
+// TEST[s/\n$/\nstartyaml\n - compare_analyzers: {index: serbian_example, first: serbian, second: rebuilt_serbian}\nendyaml\n/]
+
+<1> The default stopwords can be overridden with the `stopwords`
+or `stopwords_path` parameters.
+<2> This filter should be removed unless there are words which should
+be excluded from stemming.
+
[[sorani-analyzer]]
===== `sorani` analyzer
diff --git a/docs/reference/analysis/tokenfilters/snowball-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/snowball-tokenfilter.asciidoc
index a76bc6f6c5254..57e402988cc5a 100644
--- a/docs/reference/analysis/tokenfilters/snowball-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/snowball-tokenfilter.asciidoc
@@ -9,7 +9,7 @@ A filter that stems words using a Snowball-generated stemmer. The
values: `Arabic`, `Armenian`, `Basque`, `Catalan`, `Danish`, `Dutch`, `English`,
`Estonian`, `Finnish`, `French`, `German`, `German2`, `Hungarian`, `Italian`, `Irish`, `Kp`,
`Lithuanian`, `Lovins`, `Norwegian`, `Porter`, `Portuguese`, `Romanian`,
-`Russian`, `Spanish`, `Swedish`, `Turkish`.
+`Russian`, `Serbian`, `Spanish`, `Swedish`, `Turkish`.
For example:
diff --git a/docs/reference/analysis/tokenfilters/stemmer-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/stemmer-tokenfilter.asciidoc
index 162164e12872d..b8d883b057823 100644
--- a/docs/reference/analysis/tokenfilters/stemmer-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/stemmer-tokenfilter.asciidoc
@@ -230,6 +230,9 @@ Russian::
https://snowballstem.org/algorithms/russian/stemmer.html[*`russian`*],
https://doc.rero.ch/lm.php?url=1000%2C43%2C4%2C20091209094227-CA%2FDolamic_Ljiljana_-_Indexing_and_Searching_Strategies_for_the_Russian_20091209.pdf[`light_russian`]
+Serbian::
+https://snowballstem.org/algorithms/serbian/stemmer.html[*`serbian`*]
+
Spanish::
https://www.ercim.eu/publication/ws-proceedings/CLEF2/savoy.pdf[*`light_spanish`*],
https://snowballstem.org/algorithms/spanish/stemmer.html[`spanish`]
diff --git a/docs/reference/analysis/tokenfilters/stop-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/stop-tokenfilter.asciidoc
index 12e0d76f9901b..abba633b643dc 100644
--- a/docs/reference/analysis/tokenfilters/stop-tokenfilter.asciidoc
+++ b/docs/reference/analysis/tokenfilters/stop-tokenfilter.asciidoc
@@ -356,6 +356,10 @@ parameter and a link to their predefined stop words in Lucene.
`_russian_`::
{lucene-stop-word-link}/snowball/russian_stop.txt[Russian stop words]
+[[serbian-stop-words]]
+`_serbian_`::
+{lucene-stop-word-link}/sr/stopwords.txt[Serbian stop words]
+
[[sorani-stop-words]]
`_sorani_`::
{lucene-stop-word-link}/ckb/stopwords.txt[Sorani stop words]
diff --git a/docs/reference/data-streams/change-mappings-and-settings.asciidoc b/docs/reference/data-streams/change-mappings-and-settings.asciidoc
index 3922ef018a713..86d72cf52c9e9 100644
--- a/docs/reference/data-streams/change-mappings-and-settings.asciidoc
+++ b/docs/reference/data-streams/change-mappings-and-settings.asciidoc
@@ -601,7 +601,7 @@ stream's oldest backing index.
// TESTRESPONSE[s/"index_uuid": "_eEfRrFHS9OyhqWntkgHAQ"/"index_uuid": $body.data_streams.0.indices.1.index_uuid/]
// TESTRESPONSE[s/"index_name": ".ds-my-data-stream-2099.03.07-000001"/"index_name": $body.data_streams.0.indices.0.index_name/]
// TESTRESPONSE[s/"index_name": ".ds-my-data-stream-2099.03.08-000002"/"index_name": $body.data_streams.0.indices.1.index_name/]
-// TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW"/]
+// TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW","failure_indices":[],"failure_store":false/]
<1> First item in the `indices` array for `my-data-stream`. This item contains
information about the stream's oldest backing index,
@@ -704,4 +704,4 @@ Use the <> to update an existing data stream's
aliases. Changing an existing data stream's aliases in its index pattern has no
effect.
-include::../alias.asciidoc[tag=alias-multiple-actions-example]
\ No newline at end of file
+include::../alias.asciidoc[tag=alias-multiple-actions-example]
diff --git a/docs/reference/data-streams/data-stream-apis.asciidoc b/docs/reference/data-streams/data-stream-apis.asciidoc
index d3580ca4448a7..3c2e703d264ff 100644
--- a/docs/reference/data-streams/data-stream-apis.asciidoc
+++ b/docs/reference/data-streams/data-stream-apis.asciidoc
@@ -25,6 +25,8 @@ preview:[]
preview:[]
* <>
preview:[]
+* <>
+preview:[]
The following API is available for <>:
@@ -55,4 +57,6 @@ include::{es-repo-dir}/data-streams/lifecycle/apis/delete-lifecycle.asciidoc[]
include::{es-repo-dir}/data-streams/lifecycle/apis/explain-lifecycle.asciidoc[]
+include::{es-repo-dir}/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc[]
+
include::{es-repo-dir}/indices/downsample-data-stream.asciidoc[]
diff --git a/docs/reference/data-streams/downsampling-manual.asciidoc b/docs/reference/data-streams/downsampling-manual.asciidoc
index b7d46b6301884..5bdfaf428d169 100644
--- a/docs/reference/data-streams/downsampling-manual.asciidoc
+++ b/docs/reference/data-streams/downsampling-manual.asciidoc
@@ -17,7 +17,7 @@ DELETE _ingest/pipeline/my-timestamp-pipeline
The recommended way to downsample a time series data stream (TSDS) is
<>. However, if
you're not using ILM, you can downsample a TSDS manually. This guide shows you
-how, using typical Kubernetes cluster monitoring data.
+how, using typical Kubernetes cluster monitoring data.
To test out manual downsampling, follow these steps:
@@ -32,13 +32,13 @@ To test out manual downsampling, follow these steps:
==== Prerequisites
* Refer to the <>.
-* It is not possible to downsample a data stream directly, nor
+* It is not possible to downsample a data stream directly, nor
multiple indices at once. It's only possible to downsample one time series index
(TSDS backing index).
* In order to downsample an index, it needs to be read-only. For a TSDS write
index, this means it needs to be rolled over and made read-only first.
* Downsampling uses UTC timestamps.
-* Downsampling needs at least one metric field to exist in the time series
+* Downsampling needs at least one metric field to exist in the time series
index.
[discrete]
@@ -51,8 +51,8 @@ First, you'll create a TSDS. For simplicity, in the time series mapping all
be used. The `time_series_metric` values determine the kind of statistical
representations that are used during downsampling.
-The index template includes a set of static
-<>: `host`, `namespace`,
+The index template includes a set of static
+<>: `host`, `namespace`,
`node`, and `pod`. The time series dimensions are not changed by the
downsampling process.
@@ -388,6 +388,7 @@ This returns:
// TESTRESPONSE[s/"ltOJGmqgTVm4T-Buoe7Acg"/$body.data_streams.0.indices.0.index_uuid/]
// TESTRESPONSE[s/"2023-07-26T09:26:42.000Z"/$body.data_streams.0.time_series.temporal_ranges.0.start/]
// TESTRESPONSE[s/"2023-07-26T13:26:42.000Z"/$body.data_streams.0.time_series.temporal_ranges.0.end/]
+// TESTRESPONSE[s/"replicated": false/"replicated": false,"failure_indices":[],"failure_store":false/]
<1> The backing index for this data stream.
Before a backing index can be downsampled, the TSDS needs to be rolled over and
diff --git a/docs/reference/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc b/docs/reference/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc
new file mode 100644
index 0000000000000..6fa82dc2a810c
--- /dev/null
+++ b/docs/reference/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc
@@ -0,0 +1,93 @@
+[[data-streams-get-lifecycle-stats]]
+=== Get data stream lifecycle stats
+++++
+Get Data Stream Lifecycle
+++++
+
+preview::[]
+
+Gets stats about the execution of data stream lifecycle.
+
+[[get-lifecycle-stats-api-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have the `monitor` or
+`manage` <> to use this API.
+
+[[data-streams-get-lifecycle-stats-request]]
+==== {api-request-title}
+
+`GET _lifecycle/stats`
+
+[[data-streams-get-lifecycle-stats-desc]]
+==== {api-description-title}
+
+Gets stats about the execution of the data stream lifecycle. The data stream level stats include only stats about data streams
+managed by the data stream lifecycle.
+
+[[get-lifecycle-stats-api-response-body]]
+==== {api-response-body-title}
+
+`last_run_duration_in_millis`::
+(Optional, long)
+The duration of the last data stream lifecycle execution.
+`time_between_starts_in_millis`::
+(Optional, long)
+The time passed between the start of the last two data stream lifecycle executions. This should amount approximately to
+<>.
+`data_stream_count`::
+(integer)
+The count of data streams currently being managed by the data stream lifecycle.
+`data_streams`::
+(array of objects)
+Contains information about the retrieved data stream lifecycles.
++
+.Properties of objects in `data_streams`
+[%collapsible%open]
+====
+`name`::
+(string)
+The name of the data stream.
+`backing_indices_in_total`::
+(integer)
+The count of the backing indices of this data stream that are managed by the data stream lifecycle.
+`backing_indices_in_error`::
+(integer)
+The count of the backing indices of this data stream that are managed by the data stream lifecycle and have encountered an error.
+====
+
+[[data-streams-get-lifecycle-stats-example]]
+==== {api-examples-title}
+
+Let's retrieve the data stream lifecycle stats of a cluster that has already executed the lifecycle more than once:
+
+[source,console]
+--------------------------------------------------
+GET _lifecycle/stats?human&pretty
+--------------------------------------------------
+// TEST[skip:this is for demonstration purposes only, we cannot ensure that DSL has run]
+
+The response will look like the following:
+
+[source,console-result]
+--------------------------------------------------
+{
+ "last_run_duration_in_millis": 2,
+ "last_run_duration": "2ms",
+ "time_between_starts_in_millis": 9998,
+ "time_between_starts": "9.99s",
+ "data_streams_count": 2,
+ "data_streams": [
+ {
+ "name": "my-data-stream",
+ "backing_indices_in_total": 2,
+ "backing_indices_in_error": 0
+ },
+ {
+ "name": "my-other-stream",
+ "backing_indices_in_total": 2,
+ "backing_indices_in_error": 1
+ }
+ ]
+}
+--------------------------------------------------
\ No newline at end of file
diff --git a/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc b/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc
index de11bbcfc2d4e..aa598b010badc 100644
--- a/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc
+++ b/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc
@@ -1,20 +1,36 @@
[role="xpack"]
[[tutorial-migrate-data-stream-from-ilm-to-dsl]]
-=== Tutorial: Migrate ILM managed data stream to Data stream lifecycle
+=== Tutorial: Migrate ILM managed data stream to data stream lifecycle
preview::[]
-In this tutorial we'll look at migrating an existing data stream from {ilm-init} to
-Data stream lifecycle. The existing {ilm-init} managed backing indices will continue
+In this tutorial we'll look at migrating an existing data stream from Index Lifecycle Management ({ilm-init}) to
+data stream lifecycle. The existing {ilm-init} managed backing indices will continue
to be managed by {ilm-init} until they age out and get deleted by {ilm-init}; however,
-the new backing indices will be managed by Data stream lifecycle.
-This way, a data stream is gradually migrated away from being managed by {ilm-cap} to
-being managed by Data stream lifecycle. As we'll see, {ilm-cap} and Data stream lifecycle
+the new backing indices will be managed by data stream lifecycle.
+This way, a data stream is gradually migrated away from being managed by {ilm-init} to
+being managed by data stream lifecycle. As we'll see, {ilm-init} and data stream lifecycle
can co-manage a data stream; however, an index can only be managed by one system at
a time.
-Let's first create a data stream with two backing indices managed by {ilm-cap}.
-We first create an {ilm-cap} policy:
+[discrete]
+[[migrate-dsl-ilm-tldr]]
+==== TL;DR
+To migrate a data stream from {ilm-init} to data stream lifecycle we'll have to execute
+two steps:
+
+1. Update the index template that's backing the data stream to set <>
+to `false`, and to configure data stream lifecycle.
+2. Configure the data stream lifecycle for the _existing_ data stream using
+the <>.
+
+For more details see the <> section.
+
+[discrete]
+[[setup-test-data]]
+==== Setup ILM managed data stream
+Let's first create a data stream with two backing indices managed by {ilm-init}.
+We first create an {ilm-init} policy:
[source,console]
----
@@ -40,7 +56,7 @@ PUT _ilm/policy/pre-dsl-ilm-policy
}
----
-And let's create an index template that'll back the data stream and configures {ilm-cap}:
+And let's create an index template that'll back the data stream and configures {ilm-init}:
[source,console]
----
@@ -77,7 +93,7 @@ POST dsl-data-stream/_rollover
----
// TEST[continued]
-We'll use the <> API to inspect the state of
+We'll use the <> API to inspect the state of
the data stream:
[source,console]
@@ -87,7 +103,7 @@ GET _data_stream/dsl-data-stream
// TEST[continued]
Inspecting the response we'll see that both backing indices are managed by {ilm-init}
-and that the next generation index will also be managed by {ilm-init}:
+and that the next generation index will also be managed by {ilm-init}:
[source,console-result]
----
@@ -100,7 +116,7 @@ and that the next generation index will also be managed by {ilm-init}:
},
"indices": [
{
- "index_name": ".ds-dsl-data-stream-2023.10.19-000001", <1>
+ "index_name": ".ds-dsl-data-stream-2023.10.19-000001", <1>
"index_uuid": "xCEhwsp8Tey0-FLNFYVwSg",
"prefer_ilm": true, <2>
"ilm_policy": "pre-dsl-ilm-policy", <3>
@@ -132,37 +148,40 @@ and that the next generation index will also be managed by {ilm-init}:
// TESTRESPONSE[s/"index_uuid": "xCEhwsp8Tey0-FLNFYVwSg"/"index_uuid": $body.data_streams.0.indices.0.index_uuid/]
// TESTRESPONSE[s/"index_name": ".ds-dsl-data-stream-2023.10.19-000002"/"index_name": $body.data_streams.0.indices.1.index_name/]
// TESTRESPONSE[s/"index_uuid": "PA_JquKGSiKcAKBA8DJ5gw"/"index_uuid": $body.data_streams.0.indices.1.index_uuid/]
-// TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW"/]
+// TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW","failure_indices":[],"failure_store":false/]
<1> The name of the backing index.
-<2> For each backing index we display the value of the <>
+<2> For each backing index we display the value of the <>
configuration which will indicate if {ilm-init} takes precedence over data stream lifecycle in case
both systems are configured for an index.
-<3> The {ilm-ini} policy configured for this index.
-<4> The system that manages this index (possible values are "Index Lifecycle Management",
+<3> The {ilm-init} policy configured for this index.
+<4> The system that manages this index (possible values are "Index Lifecycle Management",
"Data stream lifecycle", or "Unmanaged")
-<5> The system that will manage the next generation index (the new write index of this
-data stream, once the data stream is rolled over). The possible values are
+<5> The system that will manage the next generation index (the new write index of this
+data stream, once the data stream is rolled over). The possible values are
"Index Lifecycle Management", "Data stream lifecycle", or "Unmanaged".
<6> The <> value configured in the index template
that's backing the data stream. This value will be configured for all the new backing indices.
If it's not configured in the index template the backing indices will receive the `true`
-default value ({ilm-init} takes precedence over data stream lifecycle by default as it's
+default value ({ilm-init} takes precedence over data stream lifecycle by default as it's
currently richer in features).
-<7> The {ilm-init} policy configured in the index template that's backing this data
-stream (which will be configured on all the new backing indices, as long as it exists
+<7> The {ilm-init} policy configured in the index template that's backing this data
+stream (which will be configured on all the new backing indices, as long as it exists
in the index template).
+[discrete]
+[[migrate-from-ilm-to-dsl]]
+==== Migrate data stream to data stream lifecycle
To migrate the `dsl-data-stream` to data stream lifecycle we'll have to execute
two steps:
-1. Update the index template that's backing the index template to configure <>
+1. Update the index template that's backing the data stream to set <>
to `false`, and to configure data stream lifecycle.
2. Configure the data stream lifecycle for the _existing_ `dsl-data-stream` using
the <>.
IMPORTANT: The data stream lifecycle configuration that's added to the index template,
-being a data stream configuration, will only apply to **new** data streams.
+being a data stream configuration, will only apply to **new** data streams.
Our data stream exists already, so even though we added a data stream lifecycle
configuration in the index template it will not be applied to `dsl-data-stream`.
@@ -192,13 +211,13 @@ PUT _index_template/dsl-data-stream-template
<1> The `prefer_ilm` setting will now be configured on the **new** backing indices
(created by rolling over the data stream) such that {ilm-init} does _not_ take
-precedence over Data stream lifecycle.
+precedence over data stream lifecycle.
<2> We're configuring the data stream lifecycle so _new_ data streams will be
-managed by Data stream lifecycle.
+managed by data stream lifecycle.
-We've now make sure that new data streams will be managed by Data stream lifecycle.
+We've now made sure that new data streams will be managed by data stream lifecycle.
-Let's update our existing `dsl-data-stream` and configure Data stream lifecycle:
+Let's update our existing `dsl-data-stream` and configure data stream lifecycle:
[source,console]
----
@@ -210,7 +229,7 @@ PUT _data_stream/dsl-data-stream/_lifecycle
// TEST[continued]
We can inspect the data stream to check that the next generation will indeed be
-managed by Data stream lifecycle:
+managed by data stream lifecycle:
[source,console]
--------------------------------------------------
@@ -229,10 +248,10 @@ GET _data_stream/dsl-data-stream
},
"indices": [
{
- "index_name": ".ds-dsl-data-stream-2023.10.19-000001",
+ "index_name": ".ds-dsl-data-stream-2023.10.19-000001",
"index_uuid": "xCEhwsp8Tey0-FLNFYVwSg",
- "prefer_ilm": true,
- "ilm_policy": "pre-dsl-ilm-policy",
+ "prefer_ilm": true,
+ "ilm_policy": "pre-dsl-ilm-policy",
"managed_by": "Index Lifecycle Management" <1>
},
{
@@ -250,7 +269,7 @@ GET _data_stream/dsl-data-stream
"enabled": true,
"data_retention": "7d"
},
- "ilm_policy": "pre-dsl-ilm-policy",
+ "ilm_policy": "pre-dsl-ilm-policy",
"next_generation_managed_by": "Data stream lifecycle", <3>
"prefer_ilm": false, <4>
"hidden": false,
@@ -265,7 +284,7 @@ GET _data_stream/dsl-data-stream
// TESTRESPONSE[s/"index_uuid": "xCEhwsp8Tey0-FLNFYVwSg"/"index_uuid": $body.data_streams.0.indices.0.index_uuid/]
// TESTRESPONSE[s/"index_name": ".ds-dsl-data-stream-2023.10.19-000002"/"index_name": $body.data_streams.0.indices.1.index_name/]
// TESTRESPONSE[s/"index_uuid": "PA_JquKGSiKcAKBA8DJ5gw"/"index_uuid": $body.data_streams.0.indices.1.index_uuid/]
-// TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW"/]
+// TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW","failure_indices":[],"failure_store":false/]
<1> The existing backing index will continue to be managed by {ilm-init}
<2> The existing backing index will continue to be managed by {ilm-init}
@@ -274,7 +293,7 @@ GET _data_stream/dsl-data-stream
and will be configured accordingly for new backing indices.
We'll now rollover the data stream to see the new generation index being managed by
-Data stream lifecycle:
+data stream lifecycle:
[source,console]
----
@@ -299,11 +318,11 @@ GET _data_stream/dsl-data-stream
},
"indices": [
{
- "index_name": ".ds-dsl-data-stream-2023.10.19-000001",
+ "index_name": ".ds-dsl-data-stream-2023.10.19-000001",
"index_uuid": "xCEhwsp8Tey0-FLNFYVwSg",
- "prefer_ilm": true,
- "ilm_policy": "pre-dsl-ilm-policy",
- "managed_by": "Index Lifecycle Management" <1>
+ "prefer_ilm": true,
+ "ilm_policy": "pre-dsl-ilm-policy",
+ "managed_by": "Index Lifecycle Management" <1>
},
{
"index_name": ".ds-dsl-data-stream-2023.10.19-000002",
@@ -327,9 +346,9 @@ GET _data_stream/dsl-data-stream
"enabled": true,
"data_retention": "7d"
},
- "ilm_policy": "pre-dsl-ilm-policy",
- "next_generation_managed_by": "Data stream lifecycle",
- "prefer_ilm": false,
+ "ilm_policy": "pre-dsl-ilm-policy",
+ "next_generation_managed_by": "Data stream lifecycle",
+ "prefer_ilm": false,
"hidden": false,
"system": false,
"allow_custom_routing": false,
@@ -344,7 +363,7 @@ GET _data_stream/dsl-data-stream
// TESTRESPONSE[s/"index_uuid": "PA_JquKGSiKcAKBA8DJ5gw"/"index_uuid": $body.data_streams.0.indices.1.index_uuid/]
// TESTRESPONSE[s/"index_name": ".ds-dsl-data-stream-2023.10.19-000003"/"index_name": $body.data_streams.0.indices.2.index_name/]
// TESTRESPONSE[s/"index_uuid": "PA_JquKGSiKcAKBA8abcd1"/"index_uuid": $body.data_streams.0.indices.2.index_uuid/]
-// TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW"/]
+// TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW","failure_indices":[],"failure_store":false/]
<1> The backing indices that existed before rollover will continue to be managed by {ilm-init}
<2> The backing indices that existed before rollover will continue to be managed by {ilm-init}
@@ -352,27 +371,30 @@ GET _data_stream/dsl-data-stream
in the index template
<4> The new write index is managed by `Data stream lifecycle`
-We can easily change this data stream to be managed by {ilm-cap} because we didn't remove
-the {ilm-cap} policy when we <>.
We can achieve this in two ways:
1. <> from the data streams
-2. Disable Data stream lifecycle by configured the `enabled` flag to `false`.
+2. Disable data stream lifecycle by configuring the `enabled` flag to `false`.
-Let's implement option 2 and disable the data stream lifecycle:
+Let's implement option 2 and disable the data stream lifecycle:
[source,console]
----
PUT _data_stream/dsl-data-stream/_lifecycle
{
"data_retention": "7d",
- "enabled": false <1>
+ "enabled": false <1>
}
----
// TEST[continued]
-<1> The `enabled` flag can be ommitted and defaults to `true` however, here we
+<1> The `enabled` flag can be ommitted and defaults to `true` however, here we
explicitly configure it to `false`
Let's check the state of the data stream:
@@ -393,23 +415,23 @@ GET _data_stream/dsl-data-stream
},
"indices": [
{
- "index_name": ".ds-dsl-data-stream-2023.10.19-000001",
+ "index_name": ".ds-dsl-data-stream-2023.10.19-000001",
"index_uuid": "xCEhwsp8Tey0-FLNFYVwSg",
- "prefer_ilm": true,
- "ilm_policy": "pre-dsl-ilm-policy",
- "managed_by": "Index Lifecycle Management"
+ "prefer_ilm": true,
+ "ilm_policy": "pre-dsl-ilm-policy",
+ "managed_by": "Index Lifecycle Management"
},
{
"index_name": ".ds-dsl-data-stream-2023.10.19-000002",
"index_uuid": "PA_JquKGSiKcAKBA8DJ5gw",
"prefer_ilm": true,
"ilm_policy": "pre-dsl-ilm-policy",
- "managed_by": "Index Lifecycle Management"
+ "managed_by": "Index Lifecycle Management"
},
{
"index_name": ".ds-dsl-data-stream-2023.10.19-000003",
"index_uuid": "PA_JquKGSiKcAKBA8abcd1",
- "prefer_ilm": false,
+ "prefer_ilm": false,
"ilm_policy": "pre-dsl-ilm-policy",
"managed_by": "Index Lifecycle Management" <1>
}
@@ -421,9 +443,9 @@ GET _data_stream/dsl-data-stream
"enabled": false, <2>
"data_retention": "7d"
},
- "ilm_policy": "pre-dsl-ilm-policy",
- "next_generation_managed_by": "Index Lifecycle Management", <3>
- "prefer_ilm": false,
+ "ilm_policy": "pre-dsl-ilm-policy",
+ "next_generation_managed_by": "Index Lifecycle Management", <3>
+ "prefer_ilm": false,
"hidden": false,
"system": false,
"allow_custom_routing": false,
@@ -438,14 +460,14 @@ GET _data_stream/dsl-data-stream
// TESTRESPONSE[s/"index_uuid": "PA_JquKGSiKcAKBA8DJ5gw"/"index_uuid": $body.data_streams.0.indices.1.index_uuid/]
// TESTRESPONSE[s/"index_name": ".ds-dsl-data-stream-2023.10.19-000003"/"index_name": $body.data_streams.0.indices.2.index_name/]
// TESTRESPONSE[s/"index_uuid": "PA_JquKGSiKcAKBA8abcd1"/"index_uuid": $body.data_streams.0.indices.2.index_uuid/]
-// TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW"/]
-<1> The write index is now managed by {ilm-cap}
-<2> The `lifecycle` configured on the data stream is now disabled.
-<3> The next write index will be managed by {ilm-cap}
-
-Had we removed the {ilm-cap} policy from the index template when we <>
-it, the write index of the data stream will now be `Unmanaged` because the index
-wouldn't have the {ilm-cap} policy configured to fallback onto.
+// TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW","failure_indices":[],"failure_store":false/]
+<1> The write index is now managed by {ilm-init}
+<2> The `lifecycle` configured on the data stream is now disabled.
+<3> The next write index will be managed by {ilm-init}
+
+Had we removed the {ilm-init} policy from the index template when we <>
+it, the write index of the data stream will now be `Unmanaged` because the index
+wouldn't have the {ilm-init} policy configured to fallback onto.
//////////////////////////
[source,console]
diff --git a/docs/reference/docs/data-replication.asciidoc b/docs/reference/docs/data-replication.asciidoc
index 9503b6b6bb29d..2c1a16c81d011 100644
--- a/docs/reference/docs/data-replication.asciidoc
+++ b/docs/reference/docs/data-replication.asciidoc
@@ -28,6 +28,8 @@ Every indexing operation in Elasticsearch is first resolved to a replication gro
typically based on the document ID. Once the replication group has been determined, the operation is forwarded
internally to the current _primary shard_ of the group. This stage of indexing is referred to as the _coordinating stage_.
+image::images/data_processing_flow.png[An example of a basic write model.]
+
The next stage of indexing is the _primary stage_, performed on the primary shard. The primary shard is responsible
for validating the operation and forwarding it to the other replicas. Since replicas can be offline, the primary
is not required to replicate to all replicas. Instead, Elasticsearch maintains a list of shard copies that should
diff --git a/docs/reference/esql/esql-get-started.asciidoc b/docs/reference/esql/esql-get-started.asciidoc
index 82831ef943398..e54825406257f 100644
--- a/docs/reference/esql/esql-get-started.asciidoc
+++ b/docs/reference/esql/esql-get-started.asciidoc
@@ -7,50 +7,14 @@
This guide shows how you can use {esql} to query and aggregate your data.
-TIP: To get started with {esql} without setting up your own deployment, visit
-the public {esql} demo environment at
-https://esql.demo.elastic.co/[esql.demo.elastic.co]. It comes with preloaded
-data sets and sample queries.
-
[discrete]
[[esql-getting-started-prerequisites]]
=== Prerequisites
-To follow along with the queries in this getting started guide, first ingest
-some sample data using the following requests:
-
-[source,console]
-----
-PUT sample_data
-{
- "mappings": {
- "properties": {
- "client.ip": {
- "type": "ip"
- },
- "message": {
- "type": "keyword"
- }
- }
- }
-}
-
-PUT sample_data/_bulk
-{"index": {}}
-{"@timestamp": "2023-10-23T12:15:03.360Z", "client.ip": "172.21.2.162", "message": "Connected to 10.1.0.3", "event.duration": 3450233}
-{"index": {}}
-{"@timestamp": "2023-10-23T12:27:28.948Z", "client.ip": "172.21.2.113", "message": "Connected to 10.1.0.2", "event.duration": 2764889}
-{"index": {}}
-{"@timestamp": "2023-10-23T13:33:34.937Z", "client.ip": "172.21.0.5", "message": "Disconnected", "event.duration": 1232382}
-{"index": {}}
-{"@timestamp": "2023-10-23T13:51:54.732Z", "client.ip": "172.21.3.15", "message": "Connection error", "event.duration": 725448}
-{"index": {}}
-{"@timestamp": "2023-10-23T13:52:55.015Z", "client.ip": "172.21.3.15", "message": "Connection error", "event.duration": 8268153}
-{"index": {}}
-{"@timestamp": "2023-10-23T13:53:55.832Z", "client.ip": "172.21.3.15", "message": "Connection error", "event.duration": 5033755}
-{"index": {}}
-{"@timestamp": "2023-10-23T13:55:01.543Z", "client.ip": "172.21.3.15", "message": "Connected to 10.1.0.1", "event.duration": 1756467}
-----
+To follow along with the queries in this guide, you can either set up your own
+deployment, or use Elastic's public {esql} demo environment.
+
+include::{es-repo-dir}/tab-widgets/esql/esql-getting-started-widget-sample-data.asciidoc[]
[discrete]
[[esql-getting-started-running-queries]]
@@ -58,7 +22,7 @@ PUT sample_data/_bulk
In {kib}, you can use Console or Discover to run {esql} queries:
-include::{es-repo-dir}/tab-widgets/esql/esql-getting-started-widget.asciidoc[]
+include::{es-repo-dir}/tab-widgets/esql/esql-getting-started-widget-discover-console.asciidoc[]
[discrete]
[[esql-getting-started-first-query]]
@@ -300,57 +264,9 @@ image::images/esql/esql-enrich.png[align="center"]
Before you can use `ENRICH`, you first need to
<> and <>
-an <