Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/main' into nested_mapper_context
Browse files Browse the repository at this point in the history
  • Loading branch information
jimczi committed Jun 6, 2024
2 parents 6e89754 + 8619713 commit b0bb80e
Show file tree
Hide file tree
Showing 141 changed files with 5,778 additions and 995 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@ tasks.register("bwcTest") {

plugins.withType(ElasticsearchTestBasePlugin) {
tasks.withType(Test).matching { it.name ==~ /v[0-9\.]+#.*/ }.configureEach {
onlyIf("BWC tests enabled") { project.bwc_tests_enabled }
boolean bwcEnabled = project.bwc_tests_enabled
onlyIf("BWC tests enabled") { bwcEnabled }
nonInputProperties.systemProperty 'tests.bwc', 'true'
}
}
Expand All @@ -50,5 +51,5 @@ plugins.withType(InternalJavaRestTestPlugin) {
}
}

tasks.matching { it.name.equals("check") }.configureEach {dependsOn(bwcTestSnapshots) }
tasks.matching { it.name.equals("test") }.configureEach {enabled = false}
tasks.matching { it.name.equals("check") }.configureEach { dependsOn(bwcTestSnapshots) }
tasks.matching { it.name.equals("test") }.configureEach { enabled = false }
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@
import org.gradle.api.model.ObjectFactory;
import org.gradle.api.provider.Property;
import org.gradle.api.provider.Provider;
import org.gradle.api.provider.ProviderFactory;
import org.gradle.api.provider.ValueSource;
import org.gradle.api.provider.ValueSourceParameters;
import org.gradle.api.tasks.TaskProvider;
import org.gradle.jvm.toolchain.JavaLanguageVersion;
import org.gradle.jvm.toolchain.JavaToolchainService;
Expand All @@ -41,6 +44,7 @@ public class BwcSetupExtension {
private static final Version BUILD_TOOL_MINIMUM_VERSION = Version.fromString("7.14.0");
private final Project project;
private final ObjectFactory objectFactory;
private final ProviderFactory providerFactory;
private final JavaToolchainService toolChainService;
private final Provider<BwcVersions.UnreleasedVersionInfo> unreleasedVersionInfo;

Expand All @@ -49,12 +53,14 @@ public class BwcSetupExtension {
public BwcSetupExtension(
Project project,
ObjectFactory objectFactory,
ProviderFactory providerFactory,
JavaToolchainService toolChainService,
Provider<BwcVersions.UnreleasedVersionInfo> unreleasedVersionInfo,
Provider<File> checkoutDir
) {
this.project = project;
this.objectFactory = objectFactory;
this.providerFactory = providerFactory;
this.toolChainService = toolChainService;
this.unreleasedVersionInfo = unreleasedVersionInfo;
this.checkoutDir = checkoutDir;
Expand All @@ -65,11 +71,26 @@ TaskProvider<LoggedExec> bwcTask(String name, Action<LoggedExec> configuration)
}

TaskProvider<LoggedExec> bwcTask(String name, Action<LoggedExec> configuration, boolean useUniqueUserHome) {
return createRunBwcGradleTask(project, name, configuration, useUniqueUserHome);
return createRunBwcGradleTask(
project,
checkoutDir,
providerFactory,
unreleasedVersionInfo,
objectFactory,
toolChainService,
name,
configuration,
useUniqueUserHome
);
}

private TaskProvider<LoggedExec> createRunBwcGradleTask(
private static TaskProvider<LoggedExec> createRunBwcGradleTask(
Project project,
Provider<File> checkoutDir,
ProviderFactory providerFactory,
Provider<BwcVersions.UnreleasedVersionInfo> unreleasedVersionInfo,
ObjectFactory objectFactory,
JavaToolchainService toolChainService,
String name,
Action<LoggedExec> configAction,
boolean useUniqueUserHome
Expand All @@ -78,10 +99,10 @@ private TaskProvider<LoggedExec> createRunBwcGradleTask(
loggedExec.dependsOn("checkoutBwcBranch");
loggedExec.getWorkingDir().set(checkoutDir.get());

loggedExec.getEnvironment().put("JAVA_HOME", unreleasedVersionInfo.zip(checkoutDir, (version, checkoutDir) -> {
String minimumCompilerVersion = readFromFile(new File(checkoutDir, minimumCompilerVersionPath(version.version())));
return getJavaHome(Integer.parseInt(minimumCompilerVersion));
}));
loggedExec.getNonTrackedEnvironment().put("JAVA_HOME", providerFactory.of(JavaHomeValueSource.class, spec -> {
spec.getParameters().getVersion().set(unreleasedVersionInfo.map(it -> it.version()));
spec.getParameters().getCheckoutDir().set(checkoutDir);
}).flatMap(s -> getJavaHome(objectFactory, toolChainService, Integer.parseInt(s))));

if (BuildParams.isCi() && OS.current() != OS.WINDOWS) {
// TODO: Disabled for now until we can figure out why files are getting corrupted
Expand Down Expand Up @@ -137,10 +158,13 @@ private TaskProvider<LoggedExec> createRunBwcGradleTask(
});
}

private String minimumCompilerVersionPath(Version bwcVersion) {
return (bwcVersion.onOrAfter(BUILD_TOOL_MINIMUM_VERSION))
? "build-tools-internal/" + MINIMUM_COMPILER_VERSION_PATH
: "buildSrc/" + MINIMUM_COMPILER_VERSION_PATH;
/** A convenience method for getting java home for a version of java and requiring that version for the given task to execute */
private static Provider<String> getJavaHome(ObjectFactory objectFactory, JavaToolchainService toolChainService, final int version) {
Property<JavaLanguageVersion> value = objectFactory.property(JavaLanguageVersion.class).value(JavaLanguageVersion.of(version));
return toolChainService.launcherFor(javaToolchainSpec -> {
javaToolchainSpec.getLanguageVersion().value(value);
javaToolchainSpec.getVendor().set(JvmVendorSpec.ORACLE);
}).map(launcher -> launcher.getMetadata().getInstallationPath().getAsFile().getAbsolutePath());
}

private static String readFromFile(File file) {
Expand All @@ -151,13 +175,25 @@ private static String readFromFile(File file) {
}
}

/** A convenience method for getting java home for a version of java and requiring that version for the given task to execute */
public String getJavaHome(final int version) {
Property<JavaLanguageVersion> value = objectFactory.property(JavaLanguageVersion.class).value(JavaLanguageVersion.of(version));
return toolChainService.launcherFor(javaToolchainSpec -> {
javaToolchainSpec.getLanguageVersion().value(value);
javaToolchainSpec.getVendor().set(JvmVendorSpec.ORACLE);
}).get().getMetadata().getInstallationPath().getAsFile().getAbsolutePath();
}
public static abstract class JavaHomeValueSource implements ValueSource<String, JavaHomeValueSource.Params> {

private String minimumCompilerVersionPath(Version bwcVersion) {
return (bwcVersion.onOrAfter(BUILD_TOOL_MINIMUM_VERSION))
? "build-tools-internal/" + MINIMUM_COMPILER_VERSION_PATH
: "buildSrc/" + MINIMUM_COMPILER_VERSION_PATH;
}

@Override
public String obtain() {
return readFromFile(
new File(getParameters().getCheckoutDir().get(), minimumCompilerVersionPath(getParameters().getVersion().get()))
);
}

public interface Params extends ValueSourceParameters {
Property<Version> getVersion();

Property<File> getCheckoutDir();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -93,13 +93,6 @@ public void execute(Task task) {
String remoteRepo = remote.get();
// for testing only we can override the base remote url
String remoteRepoUrl = providerFactory.systemProperty("testRemoteRepo")
.orElse(
providerFactory.provider(
() -> addRemote.getExtensions().getExtraProperties().has("remote")
? addRemote.getExtensions().getExtraProperties().get("remote").toString()
: null
)
)
.getOrElse("https://github.com/" + remoteRepo + "/" + rootProjectName);
spec.commandLine("git", "remote", "add", remoteRepo, remoteRepoUrl);
});
Expand Down Expand Up @@ -213,6 +206,7 @@ private String maybeAlignedRefSpec(Logger logger, String defaultRefSpec) {

private void writeFile(File file, String content) {
try {
file.getParentFile().mkdirs();
Files.writeString(file.toPath(), content, CREATE, TRUNCATE_EXISTING);
} catch (IOException e) {
throw new UncheckedIOException(e);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import org.gradle.api.Plugin;
import org.gradle.api.Project;
import org.gradle.api.Task;
import org.gradle.api.file.ProjectLayout;
import org.gradle.api.model.ObjectFactory;
import org.gradle.api.plugins.JvmToolchainsPlugin;
import org.gradle.api.provider.Provider;
Expand Down Expand Up @@ -63,15 +64,39 @@ public void apply(Project project) {
project.getPlugins().apply(JvmToolchainsPlugin.class);
toolChainService = project.getExtensions().getByType(JavaToolchainService.class);
BuildParams.getBwcVersions().forPreviousUnreleased((BwcVersions.UnreleasedVersionInfo unreleasedVersion) -> {
configureBwcProject(project.project(unreleasedVersion.gradleProjectPath()), unreleasedVersion);
configureBwcProject(
project.project(unreleasedVersion.gradleProjectPath()),
unreleasedVersion,
providerFactory,
objectFactory,
toolChainService
);
});
}

private void configureBwcProject(Project project, BwcVersions.UnreleasedVersionInfo versionInfo) {
private static void configureBwcProject(
Project project,
BwcVersions.UnreleasedVersionInfo versionInfo,
ProviderFactory providerFactory,
ObjectFactory objectFactory,
JavaToolchainService toolChainService
) {
ProjectLayout layout = project.getLayout();
Provider<BwcVersions.UnreleasedVersionInfo> versionInfoProvider = providerFactory.provider(() -> versionInfo);
Provider<File> checkoutDir = versionInfoProvider.map(info -> new File(project.getBuildDir(), "bwc/checkout-" + info.branch()));
Provider<File> checkoutDir = versionInfoProvider.map(
info -> new File(layout.getBuildDirectory().get().getAsFile(), "bwc/checkout-" + info.branch())
);
BwcSetupExtension bwcSetupExtension = project.getExtensions()
.create("bwcSetup", BwcSetupExtension.class, project, objectFactory, toolChainService, versionInfoProvider, checkoutDir);
.create(
"bwcSetup",
BwcSetupExtension.class,
project,
objectFactory,
providerFactory,
toolChainService,
versionInfoProvider,
checkoutDir
);
BwcGitExtension gitExtension = project.getPlugins().apply(InternalBwcGitPlugin.class).getGitExtension();
Provider<Version> bwcVersion = versionInfoProvider.map(info -> info.version());
gitExtension.setBwcVersion(versionInfoProvider.map(info -> info.version()));
Expand Down Expand Up @@ -157,7 +182,7 @@ private void configureBwcProject(Project project, BwcVersions.UnreleasedVersionI
}
}

private void registerBwcDistributionArtifacts(Project bwcProject, DistributionProject distributionProject) {
private static void registerBwcDistributionArtifacts(Project bwcProject, DistributionProject distributionProject) {
String projectName = distributionProject.name;
String buildBwcTask = buildBwcTaskName(projectName);

Expand All @@ -174,7 +199,11 @@ private void registerBwcDistributionArtifacts(Project bwcProject, DistributionPr
}
}

private void registerDistributionArchiveArtifact(Project bwcProject, DistributionProject distributionProject, String buildBwcTask) {
private static void registerDistributionArchiveArtifact(
Project bwcProject,
DistributionProject distributionProject,
String buildBwcTask
) {
File distFile = distributionProject.expectedBuildArtifact.distFile;
String artifactFileName = distFile.getName();
String artifactName = artifactFileName.contains("oss") ? "elasticsearch-oss" : "elasticsearch";
Expand Down Expand Up @@ -363,5 +392,4 @@ private static class DistributionProjectArtifact {
this.expandedDistDir = expandedDistDir;
}
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,9 @@ public abstract class LoggedExec extends DefaultTask implements FileSystemOperat
@Optional
abstract public MapProperty<String, String> getEnvironment();

@Internal
abstract public MapProperty<String, String> getNonTrackedEnvironment();

@Input
abstract public Property<String> getExecutable();

Expand Down Expand Up @@ -139,7 +142,8 @@ public void run() {
execSpec.setStandardOutput(finalOutputStream);
execSpec.setErrorOutput(finalOutputStream);
execSpec.setExecutable(getExecutable().get());
execSpec.setEnvironment(getEnvironment().get());
execSpec.environment(getEnvironment().get());
execSpec.environment(getNonTrackedEnvironment().get());
if (getArgs().isPresent()) {
execSpec.setArgs(getArgs().get());
}
Expand Down
5 changes: 5 additions & 0 deletions docs/changelog/108538.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pr: 108538
summary: Adding RankFeature search phase implementation
area: Search
type: feature
issues: []
5 changes: 5 additions & 0 deletions docs/changelog/109357.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pr: 109357
summary: Fix task cancellation authz on fulfilling cluster
area: Authorization
type: bug
issues: []
15 changes: 9 additions & 6 deletions muted-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,7 @@ tests:
method: "testGuessIsDayFirstFromLocale"
- class: "org.elasticsearch.test.rest.ClientYamlTestSuiteIT"
issue: "https://github.com/elastic/elasticsearch/issues/108857"
method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale\
\ dependent mappings / dates}"
method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}"
- class: "org.elasticsearch.upgrades.SearchStatesIT"
issue: "https://github.com/elastic/elasticsearch/issues/108991"
method: "testCanMatch"
Expand All @@ -29,8 +28,7 @@ tests:
method: "testTrainedModelInference"
- class: "org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT"
issue: "https://github.com/elastic/elasticsearch/issues/109188"
method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale\
\ dependent mappings / dates}"
method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}"
- class: "org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT"
issue: "https://github.com/elastic/elasticsearch/issues/109189"
method: "test {p0=esql/70_locale/Date format with Italian locale}"
Expand All @@ -45,8 +43,7 @@ tests:
method: "testTimestampFieldTypeExposedByAllIndicesServices"
- class: "org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT"
issue: "https://github.com/elastic/elasticsearch/issues/109318"
method: "test {yaml=analysis-common/50_char_filters/pattern_replace error handling\
\ (too complex pattern)}"
method: "test {yaml=analysis-common/50_char_filters/pattern_replace error handling (too complex pattern)}"
- class: "org.elasticsearch.xpack.ml.integration.ClassificationHousePricingIT"
issue: "https://github.com/elastic/elasticsearch/issues/101598"
method: "testFeatureImportanceValues"
Expand All @@ -59,6 +56,12 @@ tests:
- class: "org.elasticsearch.xpack.inference.InferenceCrudIT"
issue: "https://github.com/elastic/elasticsearch/issues/109391"
method: "testDeleteEndpointWhileReferencedByPipeline"
- class: org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvAppendTests
method: testEvaluateBlockWithoutNulls {TestCase=<cartesian_shape>, <cartesian_shape>}
issue: https://github.com/elastic/elasticsearch/issues/109409
- class: org.elasticsearch.search.rank.RankFeatureShardPhaseTests
method: testProcessFetch
issue: https://github.com/elastic/elasticsearch/issues/109428

# Examples:
#
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -100,10 +100,10 @@ public void testUnassignedPrimaryWithExistingIndex() throws Exception {

// verify unassigned info
assertNotNull(unassignedInfo);
assertEquals(Reason.NODE_LEFT, unassignedInfo.getReason());
assertEquals(Reason.NODE_LEFT, unassignedInfo.reason());
assertTrue(
unassignedInfo.getLastAllocationStatus() == AllocationStatus.FETCHING_SHARD_DATA
|| unassignedInfo.getLastAllocationStatus() == AllocationStatus.NO_VALID_SHARD_COPY
unassignedInfo.lastAllocationStatus() == AllocationStatus.FETCHING_SHARD_DATA
|| unassignedInfo.lastAllocationStatus() == AllocationStatus.NO_VALID_SHARD_COPY
);

// verify cluster info
Expand Down Expand Up @@ -190,8 +190,8 @@ public void testUnassignedReplicaDelayedAllocation() throws Exception {

// verify unassigned info
assertNotNull(unassignedInfo);
assertEquals(Reason.NODE_LEFT, unassignedInfo.getReason());
assertEquals(AllocationStatus.NO_ATTEMPT, unassignedInfo.getLastAllocationStatus());
assertEquals(Reason.NODE_LEFT, unassignedInfo.reason());
assertEquals(AllocationStatus.NO_ATTEMPT, unassignedInfo.lastAllocationStatus());

// verify cluster info
verifyClusterInfo(clusterInfo, includeDiskInfo, 2);
Expand Down Expand Up @@ -320,8 +320,8 @@ public void testUnassignedReplicaWithPriorCopy() throws Exception {

// verify unassigned info
assertNotNull(unassignedInfo);
assertEquals(Reason.NODE_LEFT, unassignedInfo.getReason());
assertEquals(AllocationStatus.NO_ATTEMPT, unassignedInfo.getLastAllocationStatus());
assertEquals(Reason.NODE_LEFT, unassignedInfo.reason());
assertEquals(AllocationStatus.NO_ATTEMPT, unassignedInfo.lastAllocationStatus());

// verify cluster info
verifyClusterInfo(clusterInfo, includeDiskInfo, 3);
Expand Down Expand Up @@ -432,8 +432,8 @@ public void testAllocationFilteringOnIndexCreation() throws Exception {

// verify unassigned info
assertNotNull(unassignedInfo);
assertEquals(Reason.INDEX_CREATED, unassignedInfo.getReason());
assertEquals(AllocationStatus.DECIDERS_NO, unassignedInfo.getLastAllocationStatus());
assertEquals(Reason.INDEX_CREATED, unassignedInfo.reason());
assertEquals(AllocationStatus.DECIDERS_NO, unassignedInfo.lastAllocationStatus());

// verify cluster info
verifyClusterInfo(clusterInfo, includeDiskInfo, 2);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -386,9 +386,9 @@ public void testCreateShrinkIndexFails() throws Exception {
assertTrue(routingTables.index("target").shard(0).shard(0).unassigned());
assertEquals(
UnassignedInfo.Reason.ALLOCATION_FAILED,
routingTables.index("target").shard(0).shard(0).unassignedInfo().getReason()
routingTables.index("target").shard(0).shard(0).unassignedInfo().reason()
);
assertEquals(1, routingTables.index("target").shard(0).shard(0).unassignedInfo().getNumFailedAllocations());
assertEquals(1, routingTables.index("target").shard(0).shard(0).unassignedInfo().failedAllocations());
});
// now relocate them all to the right node
updateIndexSettings(Settings.builder().put("index.routing.allocation.require._name", mergeNode), "source");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ public void testFailedRecoveryOnAllocateStalePrimaryRequiresAnotherAllocateStale
final ClusterState state = clusterAdmin().prepareState().get().getState();
final ShardRouting shardRouting = state.routingTable().index(indexName).shard(shardId.id()).primaryShard();
assertThat(shardRouting.state(), equalTo(ShardRoutingState.UNASSIGNED));
assertThat(shardRouting.unassignedInfo().getReason(), equalTo(UnassignedInfo.Reason.ALLOCATION_FAILED));
assertThat(shardRouting.unassignedInfo().reason(), equalTo(UnassignedInfo.Reason.ALLOCATION_FAILED));
});

internalCluster().stopNode(node1);
Expand Down
Loading

0 comments on commit b0bb80e

Please sign in to comment.