forked from elastic/elasticsearch
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'main' into fix_testCannotFollowLeaderInUpgradedCluster
- Loading branch information
Showing
256 changed files
with
6,935 additions
and
5,205 deletions.
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
93 changes: 0 additions & 93 deletions
93
build-tools-internal/src/main/groovy/elasticsearch.build-complete.gradle
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
219 changes: 219 additions & 0 deletions
219
...nal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,219 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License | ||
| * 2.0 and the Server Side Public License, v 1; you may not use this file except | ||
| * in compliance with, at your election, the Elastic License 2.0 or the Server | ||
| * Side Public License, v 1. | ||
| */ | ||
|
|
||
| package org.elasticsearch.gradle.internal; | ||
|
|
||
| import com.gradle.scan.plugin.BuildScanExtension; | ||
|
|
||
| import org.apache.commons.compress.archivers.tar.TarArchiveEntry; | ||
| import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; | ||
| import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; | ||
| import org.apache.commons.io.IOUtils; | ||
| import org.elasticsearch.gradle.util.GradleUtils; | ||
| import org.gradle.api.Plugin; | ||
| import org.gradle.api.Project; | ||
| import org.gradle.api.file.FileSystemOperations; | ||
| import org.gradle.api.flow.FlowAction; | ||
| import org.gradle.api.flow.FlowParameters; | ||
| import org.gradle.api.flow.FlowProviders; | ||
| import org.gradle.api.flow.FlowScope; | ||
| import org.gradle.api.internal.file.FileOperations; | ||
| import org.gradle.api.provider.ListProperty; | ||
| import org.gradle.api.provider.Property; | ||
| import org.gradle.api.tasks.Input; | ||
| import org.jetbrains.annotations.NotNull; | ||
|
|
||
| import java.io.*; | ||
| import java.nio.file.Files; | ||
| import java.nio.file.Path; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import java.util.stream.Collectors; | ||
|
|
||
| import javax.inject.Inject; | ||
|
|
||
| public abstract class ElasticsearchBuildCompletePlugin implements Plugin<Project> { | ||
|
|
||
| @Inject | ||
| protected abstract FlowScope getFlowScope(); | ||
|
|
||
| @Inject | ||
| protected abstract FlowProviders getFlowProviders(); | ||
|
|
||
| @Inject | ||
| protected abstract FileOperations getFileOperations(); | ||
|
|
||
| @Override | ||
| public void apply(Project target) { | ||
| String buildNumber = System.getenv("BUILD_NUMBER") != null | ||
| ? System.getenv("BUILD_NUMBER") | ||
| : System.getenv("BUILDKITE_BUILD_NUMBER"); | ||
| String performanceTest = System.getenv("BUILD_PERFORMANCE_TEST"); | ||
| if (buildNumber != null && performanceTest == null && GradleUtils.isIncludedBuild(target) == false) { | ||
| File targetFile = target.file("build/" + buildNumber + ".tar.bz2"); | ||
| File projectDir = target.getProjectDir(); | ||
| File gradleWorkersDir = new File(target.getGradle().getGradleUserHomeDir(), "workers/"); | ||
| BuildScanExtension extension = target.getExtensions().getByType(BuildScanExtension.class); | ||
| File daemonsLogDir = new File(target.getGradle().getGradleUserHomeDir(), "daemon/" + target.getGradle().getGradleVersion()); | ||
|
|
||
| getFlowScope().always(BuildFinishedFlowAction.class, spec -> { | ||
| spec.getParameters().getBuildScan().set(extension); | ||
| spec.getParameters().getUploadFile().set(targetFile); | ||
| spec.getParameters().getProjectDir().set(projectDir); | ||
| spec.getParameters().getFilteredFiles().addAll(getFlowProviders().getBuildWorkResult().map((result) -> { | ||
| List<File> files = new ArrayList<>(); | ||
| files.addAll(resolveProjectLogs(projectDir)); | ||
| if (files.isEmpty() == false) { | ||
| files.addAll(resolveDaemonLogs(daemonsLogDir)); | ||
| files.addAll(getFileOperations().fileTree(gradleWorkersDir).getFiles()); | ||
| files.addAll(getFileOperations().fileTree(new File(projectDir, ".gradle/reaper/")).getFiles()); | ||
| } | ||
| return files; | ||
| })); | ||
| }); | ||
| } | ||
| } | ||
|
|
||
| private List<File> resolveProjectLogs(File projectDir) { | ||
| var projectDirFiles = getFileOperations().fileTree(projectDir); | ||
| projectDirFiles.include("**/*.hprof"); | ||
| projectDirFiles.include("**/build/test-results/**/*.xml"); | ||
| projectDirFiles.include("**/build/testclusters/**"); | ||
| projectDirFiles.include("**/build/testrun/*/temp/**"); | ||
| projectDirFiles.include("**/build/**/hs_err_pid*.log"); | ||
| projectDirFiles.exclude("**/build/testclusters/**/data/**"); | ||
| projectDirFiles.exclude("**/build/testclusters/**/distro/**"); | ||
| projectDirFiles.exclude("**/build/testclusters/**/repo/**"); | ||
| projectDirFiles.exclude("**/build/testclusters/**/extract/**"); | ||
| projectDirFiles.exclude("**/build/testclusters/**/tmp/**"); | ||
| projectDirFiles.exclude("**/build/testrun/*/temp/**/data/**"); | ||
| projectDirFiles.exclude("**/build/testrun/*/temp/**/distro/**"); | ||
| projectDirFiles.exclude("**/build/testrun/*/temp/**/repo/**"); | ||
| projectDirFiles.exclude("**/build/testrun/*/temp/**/extract/**"); | ||
| projectDirFiles.exclude("**/build/testrun/*/temp/**/tmp/**"); | ||
| return projectDirFiles.getFiles().stream().filter(f -> Files.isRegularFile(f.toPath())).toList(); | ||
| } | ||
|
|
||
| private List<File> resolveDaemonLogs(File daemonsLogDir) { | ||
| var gradleDaemonFileSet = getFileOperations().fileTree(daemonsLogDir); | ||
| gradleDaemonFileSet.include("**/daemon-" + ProcessHandle.current().pid() + "*.log"); | ||
| return gradleDaemonFileSet.getFiles().stream().filter(f -> Files.isRegularFile(f.toPath())).toList(); | ||
| } | ||
|
|
||
| public abstract static class BuildFinishedFlowAction implements FlowAction<BuildFinishedFlowAction.Parameters> { | ||
| interface Parameters extends FlowParameters { | ||
| @Input | ||
| Property<File> getUploadFile(); | ||
|
|
||
| @Input | ||
| Property<File> getProjectDir(); | ||
|
|
||
| @Input | ||
| ListProperty<File> getFilteredFiles(); | ||
|
|
||
| @Input | ||
| Property<BuildScanExtension> getBuildScan(); | ||
|
|
||
| } | ||
|
|
||
| @Inject | ||
| protected abstract FileSystemOperations getFileSystemOperations(); | ||
|
|
||
| @SuppressWarnings("checkstyle:DescendantToken") | ||
| @Override | ||
| public void execute(BuildFinishedFlowAction.Parameters parameters) throws FileNotFoundException { | ||
| File uploadFile = parameters.getUploadFile().get(); | ||
| if (uploadFile.exists()) { | ||
| getFileSystemOperations().delete(spec -> spec.delete(uploadFile)); | ||
| } | ||
| uploadFile.getParentFile().mkdirs(); | ||
| createBuildArchiveTar(parameters.getFilteredFiles().get(), parameters.getProjectDir().get(), uploadFile); | ||
| if (uploadFile.exists() && System.getenv("BUILDKITE").equals("true")) { | ||
| String uploadFilePath = "build/" + uploadFile.getName(); | ||
| try { | ||
| System.out.println("Uploading buildkite artifact: " + uploadFilePath + "..."); | ||
| new ProcessBuilder("buildkite-agent", "artifact", "upload", uploadFilePath).start().waitFor(); | ||
|
|
||
| System.out.println("Generating buildscan link for artifact..."); | ||
|
|
||
| Process process = new ProcessBuilder( | ||
| "buildkite-agent", | ||
| "artifact", | ||
| "search", | ||
| uploadFilePath, | ||
| "--step", | ||
| System.getenv("BUILDKITE_JOB_ID"), | ||
| "--format", | ||
| "%i" | ||
| ).start(); | ||
| process.waitFor(); | ||
| String processOutput; | ||
| try { | ||
| processOutput = IOUtils.toString(process.getInputStream()); | ||
| } catch (IOException e) { | ||
| processOutput = ""; | ||
| } | ||
| String artifactUuid = processOutput.trim(); | ||
|
|
||
| System.out.println("Artifact UUID: " + artifactUuid); | ||
| if (artifactUuid.isEmpty() == false) { | ||
| String buildkitePipelineSlug = System.getenv("BUILDKITE_PIPELINE_SLUG"); | ||
| String targetLink = "https://buildkite.com/organizations/elastic/pipelines/" | ||
| + buildkitePipelineSlug | ||
| + "/builds/" | ||
| + System.getenv("BUILD_NUMBER") | ||
| + "/jobs/" | ||
| + System.getenv("BUILDKITE_JOB_ID") | ||
| + "/artifacts/" | ||
| + artifactUuid; | ||
| parameters.getBuildScan().get().link("Artifact Upload", targetLink); | ||
| } | ||
| } catch (Exception e) { | ||
| System.out.println("Failed to upload buildkite artifact " + e.getMessage()); | ||
| } | ||
| } | ||
|
|
||
| } | ||
|
|
||
| private static void createBuildArchiveTar(List<File> files, File projectDir, File uploadFile) { | ||
| try ( | ||
| OutputStream fOut = Files.newOutputStream(uploadFile.toPath()); | ||
| BufferedOutputStream buffOut = new BufferedOutputStream(fOut); | ||
| BZip2CompressorOutputStream bzOut = new BZip2CompressorOutputStream(buffOut); | ||
| TarArchiveOutputStream tOut = new TarArchiveOutputStream(bzOut) | ||
| ) { | ||
| Path projectPath = projectDir.toPath(); | ||
| tOut.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); | ||
| for (Path path : files.stream().map(File::toPath).collect(Collectors.toList())) { | ||
| if (!Files.isRegularFile(path)) { | ||
| throw new IOException("Support only file!"); | ||
| } | ||
|
|
||
| TarArchiveEntry tarEntry = new TarArchiveEntry(path.toFile(), calculateArchivePath(path, projectPath)); | ||
|
|
||
| tOut.putArchiveEntry(tarEntry); | ||
|
|
||
| // copy file to TarArchiveOutputStream | ||
| Files.copy(path, tOut); | ||
| tOut.closeArchiveEntry(); | ||
|
|
||
| } | ||
| tOut.flush(); | ||
| tOut.finish(); | ||
|
|
||
| } catch (IOException e) { | ||
| throw new RuntimeException(e); | ||
| } | ||
| } | ||
|
|
||
| @NotNull | ||
| private static String calculateArchivePath(Path path, Path projectPath) { | ||
| return path.startsWith(projectPath) ? projectPath.relativize(path).toString() : path.getFileName().toString(); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.