Move CI job configuration to elasticsearch repository

wajihaparvez · May 6, 2021 · a64a6c7 · a64a6c7
1 parent 20a1862
commit a64a6c7
Show file tree

Hide file tree

Showing 69 changed files with 1,441 additions and 96 deletions.
diff --git a/.ci/build.sh b/.ci/build.sh
diff --git a/.ci/java-versions-aarch64.properties b/.ci/java-versions-aarch64.properties
@@ -5,5 +5,3 @@
 # are 'java' or 'openjdk' followed by the major release number.
 
 ES_BUILD_JAVA=jdk15
-ES_RUNTIME_JAVA=jdk15
-GRADLE_TASK=build
diff --git a/.ci/java-versions.properties b/.ci/java-versions.properties
@@ -6,4 +6,3 @@
 
 ES_BUILD_JAVA=openjdk15
 ES_RUNTIME_JAVA=openjdk11
-GRADLE_TASK=build
diff --git a/.ci/jobs.t/defaults.yml b/.ci/jobs.t/defaults.yml
@@ -0,0 +1,88 @@
+---
+
+##### GLOBAL METADATA
+
+- meta:
+    cluster: elasticsearch-ci
+
+##### JOB DEFAULTS
+
+- job:
+    vault:
+      url: https://secrets.elastic.co:8200
+      role_id: 1ba1ac3e-aee4-d040-d9a3-6ae23bd2b3db
+    node: "general-purpose"
+    concurrent: true
+    logrotate:
+      daysToKeep: 30
+      numToKeep: 90
+      artifactDaysToKeep: 7
+    parameters:
+      - string:
+          name: branch_specifier
+          default: "refs/heads/%BRANCH%"
+          description: "the Git branch specifier to build (&lt;branchName&gt;, &lt;tagName&gt;, &lt;commitId&gt;, etc.)\n"
+    scm:
+      - git:
+          name: origin
+          # master node jenkins user ~/.ssh
+          credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+          reference-repo: "/var/lib/jenkins/.git-references/elasticsearch.git"
+          branches:
+            - "${branch_specifier}"
+          url: "https://github.com/elastic/elasticsearch.git"
+          basedir: ""
+          wipe-workspace: true
+    triggers: []
+    wrappers:
+      - timeout:
+          type: absolute
+          timeout: 480
+          fail: true
+      - ansicolor
+      - timestamps
+      - gradle-build-scan
+      - inject-passwords:
+          global: false
+          job-passwords:
+            - name: VAULT_ADDR
+              password: https://secrets.elastic.co:8200
+          mask-password-params: true
+    properties:
+      - github:
+          url: https://github.com/elastic/elasticsearch/
+      - inject:
+          properties-content: |
+            HOME=$JENKINS_HOME
+            GRADLEW=./gradlew --parallel --scan --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/
+            GRADLEW_BAT=./gradlew.bat --parallel --scan --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/
+    publishers:
+      - junit:
+          results: "**/*Junit/*.xml, **/test-results/*/*.xml"
+          keep-long-stdio: true
+          allow-empty-results: true
+      # Upload additional logs
+      - google-cloud-storage:
+          credentials-id: 'elasticsearch-ci-gcs-plugin'
+          uploads:
+            - classic:
+                file-pattern: 'build/*.tar.bz2'
+                storage-location: 'gs://elasticsearch-ci-artifacts/jobs/$JOB_NAME'
+                share-publicly: false
+                upload-for-failed-jobs: true
+                show-inline: true
+      # Notify homer
+      - postbuildscript:
+          builders:
+            - role: SLAVE
+              build-on:
+                - SUCCESS
+                - FAILURE
+                - UNSTABLE
+              build-steps:
+                - http-request:
+                    url: https://homer.app.elstc.co/webhook/jenkins/build-finished
+                    mode: GET
+                    custom-headers:
+                      - name: X-Jenkins-Build
+                        value: ${BUILD_URL}
diff --git a/.ci/jobs.t/elastic+elasticsearch+branch-consistency.yml b/.ci/jobs.t/elastic+elasticsearch+branch-consistency.yml
@@ -0,0 +1,15 @@
+---
+- job:
+    name: elastic+elasticsearch+%BRANCH%+branch-consistency
+    display-name: "elastic / elasticsearch # %BRANCH% - branch consistency"
+    description: Testing of the Elasticsearch master branch version consistency.
+    triggers:
+      - timed: "H 7 * * *"
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+      - shell: |
+          #!/usr/local/bin/runbld --redirect-stderr
+          $WORKSPACE/.ci/scripts/run-gradle.sh branchConsistency
diff --git a/.ci/jobs.t/elastic+elasticsearch+branch-protection.yml b/.ci/jobs.t/elastic+elasticsearch+branch-protection.yml
@@ -0,0 +1,20 @@
+---
+- job:
+    name: elastic+elasticsearch+%BRANCH%+branch-protection
+    display-name: "elastic / elasticsearch # %BRANCH% - branch protection"
+    description: Elasticsearch %BRANCH% branch protection.
+    node: master
+    triggers:
+      - timed: "H 7 * * *"
+    scm: []
+    parameters: []
+    builders:
+      - shell: |
+          #!/bin/bash
+          set +x
+          STATUS=$(curl -s https://api.github.com/repos/elastic/elasticsearch/branches/%BRANCH% | jq '.protected')
+          echo "Branch %BRANCH% protection status is: $STATUS"
+          if [[ "$STATUS" == "false" ]]; then
+            echo "Development branch %BRANCH% is not set as protected in GitHub but should be."
+            exit 1
+          fi
diff --git a/.ci/jobs.t/elastic+elasticsearch+folder+pull-request.yml b/.ci/jobs.t/elastic+elasticsearch+folder+pull-request.yml
@@ -0,0 +1,4 @@
+- job:
+    name: elastic+elasticsearch+%BRANCH%+pull-request
+    display-name: Pull Requests
+    project-type: folder
diff --git a/.ci/jobs.t/elastic+elasticsearch+folder+triggers.yml b/.ci/jobs.t/elastic+elasticsearch+folder+triggers.yml
@@ -0,0 +1,4 @@
+- job:
+    name: elastic+elasticsearch+%BRANCH%+triggers
+    display-name: Periodic Triggers
+    project-type: folder
diff --git a/.ci/jobs.t/elastic+elasticsearch+intake+multijob+bwc.yml b/.ci/jobs.t/elastic+elasticsearch+intake+multijob+bwc.yml
@@ -0,0 +1,7 @@
+---
+jjbb-template: generic-gradle-unix.yml
+vars:
+  - job-name: elastic+elasticsearch+%BRANCH%+intake+multijob+bwc
+  - job-display-name: "elastic / elasticsearch # %BRANCH% - intake bwc"
+  - job-description: Elasticsearch %BRANCH% branch intake backwards compatibility checks.
+  - gradle-args: "-Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files bwcTestSnapshots"
diff --git a/.ci/jobs.t/elastic+elasticsearch+intake+multijob+part1.yml b/.ci/jobs.t/elastic+elasticsearch+intake+multijob+part1.yml
@@ -0,0 +1,7 @@
+---
+jjbb-template: generic-gradle-unix.yml
+vars:
+  - job-name: elastic+elasticsearch+%BRANCH%+intake+multijob+part1
+  - job-display-name: "elastic / elasticsearch # %BRANCH% - intake part 1"
+  - job-description: Elasticsearch %BRANCH% branch intake check part 1.
+  - gradle-args: "-Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart1"
diff --git a/.ci/jobs.t/elastic+elasticsearch+intake+multijob+part2.yml b/.ci/jobs.t/elastic+elasticsearch+intake+multijob+part2.yml
@@ -0,0 +1,7 @@
+---
+jjbb-template: generic-gradle-unix.yml
+vars:
+  - job-name: elastic+elasticsearch+%BRANCH%+intake+multijob+part2
+  - job-display-name: "elastic / elasticsearch # %BRANCH% - intake part 2"
+  - job-description: Elasticsearch %BRANCH% branch intake check part 2.
+  - gradle-args: "-Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2"
diff --git a/.ci/jobs.t/elastic+elasticsearch+intake+multijob+rest-compat.yml b/.ci/jobs.t/elastic+elasticsearch+intake+multijob+rest-compat.yml
@@ -0,0 +1,7 @@
+---
+jjbb-template: generic-gradle-unix.yml
+vars:
+  - job-name: elastic+elasticsearch+%BRANCH%+intake+multijob+rest-compat
+  - job-display-name: "elastic / elasticsearch # %BRANCH% - intake rest compatibility"
+  - job-description: Elasticsearch %BRANCH% branch intake REST compatibility checks.
+  - gradle-args: "-Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkRestCompat"
diff --git a/.ci/jobs.t/elastic+elasticsearch+intake+multijob+sanity-check.yml b/.ci/jobs.t/elastic+elasticsearch+intake+multijob+sanity-check.yml
@@ -0,0 +1,7 @@
+---
+jjbb-template: generic-gradle-unix.yml
+vars:
+  - job-name: elastic+elasticsearch+%BRANCH%+intake+multijob+sanity-check
+  - job-display-name: "elastic / elasticsearch # %BRANCH% - intake sanity check"
+  - job-description: Elasticsearch %BRANCH% branch intake sanity check.
+  - gradle-args: "-Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files precommit"
diff --git a/.ci/jobs.t/elastic+elasticsearch+intake+multijob+update-last-good-commit.yml b/.ci/jobs.t/elastic+elasticsearch+intake+multijob+update-last-good-commit.yml
@@ -0,0 +1,11 @@
+---
+- job:
+    name: elastic+elasticsearch+%BRANCH%+intake+multijob+update-last-good-commit
+    display-name: "elastic / elasticsearch # %BRANCH% - update last good commit"
+    description: Elasticsearch %BRANCH% branch update last good commit in build-stats.
+    node: light
+    properties: []
+    builders:
+      - shell: |
+          #!/usr/local/bin/runbld --job-name elastic+elasticsearch+%BRANCH%+git+push
+          /usr/bin/true
diff --git a/.ci/jobs.t/elastic+elasticsearch+intake.yml b/.ci/jobs.t/elastic+elasticsearch+intake.yml
@@ -0,0 +1,59 @@
+---
+- job:
+    name: elastic+elasticsearch+%BRANCH%+intake
+    display-name: "elastic / elasticsearch # %BRANCH% - intake"
+    description: "Testing of the Elasticsearch %BRANCH% branch on every push.\n"
+    project-type: multijob
+    node: master
+    vault: []
+    triggers:
+      # We use this trigger instead of the provided "github" webhook trigger because it's more robust.
+      # Here we only trigger builds for pushes to the corresponding branch, rather than a push to any branch of the
+      # configured git repository. This avoids duplicate builds being triggered when pushes to multiple branches are
+      # done in quick succession.
+      - generic-webhook-trigger:
+          post-content-params:
+            - type: JSONPath
+              key: ref
+              value: '$.ref'
+          regex-filter-text: '$ref'
+          regex-filter-expression: "^refs/heads/%BRANCH%$"
+          cause: Push to GitHub (refs/heads/%BRANCH%)
+          silent-response: true
+    scm:
+      - git:
+          wipe-workspace: false
+    builders:
+      - multijob:
+          name: Sanity Check
+          projects:
+            - name: elastic+elasticsearch+%BRANCH%+intake+multijob+sanity-check
+              kill-phase-on: NEVER
+              current-parameters: true
+              git-revision: true
+      - multijob:
+          name: Verification
+          projects:
+            - name: elastic+elasticsearch+%BRANCH%+intake+multijob+part1
+              kill-phase-on: NEVER
+              current-parameters: true
+              git-revision: true
+            - name: elastic+elasticsearch+%BRANCH%+intake+multijob+part2
+              kill-phase-on: NEVER
+              current-parameters: true
+              git-revision: true
+            - name: elastic+elasticsearch+%BRANCH%+intake+multijob+bwc
+              kill-phase-on: NEVER
+              current-parameters: true
+              git-revision: true
+            - name: elastic+elasticsearch+%BRANCH%+intake+multijob+rest-compat
+              kill-phase-on: NEVER
+              current-parameters: true
+              git-revision: true
+      - multijob:
+          name: Update last good commit
+          projects:
+            - name: elastic+elasticsearch+%BRANCH%+intake+multijob+update-last-good-commit
+              kill-phase-on: NEVER
+              current-parameters: true
+              git-revision: true
diff --git a/.ci/jobs.t/elastic+elasticsearch+multijob+packaging-tests-unix.yml b/.ci/jobs.t/elastic+elasticsearch+multijob+packaging-tests-unix.yml
@@ -0,0 +1,36 @@
+---
+- job:
+    name: elastic+elasticsearch+%BRANCH%+multijob+packaging-tests-unix
+    display-name: "elastic / elasticsearch # %BRANCH% - unix packaging tests"
+    description: "Testing of the Elasticsearch %BRANCH% branch unix packaging test support matrix.\n"
+    project-type: matrix
+    node: master
+    scm:
+      - git:
+          wipe-workspace: false
+    axes:
+      - axis:
+          type: label-expression
+          name: os
+          values:
+            - centos-7-packaging
+            - centos-8-packaging
+            - debian-9-packaging
+            - debian-10-packaging
+            - fedora-32-packaging
+            - opensuse-15-1-packaging
+            - oraclelinux-7-packaging
+            - oraclelinux-8-packaging
+            - sles-12-packaging
+            - sles-15-packaging
+            - ubuntu-18.04-packaging
+            - ubuntu-20.04-packaging
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+      - shell: |
+          #!/usr/local/bin/runbld --redirect-stderr
+          ./.ci/os.sh --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ destructivePackagingTest
diff --git a/.ci/jobs.t/elastic+elasticsearch+multijob+packaging-tests-upgrade.yml b/.ci/jobs.t/elastic+elasticsearch+multijob+packaging-tests-upgrade.yml
@@ -0,0 +1,30 @@
+---
+- job:
+    name: elastic+elasticsearch+%BRANCH%+multijob+packaging-tests-upgrade
+    display-name: "elastic / elasticsearch # %BRANCH% - packaging upgrade tests"
+    description: "Testing of the Elasticsearch %BRANCH% branch packaging test upgrade support matrix.\n"
+    project-type: matrix
+    node: master
+    scm:
+      - git:
+          wipe-workspace: false
+    axes:
+      - axis:
+          type: label-expression
+          name: os
+          values:
+            - centos-8-packaging
+            - ubuntu-20.04-packaging
+      - axis:
+          type: yaml
+          filename: ".ci/bwcVersions"
+          name: BWC_VERSION
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+      - shell: |
+          #!/usr/local/bin/runbld --redirect-stderr
+          ./.ci/os.sh --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ destructiveDistroUpgradeTest.v$BWC_VERSION
diff --git a/.ci/jobs.t/elastic+elasticsearch+multijob+packaging-tests-windows.yml b/.ci/jobs.t/elastic+elasticsearch+multijob+packaging-tests-windows.yml
@@ -0,0 +1,31 @@
+---
+- job:
+    name: elastic+elasticsearch+%BRANCH%+multijob+packaging-tests-windows
+    display-name: "elastic / elasticsearch # %BRANCH% - windows packaging tests"
+    description: "Testing of the Elasticsearch %BRANCH% branch windows packaging test support matrix.\n"
+    project-type: matrix
+    node: master
+    scm:
+      - git:
+          wipe-workspace: false
+    axes:
+      - axis:
+          type: label-expression
+          name: os
+          values:
+            - "windows-2012-r2"
+            - "windows-2016"
+            - "windows-2019"
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$USERPROFILE\\.java\\$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$USERPROFILE\\.java\\$ES_RUNTIME_JAVA
+      - batch: |
+          del /f /s /q %USERPROFILE%\.gradle\init.d\*.*
+          mkdir %USERPROFILE%\.gradle\init.d
+          copy .ci\init.gradle %USERPROFILE%\.gradle\init.d\
+          (
+             echo powershell.exe .\.ci\os.ps1   ^|^| exit /b 1
+          ) | java -jar "C:\Program Files\infra\bin\runbld" --redirect-stderr -
Original file line number	Diff line number	Diff line change
Expand Up		@@ -6,4 +6,3 @@

		ES_BUILD_JAVA=openjdk15
		ES_RUNTIME_JAVA=openjdk11
		GRADLE_TASK=build