From d03547da3e3af9cd0191bdd8a4bc3bba0a13bd15 Mon Sep 17 00:00:00 2001 From: James McMullan Date: Thu, 1 Feb 2024 11:03:19 -0500 Subject: [PATCH] HPCC4J-553 Add baremetal regression suite - Back ported baremetal regression suite and various other github action workflow changes to 9.2.x Signed-off-by: James McMullan James.McMullan@lexisnexis.com --- .github/workflows/Jirabot.yml | 6 +- .github/workflows/JirabotMerge.yml | 24 ++- .../workflows/baremetal-regression-suite.yml | 168 ++++++++++++++++++ ...UnitTests.yml => k8s-regression-suite.yml} | 27 ++- .../workflows/publish-snapshots-on-merge.yml | 2 +- 5 files changed, 218 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/baremetal-regression-suite.yml rename .github/workflows/{httpsUnitTests.yml => k8s-regression-suite.yml} (88%) diff --git a/.github/workflows/Jirabot.yml b/.github/workflows/Jirabot.yml index 029ce605b..b03530464 100644 --- a/.github/workflows/Jirabot.yml +++ b/.github/workflows/Jirabot.yml @@ -24,6 +24,8 @@ jobs: python -m site python -m pip install --upgrade pip setuptools wheel python -m pip install --upgrade jira + python -m pip --version + python -m pip freeze | grep jira - name: "Run" env: JIRABOT_USERNAME : ${{ secrets.JIRABOT_USERNAME }} @@ -60,9 +62,9 @@ jobs: try: jira.transition_issue(issue, transition) result += 'Workflow Transition: ' + transition + '\n' - except: + except Exception as error: transitions = jira.transitions(issue) - result += 'Error: Transition: "' + transition + '" failed. Valid transitions=[' + (', '.join(transitions)) + ']\n' + result += 'Error: Transition: "' + transition + '" failed with: "' + str(error) + '" Valid transitions=' + str(transitions) + '\n' if issue.fields.customfield_10010 is None: issue.update(fields={'customfield_10010': pull_url}) diff --git a/.github/workflows/JirabotMerge.yml b/.github/workflows/JirabotMerge.yml index 3fc3dadf0..634c408f4 100644 --- a/.github/workflows/JirabotMerge.yml +++ b/.github/workflows/JirabotMerge.yml @@ -1,19 +1,26 @@ name: Jirabot - Merge on: - pull_request: + pull_request_target: types: [closed] branches: - "master" - "candidate-*" -permissions: write-all - jobs: jirabot: runs-on: ubuntu-latest if: github.event.pull_request.merged == true steps: + - name: "Debug Vars" + run: | + echo "JIRA_URL: ${{ vars.JIRA_URL }}" + echo "Pull Request Number: ${{ github.event.pull_request.number }}" + echo "Pull Request Title: ${{ github.event.pull_request.title }}" + echo "Pull Request Author Name: ${{ github.event.pull_request.user.login }}" + echo "Pull Request URL: ${{ github.event.pull_request.html_url }}" + echo "Comments URL: ${{ github.event.pull_request.comments_url }}" + echo "Branch Name: ${{ github.ref_name }}" - uses: "actions/setup-python@v2" with: python-version: "3.8" @@ -24,6 +31,12 @@ jobs: python -m site python -m pip install --upgrade pip setuptools wheel python -m pip install --upgrade jira + - name: "Checkout" + uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.base.ref }} + fetch-depth: 0 + fetch-tags: true - name: "Run" env: JIRABOT_USERNAME : ${{ secrets.JIRABOT_USERNAME }} @@ -176,6 +189,11 @@ jobs: jirabot_user = os.environ['JIRABOT_USERNAME'] jirabot_pass = os.environ['JIRABOT_PASSWORD'] jira_url = os.environ['JIRA_URL'] + + if not jira_url: + jira_url = 'https://track.hpccsystems.com' + print('Jira URL us empty defaulting to: ' + jira_url) + pr = os.environ['PULL_REQUEST_NUMBER'] title = os.environ['PULL_REQUEST_TITLE'] user = os.environ['PULL_REQUEST_AUTHOR_NAME'] diff --git a/.github/workflows/baremetal-regression-suite.yml b/.github/workflows/baremetal-regression-suite.yml new file mode 100644 index 000000000..56fcc4428 --- /dev/null +++ b/.github/workflows/baremetal-regression-suite.yml @@ -0,0 +1,168 @@ +name: Baremetal Regression Suite + +on: + pull_request: + branches: + - "master" + - "candidate-*" + + workflow_dispatch: + +jobs: + test-against-platform: + runs-on: ubuntu-latest + + steps: + - name: Setup JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + + - uses: "actions/setup-python@v2" + with: + python-version: "3.8" + + - name: "Install Python dependencies" + run: | + set -xe + python -VV + python -m site + python -m pip install --upgrade pip setuptools wheel + + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + fetch-tags: true + + - name: Extract Latest Tagged Version + id: extract_version + env: + PULL_REQUEST_NUMBER : ${{ github.event.pull_request.number }} + PULL_REQUEST_TITLE : ${{ github.event.pull_request.title }} + PULL_REQUEST_AUTHOR_NAME : ${{ github.event.pull_request.user.login }} + PULL_URL: ${{ github.event.pull_request.html_url }} + COMMENTS_URL: ${{ github.event.pull_request.comments_url }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BRANCH_NAME: ${{ github.base_ref }} + shell: python + run: | + import os + import re + import subprocess + import time + import sys + + def extractVersion(versionStr): + parts = versionStr.split('.') + if len(parts) != 3: + print('Invalid version: ' + versionStr) + sys.exit(1) + if parts[2].lower() == 'x': + parts[2] = '0' + + major, minor, point = map(int, parts) + return [major, minor, point] + + def getTagVersionForCmd(cmd): + versionPattern = re.compile(r'.*([0-9]+\.[0-9]+\.[0-9]+).*') + + # Get latest release version + gitTagProcess = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) + (output, err) = gitTagProcess.communicate() + gitTagProcessStatus = gitTagProcess.wait() + + if gitTagProcessStatus != 0: + print('Unable to retrieve latest git tag. With error: ' + str(err)) + sys.exit(1) + + latestGitTag = str(output) + + versionMatch = versionPattern.match(latestGitTag) + if versionMatch: + return extractVersion(versionMatch.group(1)) + else: + print('Unable to extract version from git tag: ' + latestGitTag) + sys.exit(2) + + def getLatestBranchVersion(branchName): + + latestVersion = getTagVersionForCmd("git tag --list 'hpcc4j_*-release' --sort=-v:refname | head -n 1") + + if branchName == 'master': + return [latestVersion[0], latestVersion[1], latestVersion[2]] + else: + # Extract candidate branch major / minor version + candidateBranchPattern = re.compile(r'candidate-([0-9]+\.[0-9]+\.([0-9]+|x)).*') + branchVersionMatch = candidateBranchPattern.match(branchName) + if branchVersionMatch is None: + print('Unable to extract version from branch name: ' + branchName) + sys.exit(3) + + branchVersion = extractVersion(branchVersionMatch.group(1)) + + # Get latest release in branch + findLatestBranchVer = "git tag --list 'hpcc4j_" + str(branchVersion[0]) + "." + str(branchVersion[1]) + "*-release' --sort=-v:refname | head -n 1" + return getTagVersionForCmd(findLatestBranchVer) + + branch_name = os.environ['BRANCH_NAME'] + + latestVersion = getLatestBranchVersion(branch_name) + + if latestVersion[2] == 0: + print('Latest version is a new minor. Setting previous version to latest version') + previousVersion = latestVersion + else: + previousVersion = [latestVersion[0], latestVersion[1], latestVersion[2] - 2] + + previousVersionStr = ".".join(map(str, previousVersion)) + previousVersionURL = 'https://cdn.hpccsystems.com/releases/CE-Candidate-' + previousVersionStr \ + + '/bin/platform/hpccsystems-platform-community_' + previousVersionStr + '-1jammy_amd64_withsymbols.deb' + + latestVersionStr = ".".join(map(str, latestVersion)) + latestVersionURL = 'https://cdn.hpccsystems.com/releases/CE-Candidate-' + latestVersionStr \ + + '/bin/platform/hpccsystems-platform-community_' + latestVersionStr + '-1jammy_amd64_withsymbols.deb' + + print(f"::set-output name=previousVersion::{previousVersionStr}") + print(f"::set-output name=previousVersionURL::{previousVersionURL}") + print(f"::set-output name=latestVersion::{latestVersionStr}") + print(f"::set-output name=latestVersionURL::{latestVersionURL}") + + - name: Install latest version + run: | + if wget -q --spider ${{ steps.extract_version.outputs.latestVersionURL }}; then + wget -q ${{ steps.extract_version.outputs.latestVersionURL }} + elif wget -q --spider ${{ steps.extract_version.outputs.previousVersionURL }}; then + wget -q ${{ steps.extract_version.outputs.previousVersionURL }} + else + echo "Unable to find HPCC version to install" + exit 1 + fi + + sudo apt-get update + sudo apt-get install -y expect + sudo dpkg -i hpccsystems-platform-community_*.deb + sudo apt-get -f install -y + + - name: Start HPCC-Platform + shell: "bash" + run: | + export LANG="en_US.UTF-8" + sudo update-locale + sudo /etc/init.d/hpcc-init start + + - name: Add Host File Entries + run: | + sudo -- sh -c -e "echo '127.0.0.1 eclwatch.default' >> /etc/hosts"; + sudo -- sh -c -e "echo '127.0.0.1 rowservice.default' >> /etc/hosts"; + sudo -- sh -c -e "echo '127.0.0.1 sql2ecl.default' >> /etc/hosts"; + + # speed things up with caching from https://docs.github.com/en/actions/guides/building-and-testing-java-with-maven + - name: Cache Maven packages + uses: actions/cache@v2 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2 + + - name: Build with Maven + run: mvn -B --activate-profiles jenkins-on-demand -Dmaven.gpg.skip=true -Dmaven.javadoc.skip=true -Dmaven.test.failure.ignore=false -Dhpccconn=http://eclwatch.default:8010 -Dwssqlconn=http://sql2ecl.default:8510 -DHPCC30117=open install diff --git a/.github/workflows/httpsUnitTests.yml b/.github/workflows/k8s-regression-suite.yml similarity index 88% rename from .github/workflows/httpsUnitTests.yml rename to .github/workflows/k8s-regression-suite.yml index 7382205db..b37ddecba 100644 --- a/.github/workflows/httpsUnitTests.yml +++ b/.github/workflows/k8s-regression-suite.yml @@ -1,4 +1,4 @@ -name: https unit tests +name: K8s Regression Suite on: pull_request: @@ -66,10 +66,31 @@ jobs: - name: Install HPCC Cluster run: | - echo -e "certificates:\n enabled: true\ndafilesrv:\n - name: rowservice\n disabled: false\n application: stream\n service:\n servicePort: 7600\n visibility: global" > values.yaml + cat < values.yaml + certificates: + enabled: true + dafilesrv: + - name: rowservice + disabled: false + application: stream + service: + servicePort: 7600 + visibility: global + - name: direct-access + disabled: true + application: directio + service: + servicePort: 7200 + visibility: local + - name: spray-service + application: spray + service: + servicePort: 7300 + visibility: cluster + EOF helm repo add hpcc https://hpcc-systems.github.io/helm-chart helm repo update - helm install myhpcc hpcc/hpcc --set global.image.version=latest -f values.yaml + helm install myhpcc hpcc/hpcc -f values.yaml - uses: actions/checkout@v3 with: diff --git a/.github/workflows/publish-snapshots-on-merge.yml b/.github/workflows/publish-snapshots-on-merge.yml index 0b6018545..dc50b7b92 100644 --- a/.github/workflows/publish-snapshots-on-merge.yml +++ b/.github/workflows/publish-snapshots-on-merge.yml @@ -15,7 +15,7 @@ jobs: - name: Set up Maven Central Repository uses: actions/setup-java@v3 with: - java-version: '11' + java-version: '8' distribution: 'adopt' server-id: ossrh server-username: MAVEN_USERNAME