HPCC4J-553 Add baremetal regression suite

.github/workflows/Jirabot.yml

@@ -24,6 +24,8 @@ jobs:
           python -m site
           python -m pip install --upgrade pip setuptools wheel
           python -m pip install --upgrade jira
+          python -m pip --version
+          python -m pip freeze | grep jira
       - name: "Run"
         env:
           JIRABOT_USERNAME : ${{ secrets.JIRABOT_USERNAME }}
@@ -60,9 +62,9 @@ jobs:
                     try:
                         jira.transition_issue(issue, transition)
                         result += 'Workflow Transition: ' + transition + '\n'
-                    except:
+                    except Exception as error:
                         transitions = jira.transitions(issue)
-                        result += 'Error: Transition: "' + transition + '" failed. Valid transitions=[' + (', '.join(transitions)) + ']\n'
+                        result += 'Error: Transition: "' + transition + '" failed with: "' + str(error) + '" Valid transitions=' + str(transitions) + '\n'
 
                 if issue.fields.customfield_10010 is None:
                     issue.update(fields={'customfield_10010': pull_url})

.github/workflows/JirabotMerge.yml

@@ -1,19 +1,26 @@
 name: Jirabot - Merge
 
 on:
-  pull_request:
+  pull_request_target:
     types: [closed]
     branches:
       - "master"
       - "candidate-*"
 
-permissions: write-all
-
 jobs:
   jirabot:
     runs-on: ubuntu-latest
     if: github.event.pull_request.merged == true
     steps:
+      - name: "Debug Vars"
+        run: |
+          echo "JIRA_URL: ${{ vars.JIRA_URL }}"
+          echo "Pull Request Number: ${{ github.event.pull_request.number }}"
+          echo "Pull Request Title: ${{ github.event.pull_request.title }}"
+          echo "Pull Request Author Name: ${{ github.event.pull_request.user.login }}"
+          echo "Pull Request URL: ${{ github.event.pull_request.html_url }}"
+          echo "Comments URL: ${{ github.event.pull_request.comments_url }}"
+          echo "Branch Name: ${{ github.ref_name }}"
       - uses: "actions/setup-python@v2"
         with:
           python-version: "3.8"
@@ -24,6 +31,12 @@ jobs:
           python -m site
           python -m pip install --upgrade pip setuptools wheel
           python -m pip install --upgrade jira
+      - name: "Checkout"
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.pull_request.base.ref }}
+          fetch-depth: 0
+          fetch-tags: true
       - name: "Run"
         env:
           JIRABOT_USERNAME : ${{ secrets.JIRABOT_USERNAME }}
@@ -176,6 +189,11 @@ jobs:
             jirabot_user = os.environ['JIRABOT_USERNAME']
             jirabot_pass = os.environ['JIRABOT_PASSWORD']
             jira_url = os.environ['JIRA_URL']
+
+            if not jira_url:
+                jira_url = 'https://track.hpccsystems.com'
+                print('Jira URL us empty defaulting to: ' + jira_url)
+
             pr = os.environ['PULL_REQUEST_NUMBER']
             title = os.environ['PULL_REQUEST_TITLE']
             user = os.environ['PULL_REQUEST_AUTHOR_NAME']

.github/workflows/baremetal-regression-suite.yml

@@ -0,0 +1,168 @@
+name: Baremetal Regression Suite
+
+on:
+  pull_request:
+    branches:
+      - "master"
+      - "candidate-*"
+
+  workflow_dispatch:
+
+jobs:
+  test-against-platform:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Setup JDK 11
+      uses: actions/setup-java@v1
+      with:
+        java-version: 11
+
+    - uses: "actions/setup-python@v2"
+      with:
+        python-version: "3.8"
+
+    - name: "Install Python dependencies"
+      run: |
+        set -xe
+        python -VV
+        python -m site
+        python -m pip install --upgrade pip setuptools wheel
+
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        fetch-tags: true
+
+    - name: Extract Latest Tagged Version
+      id: extract_version
+      env:
+        PULL_REQUEST_NUMBER : ${{ github.event.pull_request.number }}
+        PULL_REQUEST_TITLE : ${{ github.event.pull_request.title }}
+        PULL_REQUEST_AUTHOR_NAME : ${{ github.event.pull_request.user.login }}
+        PULL_URL: ${{ github.event.pull_request.html_url }}
+        COMMENTS_URL: ${{ github.event.pull_request.comments_url }}
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        BRANCH_NAME: ${{ github.base_ref }}
+      shell: python
+      run: |
+        import os
+        import re
+        import subprocess
+        import time
+        import sys
+
+        def extractVersion(versionStr):
+            parts = versionStr.split('.')
+            if len(parts) != 3:
+                print('Invalid version: ' + versionStr)
+                sys.exit(1)
+            if parts[2].lower() == 'x':
+                parts[2] = '0'
+
+            major, minor, point = map(int, parts)
+            return [major, minor, point]
+
+        def getTagVersionForCmd(cmd):
+            versionPattern = re.compile(r'.*([0-9]+\.[0-9]+\.[0-9]+).*')
+
+            # Get latest release version
+            gitTagProcess = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
+            (output, err) = gitTagProcess.communicate()
+            gitTagProcessStatus = gitTagProcess.wait()
+
+            if gitTagProcessStatus != 0:
+                print('Unable to retrieve latest git tag. With error: ' + str(err))
+                sys.exit(1)
+
+            latestGitTag = str(output)
+
+            versionMatch = versionPattern.match(latestGitTag)
+            if versionMatch:
+                return extractVersion(versionMatch.group(1))
+            else:
+                print('Unable to extract version from git tag: ' + latestGitTag)
+                sys.exit(2)
+
+        def getLatestBranchVersion(branchName):
+
+            latestVersion = getTagVersionForCmd("git tag --list 'hpcc4j_*-release' --sort=-v:refname | head -n 1")
+
+            if branchName == 'master':
+                return [latestVersion[0], latestVersion[1], latestVersion[2]]
+            else:
+                # Extract candidate branch major / minor version
+                candidateBranchPattern = re.compile(r'candidate-([0-9]+\.[0-9]+\.([0-9]+|x)).*')
+                branchVersionMatch = candidateBranchPattern.match(branchName)
+                if branchVersionMatch is None:
+                    print('Unable to extract version from branch name: ' + branchName)
+                    sys.exit(3)
+
+                branchVersion = extractVersion(branchVersionMatch.group(1))
+
+                # Get latest release in branch
+                findLatestBranchVer = "git tag --list 'hpcc4j_" + str(branchVersion[0]) + "." + str(branchVersion[1]) + "*-release' --sort=-v:refname | head -n 1"
+                return getTagVersionForCmd(findLatestBranchVer)
+
+        branch_name = os.environ['BRANCH_NAME']
+
+        latestVersion = getLatestBranchVersion(branch_name)
+
+        if latestVersion[2] == 0:
+          print('Latest version is a new minor. Setting previous version to latest version')
+          previousVersion = latestVersion
+        else:
+          previousVersion = [latestVersion[0], latestVersion[1], latestVersion[2] - 2]
+
+        previousVersionStr = ".".join(map(str, previousVersion))
+        previousVersionURL = 'https://cdn.hpccsystems.com/releases/CE-Candidate-' + previousVersionStr \
+                           + '/bin/platform/hpccsystems-platform-community_' + previousVersionStr + '-1jammy_amd64_withsymbols.deb'
+
+        latestVersionStr = ".".join(map(str, latestVersion))
+        latestVersionURL = 'https://cdn.hpccsystems.com/releases/CE-Candidate-' + latestVersionStr \
+                         + '/bin/platform/hpccsystems-platform-community_' + latestVersionStr + '-1jammy_amd64_withsymbols.deb'
+
+        print(f"::set-output name=previousVersion::{previousVersionStr}")
+        print(f"::set-output name=previousVersionURL::{previousVersionURL}")
+        print(f"::set-output name=latestVersion::{latestVersionStr}")
+        print(f"::set-output name=latestVersionURL::{latestVersionURL}")
+
+    - name: Install latest version
+      run: |
+        if wget -q --spider ${{ steps.extract_version.outputs.latestVersionURL }}; then
+          wget -q ${{ steps.extract_version.outputs.latestVersionURL }}
+        elif wget -q --spider ${{ steps.extract_version.outputs.previousVersionURL }}; then
+          wget -q ${{ steps.extract_version.outputs.previousVersionURL }}
+        else
+          echo "Unable to find HPCC version to install"
+          exit 1
+        fi
+
+        sudo apt-get update
+        sudo apt-get install -y expect
+        sudo dpkg -i hpccsystems-platform-community_*.deb
+        sudo apt-get -f install -y
+
+    - name: Start HPCC-Platform
+      shell: "bash"
+      run: |
+        export LANG="en_US.UTF-8"
+        sudo update-locale
+        sudo /etc/init.d/hpcc-init start
+
+    - name: Add Host File Entries
+      run: |
+        sudo -- sh -c -e "echo '127.0.0.1 eclwatch.default' >> /etc/hosts";
+        sudo -- sh -c -e "echo '127.0.0.1 rowservice.default' >> /etc/hosts";
+        sudo -- sh -c -e "echo '127.0.0.1 sql2ecl.default' >> /etc/hosts";
+
+    # speed things up with caching from https://docs.github.com/en/actions/guides/building-and-testing-java-with-maven
+    - name: Cache Maven packages
+      uses: actions/cache@v2
+      with:
+        path: ~/.m2
+        key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+        restore-keys: ${{ runner.os }}-m2
+
+    - name: Build with Maven
+      run: mvn -B --activate-profiles jenkins-on-demand -Dmaven.gpg.skip=true -Dmaven.javadoc.skip=true -Dmaven.test.failure.ignore=false -Dhpccconn=http://eclwatch.default:8010 -Dwssqlconn=http://sql2ecl.default:8510 -DHPCC30117=open install

.github/workflows/httpsUnitTests.yml → .github/workflows/k8s-regression-suite.yml

@@ -1,4 +1,4 @@
-name: https unit tests
+name: K8s Regression Suite
 
 on:
   pull_request:
@@ -66,10 +66,31 @@ jobs:
 
     - name: Install HPCC Cluster
       run: |
-        echo -e "certificates:\n  enabled: true\ndafilesrv:\n  - name: rowservice\n    disabled: false\n    application: stream\n    service:\n      servicePort: 7600\n      visibility: global" > values.yaml
+        cat <<EOF > values.yaml
+        certificates:
+          enabled: true
+        dafilesrv:
+          - name: rowservice
+            disabled: false
+            application: stream
+            service:
+              servicePort: 7600
+              visibility: global
+          - name: direct-access
+            disabled: true
+            application: directio
+            service:
+              servicePort: 7200
+              visibility: local
+          - name: spray-service
+            application: spray
+            service:
+              servicePort: 7300
+              visibility: cluster
+        EOF
         helm repo add hpcc https://hpcc-systems.github.io/helm-chart
         helm repo update
-        helm install myhpcc hpcc/hpcc --set global.image.version=latest -f values.yaml
+        helm install myhpcc hpcc/hpcc -f values.yaml
 
     - uses: actions/checkout@v3
       with:

.github/workflows/publish-snapshots-on-merge.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Set up Maven Central Repository
         uses: actions/setup-java@v3
         with:
-          java-version: '11'
+          java-version: '8'
           distribution: 'adopt'
           server-id: ossrh
           server-username: MAVEN_USERNAME