diff --git a/.github/workflows/build-containers-pr.yml b/.github/workflows/build-containers-pr.yml deleted file mode 100644 index c5cca9badd6..00000000000 --- a/.github/workflows/build-containers-pr.yml +++ /dev/null @@ -1,141 +0,0 @@ -name: Docker smoketest build -on: - pull_request: - branches: - - "master" - - "candidate-*.x" - - "!candidate-8.2.*" - - "!candidate-8.0.*" - - "!candidate-7.12.*" - - "!candidate-7.10.*" - - "!candidate-7.8.*" - - "!candidate-7.6.*" - - "!candidate-7.4.*" - - "!candidate-7.2.*" - - "!candidate-7.0.*" - - "!candidate-6.*" - -jobs: - check-skip: - # continue-on-error: true # Uncomment once integration is finished - runs-on: ubuntu-20.04 - # Map a step output to a job output - outputs: - changed: ${{ steps.skip_check.outputs.dockerfiles || steps.skip_check.outputs.platform }} - steps: - - id: skip_check - uses: hpcc-systems/github-actions/changed-modules@main - with: - github_token: ${{ github.token }} - - build-images: - needs: check-skip - if: ${{ needs.check-skip.outputs.changed }} - runs-on: ubuntu-20.04 - steps: - - name: vars - id: vars - run: | - # echo ::set-output name=container_registry::ghcr.io - # echo ::set-output name=cr_user::${{ github.repository_owner }} - echo ::set-output name=container_registry::docker.io - echo ::set-output name=cr_user::hpccbuilds - echo ::set-output name=build_prbase_sha::${{ github.event.pull_request.base.sha }} - echo ::set-output name=build_prbase_label::${{ github.base_ref }} - echo ::set-output name=build_user::${{ github.actor }} - echo ::set-output name=build_type::RelWithDebInfo - echo ::set-output name=use_cppunit::1 - echo ::set-output name=platform_build::smoketest-platform-build-vcpkg - - - name: tracing - run: | - echo "Base ref = ${{ github.ref }}" - echo "Action = ${{ github.action }}" - echo "Event = ${{ github.event_name }}" - echo "Actor = ${{ github.actor }}" - echo "Ref = ${{ github.ref }}" - echo "base sha = ${{ github.event.pull_request.base.sha }}" - echo "Sha = ${{ github.sha }}" - echo "github.repository = ${{ github.repository }}" - echo "repository_owner = ${{ github.repository_owner }}" - echo "github.workspace = ${{ github.workspace }}" - echo "runner.workspace = ${{ runner.workspace }}" - echo "github.event.pull_request.head.repo.owner.login = ${{ github.event.pull_request.head.repo.owner.login }}" - echo "build_prbase_label = ${{ steps.vars.outputs.build_prbase_label }}" - echo "platform_build = ${{ steps.vars.outputs.platform_build }}" - - - name: Checkout PR - uses: actions/checkout@v3 - - - name: Calculate vcpkg vars - id: vcpkg_vars - run: | - echo "base_ver=$(git submodule status vcpkg | cut -c2-9)" >> $GITHUB_OUTPUT - - - name: tracing (vcpkg_vars) - run: | - echo "vcpkg_vars.base_ver = ${{ steps.vcpkg_vars.outputs.base_ver }}" - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - with: - driver: docker - - - name: Check if PR-Base prebuilt - id: check-images - run: | - prbase_missing=$(docker manifest inspect ${{ steps.vars.outputs.container_registry }}/${{ steps.vars.outputs.cr_user }}/${{ steps.vars.outputs.platform_build }}:${{ steps.vars.outputs.build_prbase_label }}-${{ github.event.pull_request.base.sha }} > /dev/null ; echo $?) - echo prbase_missing=${prbase_missing} - if [[ "${prbase_missing}" -eq 1 ]] - then - echo "Current PR target branch image cannot be found, using latest" - prbase_missing=$(docker manifest inspect ${{ steps.vars.outputs.container_registry }}/${{ steps.vars.outputs.cr_user }}/${{ steps.vars.outputs.platform_build }}:${{ steps.vars.outputs.build_prbase_label }}-latest > /dev/null ; echo $?) - if [[ "${prbase_missing}" -eq 1 ]] - then - echo "Cannot find the 'latest' target branch image" - echo ::set-output name=prbase_missing::${prbase_missing} - fi - echo ::set-output name=platform_prbase_ver::${{ steps.vars.outputs.build_prbase_label }}-latest - else - echo ::set-output name=platform_prbase_ver::${{ steps.vars.outputs.build_prbase_label }}-${{ github.event.pull_request.base.sha }} - fi - - # Normal expectation is that the following step would normally be skipped - # NB: This is only for the case where the target branch image doesn't already exist. - # The build-containers-target-branch.yml action would normally have built/published this image, - # or they'll be a "latest" which will have been picked up by the check-images step. - - name: branch image - if: ${{ steps.check-images.outputs.prbase_missing == '1' }} - uses: docker/build-push-action@v2 - with: - context: . - file: ./dockerfiles/platform-build/Dockerfile - builder: ${{ steps.buildx.outputs.name }} - tags: | - ${{ steps.vars.outputs.container_registry }}/${{ steps.vars.outputs.cr_user }}/${{ steps.vars.outputs.platform_build }}:${{ steps.vars.outputs.build_prbase_label }}-${{ steps.vars.outputs.build_prbase_sha }} - ${{ steps.vars.outputs.container_registry }}/${{ steps.vars.outputs.cr_user }}/${{ steps.vars.outputs.platform_build }}:${{ steps.vars.outputs.build_prbase_label }}-latest - build-args: | - CR_REPO=${{ steps.vars.outputs.container_registry }} - BASE_VER=${{ steps.vcpkg_vars.outputs.base_ver }} - BUILD_USER=${{ github.repository_owner }} - GITHUB_ACTOR=${{ github.actor }} - BUILD_TAG=${{ steps.vars.outputs.build_prbase_sha }} - BUILD_TYPE=${{ steps.vars.outputs.build_type }} - GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} - USE_CPPUNIT=${{ steps.vars.outputs.use_cppunit }} - BUILD_THREADS=${{ steps.vars.outputs.build_threads }} - - - name: PR image - uses: docker/build-push-action@v2 - with: - context: . - file: ./dockerfiles/platform-build-incremental-container/Dockerfile - builder: ${{ steps.buildx.outputs.name }} - build-args: | - CR_USER=${{ steps.vars.outputs.cr_user }} - CR_REPO=${{ steps.vars.outputs.container_registry }} - PLATFORM_PRBASE_VER=${{ steps.check-images.outputs.platform_prbase_ver }} - GITHUB_REPO=${{ github.repository }} - GITHUB_PRREF=${{ github.ref }} - BUILD_THREADS=${{ steps.vars.outputs.build_threads }} diff --git a/.github/workflows/build-containers-target-branch.yml b/.github/workflows/build-containers-target-branch.yml deleted file mode 100644 index dafe221a23e..00000000000 --- a/.github/workflows/build-containers-target-branch.yml +++ /dev/null @@ -1,112 +0,0 @@ -name: Docker target branch build -on: - push: - branches: - - "master" - - "candidate-*.x" - - "!candidate-8.2.*" - - "!candidate-8.0.*" - - "!candidate-7.12.*" - - "!candidate-7.10.*" - - "!candidate-7.8.*" - - "!candidate-7.6.*" - - "!candidate-7.4.*" - - "!candidate-7.2.*" - - "!candidate-7.0.*" - - "!candidate-6.*" - -jobs: - check-skip: - # continue-on-error: true # Uncomment once integration is finished - runs-on: ubuntu-20.04 - # Map a step output to a job output - outputs: - changed: ${{ steps.skip_check.outputs.dockerfiles || steps.skip_check.outputs.platform }} - steps: - - id: skip_check - uses: hpcc-systems/github-actions/changed-modules@main - with: - github_token: ${{ github.token }} - - build-images: - needs: check-skip - if: ${{ needs.check-skip.outputs.changed }} - runs-on: ubuntu-20.04 - steps: - - name: vars - id: vars - run: | - # echo ::set-output name=container_registry::ghcr.io - # echo ::set-output name=cr_user::${{ github.repository_owner }} - echo ::set-output name=container_registry::docker.io - echo ::set-output name=cr_user::hpccbuilds - echo ::set-output name=build_base_sha::${{ github.sha }} - echo ::set-output name=build_base_label::${GITHUB_REF##*/} - echo ::set-output name=build_user::${{ github.actor }} - echo ::set-output name=build_type::RelWithDebInfo - echo ::set-output name=use_cppunit::1 - echo ::set-output name=platform_build::smoketest-platform-build-vcpkg - - - name: tracing - run: | - echo "Action = ${{ github.action }}" - echo "Event = ${{ github.event_name }}" - echo "Actor = ${{ github.actor }}" - echo "Ref = ${{ github.ref }}" - echo "Sha = ${{ github.sha }}" - echo "github.repository = ${{ github.repository }}" - echo "repository_owner = ${{ github.repository_owner }}" - echo "github.workspace = ${{ github.workspace }}" - echo "runner.workspace = ${{ runner.workspace }}" - echo "build_base_sha = ${{ steps.vars.outputs.build_base_sha }}" - echo "build_base_label = ${{ steps.vars.outputs.build_base_label }}" - - - name: Checkout - uses: actions/checkout@v2 - - - name: Calculate vcpkg vars - id: vcpkg_vars - run: | - echo "base_ver=$(git submodule status vcpkg | cut -c2-9)" >> $GITHUB_OUTPUT - - - name: tracing (vcpkg_vars) - run: | - echo "vcpkg_vars.base_ver = ${{ steps.vcpkg_vars.outputs.base_ver }}" - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - - # - name: Login to GitHub Container Registry - # uses: docker/login-action@v1 - # with: - # registry: ghcr.io - # username: ${{ github.repository_owner }} - # password: ${{ secrets.GITHUB_TOKEN }} - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - # build branch image - - name: branch image - uses: docker/build-push-action@v2 - with: - context: . - file: ./dockerfiles/platform-build/Dockerfile - builder: ${{ steps.buildx.outputs.name }} - tags: | - ${{ steps.vars.outputs.container_registry }}/${{ steps.vars.outputs.cr_user }}/${{ steps.vars.outputs.platform_build }}:${{ steps.vars.outputs.build_base_label }}-${{ steps.vars.outputs.build_base_sha }} - ${{ steps.vars.outputs.container_registry }}/${{ steps.vars.outputs.cr_user }}/${{ steps.vars.outputs.platform_build }}:${{ steps.vars.outputs.build_base_label }}-latest - push: true - build-args: | - CR_REPO=${{ steps.vars.outputs.container_registry }} - BASE_VER=${{ steps.vcpkg_vars.outputs.base_ver }} - BUILD_USER=${{ github.repository_owner }} - BUILD_TAG=${{ steps.vars.outputs.build_base_sha }} - BUILD_TYPE=${{ steps.vars.outputs.build_type }} - GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} - USE_CPPUNIT=${{ steps.vars.outputs.use_cppunit }} - BUILD_THREADS=${{ steps.vars.outputs.build_threads }} diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml new file mode 100644 index 00000000000..e7bfab4a02f --- /dev/null +++ b/.github/workflows/build-docker.yml @@ -0,0 +1,179 @@ +name: Build Package (Docker) + +on: + workflow_call: + inputs: + os: + type: string + description: 'Operating System' + required: false + default: 'ubuntu-22.04' + ln: + type: boolean + description: 'Internal Build' + required: false + default: false + single-package: + type: boolean + description: 'Single Package' + required: false + default: true + build-type: + type: string + description: 'CMake Build Type' + required: false + default: 'RelWithDebInfo' + containerized: + type: boolean + description: 'Containerized Build' + required: false + default: false + strip-files: + type: boolean + description: 'Strip Debug Symbols' + required: false + default: true + cmake-configuration: + type: string + description: 'CMake Configuration' + required: false + default: '-DVCPKG_FILES_DIR=/hpcc-dev -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF -DUSE_CPPUNIT=ON -DSUPPRESS_V8EMBED=ON -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache' + cmake-configuration-ex: + type: string + description: 'CMake Configuration Extra' + required: false + default: '' + update-cache: + type: boolean + description: 'Update Cache' + required: false + default: true + upload-package: + type: boolean + description: 'Upload Package as Asset' + required: false + default: false + asset-name: + type: string + description: 'Asset Name (if upload-package is true)' + required: false + default: 'build-docker-package' + secrets: + LNB_TOKEN: + required: false + +jobs: + + build-docker: + runs-on: ubuntu-22.04 + steps: + - name: Free additional disk space (remove Android SDK + Tools) + run: | + sudo rm -rf /usr/local/lib/android + + - name: Checkout HPCC-Platform + uses: actions/checkout@v3 + with: + submodules: recursive + path: ${{ github.workspace }}/HPCC-Platform + + - name: Checkout LN + if: ${{ inputs.ln == true }} + uses: actions/checkout@v3 + with: + repository: ${{ github.repository_owner }}/LN + token: ${{ secrets.LNB_TOKEN }} + ref: ${{ github.base_ref }} + submodules: recursive + path: ${{ github.workspace }}/LN + + - name: Calculate vars + id: vars + working-directory: ${{ github.workspace }}/HPCC-Platform/vcpkg + shell: "bash" + run: | + branch_label_1=${{ github.base_ref }} + branch_label_2=$(echo ${{ github.ref }} | cut -d'/' -f3) + echo "branch_label=${branch_label_1:-$branch_label_2}" >> $GITHUB_OUTPUT + vcpkg_sha_short=$(git rev-parse --short=8 HEAD) + echo "vcpkg_sha_short=$vcpkg_sha_short" >> $GITHUB_OUTPUT + docker_build_label=hpccsystems/platform-build-base-${{ inputs.os }} + echo "docker_tag=$docker_build_label:$vcpkg_sha_short" >> $GITHUB_OUTPUT + + - name: Print vars + shell: "bash" + run: | + echo "${{ toJSON(steps.vars.outputs) }}" + + - uses: docker/setup-buildx-action@v2 + + - name: Pull Build Image + run: | + docker pull ${{ steps.vars.outputs.docker_tag }} || true + + - uses: hendrikmuhs/ccache-action@v1.2 + with: + save: ${{ inputs.update-cache == true }} + key: docker-${{ inputs.os }}-${{ inputs.build-type }}-${{ steps.vars.outputs.branch_label }}-${{ inputs.containerized == true && 'k8s' || 'bare-metal' }}-${{ inputs.ln == true && 'LN' || 'HPCC-Platform' }} + restore-keys: | + docker-${{ inputs.os }}-${{ inputs.build-type }}-${{ steps.vars.outputs.branch_label }}-${{ inputs.containerized == true && 'k8s' || 'bare-metal' }}- + docker-${{ inputs.os }}-${{ inputs.build-type }}-${{ steps.vars.outputs.branch_label }}- + docker-${{ inputs.os }}-${{ inputs.build-type }}- + docker-${{ inputs.os }}- + + - name: CMake Configure and Build + shell: "bash" + run: | + mkdir -p ${{ github.workspace }}/LN + mkdir -p ${{ github.workspace }}/build + mkdir -p ${{ github.workspace }}/.ccache + declare -a plugins + plugins=(${{ inputs.single-package == true && '"PLATFORM"' || '"PLATFORM" "CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "PARQUETEMBED" "REDIS" "REMBED" "SQLITE3EMBED" "SQS"' }}) + for plugin in "${plugins[@]}"; do + sudo rm -f ${{ github.workspace }}/build/CMakeCache.txt + sudo rm -rf ${{ github.workspace }}/build/CMakeFiles + docker run --rm \ + --mount source="${{ github.workspace }}/HPCC-Platform",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=delegated \ + --mount source="${{ github.workspace }}/LN",target=/hpcc-dev/LN,type=bind,consistency=delegated \ + --mount source="${{ github.workspace }}/build",target=/hpcc-dev/build,type=bind,consistency=delegated \ + --mount source="${{ github.workspace }}/.ccache",target=/root/.ccache,type=bind,consistency=delegated \ + ${{ steps.vars.outputs.docker_tag }} "\ + cmake -G Ninja -S /hpcc-dev/${{ inputs.ln == true && 'LN' || 'HPCC-Platform' }} -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform -DCMAKE_BUILD_TYPE=${{ inputs.build-type }} -DCONTAINERIZED=${{ inputs.containerized == true && 'ON' || 'OFF' }} -DCPACK_STRIP_FILES=${{ inputs.strip-files == true && 'ON' || 'OFF' }} ${{ inputs.single-package == true && '-DINCLUDE_PLUGINS=ON' || '-D$plugin=ON' }} ${{ inputs.cmake-configuration }} ${{ inputs.cmake-configuration-ex }} && \ + cmake --build /hpcc-dev/build --parallel ${{ inputs.upload-package == true && '--target package' || ''}}" + done + + - name: Upload Package + if: ${{ inputs.upload-package == true }} + uses: actions/upload-artifact@v3 + with: + name: ${{ inputs.asset-name }} + path: | + ${{ github.workspace }}/build/*.deb + ${{ github.workspace }}/build/*.rpm + if-no-files-found: error + + - name: Upload Support Files + if: ${{ inputs.upload-package == true }} + uses: actions/upload-artifact@v3 + with: + name: ${{ inputs.asset-name }}-support-files + path: | + ${{ github.workspace }}/HPCC-Platform/.github/workflows/smoketest-preabort.sh + ${{ github.workspace }}/HPCC-Platform/.github/workflows/timeoutcmd + if-no-files-found: error + + - name: Upload UI Test Files + if: ${{ inputs.upload-package == true }} + uses: actions/upload-artifact@v3 + with: + name: ${{ inputs.asset-name }}-ui_test-files + path: | + ${{ github.workspace }}/HPCC-Platform/esp/src/test-ui/**/* + if-no-files-found: error + + - name: Upload Error Logs + if: ${{ failure() || cancelled() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ inputs.os }}-${{ inputs.ln == true && 'LN' || 'HPCC-Platform' }}-logs + path: ${{ github.workspace }}/build/**/*.log diff --git a/.github/workflows/build-gh_runner.yml b/.github/workflows/build-gh_runner.yml new file mode 100644 index 00000000000..48823c004c5 --- /dev/null +++ b/.github/workflows/build-gh_runner.yml @@ -0,0 +1,250 @@ +name: Build Package (gh-runner) + +env: + VCPKG_BINARY_SOURCES: "clear;nuget,GitHub,read" + VCPKG_NUGET_REPOSITORY: "https://github.com/hpcc-systems/vcpkg" + +on: + workflow_call: + inputs: + os: + type: string + description: 'Operating System' + required: false + default: 'ubuntu-22.04' + ln: + type: boolean + description: 'Internal Build' + required: false + default: false + single-package: + type: boolean + description: 'Single Package' + required: false + default: true + build-type: + type: string + description: 'CMake Build Type' + required: false + default: 'RelWithDebInfo' + containerized: + type: boolean + description: 'Containerized Build' + required: false + default: false + strip-files: + type: boolean + description: 'Strip Debug Symbols' + required: false + default: true + cmake-configuration: + type: string + description: 'CMake Configuration' + required: false + default: '-DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF -DUSE_CPPUNIT=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_REMBED=ON -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache' + cmake-configuration-ex: + type: string + description: 'CMake Configuration Extra' + required: false + default: '' + update-cache: + type: boolean + description: 'Update Cache' + required: false + default: true + upload-package: + type: boolean + description: 'Upload Package as Asset' + required: false + default: false + asset-name: + type: string + description: 'Asset Name (if upload-package is true)' + required: false + default: 'build-docker-package' + secrets: + LNB_TOKEN: + required: false + +jobs: + + build-gh_runner: + runs-on: ${{ inputs.os }} + steps: + - name: Ubuntu Dependencies + if: ${{ contains(inputs.os, 'ubuntu') }} + shell: "bash" + run: | + sudo apt-get update -y + sudo apt-get install -yq --no-install-recommends \ + apt-transport-https \ + autoconf \ + autoconf-archive \ + automake \ + autotools-dev \ + binutils-dev \ + bison \ + build-essential \ + ca-certificates \ + curl \ + dirmngr \ + flex \ + git \ + gnupg \ + groff-base \ + libtool \ + pkg-config \ + software-properties-common \ + tar \ + unzip \ + uuid-dev \ + zip + + - name: Ubuntu Dependencies 2 + if: ${{ contains(inputs.os, 'ubuntu') }} + shell: "bash" + run: | + sudo apt-get install -yq --no-install-recommends \ + ccache \ + default-jdk \ + ninja-build \ + python3-dev \ + rsync \ + fop \ + libsaxonb-java + + - name: Ubuntu Mono Dependencies + if: ${{ inputs.os == 'ubuntu-22.04' }} + shell: "bash" + run: | + sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF + echo "deb https://download.mono-project.com/repo/ubuntu stable-focal main" | sudo tee /etc/apt/sources.list.d/mono-official-stable.list + sudo apt-get update -y + sudo apt-get -yq install -f mono-complete + + - name: OSX Dependencies + if: ${{ contains(inputs.os, 'macos') }} + shell: "bash" + run: | + brew install bison flex pkg-config autoconf autoconf-archive automake libtool cmake openjdk@11 ninja + + - name: "Remove builtin vcpkg (old)" + working-directory: . + shell: "bash" + run: | + ${{ !contains(inputs.os, 'windows') && 'sudo' || '' }} rm -rf "$VCPKG_INSTALLATION_ROOT" + + - name: Checkout HPCC-Platform + uses: actions/checkout@v3 + with: + submodules: recursive + path: ${{ github.workspace }}/HPCC-Platform + + - name: Checkout LN + if: ${{ inputs.ln == true }} + uses: actions/checkout@v3 + with: + repository: ${{ github.repository_owner }}/LN + token: ${{ secrets.LNB_TOKEN }} + ref: ${{ github.base_ref }} + submodules: recursive + path: ${{ github.workspace }}/LN + + - name: "vcpkg Bootstrap" + shell: "bash" + run: | + ./HPCC-Platform/vcpkg/bootstrap-vcpkg.sh + + - name: "Setup NuGet credentials" + shell: "bash" + run: | + ${{ !contains(inputs.os, 'windows') && 'mono' || '' }} `./HPCC-Platform/vcpkg/vcpkg fetch nuget | tail -n 1` \ + sources add \ + -name "GitHub" \ + -source "https://nuget.pkg.github.com/hpcc-systems/index.json" \ + -storepasswordincleartext \ + -username "${{ github.actor }}" \ + -password "${{ secrets.GITHUB_TOKEN }}" + + - name: Calculate vars + id: vars + working-directory: ${{ github.workspace }}/HPCC-Platform/vcpkg + shell: "bash" + run: | + branch_label_1=${{ github.base_ref }} + branch_label_2=$(echo ${{ github.ref }} | cut -d'/' -f3) + echo "branch_label=${branch_label_1:-$branch_label_2}" >> $GITHUB_OUTPUT + vcpkg_sha_short=$(git rev-parse --short=8 HEAD) + echo "vcpkg_sha_short=$vcpkg_sha_short" >> $GITHUB_OUTPUT + + - name: Print vars + shell: "bash" + run: | + echo "${{ toJSON(steps.vars.outputs) }}" + + - uses: actions/cache@v3 + id: cache + with: + path: | + ${{github.workspace}}/build/vcpkg_installed + key: vcpkg-${{ inputs.os }}-${{ steps.vars.outputs.vcpkg_sha_short }} + + - uses: hendrikmuhs/ccache-action@v1.2 + with: + save: ${{ inputs.update-cache == true }} + key: gh-runner-${{ inputs.os }}-${{ inputs.build-type }}-${{ steps.vars.outputs.branch_label }}-${{ inputs.containerized == true && 'k8s' || 'bare-metal' }}-${{ inputs.ln == true && 'LN' || 'HPCC-Platform' }} + restore-keys: | + gh-runner-${{ inputs.os }}-${{ inputs.build-type }}-${{ steps.vars.outputs.branch_label }}-${{ inputs.containerized == true && 'k8s' || 'bare-metal' }}- + gh-runner-${{ inputs.os }}-${{ inputs.build-type }}-${{ steps.vars.outputs.branch_label }}- + gh-runner-${{ inputs.os }}-${{ inputs.build-type }}- + gh-runner-${{ inputs.os }}- + + - name: CMake Configure and Build + shell: "bash" + run: | + mkdir -p ${{ github.workspace }}/LN + mkdir -p ${{ github.workspace }}/build + declare -a plugins + plugins=(${{ inputs.single-package == true && '"PLATFORM"' || '"PLATFORM" "CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "PARQUETEMBED" "REDIS" "REMBED" "SQLITE3EMBED" "SQS"' }}) + for plugin in "${plugins[@]}"; do + rm -f ./build/CMakeCache.txt + rm -rf ./build/CMakeFiles + cmake ${{ !contains(inputs.os, 'windows') && '-G Ninja' || '' }} -S ./${{ inputs.ln == true && 'LN' || 'HPCC-Platform' }} -B ./build -DCMAKE_BUILD_TYPE=${{ inputs.build-type }} -DCONTAINERIZED=${{ inputs.containerized == true && 'ON' || 'OFF' }} -DCPACK_STRIP_FILES=${{ inputs.strip-files == true && 'ON' || 'OFF' }} ${{ inputs.single-package == true && '-DINCLUDE_PLUGINS=ON' || '-D$plugin=ON' }} ${{ inputs.cmake-configuration }} ${{ inputs.cmake-configuration-ex }} + cmake --build ./build ${{ contains(inputs.os, 'windows') && '--config RelWithDebInfo' || ''}} --parallel ${{ inputs.upload-package == true && '--target package' || ''}} + done + + - name: Upload Package + if: ${{ inputs.upload-package == true }} + uses: actions/upload-artifact@v3 + with: + name: ${{ inputs.asset-name }} + path: | + ${{ github.workspace }}/build/*.deb + ${{ github.workspace }}/build/*.rpm + if-no-files-found: error + + - name: Upload Support Files + if: ${{ inputs.upload-package == true }} + uses: actions/upload-artifact@v3 + with: + name: ${{ inputs.asset-name }}-support-files + path: | + ${{ github.workspace }}/HPCC-Platform/.github/workflows/smoketest-preabort.sh + ${{ github.workspace }}/HPCC-Platform/.github/workflows/timeoutcmd + if-no-files-found: error + + - name: Upload UI Test Files + if: ${{ inputs.upload-package == true }} + uses: actions/upload-artifact@v3 + with: + name: ${{ inputs.asset-name }}-ui_test-files + path: | + ${{ github.workspace }}/HPCC-Platform/esp/src/test-ui/**/* + if-no-files-found: error + + - name: Upload Error Logs + if: ${{ failure() || cancelled() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ inputs.os }}-${{ inputs.ln == true && 'LN' || 'HPCC-Platform' }}-logs + path: ${{ github.workspace }}/build/**/*.log diff --git a/.github/workflows/build-test-eclwatch.yml b/.github/workflows/build-test-eclwatch.yml index 13eee0bfa2a..520fea47ba2 100644 --- a/.github/workflows/build-test-eclwatch.yml +++ b/.github/workflows/build-test-eclwatch.yml @@ -38,13 +38,6 @@ jobs: - uses: actions/setup-node@v2 with: node-version: ${{ matrix.node }} - - name: Cache node modules - uses: actions/cache@v2 - with: - path: ./esp/src/node_modules - key: npm-deps-${{ hashFiles('./esp/src/package-lock.json') }} - restore-keys: | - npm-deps-${{ hashFiles('./esp/src/package-lock.json') }} - name: Install Dependencies working-directory: ./esp/src run: npm ci diff --git a/.github/workflows/build-vcpkg.yml b/.github/workflows/build-vcpkg.yml index 0bd873d4219..583cecc5d4e 100644 --- a/.github/workflows/build-vcpkg.yml +++ b/.github/workflows/build-vcpkg.yml @@ -1,10 +1,34 @@ name: Test Build -env: - VCPKG_BINARY_SOURCES: "clear;nuget,GitHub,read" - VCPKG_NUGET_REPOSITORY: "https://github.com/hpcc-systems/vcpkg" - on: + workflow_dispatch: + inputs: + os: + type: choice + options: + - 'ubuntu-23.10' + - 'ubuntu-22.04' + - 'ubuntu-20.04' + - 'centos-8' + - 'centos-7' + - 'amazonlinux' + description: 'Operating System' + required: false + default: 'ubuntu-22.04' + ln: + type: boolean + description: 'Internal Build' + required: false + default: false + smoketest: + type: boolean + description: 'Run Smoketest' + required: false + default: false + + # Do not include "push" in final version (uncomment for testing on users own repo) + # push: + pull_request: branches: - "master" @@ -21,348 +45,143 @@ on: - cron: "0 0 * * *" jobs: - preamble: - name: Check Skip - runs-on: ubuntu-22.04 - outputs: - folder_platform: ${{ steps.vars.outputs.folder_platform }} - mount_platform: ${{ steps.vars.outputs.mount_platform }} - folder_ln: ${{ steps.vars.outputs.folder_ln }} - mount_ln: ${{ steps.vars.outputs.mount_ln }} - folder_build: ${{ steps.vars.outputs.folder_build }} - mount_build: ${{ steps.vars.outputs.mount_build }} - community_ref: ${{ steps.vars.outputs.community_ref }} - internal_ref: ${{ steps.vars.outputs.internal_ref }} - community_tag: ${{ steps.vars.outputs.community_tag }} - internal_tag: ${{ steps.vars.outputs.internal_tag }} - candidate_base_branch: ${{ steps.vars.outputs.candidate_base_branch }} - cmake_docker_config: ${{ steps.vars.outputs.cmake_docker_config }} - platform: "1" - platform_testing_do_not_release: ${{ steps.skip_check.outputs.platform }} - include_plugins: ${{ (steps.skip_check.outputs.plugins && 'ON') || 'OFF' }} - steps: - - name: Calculate vars - id: vars - run: | - echo "folder_platform=${{ github.workspace }}/HPCC-Platform" >> $GITHUB_OUTPUT - echo 'mount_platform=source="${{ github.workspace }}/HPCC-Platform",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached' >> $GITHUB_OUTPUT - echo "folder_ln=${{ github.workspace }}/LN" >> $GITHUB_OUTPUT - echo 'mount_ln=source="${{ github.workspace }}/LN",target=/hpcc-dev/LN,type=bind,consistency=cached' >> $GITHUB_OUTPUT - echo "folder_build=${{ github.workspace }}/build" >> $GITHUB_OUTPUT - echo 'mount_build=source="${{ github.workspace }}/build",target=/hpcc-dev/build,type=bind,consistency=cached' >> $GITHUB_OUTPUT - community_ref=${{ github.ref }} - echo "community_ref=$community_ref" >> $GITHUB_OUTPUT - echo "internal_ref=${{ github.base_ref }}" >> $GITHUB_OUTPUT - community_tag=$(echo $community_ref | cut -d'/' -f3) - echo "community_tag=$community_tag" >> $GITHUB_OUTPUT - echo "internal_tag=$(echo $community_tag | sed 's/community/internal/')" >> $GITHUB_OUTPUT - echo "candidate_base_branch=${{ github.base_ref }}" >> $GITHUB_OUTPUT - echo "cmake_docker_config=-DCMAKE_BUILD_TYPE=RelWithDebInfo -DVCPKG_FILES_DIR=/hpcc-dev -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF" >> $GITHUB_OUTPUT - - - id: skip_check - uses: hpcc-systems/github-actions/changed-modules@main - with: - github_token: ${{ github.token }} - - - name: Print vars - run: | - echo "${{ toJSON(steps.vars.outputs) }})" - echo "${{ toJSON(steps.skip_check.outputs) }})" - - build-docker: - name: Build Docker - needs: preamble - runs-on: ubuntu-22.04 - strategy: - matrix: - include: - - os: ubuntu-23.10 - event_name: "pull_request" - - os: ubuntu-22.04 - name: k8s - container: true - event_name: "schedule" - - os: ubuntu-22.04 - event_name: "schedule" - - os: ubuntu-20.04 - event_name: "schedule" - - os: centos-8 - event_name: "schedule" - - os: centos-7 - event_name: "pull_request" - cmake_options_extra: "-DVCPKG_TARGET_TRIPLET=x64-centos-7-dynamic" - - os: amazonlinux - event_name: "schedule" - cmake_options_extra: "-DVCPKG_TARGET_TRIPLET=x64-amazonlinux-dynamic" - - os: centos-7 - name: LN - cmake_options_extra: "-DVCPKG_TARGET_TRIPLET=x64-centos-7-dynamic" - ln: true - event_name: "schedule" - fail-fast: false - - steps: - - name: Free additional disk space (remove Android SDK + Tools) - run: | - sudo rm -rf /usr/local/lib/android - - - name: Checkout HPCC-Platform - if: ${{ contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform }} - uses: actions/checkout@v3 - with: - ref: ${{ needs.preamble.outputs.community_ref }} - submodules: recursive - path: ${{ needs.preamble.outputs.folder_platform }} - - - name: Checkout LN - if: ${{ matrix.ln && contains(matrix.event_name, github.event_name) }} - uses: actions/checkout@v3 - with: - repository: ${{ github.repository_owner }}/LN - token: ${{ secrets.LNB_TOKEN }} - ref: ${{ needs.preamble.outputs.internal_ref }} - submodules: recursive - path: ${{ needs.preamble.outputs.folder_ln }} - - - name: Calculate vars - if: ${{ contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform }} - id: vars - working-directory: ${{ needs.preamble.outputs.folder_platform }}/vcpkg - run: | - vcpkg_sha_short=$(git rev-parse --short=8 HEAD) - echo "vcpkg_sha_short=$vcpkg_sha_short" >> $GITHUB_OUTPUT - docker_build_label=hpccsystems/platform-build-${{ matrix.os }} - echo "docker_build_label=$docker_build_label" >> $GITHUB_OUTPUT - echo "docker_tag=$docker_build_label:$vcpkg_sha_short" >> $GITHUB_OUTPUT - echo "docker_tag_candidate_base=$docker_build_label:${{ needs.preamble.outputs.candidate_base_branch }}" >> $GITHUB_OUTPUT - - - name: Print vars - run: | - echo "${{ toJSON(needs.preamble.outputs) }})" - echo "${{ toJSON(steps.vars.outputs) }})" - - - name: Set up Docker Buildx - if: ${{ contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform }} - id: buildx - uses: docker/setup-buildx-action@v2 - - - name: Pull previous images - run: | - docker pull ${{ steps.vars.outputs.docker_tag_candidate_base }} || true - docker pull ${{ steps.vars.outputs.docker_tag }} || true - - - name: Create Build Image - if: ${{ contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform }} - uses: docker/build-push-action@v4 - with: - builder: ${{ steps.buildx.outputs.name }} - file: ${{ needs.preamble.outputs.folder_platform }}/dockerfiles/vcpkg/${{ matrix.os }}.dockerfile - context: ${{ needs.preamble.outputs.folder_platform }}/dockerfiles/vcpkg - load: true - build-args: | - VCPKG_REF=${{ steps.vars.outputs.vcpkg_sha_short }} - tags: | - ${{ steps.vars.outputs.docker_tag_candidate_base }} - cache-from: | - type=registry,ref=${{ steps.vars.outputs.docker_tag_candidate_base }} - type=registry,ref=${{ steps.vars.outputs.docker_tag }} - cache-to: type=inline - - # Communtiy Build - - name: CMake Packages - if: ${{ !matrix.ln && !matrix.container && contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform }} - run: | - mkdir -p ${{ needs.preamble.outputs.folder_build }} - declare -a plugins - if [ ${{ needs.preamble.outputs.include_plugins }} == "ON" ]; then - plugins=("CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "PARQUETEMBED" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "PLATFORM") - else - plugins=("PLATFORM") - fi - for plugin in "${plugins[@]}"; do - sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt - sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles - docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "\ - cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -D$plugin=ON -DCONTAINERIZED=OFF -DCPACK_STRIP_FILES=OFF ${{ matrix.cmake_options_extra }} && \ - cmake --build /hpcc-dev/build --parallel $(nproc) --target package" - # Disabled as not currently needed --- - # docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -D$plugin=ON -DCONTAINERIZED=OFF -DCPACK_STRIP_FILES=ON" - # docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cmake --build /hpcc-dev/build --parallel $(nproc) --target package" - done - - - name: CMake Containerized Packages - if: ${{ matrix.container && contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform }} - run: | - mkdir -p ${{ needs.preamble.outputs.folder_build }} - sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt - sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles - docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "\ - cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/build ${{ needs.preamble.outputs.cmake_docker_config }} -DINCLUDE_PLUGINS=${{ needs.preamble.outputs.include_plugins }} -DCONTAINERIZED=ON -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=OFF ${{ matrix.cmake_options_extra }} && \ - cmake --build /hpcc-dev/build --parallel $(nproc) --target package" - - # Internal Build --- - - name: CMake LN Packages - if: ${{ matrix.ln && !matrix.container && contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform }} - run: | - mkdir -p ${{ needs.preamble.outputs.folder_build }} - sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt - sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles - docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "\ - cmake -S /hpcc-dev/LN -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform ${{ needs.preamble.outputs.cmake_docker_config }} -DINCLUDE_PLUGINS=${{ needs.preamble.outputs.include_plugins }} -DCONTAINERIZED=OFF -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=OFF ${{ matrix.cmake_options_extra }} && \ - cmake --build /hpcc-dev/build --parallel $(nproc) --target package" - # Disabled as not currently needed --- - # docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cmake -S /hpcc-dev/LN -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform ${{ needs.preamble.outputs.cmake_docker_config }} -DINCLUDE_PLUGINS=${{ needs.preamble.outputs.include_plugins }} -DCONTAINERIZED=OFF -DSUPPRESS_REMBED=ON -DSUPPRESS_V8EMBED=ON -DSUPPRESS_SPARK=ON -DCPACK_STRIP_FILES=ON" - # docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_ln }} --mount ${{ needs.preamble.outputs.mount_build }} ${{ steps.vars.outputs.docker_tag_candidate_base }} "cmake --build /hpcc-dev/build --parallel $(nproc) --target package" - - # Common --- - - name: Upload error logs - if: ${{ failure() || cancelled() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ matrix.os }}-${{ matrix.ln }}-${{ matrix.container }}-logs - path: ${{ needs.preamble.outputs.folder_build }}/**/*.log - - build-bare-metal: - name: Build Bare Metal - # if: github.repository == 'hpcc-systems/HPCC-Platform' - needs: preamble - strategy: - matrix: - include: - - os: "ubuntu-20.04" - mono: "mono" - sudo: "sudo" - cmake_config_options: "-DCMAKE_BUILD_TYPE=RelWithDebInfo -DINCLUDE_PLUGINS=${{ needs.preamble.outputs.include_plugins }} -DSUPPRESS_V8EMBED=ON -DSUPPRESS_REMBED=ON" - cmake_build_options: "-- -j$(nproc) -k" - event_name: "schedule" - - os: "ubuntu-22.04" - mono: "mono" - sudo: "sudo" - cmake_config_options: "-DCMAKE_BUILD_TYPE=RelWithDebInfo -DINCLUDE_PLUGINS=${{ needs.preamble.outputs.include_plugins }} -DSUPPRESS_V8EMBED=ON -DSUPPRESS_REMBED=ON" - cmake_build_options: "-- -j$(nproc) -k" - event_name: "pull_request" - - os: "windows-2019" - mono: "" - sudo: "" - cmake_config_options: '-T host=x64 -A x64 -DUSE_OPTIONAL=OFF -DCLIENTTOOLS_ONLY=ON -DUSE_AZURE=OFF -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF' - cmake_build_options: "--config Release --parallel $NUMBER_OF_PROCESSORS" - event_name: "schedule" - - os: "windows-2022" - mono: "" - sudo: "" - cmake_config_options: '-T host=x64 -A x64 -DUSE_OPTIONAL=OFF -DCLIENTTOOLS_ONLY=ON -DUSE_AZURE=OFF -DUSE_CASSANDRA=OFF -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF' - cmake_build_options: "--config Release --parallel $NUMBER_OF_PROCESSORS" - event_name: "pull_request" - - os: "macos-11" - mono: "mono" - sudo: "sudo" - cmake_config_options: "-DCMAKE_BUILD_TYPE=Release -DUSE_OPTIONAL=OFF -DCLIENTTOOLS_ONLY=ON -DUSE_AZURE=OFF -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF" - cmake_build_options: "--parallel $(nproc)" - event_name: "schedule" - - os: "macos-12" - mono: "mono" - sudo: "sudo" - cmake_config_options: "-DCMAKE_BUILD_TYPE=Release -DUSE_OPTIONAL=OFF -DCLIENTTOOLS_ONLY=ON -DUSE_AZURE=OFF -DUSE_CASSANDRA=OFF -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF" - cmake_build_options: "--parallel $(nproc)" - event_name: "pull_request" - fail-fast: false - - runs-on: ${{ matrix.os }} - steps: - - name: Print vars - shell: "bash" - run: | - echo ${{ matrix.os }} - echo "${{ toJSON(needs.preamble.outputs) }})" - - - name: Ubuntu Dependencies - if: ${{ contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform && contains(matrix.os, 'ubuntu') }} - shell: "bash" - run: | - sudo apt-get update -y - sudo apt-get install -yq --no-install-recommends \ - apt-transport-https \ - autoconf \ - autoconf-archive \ - automake \ - autotools-dev \ - binutils-dev \ - bison \ - build-essential \ - ca-certificates \ - curl \ - dirmngr \ - flex \ - git \ - gnupg \ - groff-base \ - libtool \ - pkg-config \ - software-properties-common \ - tar \ - unzip \ - uuid-dev \ - zip - - - name: Ubuntu Mono Dependencies - if: ${{ matrix.os == 'ubuntu-22.04' && needs.preamble.outputs.platform && contains(matrix.os, 'ubuntu') }} - shell: "bash" - run: | - sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF - echo "deb https://download.mono-project.com/repo/ubuntu stable-focal main" | sudo tee /etc/apt/sources.list.d/mono-official-stable.list - sudo apt-get update -y - sudo apt-get -yq install -f mono-complete - - - name: OSX Dependencies - if: ${{ contains(matrix.os, 'macos') && contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform }} - shell: "bash" - run: | - brew install bison flex pkg-config autoconf autoconf-archive automake libtool cmake openjdk@11 - - - name: "Remove builtin vcpkg (old)" - if: ${{ contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform }} - working-directory: . - shell: "bash" - run: | - ${{ matrix.sudo }} rm -rf "$VCPKG_INSTALLATION_ROOT" - - - name: Checkout HPCC-Platform - if: ${{ contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform }} - uses: actions/checkout@v3 - with: - ref: ${{ needs.preamble.outputs.community_ref }} - submodules: recursive - path: ./HPCC-Platform - - - name: "vcpkg Bootstrap" - if: ${{ contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform }} - shell: "bash" - run: | - ./HPCC-Platform/vcpkg/bootstrap-vcpkg.sh - - - name: "Setup NuGet credentials" - if: ${{ contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform }} - shell: "bash" - run: | - ${{ matrix.mono }} `./HPCC-Platform/vcpkg/vcpkg fetch nuget | tail -n 1` \ - sources add \ - -name "GitHub" \ - -source "https://nuget.pkg.github.com/hpcc-systems/index.json" \ - -storepasswordincleartext \ - -username "${{ github.actor }}" \ - -password "${{ secrets.GITHUB_TOKEN }}" - - - name: CMake Packages - if: ${{ contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform }} - shell: "bash" - run: | - mkdir -p ./build - cmake -S ./HPCC-Platform -B ./build ${{ matrix.cmake_config_options }} - cmake --build ./build ${{ matrix.cmake_build_options }} - - name: Upload error logs - if: ${{ failure() || cancelled() }} - uses: actions/upload-artifact@v3 - with: - name: ${{ matrix.os }}--${{ matrix.ln }}-${{ matrix.container }}-logs - path: ./build/**/*.log + build-workflow-dispatch: + if: ${{ contains('workflow_dispatch', github.event_name) }} + uses: ./.github/workflows/build-docker.yml + with: + os: ${{ inputs.os }} + ln: ${{ inputs.ln }} + upload-package: true + asset-name: 'docker-package' + secrets: inherit + + test-workflow-dispatch: + if: ${{ contains('workflow_dispatch', github.event_name) && inputs.smoketest == true }} + needs: build-workflow-dispatch + uses: ./.github/workflows/test-smoke-gh_runner.yml + with: + os: ${{ inputs.os }} + asset-name: 'docker-package' + secrets: inherit + + build-docker-ubuntu-23_10: + if: ${{ contains('pull_request,push', github.event_name) }} + uses: ./.github/workflows/build-docker.yml + with: + os: ubuntu-23.10 + secrets: inherit + + build-docker-ubuntu-22_04: + if: ${{ contains('pull_request,push', github.event_name) }} + uses: ./.github/workflows/build-docker.yml + with: + os: ubuntu-22.04 + upload-package: true + asset-name: 'docker-ubuntu-22_04' + secrets: inherit + + test-smoke-docker-ubuntu-22_04: + if: ${{ contains('pull_request,push', github.event_name) }} + needs: build-docker-ubuntu-22_04 + uses: ./.github/workflows/test-smoke-gh_runner.yml + with: + os: ubuntu-22.04 + asset-name: 'docker-ubuntu-22_04' + secrets: inherit + + test-unit-docker-ubuntu-22_04: + if: ${{ contains('pull_request,push', github.event_name) }} + needs: build-docker-ubuntu-22_04 + uses: ./.github/workflows/test-unit-gh_runner.yml + with: + os: ubuntu-22.04 + asset-name: 'docker-ubuntu-22_04' + secrets: inherit + + test-ui-docker-ubuntu-22_04: + if: ${{ contains('pull_request,push', github.event_name) }} + needs: build-docker-ubuntu-22_04 + uses: ./.github/workflows/test-ui-gh_runner.yml + with: + os: ubuntu-22.04 + asset-name: 'docker-ubuntu-22_04' + secrets: inherit + + build-docker-ubuntu-20_04: + if: ${{ contains('schedule,push', github.event_name) }} + uses: ./.github/workflows/build-docker.yml + with: + os: ubuntu-20.04 + secrets: inherit + + build-docker-centos-8: + if: ${{ contains('schedule,push', github.event_name) }} + uses: ./.github/workflows/build-docker.yml + with: + os: centos-8 + secrets: inherit + + build-docker-centos-7: + if: ${{ contains('pull_request,push', github.event_name) }} + uses: ./.github/workflows/build-docker.yml + with: + os: centos-7 + cmake-configuration-ex: "-DVCPKG_TARGET_TRIPLET=x64-centos-7-dynamic" + secrets: inherit + + build-docker-amazonlinux: + if: ${{ contains('schedule,push', github.event_name) }} + uses: ./.github/workflows/build-docker.yml + with: + os: amazonlinux + cmake-configuration-ex: "-DVCPKG_TARGET_TRIPLET=x64-amazonlinux-dynamic" + secrets: inherit + + build-gh_runner-ubuntu-22_04: + if: ${{ contains('schedule,push', github.event_name) }} + uses: ./.github/workflows/build-gh_runner.yml + with: + os: ubuntu-22.04 + secrets: inherit + + build-gh_runner-ubuntu-20_04: + if: ${{ contains('schedule,push', github.event_name) }} + uses: ./.github/workflows/build-gh_runner.yml + with: + os: ubuntu-20.04 + secrets: inherit + + build-gh_runner-windows-2022: + if: ${{ contains('pull_request,push', github.event_name) }} + uses: ./.github/workflows/build-gh_runner.yml + with: + os: windows-2022 + cmake-configuration-ex: '-T host=x64 -A x64 -DUSE_CPPUNIT=OFF -DCLIENTTOOLS_ONLY=ON -DUSE_AZURE=OFF -DINCLUDE_PLUGINS=OFF -DUSE_CASSANDRA=OFF -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF' + secrets: inherit + + build-gh_runner-windows-2019: + if: ${{ contains('schedule,push', github.event_name) }} + uses: ./.github/workflows/build-gh_runner.yml + with: + os: windows-2019 + cmake-configuration-ex: '-T host=x64 -A x64 -DUSE_CPPUNIT=OFF -DCLIENTTOOLS_ONLY=ON -DINCLUDE_PLUGINS=OFF -DUSE_AZURE=OFF -DUSE_CASSANDRA=OFF -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF' + secrets: inherit + + build-gh_runner-macos-12: + if: ${{ contains('pull_request,push', github.event_name) }} + uses: ./.github/workflows/build-gh_runner.yml + with: + os: macos-12 + build-type: 'Release' + cmake-configuration-ex: '-DUSE_CPPUNIT=OFF -DCLIENTTOOLS_ONLY=ON -DINCLUDE_PLUGINS=OFF -DUSE_AZURE=OFF -DUSE_CASSANDRA=OFF -DSUPPRESS_CASSANDRAEMBED=ON -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF' + secrets: inherit + + build-gh_runner-macos-11: + if: ${{ contains('schedule,push', github.event_name) }} + uses: ./.github/workflows/build-gh_runner.yml + with: + os: macos-11 + build-type: 'Release' + cmake-configuration-ex: '-DUSE_CPPUNIT=OFF -DCLIENTTOOLS_ONLY=ON -DINCLUDE_PLUGINS=OFF -DUSE_AZURE=OFF -DUSE_CASSANDRA=OFF -DSUPPRESS_CASSANDRAEMBED=ON -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF' + secrets: inherit diff --git a/.github/workflows/smoketest.yml b/.github/workflows/smoketest.yml deleted file mode 100644 index 0428925cbaf..00000000000 --- a/.github/workflows/smoketest.yml +++ /dev/null @@ -1,592 +0,0 @@ -# Smoketest github action -# ======================= -# -# Uses cached instances of previous builds to perform partial (and therefore quicker) incremental builds. -# Once the platform is built and installed (make install), -# it runs the regression suite setup stage within the same github job ('build-and-setup'). -# -# On success, an artifact is built, that contains the install binaries and -# the HPCC binaries that have accumulated from the setup stage. -# -# Once the 'build-and-setup' job is complete, the dependent regression suite jobs and unittest job are launched in parallel. -# The regression suite queries are manually sub-divided into chunks (alphabeticaly), and run via a job matrix for parallism. -# If anything fails, all jobs are aborted (fail-fast=true), and the logs are captured into a published artifact. -# -# NOTES: -# + pre-requisite build dependencies (and runtime dependencies) are listed manually, and must be kept up to date with the -# requirements of the platform. MORE: a list of required build dependencies could be kept within the platform source and picked up -# + 'cacheversion' is purely in case it is necessary to force a cache-miss, i.e. all cached items are based on this version -# + Caching is via github's actions/cache, and is limited to 5GB total per repository, with oldest ejected first, and/or >7 days -# cached builds are tagged with: -# 1) base+ref+SHA (exact match, e.g. hpccbuild-1-7.12.10-6c981c48ae6e35b62d86d8e59e42799c5cd14812) -# 2) base_ref (branch match, e.g. hpccbuild-1-7.12.10) -# 3) base-ref major-minor only (e.g. hpccbuild-1-7.12.) -# 4) base-ref major only (e.g. hpccbuild-1-7.) -# 5) generic cacheversion only (e.g. hpccbuild-1-) -# The cache will initially try to match an exact key match (only true if rerunning PR and SHA same), -# and will then failover to trying to find a match of 2-5, in that order. - - -name: smoketest -env: - cacheversion: 3 - VCPKG_BINARY_SOURCES: "clear;nuget,GitHub,read" - OS_DEPENDENCIES: "bison flex build-essential binutils-dev curl lsb-release libcppunit-dev python3-dev default-jdk - r-base-dev r-cran-rcpp r-cran-rinside r-cran-inline pkg-config libtool autotools-dev automake git cmake" - -on: - pull_request: - branches: - - "master" - - "newbuild" - - "candidate-*" - - "!candidate-7.6.*" - - "!candidate-7.4.*" - - "!candidate-7.2.*" - - "!candidate-7.0.*" - - "!candidate-6.*" - -# NB: this fails to cancel in-flight actions, with error 'Resource not accessible by integration', -# but it's non-fatal. Appears to be dependent on what permissions the PR owner has, i.e. the github token used -# does not have permission to cancel actions. -jobs: - check-skip: - # continue-on-error: true # Uncomment once integration is finished - runs-on: ubuntu-22.04 - # Map a step output to a job output - outputs: - platform: ${{ contains(github.event.pull_request.labels.*.name, 'smoketest-force-rerun') || steps.skip_check.outputs.platform }} - eclwatch: ${{ contains(github.event.pull_request.labels.*.name, 'smoketest-force-rerun') || steps.skip_check.outputs.eclwatch }} - - steps: - - id: skip_check - uses: hpcc-systems/github-actions/changed-modules@main - with: - github_token: ${{ github.token }} - - build-and-setup: - name: Build platform and regress setup - needs: check-skip - if: ${{ needs.check-skip.outputs.platform || needs.check-skip.outputs.eclwatch }} - runs-on: ubuntu-22.04 - timeout-minutes: 150 # the build step has its own timeout, however the job can sometimes take time to download dependencies - outputs: - cache-state: ${{ steps.check-cache.outputs.state }} - - steps: - - name: Setup and git clone - id: vars - run: | - if [[ "${{ github.event_name }}" = 'pull_request' ]] - then - echo ::set-output name=branch::${{ github.head_ref }} - echo ::set-output name=base_ref::${{ github.base_ref }} - versioned=$(echo ${{ github.base_ref }} | sed -E -n 's/^(.*)-([0-9]+)\.([0-9]+)\.([0-9]+|x)(-[0-9]+|).*$/\1 \2 \3 \4 \5/p') - if [[ "${versioned}" ]] - then - echo "matched version = ${versioned}" - IFS=' ' read prefix major minor point cand <<< ${versioned} - echo "name=base-majorminor-restore-key::hpccbuild-${{ env.cacheversion }}-${prefix}-${major}.${minor}." - echo "name=base-majoronly-restore-key::hpccbuild-${{ env.cacheversion }}-${prefix}-${major}." - echo ::set-output name=base-majorminor-restore-key::hpccbuild-${{ env.cacheversion }}-${prefix}-${major}.${minor}. - echo ::set-output name=base-majoronly-restore-key::hpccbuild-${{ env.cacheversion }}-${prefix}-${major}. - fi - else - echo ::set-output name=branch::${GITHUB_REF#refs/heads/} - echo ::set-output name=base_ref::${GITHUB_REF#refs/heads/} - fi - if [[ "${{ github.event.pull_request.head.repo.owner.login }}" = "${{ github.repository_owner }}" ]] - then - repository=${{ github.repository }} - else - repository=$(echo -n ${{ github.event.pull_request.head.repo.owner.login }}/ ; echo -n ${{ github.repository }} | sed -E -n 's@^[^/]+/(.+)$@\1@p') - fi - echo ::set-output name=repository::${repository} - echo "Cloning https://github.com/${repository}" - git clone -n https://github.com/${repository} src - if [[ "${{ github.event_name }}" = 'pull_request' ]] - then - cd src - git remote add upstream https://github.com/${{ github.repository }} - git fetch upstream +${{ github.ref }} - fetch_sha=$(git rev-parse --verify FETCH_HEAD) - git fetch upstream ${{ github.base_ref}} - base_ref_sha=$(git rev-parse upstream/${{ github.base_ref }}) - echo ::set-output name=base_ref_sha::${base_ref_sha} - echo ::set-output name=sha::${fetch_sha} - echo "base_ref_sha = ${base_ref_sha}" - echo "fetch_sha = ${fetch_sha}" - else - echo ::set-output name=sha::${{ github.sha }} - echo ::set-output name=base_ref_sha::${{ github.sha }} - fi - - # NB: restore-keys set to e.g. hpccbuild-2-7.12.10, hpccbuild-2-7.12, hpccbuild-2-7, hpccbuild-2- - - name: Fetch cache - id: cache - uses: actions/cache@v2 - with: - path: | - build-cache - merge-patch - src-cache - key: hpccbuild-${{ env.cacheversion }}-${{ steps.vars.outputs.base_ref }}-${{ steps.vars.outputs.sha }} - restore-keys: | - hpccbuild-${{ env.cacheversion }}-${{ steps.vars.outputs.base_ref }} - ${{ steps.vars.outputs.base-majorminor-restore-key }} - ${{ steps.vars.outputs.base-majoronly-restore-key }} - hpccbuild-${{ env.cacheversion }}- - - - name: tracing - run: | - echo "Branch = ${{ steps.vars.outputs.branch }}" - echo "Base ref = ${{ steps.vars.outputs.base_ref }}" - echo "SHA = ${{ steps.vars.outputs.sha}}" - echo "Action = ${{ github.action }}" - echo "Event = ${{ github.event_name }}" - echo "Actor = ${{ github.actor }}" - echo "Ref = ${{ github.ref }}" - echo "Sha = ${{ github.sha }}" - echo "github.repository = ${{ github.repository }}" - echo "repository = ${{ steps.vars.outputs.repository }}" - echo "repository_owner = ${{ github.repository_owner }}" - echo "github.workspace = ${{ github.workspace }}" - echo "runner.workspace = ${{ runner.workspace }}" - echo "github.event.pull_request.head.repo.owner.login = ${{ github.event.pull_request.head.repo.owner.login }}" - - if [[ "${{ github.event_name }}" = 'pull_request' ]] - then - echo "PR base_ref = ${{ github.base_ref }}" - echo "PR head_ref = ${{ github.head_ref }}" - echo "PR base SHA = ${{ github.event.pull_request.base.sha }}" - fi - echo "restore-key1: hpccbuild-${{ env.cacheversion }}-${{ steps.vars.outputs.base_ref }}" - echo "restore-key2: ${{ steps.vars.outputs.base-majorminor-restore-key }}" - echo "restore-key3: ${{ steps.vars.outputs.base-majoronly-restore-key }}" - echo "restore-key4: hpccbuild-${{ env.cacheversion }}-" - - # NB: actions/cache does not set cache-hit to true if restore-keys used. - # So here we: - # 1) check if a cache was restored - # 2) check that the SHA stored in the cache can be found in git - # 3) mv cache into place (build-cache->build) - # 4) checkout source @ base_ref - # 5) get submodules - # 6) apply git merge-patch from cache, bringing source up-to-date with the source used in the cached binary build - # 7) copy src-cache files into place - # 8) touch all files so that 'old' [before original make timestamp], i.e. so behind objects in cache - # 9) checkout pr branch - ensuring only pr files that differ are updated (and hence will rebuild) - # 10) list updated files (for tracing only) - # 11) set 'state' output variable to 'miss' if not absolute match, - # and set to 'hit' if cache is usable. i.e. if set at all, run build and tests. - - name: Check cache - id: check-cache - if: ${{ (steps.cache.outputs.cache-hit != 'true') || contains(github.event.pull_request.labels.*.name, 'smoketest-force-rerun') }} - run: | - if [[ ("${{ steps.cache.outputs.cache-hit }}" == 'true') && ("${{ contains(github.event.pull_request.labels.*.name, 'smoketest-force-rerun') }}" == 'true') ]] - then - echo "Exact cache match, but 'smoketest-force-rerun' label forced rerun" - fi - echo ::set-output name=state::miss - if [[ "${{ contains(github.event.pull_request.labels.*.name, 'smoketest-ignore-cache') }}" == 'true' ]] - then - echo "Cache found, but ignored because 'smoketest-ignore-cache' label set" - rm -rf build-cache merge-patch src-cache - else - if [[ -d build-cache ]] - then - echo "Attempting to use existing cache" - stat build-cache merge-patch src-cache - du -sh build-cache merge-patch src-cache - cd src - if [[ "commit" == "$(git cat-file -t $(cat ../build-cache/cache-sha))" ]] - then - mv ../build-cache ../build - echo "Checking out cache SHA (cached github.base_ref SHA): $(cat ../build/cache-sha)" - git checkout $(cat ../build/cache-sha) - git submodule update --init --recursive --jobs 4 - echo "Applying merge patch" - git apply ../merge-patch - # Add all mods that came from merge-patch into a commit, - # so that the final git checkout will not clash with working tree files - git add -A - git -c user.name='Smoketest' -c user.email='smoke@test.com' commit -m 'merge-patch' - cd ../src-cache - echo "Cached source tree files:" - find . -type f - echo "================" - find . -type f | cpio -p -dum ../src - cd ../src - find . -type f -exec touch -r ../build/cache-timestamp {} + - git checkout ${{ steps.vars.outputs.sha }} - git submodule update --init --recursive --jobs 4 - echo "Changed files (from SHA: $(cat ../build/cache-sha)):" - find -name '.git*' -prune -o -newer ../build/cache-timestamp -type f -print - echo ::set-output name=state::hit - else - echo "Cached SHA $(cat ../build-cache/cache-sha) could not be found). Skipping cache." - fi - fi - fi - - - name: Dependencies22.04 - if: ${{ steps.check-cache.outputs.state != '' }} # if 'state' set then step needed - run: | - sudo apt-get update - sudo apt-get install -y ${{ env.OS_DEPENDENCIES }} - sudo apt-get install -y xmlstarlet - - curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash - - sudo apt-get install -y nodejs - - echo "node.js version = $(node --version), npm version = $(npm --version), release = $(lsb_release -sc)" - - - name: Cache miss prep - if: ${{ steps.check-cache.outputs.state == 'miss' }} - run: | - rm -rf build build-cache merge-patch src-cache - mkdir build - mkdir src-cache - cd src - git checkout ${{ steps.vars.outputs.sha }} - git submodule update --init --recursive --jobs 4 - - - name: "Setup NuGet credentials" - if: ${{ steps.check-cache.outputs.state != '' }} # if 'state' set then step needed - run: | - mono `vcpkg fetch nuget | tail -n 1` \ - sources add \ - -source "https://nuget.pkg.github.com/hpcc-systems/index.json" \ - -storepasswordincleartext \ - -name "GitHub" \ - -username "${{ github.actor }}" \ - -password "${{ secrets.GITHUB_TOKEN }}" - - - name: "Remove builtin vcpkg" - if: ${{ github.event_name == matrix.event_name && needs.check-skip.outputs.platform }} - shell: "bash" - run: | - sudo rm -rf "$VCPKG_INSTALLATION_ROOT" - - - name: "vcpkg Bootstrap" - if: ${{ steps.check-cache.outputs.state != '' }} # if 'state' set then step needed - working-directory: ./src/vcpkg - run: | - ./bootstrap-vcpkg.sh - - # Subsequent cache hit builds use cache-timestamp to ensure all sources are marked 'old', - # except those changed between the cache SHA and the SHA being built. - # NB: BUILD_TAG is overridden to a constant "smoketest", this is to prevent the auto-generated build tag being - # used, which would cause it to change per PR (per SHA), and because it's in a common header, cause a lot of undesirable rebuilding. - - name: Build - timeout-minutes: 120 # Generous, typical build time from clean is < 60 - if: ${{ steps.check-cache.outputs.state != '' }} # if 'state' set to anything, then step needed - run: | - cd build - echo "removing CMakeCache.txt" - rm -f CMakeCache.txt - if [[ ! -f cache-timestamp ]]; then touch cache-timestamp; fi - touch ../cmake-timestamp - `../src/vcpkg/vcpkg fetch cmake | tail -n 1` ../src -Wno-dev -DRUNTIME_USER=$(whoami) -DRUNTIME_GROUP=$(id -gn) -DDESTDIR=$(realpath ..)/install -DINCLUDE_PLUGINS=1 -DCMAKE_BUILD_TYPE=Release -DUSE_PYTHON2=0 -DUSE_PYTHON3=1 -DSUPPRESS_SPARK=1 -DUSE_CPPUNIT=1 -DUSE_CASSANDRA=Off -DCMAKE_CXX_FLAGS="-DBUILD_TAG=\\\"dummytag\\\" -DBUILD_VERSION_MAJOR=1 -DBUILD_VERSION_MINOR=1 -DBUILD_VERSION_POINT=0" -DBUILD_TAG="smoketest" -DSTRIP_RELEASE_SYMBOLS=0 -DCMAKE_CXX_FLAGS="-DTRACE_GLOBAL_GROUP=1" - make -j$(nproc) install - - - name: build-logs-artifact - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: build-logs - path: | - build/CMakeCache.txt - build/CMakeFiles/CMakeOutput.log - build/CMakeFiles/CMakeError.log - - # Cache contains: - # - make binaries (build-cache) - # - Any sources written since cache-timestamp within the source dir (src-cache) (e.g. AWS writes a config header) - # - cache-timestamp, timestamp of cache-miss build. All files except diffs set to this time, ensuring no existing binaries are rebuilt - # - cache-sha, the SHA of the base_ref used to build. On a cache hit, used to find changed files and ensure they are only ones rebuilt. - # - merge-path, the diffs between the base_ref and the PR commit at the time of PR/cache build. - - name: Prepare cache - if: ${{ steps.check-cache.outputs.state != '' }} # if 'state' set to anything, then step needed - run: | - echo ${{ steps.vars.outputs.base_ref_sha }} > build/cache-sha - mv build build-cache - cd src - find -name '.git*' -prune -o -newer ../cmake-timestamp -type f -print | cpio -p -dum ../src-cache - echo "Merge patch files:" - git diff --name-only ${{ steps.vars.outputs.base_ref_sha }} ${{ steps.vars.outputs.sha }} - git diff --binary ${{ steps.vars.outputs.base_ref_sha }} ${{ steps.vars.outputs.sha }} > ../merge-patch - - # - alter stock ecl-test.json, to point to install dir (using jq) - # - create a clean copy of the install directory (for later reuse) - # - rm hpcc-data from copy (the post-setup hpcc-data will be moved in when done) - - name: Prepare install artifact - if: ${{ steps.check-cache.outputs.state != '' && github.event_name == 'pull_request' }} - run: | - # configure ecl-test.json with correct paths - jq --arg rootdir "${{ github.workspace }}" \ - '.Regress.dropzonePath = $rootdir + "/install" + .Regress.dropzonePath | .Regress.setupExtraParams.OriginalTextFilesOsPath = $rootdir + "/install" + .Regress.setupExtraParams.OriginalTextFilesOsPath | .Regress.setupExtraDParams.HPCCBaseDir = $rootdir + "/install" + .Regress.setupExtraDParams.HPCCBaseDir | .Regress.regressionDir = $rootdir + "/regress" | .Regress.maxAttemptCount = "1" | .Regress.logDir = $rootdir + "/regress/log"' \ - install/opt/HPCCSystems/testing/regress/ecl-test.json > ecl-test.json - mv -f ecl-test.json install/opt/HPCCSystems/testing/regress/ecl-test.json - # configure environment.xml to slavesPerNode=2, channelsPerNode=1 - xmlstarlet ed -L -u 'Environment/Software/ThorCluster/@slavesPerNode' -v 2 -u 'Environment/Software/ThorCluster/@channelsPerSlave' -v 1 install/etc/HPCCSystems/environment.xml - cp ${{ github.workspace }}/src/.github/workflows/timeoutcmd install/opt/HPCCSystems/bin/ - cp ${{ github.workspace }}/src/.github/workflows/smoketest-preabort.sh install/opt/HPCCSystems/bin/ - mkdir copy - cp -rp install copy/ - rm -rf copy/install/var/lib/HPCCSystems/hpcc-data - rm -rf copy/install/var/lib/HPCCSystems/queries - - - name: Run regression suite setup - if: ${{ steps.check-cache.outputs.state != '' && github.event_name == 'pull_request' }} - timeout-minutes: 10 - run: | - export LANG="en_US.UTF-8" - sudo update-locale - source install/opt/HPCCSystems/sbin/hpcc_setenv - install/opt/HPCCSystems/etc/init.d/hpcc-init start - mkdir -p regress - cd install/opt/HPCCSystems/testing/regress - # force regression suite to timeout after 8 minutes, so it captures ZAP report of any inflight hung queries - timeoutcmd $((8 * 60)) \ - ./ecl-test setup --preAbort 'smoketest-preabort.sh' --pq 2 --generateStackTrace - grep Failure: ${{ github.workspace }}/regress/log/setup_*.log - if [[ "$(grep -ohP '(?<=^Failure: )[0-9]+$' ${{ github.workspace }}/regress/log/setup_*.log | awk '{s+=$1} END {print s}')" -gt 0 ]]; then exit 1; fi - ${{ github.workspace }}/install/opt/HPCCSystems/etc/init.d/hpcc-init stop - - - name: regression-setup-logs-artifact - if: ${{ failure() || cancelled() }} - uses: actions/upload-artifact@v2 - with: - name: regress-setup-logs - path: | - install/var/log/HPCCSystems - regress/ - - # - mv regression suite setup created data from hpcc-data and queries into copy of install - # - create tarball of install for artifact uploading - - name: Finalize install artifact - if: ${{ steps.check-cache.outputs.state != '' && github.event_name == 'pull_request' }} - run: | - mv install/var/lib/HPCCSystems/hpcc-data copy/install/var/lib/HPCCSystems/ - mv install/var/lib/HPCCSystems/queries copy/install/var/lib/HPCCSystems/ - [[ -d ${{ github.workspace }}/src/esp/src/test-ui ]] && cp -r ${{ github.workspace }}/src/esp/src/test-ui copy/install/ - cd copy - tar --zstd -cf ../install.tgz install - - - name: Upload install artifact - if: ${{ steps.check-cache.outputs.state != '' && github.event_name == 'pull_request' }} - uses: actions/upload-artifact@v2 - with: - name: installsetup-artifact - path: | - install.tgz - - # Matrix of jobs run in parallel once build+setup above completes successfully. - # All use the post-setup install.tgz artifact, that contains binaries and setup state - # Break each engine run into sets for speed - # NB: each regression suite job, runs these steps: - # - installs dependencies (probably too many for runtime) - # - Starts HPCC - # - Runs regression suite with params { engine, match[pattern,exclude] } - # TODO: needs to process results, capture report, to be assembled by workflow when all jobs done - regression-jobs: - needs: build-and-setup - if: ${{ needs.build-and-setup.outputs.cache-state != '' && github.event_name == 'pull_request' && needs.check-skip.outputs.platform }} - timeout-minutes: 60 # each matrix step has its own timeout, however the job can sometimes take time to download dependencies - runs-on: ubuntu-22.04 - strategy: - fail-fast: true - matrix: - engine: ['hthor', 'thor', 'roxie'] - match: - - pattern: '[a-d]*.ecl' - - pattern: '[e-h]*.ecl' - - pattern: '[i-l]*.ecl' - - pattern: '[m-q]*.ecl' - - pattern: '[r-u]*.ecl' - exclude: teststdlibrary.ecl - - pattern: '[v-z]*.ecl' - include: - - engine: 'hthor' - match: - pattern: teststdlibrary.ecl - - engine: 'thor' - match: - pattern: teststdlibrary.ecl - - engine: 'roxie' - match: - pattern: teststdlibrary.ecl - - steps: - - uses: actions/download-artifact@v2 - with: - name: installsetup-artifact - - - name: vars - id: vars - run: | - echo ::set-output name=matrix-setname::$(echo -n ${{ matrix.match.pattern }} | tr -c "[:alnum:]" _) - - - name: Prerequisites - run: | - sudo apt-get update - sudo apt-get install -y ${{ env.OS_DEPENDENCIES }} - sudo apt-get install -y gdb - - curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash - - sudo apt-get install -y nodejs - - echo "node.js version = $(node --version), npm version = $(npm --version), release = $(lsb_release -sc)" - - - name: run - timeout-minutes: 30 # generous, each matrix step shouldn't take more than 10 mins - run: | - export LANG="en_US.UTF-8" - sudo update-locale - tar --zstd -xf install.tgz - source install/opt/HPCCSystems/sbin/hpcc_setenv - install/opt/HPCCSystems/etc/init.d/hpcc-init start - cd install/opt/HPCCSystems/testing/regress - - (cd ecl; ls ${{ matrix.match.pattern }}) > matches.tmp - echo queries are: - cat matches.tmp - if [[ -z "${{ matrix.match.exclude }}" ]] - then - queries="$(cat matches.tmp | tr '\n' ' ')" - else - queries="$(cd ecl; ls ${{ matrix.match.exclude }} | grep -v -f - ../matches.tmp | tr '\n' ' ')" - fi - echo queries after exclude: ${queries} - - # force regression suite to timeout after 28 minutes, so it captures ZAP report of any inflight hung queries - timeoutcmd $((28 * 60)) \ - ./ecl-test query --preAbort 'smoketest-preabort.sh' --pq 2 --target ${{ matrix.engine }} --excludeclass python2,embedded-r,embedded-js,3rdpartyservice,mongodb --generateStackTrace ${queries} - grep Failure: ${{ github.workspace }}/regress/log/${{ matrix.engine }}.*.log - if [[ "$(grep -oP '(?<=^Failure: )[0-9]+$' ${{ github.workspace }}/regress/log/${{ matrix.engine }}.*.log)" -gt 0 ]]; then exit 1; fi - - - name: regression-run-logs-artifact - if: ${{ failure() || cancelled() }} - uses: actions/upload-artifact@v2 - with: - name: regression-run-logs-${{ matrix.engine }}-${{ steps.vars.outputs.matrix-setname }} - path: | - install/var/log/HPCCSystems - regress/ - if-no-files-found: error - - - # NB: this doesn't really need the post-setup data files included in install.tgz - # but as this is relatively quick and in parallel with others, it probably doesn't matter - unittests: - needs: build-and-setup - if: ${{ needs.build-and-setup.outputs.cache-state != '' && github.event_name == 'pull_request' }} - timeout-minutes: 30 # the unittests run step has its own timeout, however the job can sometimes take time to download dependencies - runs-on: ubuntu-22.04 - steps: - - uses: actions/download-artifact@v2 - with: - name: installsetup-artifact - - - name: Prerequisites - run: | - sudo apt-get update - sudo apt-get install -y ${{ env.OS_DEPENDENCIES }} - sudo apt-get install -y gdb - - curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash - - sudo apt-get install -y nodejs - - echo "node.js version = $(node --version), npm version = $(npm --version), release = $(lsb_release -sc)" - - - name: run - timeout-minutes: 10 # generous, expected time is approx 1 min. - run: | - export LANG="en_US.UTF-8" - sudo update-locale - tar --zstd -xf install.tgz - install/opt/HPCCSystems/bin/unittests - - # NB: this doesn't really need the post-setup data files included in install.tgz - # but as this is relatively quick and in parallel with others, it probably doesn't matter - eclwatch-ui-tests: - needs: build-and-setup - if: ${{ needs.build-and-setup.outputs.cache-state != '' && github.event_name == 'pull_request' && needs.check-skip.outputs.eclwatch }} - timeout-minutes: 30 # the ui-tests run step has its own timeout, however the job can sometimes take time to download dependencies - runs-on: ubuntu-22.04 - steps: - - uses: actions/download-artifact@v2 - with: - name: installsetup-artifact - - - name: Check-ECLWatch-UI-Test-Directory - id: check - run: | - tar --zstd -xf install.tgz - if [[ ! -d install/test-ui/tests ]] - then - echo "ECLWatch UI test-ui/tests directory missing." - else - javaFilesCount=$(find install/test-ui/tests/ -iname '*.java' -type f -print | wc -l ) - echo "Number of test java files is $javaFilesCount" - if [[ ${javaFilesCount} -eq 0 ]] - then - echo "No java files, do nothing." - else - echo ::set-output name=runtests::true - fi - fi - - - name: Prerequisites - if: steps.check.outputs.runtests - run: | - sudo apt-get update - sudo apt-get install -y git wget net-tools - sudo apt-get install -y tzdata unzip xvfb libxi6 - sudo apt-get install -y default-jdk - sudo apt-get install -y ${{ env.OS_DEPENDENCIES }} - sudo apt-get install -y gdb - - curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash - - sudo apt-get install -y nodejs - - echo "node.js version = $(node --version), npm version = $(npm --version), release = $(lsb_release -sc)" - - - name: UI-Test-Prerequisites - if: steps.check.outputs.runtests - run: | - wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb - sudo apt-get install -y ./google-chrome-stable_current_amd64.deb - wget https://chromedriver.storage.googleapis.com/2.41/chromedriver_linux64.zip - unzip chromedriver_linux64.zip - sudo mv chromedriver /usr/bin/chromedriver - sudo chown root:root /usr/bin/chromedriver - sudo chmod +x /usr/bin/chromedriver - wget https://selenium-release.storage.googleapis.com/3.141/selenium-server-standalone-3.141.59.jar - wget http://www.java2s.com/Code/JarDownload/testng/testng-6.8.7.jar.zip - unzip testng-6.8.7.jar.zip - - - name: run - timeout-minutes: 10 # generous, current runtime is ~1min, this should be increased if new tests are added - if: steps.check.outputs.runtests - run: | - export LANG="en_US.UTF-8" - sudo update-locale - source install/opt/HPCCSystems/sbin/hpcc_setenv - install/opt/HPCCSystems/etc/init.d/hpcc-init start - export CLASSPATH=".:$(realpath selenium-server-standalone-3.141.59.jar):$(realpath testng-6.8.7.jar)" - pushd install/test-ui - ./run.sh tests http://localhost:8010 > eclWatchUiTest.log 2>&1 - retCode=$? - echo "UI test done" - [[ $retCode -ne 0 ]] && exit 1 - popd - - - name: eclwatch-ui-test-logs-artifact - if: ${{ failure() || cancelled() }} - uses: actions/upload-artifact@v2 - with: - name: ECLWatchUiTest - path: install/test-ui/eclWatchUiTest.log - if-no-files-found: error diff --git a/.github/workflows/test-smoke-gh_runner.yml b/.github/workflows/test-smoke-gh_runner.yml new file mode 100644 index 00000000000..b01fdd3ebbc --- /dev/null +++ b/.github/workflows/test-smoke-gh_runner.yml @@ -0,0 +1,141 @@ +name: Smoketest Package (gh-runner) + +on: + workflow_call: + inputs: + os: + type: string + description: 'Operating System' + required: false + default: 'ubuntu-22.04' + asset-name: + type: string + description: 'Asset Name' + required: false + default: 'build-docker-package' + dependencies: + type: string + description: 'Dependencies' + required: false + default: 'bison flex build-essential binutils-dev curl lsb-release libcppunit-dev python3-dev default-jdk r-base-dev r-cran-rcpp r-cran-rinside r-cran-inline pkg-config libtool autotools-dev automake git cmake xmlstarlet' + +jobs: + + main: + runs-on: ${{ inputs.os }} + strategy: + fail-fast: false + matrix: + engine: ['hthor', 'thor', 'roxie'] + match: + - pattern: '[a-d]*.ecl' + - pattern: '[e-h]*.ecl' + - pattern: '[i-l]*.ecl' + - pattern: '[m-q]*.ecl' + - pattern: '[r-u]*.ecl' + exclude: teststdlibrary.ecl + - pattern: '[v-z]*.ecl' + include: + - engine: 'hthor' + match: + pattern: teststdlibrary.ecl + - engine: 'thor' + match: + pattern: teststdlibrary.ecl + - engine: 'roxie' + match: + pattern: teststdlibrary.ecl + + steps: + + - name: Free additional disk space (remove Android SDK + Tools) + run: | + sudo rm -rf /usr/local/lib/android + + - name: Download Package + uses: actions/download-artifact@v3 + with: + name: ${{ inputs.asset-name }} + path: ${{ inputs.asset-name }} + + - name: Download Support Files + uses: actions/download-artifact@v3 + with: + name: ${{ inputs.asset-name }}-support-files + path: ${{ inputs.asset-name }}-support-files + + - name: vars + id: vars + run: | + echo "matrix-setname=$(echo -n ${{ matrix.match.pattern }} | tr -c \"[:alnum:]\" _)" >> $GITHUB_OUTPUT + + - name: Install Dependencies + shell: "bash" + run: | + sudo apt-get update + sudo apt-get install -y ${{ inputs.dependencies }} + sudo apt-get install -y gdb + + - name: Install Package + shell: "bash" + run: | + sudo apt-get install -y -f ./${{ inputs.asset-name }}/*.deb + sudo chown -R $USER:$USER /opt/HPCCSystems + + sudo xmlstarlet ed -L -u 'Environment/Software/ThorCluster/@slavesPerNode' -v 2 -u 'Environment/Software/ThorCluster/@channelsPerSlave' -v 1 /etc/HPCCSystems/environment.xml + + chmod +x ./${{ inputs.asset-name }}-support-files/* + sudo cp ./${{ inputs.asset-name }}-support-files/* /opt/HPCCSystems/bin + + - name: Start HPCC-Platform + shell: "bash" + run: | + export LANG="en_US.UTF-8" + sudo update-locale + sudo /etc/init.d/hpcc-init start + + - name: Run Setup + working-directory: /opt/HPCCSystems/testing/regress + shell: "bash" + run: | + /opt/HPCCSystems/bin/timeoutcmd $((8 * 60)) \ + ./ecl-test setup --preAbort '/opt/HPCCSystems/bin/smoketest-preabort.sh' --pq 2 --generateStackTrace + grep Failure: /home/runner/HPCCSystems-regression/log/setup_*.log + if [[ "$(grep -ohP '(?<=^Failure: )[0-9]+$' /home/runner/HPCCSystems-regression/log/setup_*.log | awk '{s+=$1} END {print s}')" -gt 0 ]]; then exit 1; fi + + - name: Select Tests + id: select-tests + working-directory: /opt/HPCCSystems/testing/regress + shell: "bash" + run: | + (cd ecl; ls ${{ matrix.match.pattern }}) > matches.tmp + echo queries are: + cat matches.tmp + if [[ -z "${{ matrix.match.exclude }}" ]] + then + queries="$(cat matches.tmp | tr '\n' ' ')" + else + queries="$(cd ecl; ls ${{ matrix.match.exclude }} | grep -v -f - ../matches.tmp | tr '\n' ' ')" + fi + echo queries after exclude: ${queries} + echo "queries=${queries}" >> $GITHUB_OUTPUT + + - name: Run Tests + working-directory: /opt/HPCCSystems/testing/regress + shell: "bash" + run: | + # force regression suite to timeout after 28 minutes, so it captures ZAP report of any inflight hung queries + /opt/HPCCSystems/bin/timeoutcmd $((28 * 60)) \ + ./ecl-test query --preAbort '/opt/HPCCSystems/bin/smoketest-preabort.sh' --pq 2 --target ${{ matrix.engine }} --excludeclass python2,embedded-r,embedded-js,3rdpartyservice,mongodb --generateStackTrace ${{ steps.select-tests.outputs.queries }} + grep Failure: /home/runner/HPCCSystems-regression/log/${{ matrix.engine }}.*.log + if [[ "$(grep -oP '(?<=^Failure: )[0-9]+$' /home/runner/HPCCSystems-regression/log/${{ matrix.engine }}.*.log)" -gt 0 ]]; then exit 1; fi + + - name: regression-run-logs-artifact + if: ${{ failure() || cancelled() }} + uses: actions/upload-artifact@v3 + with: + name: test-smoke-${{ inputs.asset-name }}-${{ matrix.engine }}-${{ steps.vars.outputs.matrix-setname }} + path: | + /var/log/HPCCSystems + /home/runner/HPCCSystems-regression + if-no-files-found: error diff --git a/.github/workflows/test-ui-gh_runner.yml b/.github/workflows/test-ui-gh_runner.yml new file mode 100644 index 00000000000..0126f2c6e39 --- /dev/null +++ b/.github/workflows/test-ui-gh_runner.yml @@ -0,0 +1,130 @@ +name: UI test Package (gh-runner) + +on: + workflow_call: + inputs: + os: + type: string + description: 'Operating System' + required: false + default: 'ubuntu-22.04' + asset-name: + type: string + description: 'Asset Name' + required: false + default: 'build-docker-package' + dependencies: + type: string + description: 'Dependencies' + required: false + default: 'bison flex build-essential binutils-dev curl lsb-release libcppunit-dev python3-dev default-jdk r-base-dev r-cran-rcpp r-cran-rinside r-cran-inline pkg-config libtool autotools-dev automake git cmake xmlstarlet' + +jobs: + + main: + timeout-minutes: 30 # the ui-tests run step has its own timeout, however the job can sometimes take time to download dependencies + runs-on: ${{ inputs.os }} + steps: + + - name: Free additional disk space (remove Android SDK + Tools) + run: | + sudo rm -rf /usr/local/lib/android + + - name: Download UI Test Files + uses: actions/download-artifact@v3 + with: + name: ${{ inputs.asset-name }}-ui_test-files + path: ${{ inputs.asset-name }}-ui_test-files + + - name: Check ECLWatch UI Test Directory + id: check + run: | + if [[ ! -d ${{ inputs.asset-name }}-ui_test-files ]] + then + echo "ECLWatch UI ${{ inputs.asset-name }}-ui_test-files directory missing." + else + javaFilesCount=$(find ${{ inputs.asset-name }}-ui_test-files/ -iname '*.java' -type f -print | wc -l ) + echo "Number of test java files is $javaFilesCount" + if [[ ${javaFilesCount} -eq 0 ]] + then + echo "No java files, do nothing." + else + echo "runtests=true" >> $GITHUB_OUTPUT + fi + fi + + - name: Install Dependencies + if: steps.check.outputs.runtests + shell: "bash" + run: | + sudo apt-get update + sudo apt-get install -y \ + git \ + wget \ + net-tools \ + tzdata \ + unzip \ + xvfb \ + libxi6 \ + default-jdk \ + gdb \ + ${{ inputs.dependencies }} + + - name: Install UI Dependencies + if: steps.check.outputs.runtests + run: | + wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb + sudo apt-get install -y ./google-chrome-stable_current_amd64.deb + wget https://chromedriver.storage.googleapis.com/2.41/chromedriver_linux64.zip + unzip chromedriver_linux64.zip + sudo mv chromedriver /usr/bin/chromedriver + sudo chown root:root /usr/bin/chromedriver + sudo chmod +x /usr/bin/chromedriver + wget https://selenium-release.storage.googleapis.com/3.141/selenium-server-standalone-3.141.59.jar + wget http://www.java2s.com/Code/JarDownload/testng/testng-6.8.7.jar.zip + unzip testng-6.8.7.jar.zip + + - name: Download Package + if: steps.check.outputs.runtests + uses: actions/download-artifact@v3 + with: + name: ${{ inputs.asset-name }} + path: ${{ inputs.asset-name }} + + - name: Install Package + if: steps.check.outputs.runtests + shell: "bash" + run: | + sudo apt-get install -y -f ./${{ inputs.asset-name }}/*.deb + sudo chown -R $USER:$USER /opt/HPCCSystems + + sudo xmlstarlet ed -L -u 'Environment/Software/ThorCluster/@slavesPerNode' -v 2 -u 'Environment/Software/ThorCluster/@channelsPerSlave' -v 1 /etc/HPCCSystems/environment.xml + + chmod +x ./${{ inputs.asset-name }}-ui_test-files/* + + - name: Start HPCC-Platform + shell: "bash" + run: | + export LANG="en_US.UTF-8" + sudo update-locale + sudo /etc/init.d/hpcc-init start + + - name: Run Tests + timeout-minutes: 10 # generous, current runtime is ~1min, this should be increased if new tests are added + if: steps.check.outputs.runtests + run: | + export CLASSPATH=".:$(realpath selenium-server-standalone-3.141.59.jar):$(realpath testng-6.8.7.jar)" + pushd ${{ inputs.asset-name }}-ui_test-files + sudo ./run.sh tests http://localhost:8010 > eclWatchUiTest.log 2>&1 + retCode=$? + echo "UI test done" + [[ $retCode -ne 0 ]] && exit 1 + popd + + - name: eclwatch-ui-test-logs-artifact + if: ${{ failure() || cancelled() }} + uses: actions/upload-artifact@v2 + with: + name: ${{ inputs.asset-name }}-ui_test-logs + path: ${{ inputs.asset-name }}-ui_test-files/eclWatchUiTest.log + if-no-files-found: error diff --git a/.github/workflows/test-unit-gh_runner.yml b/.github/workflows/test-unit-gh_runner.yml new file mode 100644 index 00000000000..69928aa0811 --- /dev/null +++ b/.github/workflows/test-unit-gh_runner.yml @@ -0,0 +1,67 @@ +name: Unittest Package (gh-runner) + +on: + workflow_call: + inputs: + os: + type: string + description: 'Operating System' + required: false + default: 'ubuntu-22.04' + asset-name: + type: string + description: 'Asset Name' + required: false + default: 'build-docker-package' + dependencies: + type: string + description: 'Dependencies' + required: false + default: 'bison flex build-essential binutils-dev curl lsb-release libcppunit-dev python3-dev default-jdk r-base-dev r-cran-rcpp r-cran-rinside r-cran-inline pkg-config libtool autotools-dev automake git cmake xmlstarlet' + +jobs: + + main: + runs-on: ${{ inputs.os }} + steps: + + - name: Free additional disk space (remove Android SDK + Tools) + run: | + sudo rm -rf /usr/local/lib/android + + - name: Download Package + uses: actions/download-artifact@v3 + with: + name: ${{ inputs.asset-name }} + path: ${{ inputs.asset-name }} + + - name: Install Dependencies + shell: "bash" + run: | + sudo apt-get update + sudo apt-get install -y ${{ inputs.dependencies }} + sudo apt-get install -y gdb + + - name: Install Package + shell: "bash" + run: | + sudo apt-get install -y -f ./${{ inputs.asset-name }}/*.deb + sudo chown -R $USER:$USER /opt/HPCCSystems + + - name: Run Tests + timeout-minutes: 10 # generous, expected time is approx 1 min. + shell: "bash" + run: | + export LANG="en_US.UTF-8" + sudo update-locale + sudo /opt/HPCCSystems/bin/unittests + + - name: unittests-logs-artifact + if: ${{ failure() || cancelled() }} + uses: actions/upload-artifact@v3 + with: + name: test-unit-${{ inputs.asset-name }} + path: | + /var/log/HPCCSystems + /home/runner/HPCCSystems-regression + if-no-files-found: error diff --git a/cmake_modules/options.cmake b/cmake_modules/options.cmake index a744818799d..0d1dbc1f677 100644 --- a/cmake_modules/options.cmake +++ b/cmake_modules/options.cmake @@ -88,7 +88,7 @@ if (NOT USE_JAVA) endif () if ((INCLUDE_PLUGINS AND NOT SUPPRESS_JAVAEMBED) OR INCLUDE_JAVAEMBED) message(WARNING "JAVAEMBED requires USE_JAVA, forcing SUPPRESS_JAVAEMBED ON") - set (SUPRESS_JAVAEMBED ON) + set (SUPPRESS_JAVAEMBED ON) endif () endif () diff --git a/dockerfiles/vcpkg/amazonlinux.dockerfile b/dockerfiles/vcpkg/amazonlinux.dockerfile index 5c1aaf941e5..190badf60f4 100644 --- a/dockerfiles/vcpkg/amazonlinux.dockerfile +++ b/dockerfiles/vcpkg/amazonlinux.dockerfile @@ -6,6 +6,7 @@ RUN amazon-linux-extras install java-openjdk11 && yum install -y \ python3-devel \ epel-release RUN yum install -y \ + ccache \ R-core-devel \ R-Rcpp-devel \ R-RInside-devel diff --git a/dockerfiles/vcpkg/build.sh b/dockerfiles/vcpkg/build.sh index 5c3926df63d..f314770edc2 100755 --- a/dockerfiles/vcpkg/build.sh +++ b/dockerfiles/vcpkg/build.sh @@ -33,7 +33,7 @@ function doBuild() { docker pull "hpccsystems/platform-build-$1:$VCPKG_REF" || true docker pull "hpccsystems/platform-build-$1:$GITHUB_BRANCH" || true - docker build --progress plain --pull --rm -f "$SCRIPT_DIR/$1.dockerfile" \ + docker build --progress plain --rm -f "$SCRIPT_DIR/$1.dockerfile" \ --build-arg DOCKER_NAMESPACE=$DOCKER_USERNAME \ --build-arg VCPKG_REF=$VCPKG_REF \ --cache-from hpccsystems/platform-build-$1:$VCPKG_REF \ @@ -51,9 +51,14 @@ function doBuild() { elif [ "$1" == "amazonlinux" ]; then CMAKE_OPTIONS_EXTRA="-DVCPKG_TARGET_TRIPLET=x64-amazonlinux-dynamic" fi - docker run --rm --mount source="$(pwd)",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached hpccsystems/platform-build-$1:$VCPKG_REF \ - "cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/HPCC-Platform/build-$1 ${CMAKE_OPTIONS} ${CMAKE_OPTIONS_EXTRA} && \ - cmake --build /hpcc-dev/HPCC-Platform/build-$1 --target package --parallel $(nproc)" + mkdir -p $HOME/.ccache + docker run --rm \ + --mount source="$(pwd)",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached \ + --mount source="$HOME/.ccache",target=/root/.ccache,type=bind,consistency=cached \ + hpccsystems/platform-build-$1:$VCPKG_REF \ + "cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/HPCC-Platform/build-$1 ${CMAKE_OPTIONS} ${CMAKE_OPTIONS_EXTRA} -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache && \ + cmake --build /hpcc-dev/HPCC-Platform/build-$1 --target install --parallel $(nproc) && \ + /etc/init.d/hpcc-init start" sudo chown -R $(id -u):$(id -g) ./build-$1 # docker run -it --mount source="$(pwd)",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached build-ubuntu-22.04:latest bash diff --git a/dockerfiles/vcpkg/centos-7.dockerfile b/dockerfiles/vcpkg/centos-7.dockerfile deleted file mode 100644 index 15524db7e21..00000000000 --- a/dockerfiles/vcpkg/centos-7.dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -ARG VCPKG_REF=latest -FROM hpccsystems/platform-build-base-centos-7:$VCPKG_REF - -RUN yum install -y \ - epel-release \ - java-11-openjdk-devel \ - python3-devel \ - wget && \ - yum update -y && yum install -y R-core-devel && \ - yum -y clean all && rm -rf /var/cache - -ENV Rcpp_package=Rcpp_0.12.19.tar.gz -ENV RInside_package=RInside_0.2.12.tar.gz - -RUN wget https://cran.r-project.org/src/contrib/Archive/Rcpp/${Rcpp_package} -RUN wget https://cran.r-project.org/src/contrib/Archive/RInside/${RInside_package} -RUN R CMD INSTALL ${Rcpp_package} ${RInside_package} -RUN rm -f ${Rcpp_package} ${RInside_package} - -WORKDIR /hpcc-dev - -ENTRYPOINT ["/bin/bash", "--login", "-c"] diff --git a/dockerfiles/vcpkg/centos-8.dockerfile b/dockerfiles/vcpkg/centos-8.dockerfile deleted file mode 100644 index 8d419588d73..00000000000 --- a/dockerfiles/vcpkg/centos-8.dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -ARG VCPKG_REF=latest -FROM hpccsystems/platform-build-base-centos-8:$VCPKG_REF - -RUN yum remove -y python3.11 java-1.* && yum install -y \ - java-11-openjdk-devel \ - python3-devel \ - epel-release - -RUN yum install -y \ - R-core-devel \ - R-Rcpp-devel \ - R-RInside-devel - -WORKDIR /hpcc-dev - -ENTRYPOINT ["/bin/bash", "--login", "-c"] diff --git a/dockerfiles/vcpkg/ubuntu-20.04.dockerfile b/dockerfiles/vcpkg/ubuntu-20.04.dockerfile deleted file mode 100644 index 7211cccebde..00000000000 --- a/dockerfiles/vcpkg/ubuntu-20.04.dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -ARG VCPKG_REF=latest -FROM hpccsystems/platform-build-base-ubuntu-20.04:$VCPKG_REF - -ENV RInside_package=RInside_0.2.14.tar.gz - -RUN apt-get update && apt-get install --no-install-recommends -y \ - default-jdk \ - python3-dev \ - wget \ - r-base \ - r-cran-rcpp -RUN wget https://cran.r-project.org/src/contrib/Archive/RInside/${RInside_package} -RUN R CMD INSTALL ${RInside_package} -RUN rm -f ${RInside_package} - -WORKDIR /hpcc-dev - -ENTRYPOINT ["/bin/bash", "--login", "-c"] diff --git a/dockerfiles/vcpkg/ubuntu-22.04.dockerfile b/dockerfiles/vcpkg/ubuntu-22.04.dockerfile deleted file mode 100644 index 768830ad711..00000000000 --- a/dockerfiles/vcpkg/ubuntu-22.04.dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -ARG VCPKG_REF=latest -FROM hpccsystems/platform-build-base-ubuntu-22.04:$VCPKG_REF - -RUN apt-get update && apt-get install --no-install-recommends -y \ - default-jdk \ - ninja-build \ - python3-dev \ - rsync \ - fop \ - libsaxonb-java \ - r-base \ - r-cran-rcpp \ - r-cran-rinside \ - r-cran-inline - -RUN git config --global --add safe.directory '*' - -WORKDIR /hpcc-dev - -ENTRYPOINT ["/bin/bash", "--login", "-c"] diff --git a/dockerfiles/vcpkg/ubuntu-23.10.dockerfile b/dockerfiles/vcpkg/ubuntu-23.10.dockerfile deleted file mode 100644 index e154dbe5bd5..00000000000 --- a/dockerfiles/vcpkg/ubuntu-23.10.dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -ARG VCPKG_REF=latest -FROM hpccsystems/platform-build-base-ubuntu-23.10:$VCPKG_REF - -RUN apt-get update && apt-get install --no-install-recommends -y \ - default-jdk \ - ninja-build \ - python3-dev \ - rsync \ - fop \ - libsaxonb-java \ - r-base \ - r-cran-rcpp \ - r-cran-rinside \ - r-cran-inline - -RUN git config --global --add safe.directory '*' - -WORKDIR /hpcc-dev - -ENTRYPOINT ["/bin/bash", "--login", "-c"] diff --git a/vcpkg b/vcpkg index 4d5501e1c57..a65a37760ac 160000 --- a/vcpkg +++ b/vcpkg @@ -1 +1 @@ -Subproject commit 4d5501e1c5796ca7b42f4ad38731f8fd2a8c8d75 +Subproject commit a65a37760ac2a85527b72b2d81aa50d882ec2e70