From cad02bb351a01b651aa7cd6028955dfe9460f295 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 12 Oct 2023 22:08:56 +0200 Subject: [PATCH 1/9] first step for CI to analyse PR --- .github/workflows/pr_analysis.yml | 84 +++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 .github/workflows/pr_analysis.yml diff --git a/.github/workflows/pr_analysis.yml b/.github/workflows/pr_analysis.yml new file mode 100644 index 0000000000..42daad4ea8 --- /dev/null +++ b/.github/workflows/pr_analysis.yml @@ -0,0 +1,84 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: Analysis of a pull request +on: [push, pull_request, workflow_dispatch] +permissions: + contents: read # to fetch code (actions/checkout) +jobs: + pilot: + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + EESSI_VERSION: + - 2023.06 + EESSI_SOFTWARE_SUBDIR: + - x86_64/generic + # - aarch64/generic + # - x86_64/amd/zen2 + # - x86_64/intel/broadwell + # - x86_64/intel/cascadelake + # - x86_64/intel/skylake_avx512 + EASYSTACK_FILE: + # - eessi-2023.06-eb-4.7.2-2021a.yml + # - eessi-2023.06-eb-4.7.2-2021b.yml + # - eessi-2023.06-eb-4.7.2-2022a.yml + # - eessi-2023.06-eb-4.7.2-2022b.yml + # - eessi-2023.06-eb-4.7.2-system.yml + # - eessi-2023.06-eb-4.8.0-system.yml + - eessi-2023.06-eb-4.8.1-2022a.yml + # - eessi-2023.06-eb-4.8.1-system.yml + python: [3.7] # ScanCode-Toolkit requires Python 3.7 or newer + steps: + - name: Install dependencies for ScanCode-Toolkit + run: | + sudo apt install python-dev bzip2 xz-utils zlib1g libxml2-dev libxslt1-dev libpopt0 + + - name: set up Python + uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0 + with: + python-version: ${{matrix.python}} + architecture: x64 + + - name: upgrade base tools + run: | + pip install --user --upgrade pip setuptools wheel + + - name: Install ScanCode-Toolkit + run: | + pip install --user scancode-toolkit + +# - name: Check out software-layer repository +# uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 +# +# - name: Mount EESSI CernVM-FS pilot repository +# uses: cvmfs-contrib/github-action-cvmfs@d4641d0d591c9a5c3be23835ced2fb648b44c04b # v3.1 +# with: +# cvmfs_config_package: https://github.com/NorESSI/filesystem-layer/releases/download/latest/cvmfs-config-nessi_latest_all.deb +# cvmfs_http_proxy: DIRECT +# cvmfs_repositories: pilot.nessi.no +# +# - name: Test check_missing_installations.sh script with EESSI_SOFTWARE_SUBDIR_OVERRIDE +# if: '!cancelled()' +# run: | +# export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} +# source /cvmfs/pilot.nessi.no/versions/${{matrix.EESSI_VERSION}}/init/bash +# module load EasyBuild +# eb --version +# export EESSI_PREFIX=/cvmfs/pilot.nessi.no/versions/${{matrix.EESSI_VERSION}} +# export EESSI_OS_TYPE=linux +# env | grep ^EESSI | sort +# echo "just run check_missing_installations.sh (should use ${{matrix.EASYSTACK_FILE}})" +# ./check_missing_installations.sh ${{matrix.EASYSTACK_FILE}} +# +# - name: Test check_missing_installations.sh script without EESSI_SOFTWARE_SUBDIR_OVERRIDE +# if: '!cancelled()' +# run: | +# source /cvmfs/pilot.nessi.no/versions/${{matrix.EESSI_VERSION}}/init/bash +# module load EasyBuild +# eb --version +# export EESSI_PREFIX=/cvmfs/pilot.nessi.no/versions/${{matrix.EESSI_VERSION}} +# export EESSI_OS_TYPE=linux +# export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} +# env | grep ^EESSI | sort +# echo "just run check_missing_installations.sh (should use ${{matrix.EASYSTACK_FILE}})" +# ./check_missing_installations.sh ${{matrix.EASYSTACK_FILE}} From ae8c8c581a9e04a446cec886f07ad70925171e17 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 12 Oct 2023 22:17:38 +0200 Subject: [PATCH 2/9] list missing installations (if any) --- .github/workflows/pr_analysis.yml | 45 +++++++++++++++---------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/.github/workflows/pr_analysis.yml b/.github/workflows/pr_analysis.yml index 42daad4ea8..5f4727c7ea 100644 --- a/.github/workflows/pr_analysis.yml +++ b/.github/workflows/pr_analysis.yml @@ -47,29 +47,28 @@ jobs: run: | pip install --user scancode-toolkit -# - name: Check out software-layer repository -# uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 -# -# - name: Mount EESSI CernVM-FS pilot repository -# uses: cvmfs-contrib/github-action-cvmfs@d4641d0d591c9a5c3be23835ced2fb648b44c04b # v3.1 -# with: -# cvmfs_config_package: https://github.com/NorESSI/filesystem-layer/releases/download/latest/cvmfs-config-nessi_latest_all.deb -# cvmfs_http_proxy: DIRECT -# cvmfs_repositories: pilot.nessi.no -# -# - name: Test check_missing_installations.sh script with EESSI_SOFTWARE_SUBDIR_OVERRIDE -# if: '!cancelled()' -# run: | -# export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} -# source /cvmfs/pilot.nessi.no/versions/${{matrix.EESSI_VERSION}}/init/bash -# module load EasyBuild -# eb --version -# export EESSI_PREFIX=/cvmfs/pilot.nessi.no/versions/${{matrix.EESSI_VERSION}} -# export EESSI_OS_TYPE=linux -# env | grep ^EESSI | sort -# echo "just run check_missing_installations.sh (should use ${{matrix.EASYSTACK_FILE}})" -# ./check_missing_installations.sh ${{matrix.EASYSTACK_FILE}} -# + - name: Check out software-layer repository + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + + - name: Mount EESSI CernVM-FS pilot repository + uses: cvmfs-contrib/github-action-cvmfs@d4641d0d591c9a5c3be23835ced2fb648b44c04b # v3.1 + with: + cvmfs_config_package: https://github.com/NorESSI/filesystem-layer/releases/download/latest/cvmfs-config-nessi_latest_all.deb + cvmfs_http_proxy: DIRECT + cvmfs_repositories: pilot.nessi.no + + - name: Determine missing installations with + if: '!cancelled()' + run: | + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} + source /cvmfs/pilot.nessi.no/versions/${{matrix.EESSI_VERSION}}/init/bash + module load EasyBuild + eb --version + export EESSI_PREFIX=/cvmfs/pilot.nessi.no/versions/${{matrix.EESSI_VERSION}} + export EESSI_OS_TYPE=linux + env | grep ^EESSI | sort + eb --missing --easystack ${{matrix.EASYSTACK_FILE}} | grep '^* ' + # - name: Test check_missing_installations.sh script without EESSI_SOFTWARE_SUBDIR_OVERRIDE # if: '!cancelled()' # run: | From 35a5a41bfa069b6d3e75cefa3f750e204944aba1 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 12 Oct 2023 22:25:56 +0200 Subject: [PATCH 3/9] configure EasyBuild for EESSI --- .github/workflows/pr_analysis.yml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/pr_analysis.yml b/.github/workflows/pr_analysis.yml index 5f4727c7ea..4ab1121c6e 100644 --- a/.github/workflows/pr_analysis.yml +++ b/.github/workflows/pr_analysis.yml @@ -57,7 +57,7 @@ jobs: cvmfs_http_proxy: DIRECT cvmfs_repositories: pilot.nessi.no - - name: Determine missing installations with + - name: Determine missing installations if: '!cancelled()' run: | export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} @@ -67,17 +67,5 @@ jobs: export EESSI_PREFIX=/cvmfs/pilot.nessi.no/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux env | grep ^EESSI | sort + source configure_easybuild eb --missing --easystack ${{matrix.EASYSTACK_FILE}} | grep '^* ' - -# - name: Test check_missing_installations.sh script without EESSI_SOFTWARE_SUBDIR_OVERRIDE -# if: '!cancelled()' -# run: | -# source /cvmfs/pilot.nessi.no/versions/${{matrix.EESSI_VERSION}}/init/bash -# module load EasyBuild -# eb --version -# export EESSI_PREFIX=/cvmfs/pilot.nessi.no/versions/${{matrix.EESSI_VERSION}} -# export EESSI_OS_TYPE=linux -# export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} -# env | grep ^EESSI | sort -# echo "just run check_missing_installations.sh (should use ${{matrix.EASYSTACK_FILE}})" -# ./check_missing_installations.sh ${{matrix.EASYSTACK_FILE}} From 7d70383970fc24ac1691a5cc35216bd8b65deac3 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 12 Oct 2023 22:46:49 +0200 Subject: [PATCH 4/9] list missing modules (if any) --- .github/workflows/pr_analysis.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr_analysis.yml b/.github/workflows/pr_analysis.yml index 4ab1121c6e..02e8436796 100644 --- a/.github/workflows/pr_analysis.yml +++ b/.github/workflows/pr_analysis.yml @@ -68,4 +68,17 @@ jobs: export EESSI_OS_TYPE=linux env | grep ^EESSI | sort source configure_easybuild - eb --missing --easystack ${{matrix.EASYSTACK_FILE}} | grep '^* ' + eb --missing --easystack ${{matrix.EASYSTACK_FILE}} > ${EB_MISSING_OUT} + grep '.* out of .* required modules missing:' ${EB_MISSING_OUT} + exit_code=$? + if [[ ${exit_code} -eq 0 ]]; then + # there may be multiple sections with the above search string + # we grab all lines listing missing modules and print a list of + # unique modules only + MISSING_MODULES=$(grep '^* ' ${EB_MISSING_OUT} | sort -u) + echo "found $(echo ${MISSING_MODULES} | wc -l) modules missing:" + echo "${MISSING_MODULES}" + else + echo "no modules missing" + fi + From 2c5cf92dbf7a0366b3baa96cf3eadde565bf8017 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 12 Oct 2023 22:57:33 +0200 Subject: [PATCH 5/9] fix undefined variable + add action to update PR description --- .github/workflows/pr_analysis.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/pr_analysis.yml b/.github/workflows/pr_analysis.yml index 02e8436796..fae072d432 100644 --- a/.github/workflows/pr_analysis.yml +++ b/.github/workflows/pr_analysis.yml @@ -68,6 +68,7 @@ jobs: export EESSI_OS_TYPE=linux env | grep ^EESSI | sort source configure_easybuild + EB_MISSING_OUT=eb_missing.out eb --missing --easystack ${{matrix.EASYSTACK_FILE}} > ${EB_MISSING_OUT} grep '.* out of .* required modules missing:' ${EB_MISSING_OUT} exit_code=$? @@ -82,3 +83,10 @@ jobs: echo "no modules missing" fi + - name: Update PR description + uses: nefrob/pr-description@eb00e54af678cf0d50a5b8fdd54b557c9cd8ddc5 + with: + content: "Hello there!" + regex: "matchuntilthenend.*" + regexFlags: i + token: ${{ secrets.GITHUB_TOKEN }} From c8e53cea85453518cf58c14532bae3f3c920b77e Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 12 Oct 2023 23:07:34 +0200 Subject: [PATCH 6/9] add a bit debug output --- .github/workflows/pr_analysis.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr_analysis.yml b/.github/workflows/pr_analysis.yml index fae072d432..46805887de 100644 --- a/.github/workflows/pr_analysis.yml +++ b/.github/workflows/pr_analysis.yml @@ -69,9 +69,13 @@ jobs: env | grep ^EESSI | sort source configure_easybuild EB_MISSING_OUT=eb_missing.out - eb --missing --easystack ${{matrix.EASYSTACK_FILE}} > ${EB_MISSING_OUT} + echo "eb start" + eb --missing --easystack ${{matrix.EASYSTACK_FILE}} > ${EB_MISSING_OUT} || true + echo "eb done" + echo "grep start" grep '.* out of .* required modules missing:' ${EB_MISSING_OUT} exit_code=$? + echo "grep done" if [[ ${exit_code} -eq 0 ]]; then # there may be multiple sections with the above search string # we grab all lines listing missing modules and print a list of From 41e4db0d9348476ea107af18ced6d521666a6d38 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 12 Oct 2023 23:20:13 +0200 Subject: [PATCH 7/9] put possibly failing grep inside if expression --- .github/workflows/pr_analysis.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr_analysis.yml b/.github/workflows/pr_analysis.yml index 46805887de..fcb159b3e8 100644 --- a/.github/workflows/pr_analysis.yml +++ b/.github/workflows/pr_analysis.yml @@ -73,10 +73,8 @@ jobs: eb --missing --easystack ${{matrix.EASYSTACK_FILE}} > ${EB_MISSING_OUT} || true echo "eb done" echo "grep start" - grep '.* out of .* required modules missing:' ${EB_MISSING_OUT} - exit_code=$? - echo "grep done" - if [[ ${exit_code} -eq 0 ]]; then + + if [[ $(grep '.* out of .* required modules missing:' ${EB_MISSING_OUT}) ]]; then # there may be multiple sections with the above search string # we grab all lines listing missing modules and print a list of # unique modules only From 689259dd21c7268de4a1f0cdbd7c82093a836823 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 12 Oct 2023 23:48:06 +0200 Subject: [PATCH 8/9] use result to update PR description --- .github/workflows/pr_analysis.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr_analysis.yml b/.github/workflows/pr_analysis.yml index fcb159b3e8..94e584d3f3 100644 --- a/.github/workflows/pr_analysis.yml +++ b/.github/workflows/pr_analysis.yml @@ -72,23 +72,27 @@ jobs: echo "eb start" eb --missing --easystack ${{matrix.EASYSTACK_FILE}} > ${EB_MISSING_OUT} || true echo "eb done" + PR_UPDATE_FILE=$(mktemp ${{matrix.EASYSTACK_FILE}}.XXX) + echo "PR_UPDATE_FILE=${PR_UPDATE_FILE}" >> "${GITHUB_ENV}" echo "grep start" - if [[ $(grep '.* out of .* required modules missing:' ${EB_MISSING_OUT}) ]]; then # there may be multiple sections with the above search string # we grab all lines listing missing modules and print a list of # unique modules only MISSING_MODULES=$(grep '^* ' ${EB_MISSING_OUT} | sort -u) echo "found $(echo ${MISSING_MODULES} | wc -l) modules missing:" + echo "found $(echo ${MISSING_MODULES} | wc -l) modules missing:" > ${PR_UPDATE_FILE} echo "${MISSING_MODULES}" + echo "${MISSING_MODULES}" >> ${PR_UPDATE_FILE} else echo "no modules missing" + echo "no modules missing" > ${PR_UPDATE_FILE} fi - name: Update PR description uses: nefrob/pr-description@eb00e54af678cf0d50a5b8fdd54b557c9cd8ddc5 with: - content: "Hello there!" + content: ${PR_UPDATE_FILE} regex: "matchuntilthenend.*" regexFlags: i token: ${{ secrets.GITHUB_TOKEN }} From 02d29597c9fdb0ac23779f509536644fb0d44a05 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 13 Oct 2023 00:23:50 +0200 Subject: [PATCH 9/9] give CI permission to update PR description --- .github/workflows/pr_analysis.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr_analysis.yml b/.github/workflows/pr_analysis.yml index 94e584d3f3..cdd4caca50 100644 --- a/.github/workflows/pr_analysis.yml +++ b/.github/workflows/pr_analysis.yml @@ -6,6 +6,9 @@ permissions: jobs: pilot: runs-on: ubuntu-20.04 + permissions: + contents: read + pull-requests: write strategy: fail-fast: false matrix: @@ -74,6 +77,7 @@ jobs: echo "eb done" PR_UPDATE_FILE=$(mktemp ${{matrix.EASYSTACK_FILE}}.XXX) echo "PR_UPDATE_FILE=${PR_UPDATE_FILE}" >> "${GITHUB_ENV}" + echo "# PR analysis for arch ${{matrix.EESSI_SOFTWARE_SUBDIR}}" > ${PR_UPDATE_FILE} echo "grep start" if [[ $(grep '.* out of .* required modules missing:' ${EB_MISSING_OUT}) ]]; then # there may be multiple sections with the above search string @@ -81,7 +85,7 @@ jobs: # unique modules only MISSING_MODULES=$(grep '^* ' ${EB_MISSING_OUT} | sort -u) echo "found $(echo ${MISSING_MODULES} | wc -l) modules missing:" - echo "found $(echo ${MISSING_MODULES} | wc -l) modules missing:" > ${PR_UPDATE_FILE} + echo "found $(echo ${MISSING_MODULES} | wc -l) modules missing:" >> ${PR_UPDATE_FILE} echo "${MISSING_MODULES}" echo "${MISSING_MODULES}" >> ${PR_UPDATE_FILE} else @@ -93,6 +97,6 @@ jobs: uses: nefrob/pr-description@eb00e54af678cf0d50a5b8fdd54b557c9cd8ddc5 with: content: ${PR_UPDATE_FILE} - regex: "matchuntilthenend.*" + regex: "# PR analysis for arch ${{matrix.EESSI_SOFTWARE_SUBDIR}}.*" regexFlags: i token: ${{ secrets.GITHUB_TOKEN }}