fix ci scripts and workflows

.github/workflows/build.yaml

@@ -25,15 +25,53 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
   cancel-in-progress: true
 
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  actions: read
+  checks: none
+  contents: read
+  deployments: none
+  discussions: none
+  id-token: write
+  issues: none
+  packages: read
+  pages: none
+  pull-requests: read
+  repository-projects: none
+  security-events: none
+  statuses: none
+
 jobs:
   python-build:
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: ${{ inputs.build_type || 'branch' }}
-      branch: ${{ inputs.branch }}
-      date: ${{ inputs.date }}
-      sha: ${{ inputs.sha }}
+    runs-on: "linux-amd64-cpu8"
+    container:
+      image: rapidsai/ci-conda:latest
+      env:
+        RAPIDS_BUILD_TYPE: ${{ inputs.build_type || 'branch' }}
+        PARALLEL_LEVEL: ${{ env.PARALLEL_LEVEL }}
+    steps:
+      - uses: aws-actions/configure-aws-credentials@v3
+        with:
+          role-to-assume: ${{ vars.AWS_ROLE_ARN }}
+          aws-region: ${{ vars.AWS_REGION }}
+          role-duration-seconds: 43200 # 12h
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.sha }}
+          fetch-depth: 0
+      - name: Standardize repository information
+        run: |
+          echo "RAPIDS_REPOSITORY=${{ github.repository }}" >> "${GITHUB_ENV}"
+          echo "RAPIDS_SHA=$(git rev-parse HEAD)" >> "${GITHUB_ENV}"
+          echo "RAPIDS_REF_NAME=${{ inputs.branch || github.ref_name }}" >> "${GITHUB_ENV}"
+          echo "RAPIDS_NIGHTLY_DATE=${{ inputs.date }}" >> "${GITHUB_ENV}"
+      - name: Python build
+        run: ci/build_python.sh
+        env:
+          GH_TOKEN: ${{ github.token }}
   upload-conda:
     needs: [python-build]
     secrets: inherit
@@ -44,15 +82,32 @@ jobs:
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
   wheel-build:
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: ${{ inputs.build_type || 'branch' }}
-      branch: ${{ inputs.branch }}
-      sha: ${{ inputs.sha }}
-      date: ${{ inputs.date }}
-      script: ci/build_wheel.sh
-      matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.10" and (.CUDA_VER == "11.8.0" or .CUDA_VER == "12.0.1")))
+    runs-on: "linux-amd64-cpu8"
+    container:
+      image: rapidsai/ci-wheel:latest
+      env:
+        RAPIDS_BUILD_TYPE: ${{ inputs.build_type || 'branch' }}
+        PARALLEL_LEVEL: ${{ env.PARALLEL_LEVEL }}
+    steps:
+      - uses: aws-actions/configure-aws-credentials@v3
+        with:
+          role-to-assume: ${{ vars.AWS_ROLE_ARN }}
+          aws-region: ${{ vars.AWS_REGION }}
+          role-duration-seconds: 43200 # 12h
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.sha }}
+          fetch-depth: 0
+      - name: Standardize repository information
+        run: |
+          echo "RAPIDS_REPOSITORY=${{ github.repository }}" >> "${GITHUB_ENV}"
+          echo "RAPIDS_SHA=$(git rev-parse HEAD)" >> "${GITHUB_ENV}"
+          echo "RAPIDS_REF_NAME=${{ inputs.branch || github.ref_name }}" >> "${GITHUB_ENV}"
+          echo "RAPIDS_NIGHTLY_DATE=${{ inputs.date }}" >> "${GITHUB_ENV}"
+      - name: Python build
+        run: ci/build_wheel.sh
+        env:
+          GH_TOKEN: ${{ github.token }}
   wheel-publish:
     needs: wheel-build
     secrets: inherit

.github/workflows/pr.yaml

@@ -32,65 +32,122 @@ jobs:
   pr-builder:
     needs:
       - checks
-      - conda-python-build-tests
-      - wheel-build-test
+      - conda-python-build
+      - conda-python-tests
+      - wheel-build
+      - wheel-tests
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
   checks:
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-  conda-python-build-tests:
+  conda-python-build:
     needs: checks
-    runs-on: "linux-amd64-gpu-v100-latest-1"
+    runs-on: "linux-amd64-cpu8"
     container:
       image: rapidsai/ci-conda:latest
       env:
-        RAPIDS_BUILD_TYPE: branch
+        RAPIDS_BUILD_TYPE: pull-request
         PARALLEL_LEVEL: ${{ env.PARALLEL_LEVEL }}
-        NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
     steps:
+      - uses: aws-actions/configure-aws-credentials@v3
+        with:
+          role-to-assume: ${{ vars.AWS_ROLE_ARN }}
+          aws-region: ${{ vars.AWS_REGION }}
+          role-duration-seconds: 43200 # 12h
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
       - name: Standardize repository information
         run: |
-          echo "RAPIDS_REPOSITORY=${{ inputs.repo || github.repository }}" >> "${GITHUB_ENV}"
+          echo "RAPIDS_REPOSITORY=${{ github.repository }}" >> "${GITHUB_ENV}"
           echo "RAPIDS_SHA=$(git rev-parse HEAD)" >> "${GITHUB_ENV}"
-          echo "RAPIDS_REF_NAME=${{ inputs.branch || github.ref_name }}" >> "${GITHUB_ENV}"
-          echo "RAPIDS_NIGHTLY_DATE=${{ inputs.date }}" >> "${GITHUB_ENV}"
+          echo "RAPIDS_REF_NAME=${{ github.ref_name }}" >> "${GITHUB_ENV}"
+          echo "RAPIDS_NIGHTLY_DATE=" >> "${GITHUB_ENV}"
       - name: Python build
         run: ci/build_python.sh
         env:
           GH_TOKEN: ${{ github.token }}
-      - name: Python test
+  conda-python-tests:
+    needs: conda-python-build
+    runs-on: "linux-amd64-gpu-v100-latest-1"
+    container:
+      image: rapidsai/ci-conda:latest
+      env:
+        RAPIDS_BUILD_TYPE: pull-request
+        PARALLEL_LEVEL: ${{ env.PARALLEL_LEVEL }}
+        NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
+    steps:
+      - uses: aws-actions/configure-aws-credentials@v3
+        with:
+          role-to-assume: ${{ vars.AWS_ROLE_ARN }}
+          aws-region: ${{ vars.AWS_REGION }}
+          role-duration-seconds: 43200 # 12h
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Standardize repository information
+        run: |
+          echo "RAPIDS_REPOSITORY=${{ github.repository }}" >> "${GITHUB_ENV}"
+          echo "RAPIDS_SHA=$(git rev-parse HEAD)" >> "${GITHUB_ENV}"
+          echo "RAPIDS_REF_NAME=${{ github.ref_name }}" >> "${GITHUB_ENV}"
+          echo "RAPIDS_NIGHTLY_DATE=" >> "${GITHUB_ENV}"
+      - name: Python tests
         run: ci/test_python.sh
         env:
           GH_TOKEN: ${{ github.token }}
-
-  wheel-build-test:
+  wheel-build:
     needs: checks
-    runs-on: "linux-amd64-gpu-v100-latest-1"
+    runs-on: "linux-amd64-cpu8"
     container:
       image: rapidsai/ci-wheel:latest
       env:
-        RAPIDS_BUILD_TYPE: branch
+        RAPIDS_BUILD_TYPE: pull-request
         PARALLEL_LEVEL: ${{ env.PARALLEL_LEVEL }}
-        NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
     steps:
+      - uses: aws-actions/configure-aws-credentials@v3
+        with:
+          role-to-assume: ${{ vars.AWS_ROLE_ARN }}
+          aws-region: ${{ vars.AWS_REGION }}
+          role-duration-seconds: 43200 # 12h
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
       - name: Standardize repository information
         run: |
-          echo "RAPIDS_REPOSITORY=${{ inputs.repo || github.repository }}" >> "${GITHUB_ENV}"
+          echo "RAPIDS_REPOSITORY=${{ github.repository }}" >> "${GITHUB_ENV}"
           echo "RAPIDS_SHA=$(git rev-parse HEAD)" >> "${GITHUB_ENV}"
-          echo "RAPIDS_REF_NAME=${{ inputs.branch || github.ref_name }}" >> "${GITHUB_ENV}"
-          echo "RAPIDS_NIGHTLY_DATE=${{ inputs.date }}" >> "${GITHUB_ENV}"
+          echo "RAPIDS_REF_NAME=${{ github.ref_name }}" >> "${GITHUB_ENV}"
+          echo "RAPIDS_NIGHTLY_DATE=" >> "${GITHUB_ENV}"
       - name: Python build
         run: ci/build_wheel.sh
         env:
           GH_TOKEN: ${{ github.token }}
-      - name: Python test
+  wheel-tests:
+    needs: wheel-build
+    runs-on: "linux-amd64-gpu-v100-latest-1"
+    container:
+      image: rapidsai/ci-wheel:latest
+      env:
+        RAPIDS_BUILD_TYPE: pull-request
+        PARALLEL_LEVEL: ${{ env.PARALLEL_LEVEL }}
+        NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}
+    steps:
+      - uses: aws-actions/configure-aws-credentials@v3
+        with:
+          role-to-assume: ${{ vars.AWS_ROLE_ARN }}
+          aws-region: ${{ vars.AWS_REGION }}
+          role-duration-seconds: 43200 # 12h
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Standardize repository information
+        run: |
+          echo "RAPIDS_REPOSITORY=${{ github.repository }}" >> "${GITHUB_ENV}"
+          echo "RAPIDS_SHA=$(git rev-parse HEAD)" >> "${GITHUB_ENV}"
+          echo "RAPIDS_REF_NAME=${{ github.ref_name }}" >> "${GITHUB_ENV}"
+          echo "RAPIDS_NIGHTLY_DATE=" >> "${GITHUB_ENV}"
+      - name: Python tests
         run: ci/test_wheel.sh
         env:
-          GH_TOKEN: ${{ github.token }}
+          GH_TOKEN: ${{ github.token }}

ci/build_python.sh

@@ -25,4 +25,7 @@ echo "__version__ = '$version'" > jupyterlab_nvdashboard/_version.py
 # TODO: Remove `--no-test` flag once importing on a CPU
 # node works correctly
 rapids-logger "Building JupyterLab NVDashboard conda package"
-RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild --no-test conda/recipes/jupyterlab-nvdashboard
+RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild --no-test conda/recipes/jupyterlab-nvdashboard --output
+
+rapids-logger "Uploading JupyterLab NVDashboard conda package to S3"
+rapids-upload-conda-to-s3 python

ci/build_wheel.sh

@@ -37,3 +37,7 @@ python -m pip install build
 
 # Build the Python package
 python -m build -s -w
+
+rapids-logger "Uploading JupyterLab NVDashboard wheels to S3"
+# Upload Python wheels to S3
+RAPIDS_PY_WHEEL_NAME="${package_name}" rapids-upload-wheels-to-s3 dist

ci/test_python.sh

@@ -18,7 +18,8 @@ set +u
 conda activate test
 set -u
 
-PYTHON_CHANNEL="/tmp/conda-bld-output/noarch"
+# rapids-logger "Downloading artifacts from previous jobs"
+PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
 
 rapids-print-env
 

ci/test_wheel.sh

@@ -6,6 +6,9 @@ set -eou pipefail
 # Set the package name
 package_name="jupyterlab-nvdashboard"
 
+rapids-logger "Downloading artifacts from previous jobs"
+RAPIDS_PY_WHEEL_NAME="${package_name}" rapids-download-wheels-from-s3 ./dist
+
 # echo to expand wildcard before adding `[extra]` required for pip
 python -m pip install $(echo ./dist/jupyterlab_nvdashboard*.whl)[test]