Testing r220 #9
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: e2e | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| - 'release-*' | |
| paths-ignore: | |
| - 'docs/**' | |
| - '**.adoc' | |
| - '**.md' | |
| - 'LICENSE' | |
| push: | |
| branches: | |
| - main | |
| - 'release-*' | |
| paths-ignore: | |
| - 'docs/**' | |
| - '**.adoc' | |
| - '**.md' | |
| - 'LICENSE' | |
| concurrency: | |
| group: ${{ github.head_ref }}-${{ github.workflow }} | |
| cancel-in-progress: true | |
| jobs: | |
| kubernetes: | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Cleanup | |
| run: | | |
| ls -lart | |
| echo "Initial status:" | |
| df -h | |
| echo "Cleaning up resources:" | |
| sudo swapoff -a | |
| sudo rm -f /swapfile | |
| sudo apt clean | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf "/usr/local/share/boost" | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| docker rmi $(docker image ls -aq) | |
| echo "Final status:" | |
| df -h | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Checkout common repo code | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: 'project-codeflare/codeflare-common' | |
| ref: 'main' | |
| path: 'common' | |
| - name: Checkout CodeFlare operator repository | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: project-codeflare/codeflare-operator | |
| path: codeflare-operator | |
| - name: Set Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: './codeflare-operator/go.mod' | |
| cache-dependency-path: "./codeflare-operator/go.sum" | |
| - name: Set up gotestfmt | |
| uses: gotesttools/gotestfmt-action@v2 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Set up specific Python version | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.9.18' | |
| cache: 'pip' # caching pip dependencies | |
| - name: Setup and start KinD cluster | |
| uses: ./common/github-actions/kind | |
| - name: Deploy CodeFlare stack | |
| id: deploy | |
| run: | | |
| cd codeflare-operator | |
| echo Setting up CodeFlare stack | |
| make setup-e2e | |
| echo Deploying CodeFlare operator | |
| IMG="${REGISTRY_ADDRESS}"/codeflare-operator | |
| make image-push -e IMG="${IMG}" | |
| make deploy -e IMG="${IMG}" -e ENV="e2e" | |
| kubectl wait --timeout=120s --for=condition=Available=true deployment -n openshift-operators codeflare-operator-manager | |
| cd .. | |
| - name: Add user to KinD | |
| uses: ./common/github-actions/kind-add-user | |
| with: | |
| user-name: sdk-user | |
| - name: Add kueue resources | |
| run: kubectl apply --server-side -f "https://github.com/kubernetes-sigs/kueue/releases/download/v0.6.2/manifests.yaml" | |
| - name: Configure RBAC for sdk user with limited permissions | |
| run: | | |
| kubectl create clusterrole list-ingresses --verb=get,list --resource=ingresses | |
| kubectl create clusterrolebinding sdk-user-list-ingresses --clusterrole=list-ingresses --user=sdk-user | |
| kubectl create clusterrole namespace-creator --verb=get,list,create,delete,patch --resource=namespaces | |
| kubectl create clusterrolebinding sdk-user-namespace-creator --clusterrole=namespace-creator --user=sdk-user | |
| kubectl create clusterrole raycluster-creator --verb=get,list,create,delete,patch --resource=rayclusters | |
| kubectl create clusterrolebinding sdk-user-raycluster-creator --clusterrole=raycluster-creator --user=sdk-user | |
| kubectl create clusterrole appwrapper-creator --verb=get,list,create,delete,patch --resource=appwrappers | |
| kubectl create clusterrolebinding sdk-user-appwrapper-creator --clusterrole=appwrapper-creator --user=sdk-user | |
| kubectl create clusterrole resourceflavor-creator --verb=get,list,create,delete --resource=resourceflavors | |
| kubectl create clusterrolebinding sdk-user-resourceflavor-creator --clusterrole=resourceflavor-creator --user=sdk-user | |
| kubectl create clusterrole clusterqueue-creator --verb=get,list,create,delete,patch --resource=clusterqueues | |
| kubectl create clusterrolebinding sdk-user-clusterqueue-creator --clusterrole=clusterqueue-creator --user=sdk-user | |
| kubectl create clusterrole localqueue-creator --verb=get,list,create,delete,patch --resource=localqueues | |
| kubectl create clusterrolebinding sdk-user-localqueue-creator --clusterrole=localqueue-creator --user=sdk-user | |
| kubectl create clusterrole list-secrets --verb=get,list --resource=secrets | |
| kubectl create clusterrolebinding sdk-user-list-secrets --clusterrole=list-secrets --user=sdk-user | |
| kubectl config use-context sdk-user | |
| - name: Run e2e tests | |
| run: | | |
| export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }} | |
| echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV | |
| python --version | |
| set -euo pipefail | |
| pip install poetry | |
| poetry install --with test,docs | |
| echo "Running e2e tests..." | |
| poetry run pytest -v -s ./tests/e2e -m kind > ${CODEFLARE_TEST_OUTPUT_DIR}/pytest_output.log 2>&1 | |
| env: | |
| GRPC_DNS_RESOLVER: "native" | |
| - name: Switch to kind-cluster context to print logs | |
| if: always() && steps.deploy.outcome == 'success' | |
| run: kubectl config use-context kind-cluster | |
| - name: Print Pytest output log | |
| if: always() && steps.deploy.outcome == 'success' | |
| run: | | |
| echo "Printing Pytest output logs" | |
| cat ${CODEFLARE_TEST_OUTPUT_DIR}/pytest_output.log | |
| - name: Print CodeFlare operator logs | |
| if: always() && steps.deploy.outcome == 'success' | |
| run: | | |
| echo "Printing CodeFlare operator logs" | |
| kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee ${CODEFLARE_TEST_OUTPUT_DIR}/codeflare-operator.log | |
| - name: Print KubeRay operator logs | |
| if: always() && steps.deploy.outcome == 'success' | |
| run: | | |
| echo "Printing KubeRay operator logs" | |
| kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kuberay.log | |
| - name: Export all KinD pod logs | |
| uses: ./common/github-actions/kind-export-logs | |
| if: always() && steps.deploy.outcome == 'success' | |
| with: | |
| output-directory: ${CODEFLARE_TEST_OUTPUT_DIR} | |
| - name: Upload logs | |
| uses: actions/upload-artifact@v4 | |
| if: always() && steps.deploy.outcome == 'success' | |
| with: | |
| name: logs | |
| retention-days: 10 | |
| path: | | |
| ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/**/*.log |