Release #60
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Release | |
| defaults: | |
| run: | |
| shell: bash -le {0} | |
| on: | |
| schedule: | |
| - cron: '0 20 * * *' | |
| release: | |
| types: [published] | |
| repository_dispatch: | |
| workflow_dispatch: | |
| inputs: | |
| repo: | |
| description: 'GitHub repo {owner}/{repo}' | |
| required: false | |
| default: '' | |
| ref: | |
| description: 'GitHub ref: Branch, Tag or Commit SHA' | |
| required: false | |
| default: '' | |
| env: | |
| CUDA_DEVICE_ORDER: PCI_BUS_ID | |
| ZEN3_SERVER: 10.0.13.0 # disable it but keep this feature | |
| ZEN4_SERVER: 10.0.14.199 | |
| TORCH_CUDA_ARCH_LIST: '6.0 6.1 6.2 7.0 7.5 8.0 8.6 8.9 9.0' | |
| CUDA_RELEASE: 1 | |
| CI: 1 | |
| repo: ${{ github.event.inputs.repo || github.repository }} | |
| ref: ${{ github.event.inputs.ref || github.ref }} | |
| concurrency: | |
| group: ${{ github.ref }}-workflow-release | |
| cancel-in-progress: true | |
| jobs: | |
| check-vm: | |
| runs-on: self-hosted | |
| container: | |
| image: modelcloud/gptqmodel:alpine-ci-v1 | |
| outputs: | |
| ip: ${{ steps.get_ip.outputs.ip }} | |
| tag: ${{ steps.get_ip.outputs.tag }} | |
| zen3_list: ${{ steps.assign.outputs.zen3_list }} | |
| zen4_list: ${{ steps.assign.outputs.zen4_list }} | |
| steps: | |
| - name: Print env | |
| run: | | |
| echo "event name: ${{ github.event_name }}" | |
| echo "repo: ${{ env.repo }}" | |
| echo "ref: ${{ env.ref }}" | |
| - name: Select server | |
| id: get_ip | |
| run: | | |
| if [[ "${{ runner.name }}" == *"zen4"* ]]; then | |
| echo "current ci is zen4" | |
| response=0 | |
| else | |
| echo "test zen4 vm status" | |
| response=$(curl --silent --fail --max-time 5 http://$ZEN4_SERVER/gpu/runner/status/zen4) || response=error | |
| if [ "$response" == "error" ]; then | |
| echo "test zen3 vm status" | |
| response=$(curl --silent --fail --max-time 5 http://${ZEN3_SERVER}/gpu/runner/status/zen4) || response=error | |
| fi | |
| fi | |
| echo "response: $response" | |
| if [ "$response" == "0" ]; then | |
| tag="zen4" | |
| elif [ "$response" == "-1" ]; then | |
| tag="zen3" | |
| else | |
| echo "Error: Unexpected result - $response" | |
| exit 1 | |
| fi | |
| echo "Runner tag: $tag" | |
| zen3_online=0 | |
| zen4_online=0 | |
| ip="" | |
| response=$(curl -s --head --fail --max-time 5 http://${ZEN4_SERVER}/gpu/status) || response="error" | |
| if echo "$response" | grep "200 OK" > /dev/null; then | |
| echo "Zen4 server is online. Set IP to $ZEN4_SERVER" | |
| ip=${ZEN4_SERVER} | |
| zen4_online=1 | |
| else | |
| echo "Zen4 server is offline." | |
| fi | |
| response=$(curl -s --head --fail --max-time 5 http://${ZEN3_SERVER}/gpu/status) || response="error" | |
| if echo "$response" | grep "200 OK" > /dev/null; then | |
| echo "ZEN3 server is online. Set IP to $ZEN3_SERVER" | |
| if [[ -z $ip ]]; then | |
| ip=${ZEN3_SERVER} | |
| fi | |
| zen3_online=1 | |
| else | |
| echo "ZEN3 server is offline." | |
| fi | |
| # 0 -> ZEN3 & Zen4 | 1 -> ZEN3 | 2 -> Zen4 | |
| if [[ $zen3_online -eq 0 ]] && [[ $zen4_online -eq 0 ]]; then | |
| exit 1 | |
| elif [[ $zen3_online -eq 1 ]] && [[ $zen4_online -eq 1 ]]; then # both zen4 & zen3 are online | |
| tag=0 | |
| elif [[ $zen3_online -eq 1 ]] && [[ $zen4_online -eq 0 ]]; then # both zen3 is online | |
| tag=1 | |
| elif [[ $zen3_online -eq 0 ]] && [[ $zen4_online -eq 1 ]]; then # both zen4 is online | |
| tag=2 | |
| fi | |
| echo "ip=$ip" >> "$GITHUB_OUTPUT" | |
| echo "tag=$tag" >> "$GITHUB_OUTPUT" | |
| echo "GPU_IP=$ip" >> $GITHUB_ENV | |
| echo "TAG=$tag" >> $GITHUB_ENV | |
| echo "tag: $tag, ip: $ip" | |
| - name: Assign tasks | |
| id: assign | |
| run: | | |
| tag=${{ env.TAG }} | |
| # tag == 0, both zen4 & zen3 are online | |
| if [[ $tag -eq 0 ]]; then | |
| divide=2 | |
| elif [[ $tag -eq 1 ]] || [[ $tag -eq 2 ]]; then | |
| divide=1 | |
| else | |
| divide=1 | |
| fi | |
| echo "divide=$divide" | |
| lists=$(curl -s "http://${{ env.GPU_IP }}/gpu/runner/docker?json=1÷=$divide") | |
| echo "lists=$lists" | |
| IFS=$'+' read -r list_1 list_2 <<< "$lists" | |
| echo "list 1: $list_1" | |
| echo "list 2: $list_2" | |
| if [[ $tag -eq 0 ]]; then | |
| echo "zen3_list=$list_1" >> "$GITHUB_OUTPUT" | |
| echo "zen4_list=$list_2" >> "$GITHUB_OUTPUT" | |
| elif [[ $tag -eq 1 ]]; then | |
| echo "zen3_list=$list_1" >> "$GITHUB_OUTPUT" | |
| elif [[ $tag -eq 2 ]]; then | |
| echo "zen4_list=$list_1" >> "$GITHUB_OUTPUT" | |
| fi | |
| release-zen3: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| tag: ${{ fromJSON(needs.check-vm.outputs.zen3_list) }} | |
| max-parallel: 6 | |
| runs-on: [ self-hosted, zen3 ] | |
| needs: | |
| - check-vm | |
| - release-source | |
| if: ((needs.check-vm.outputs.tag == '0' || needs.check-vm.outputs.tag == '1') || github.event_name == 'release') && needs.check-vm.outputs.zen3_list != '' && !cancelled() | |
| container: | |
| image: ${{ needs.check-vm.outputs.ip }}:5000/modelcloud/gptqmodel:${{ matrix.tag }} | |
| timeout-minutes: 70 | |
| steps: | |
| - name: Print Env | |
| run: | | |
| echo "== pyenv ==" | |
| pyenv versions | |
| echo "== python ==" | |
| python --version | |
| echo "== nvcc ==" | |
| #nvcc --version | |
| echo "== torch ==" | |
| pip show torch | |
| - name: Checkout Codes | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ env.repo }} | |
| ref: ${{ env.ref }} | |
| # - name: Install requirements | |
| # run: pip install -r requirements.txt -i http://${{ needs.check-vm.outputs.ip }}/simple/ --trusted-host ${{ needs.check-vm.outputs.ip }} | |
| - name: Compile | |
| run: python setup.py bdist_wheel | |
| - name: Test install | |
| run: | | |
| ls -ahl dist | |
| whl=$(ls -t dist/*.whl | head -n 1 | xargs basename) | |
| echo "WHL_NAME=$whl" >> $GITHUB_ENV | |
| twine check dist/$whl | |
| pip install dist/$whl | |
| - name: Upload wheel | |
| continue-on-error: true | |
| run: | | |
| sha256=$(sha256sum dist/${{ env.WHL_NAME }}) | |
| response=$(curl -s -F "runid=${{ github.run_id }}" -F "repo=${{ env.repo }}" -F "ref=${{ env.ref }}" -F "sha256=$sha256" -F "file=@dist/${{ env.WHL_NAME }}" http://${{ needs.check-vm.outputs.ip }}/gpu/whl/upload) | |
| if [ "$response" -eq 0 ]; then | |
| echo "UPLOADED=1" >> $GITHUB_ENV | |
| fi | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v4 | |
| continue-on-error: ${{ env.UPLOADED }} == '1' | |
| with: | |
| name: ${{ env.WHL_NAME }} | |
| path: dist/${{ env.WHL_NAME }} | |
| - name: Upload binaries to release | |
| uses: svenstaro/upload-release-action@v2 | |
| if: github.event_name == 'release' && !cancelled() | |
| with: | |
| repo_name: ${{ env.repo }} | |
| tag: ${{ env.ref }} | |
| file: dist/${{ env.WHL_NAME }} | |
| file_glob: true | |
| overwrite: true | |
| release-zen4: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| tag: ${{ fromJSON(needs.check-vm.outputs.zen4_list) }} | |
| max-parallel: 6 | |
| runs-on: [ self-hosted, zen4 ] | |
| needs: | |
| - check-vm | |
| - release-source | |
| if: ((needs.check-vm.outputs.tag == '0' || needs.check-vm.outputs.tag == '2') || github.event_name == 'release') && needs.check-vm.outputs.zen4_list != '' && !cancelled() | |
| container: | |
| image: ${{ needs.check-vm.outputs.ip }}:5000/modelcloud/gptqmodel:${{ matrix.tag }} | |
| timeout-minutes: 70 | |
| steps: | |
| - name: Print Env | |
| run: | | |
| echo "== pyenv ==" | |
| pyenv versions | |
| echo "== python ==" | |
| python --version | |
| echo "== nvcc ==" | |
| #nvcc --version | |
| echo "== torch ==" | |
| pip show torch | |
| - name: Checkout Codes | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ env.repo }} | |
| ref: ${{ env.ref }} | |
| - name: Install requirements | |
| run: | | |
| cuda_version=$(echo ${{ matrix.tag }} | grep -oP 'cuda\K[0-9.]+') | |
| torch_version=$(echo ${{ matrix.tag }} | grep -oP 'torch\K[0-9.]+') | |
| python_version=$(echo ${{ matrix.tag }} | grep -oP 'python\K[0-9.]+') | |
| /bin/bash /tmp/init/init_env.sh $cuda_version $torch_version $python_version | |
| - name: Compile | |
| run: python setup.py bdist_wheel | |
| - name: Test install | |
| run: | | |
| ls -ahl dist | |
| whl=$(ls -t dist/*.whl | head -n 1 | xargs basename) | |
| echo "WHL_NAME=$whl" >> $GITHUB_ENV | |
| twine check dist/$whl | |
| pip install dist/$whl | |
| - name: Upload wheel | |
| continue-on-error: true | |
| run: | | |
| sha256=$(sha256sum dist/${{ env.WHL_NAME }}) | |
| response=$(curl -s -F "runid=${{ github.run_id }}" -F "repo=${{ env.repo }}" -F "ref=${{ env.ref }}" -F "sha256=$sha256" -F "file=@dist/${{ env.WHL_NAME }}" http://${{ needs.check-vm.outputs.ip }}/gpu/whl/upload) | |
| if [ "$response" -eq 0 ]; then | |
| echo "UPLOADED=1" >> $GITHUB_ENV | |
| fi | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v4 | |
| continue-on-error: ${{ env.UPLOADED }} == '1' | |
| with: | |
| name: ${{ env.WHL_NAME }} | |
| path: dist/${{ env.WHL_NAME }} | |
| - name: Upload binaries to release | |
| uses: svenstaro/upload-release-action@v2 | |
| if: github.event_name == 'release' && !cancelled() | |
| with: | |
| repo_name: ${{ env.repo }} | |
| tag: ${{ env.ref }} | |
| file: dist/${{ env.WHL_NAME }} | |
| file_glob: true | |
| overwrite: true | |
| release-source: | |
| strategy: | |
| fail-fast: false | |
| runs-on: self-hosted | |
| needs: check-vm | |
| container: | |
| image: ${{ needs.check-vm.outputs.ip }}:5000/modelcloud/gptqmodel:compiler_cuda12.4-torch2.4-python3.11 | |
| env: | |
| CUDA_RELEASE: 0 | |
| steps: | |
| - name: Print Env | |
| run: | | |
| echo "== pyenv ==" | |
| pyenv versions | |
| echo "== python ==" | |
| python --version | |
| echo "== nvcc ==" | |
| #nvcc --version | |
| echo "== torch ==" | |
| pip show torch | |
| - name: Checkout Codes | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ env.repo }} | |
| ref: ${{ env.ref }} | |
| - name: Install requirements | |
| run: pip install build -i http://${{ needs.check-vm.outputs.ip }}/simple/ --trusted-host ${{ needs.check-vm.outputs.ip }} | |
| - name: Compile | |
| run: python -m build --no-isolation --sdist | |
| - name: Check dist | |
| run: | | |
| ls -ahl dist | |
| whl=$(ls -t dist/*.gz | head -n 1 | xargs basename) | |
| echo "WHL_NAME=$whl" >> $GITHUB_ENV | |
| twine check dist/$whl | |
| - name: Upload package | |
| continue-on-error: true | |
| run: | | |
| sha256=$(sha256sum dist/${{ env.WHL_NAME }}) | |
| response=$(curl -s -F "runid=${{ github.run_id }}" -F "repo=${{ env.repo }}" -F "ref=${{ env.ref }}" -F "sha256=$sha256" -F "file=@dist/${{ env.WHL_NAME }}" http://${{ needs.check-vm.outputs.ip }}/gpu/whl/upload) | |
| if [ "$response" -eq 0 ]; then | |
| echo "UPLOADED=1" >> $GITHUB_ENV | |
| fi | |
| - name: Upload to artifact | |
| uses: actions/upload-artifact@v4 | |
| continue-on-error: ${{ env.UPLOADED }} == '1' | |
| with: | |
| name: ${{ env.WHL_NAME }} | |
| path: dist/${{ env.WHL_NAME }} | |
| - name: Upload package to release | |
| uses: svenstaro/upload-release-action@v2 | |
| if: github.event_name == 'release' && !cancelled() | |
| with: | |
| file: dist/${{ env.WHL_NAME }} | |
| tag: ${{ env.ref }} | |
| file_glob: true | |
| - name: Upload sdist to pypi | |
| if: github.event_name == 'release' && !cancelled() | |
| env: | |
| TWINE_USERNAME: "__token__" | |
| TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} | |
| run: | | |
| python -m twine upload dist/*gz |