release.yml

name: Release

defaults:
  run:
    shell: bash -le {0}
on:
  schedule:
    - cron: '0 20 * * *'
  release:
    types: [published]
  repository_dispatch:
  workflow_dispatch:
    inputs:
      repo:
        description: 'GitHub repo {owner}/{repo}'
        required: false
        default: ''
      ref:
        description: 'GitHub ref: Branch, Tag or Commit SHA'
        required: false
        default: ''

env:
  CUDA_DEVICE_ORDER: PCI_BUS_ID
  AMD_SERVER: 10.0.13.31
  INTEL_SERVER: 10.0.23.35
  TORCH_CUDA_ARCH_LIST: '6.0 6.1 6.2 7.0 7.5 8.0 8.6 8.9 9.0'
  CUDA_RELEASE: 1
  repo: ${{ github.event.inputs.repo || github.repository }}
  ref: ${{ github.event.inputs.ref || github.ref }}

concurrency:
  group: ${{ github.ref }}-workflow-release
  cancel-in-progress: true

jobs:
  check-vm:
    runs-on: self-hosted
    container:
      image: modelcloud/gptqmodel:alpine-ci-v1
    outputs:
      ip: ${{ steps.get_ip.outputs.ip }}
      tag: ${{ steps.get_ip.outputs.tag }}
      amd_list: ${{ steps.assign.outputs.amd_list }}
      intel_list: ${{ steps.assign.outputs.intel_list }}
    steps:
      - name: Print env
        run: |
          echo "event name: ${{ github.event_name }}"
          echo "repo: ${{ env.repo }}"
          echo "ref: ${{ env.ref }}"

      - name: Select server
        id: get_ip
        run: |
          if [[ "${{ runner.name }}" == *"intel"* ]]; then
            echo "current ci is intel"
            response=0
          else
            echo "test intel vm status"
            response=$(curl --silent --fail --max-time 5 http://$INTEL_SERVER/gpu/runner/status/intel) || response=error
            if [ "$response" == "error" ]; then
              echo "test amd vm status"
              response=$(curl --silent --fail --max-time 5 http://${AMD_SERVER}/gpu/runner/status/intel) || response=error
            fi
          fi

          echo "response: $response"

          if [ "$response" == "0" ]; then
            tag="intel"
          elif [ "$response" == "-1" ]; then
            tag="amd"
          else
            echo "Error: Unexpected result - $response"
            exit 1
          fi

          echo "Runner tag: $tag"

          amd_online=0
          intel_online=0
          ip=""
          
          response=$(curl -s --head --fail --max-time 5 http://${INTEL_SERVER}/gpu/status) || response="error"
          if echo "$response" | grep "200 OK" > /dev/null; then
            echo "Intel server is online. Set IP to $INTEL_SERVER"
            ip=${INTEL_SERVER}
            intel_online=1
          else
            echo "Intel server is offline."
          fi
          
          response=$(curl -s --head --fail --max-time 5 http://${AMD_SERVER}/gpu/status) || response="error"
          if echo "$response" | grep "200 OK" > /dev/null; then
            echo "AMD server is online. Set IP to $AMD_SERVER"
            if [[ -z $ip ]]; then
              ip=${AMD_SERVER}
            fi
            amd_online=1
          else
            echo "AMD server is offline."
          fi

          # 0 -> AMD & Intel | 1 -> AMD | 2 -> Intel
          if [[ $amd_online -eq 0 ]] && [[ $intel_online -eq 0 ]]; then
              exit 1
          elif [[ $amd_online -eq 1 ]] && [[ $intel_online -eq 1 ]]; then # both intel & amd are online
              tag=0
          elif [[ $amd_online -eq 1 ]] && [[ $intel_online -eq 0 ]]; then # both amd is online
              tag=1
          elif [[ $amd_online -eq 0 ]] && [[ $intel_online -eq 1 ]]; then # both intel is online
              tag=2
          fi

          echo "ip=$ip" >> "$GITHUB_OUTPUT"
          echo "tag=$tag" >> "$GITHUB_OUTPUT"

          echo "GPU_IP=$ip" >> $GITHUB_ENV
          echo "TAG=$tag" >> $GITHUB_ENV

          echo "tag: $tag, ip: $ip"

      - name: Assign tasks
        id: assign
        run: |
          tag=${{ env.TAG }}

          # tag == 0, both intel & amd are online
          if [[ $tag -eq 0 ]]; then
              divide=2
          elif [[ $tag -eq 1 ]] || [[ $tag -eq 2 ]]; then
              divide=1
          fi

          lists=$(curl -s "http://${{ env.GPU_IP }}/gpu/runner/docker?json=1&divide=$divide")
          IFS=$'\n' read -r list_1 list_2 <<< "$lists"

          echo "list 1: $list_1"
          echo "list 2: $list_2"

          if [[ $tag -eq 1 ]]; then
              echo "amd_list=$list_1" >> "$GITHUB_OUTPUT"
          elif [[ $tag -eq 2 ]]; then
              echo "intel_list=$list_2" >> "$GITHUB_OUTPUT"
          fi

  release-amd:
    strategy:
      fail-fast: false
      matrix:
        tag: ${{ fromJSON(needs.check-vm.outputs.amd_list) }}
      max-parallel: 3
    runs-on: [ self-hosted, amd ]
    needs: 
      - check-vm
      - release-source
    if: (needs.check-vm.outputs.tag == '0' || needs.check-vm.outputs.tag == '1') || github.event_name == 'release'
    container:
      image: ${{ needs.check-vm.outputs.ip }}:5000/modelcloud/gptqmodel:${{ matrix.tag }}
    timeout-minutes: 50
    steps:
      - name: Print Env
        run: |
          echo "== pyenv =="
          pyenv versions
          echo "== python =="
          python --version
          echo "== nvcc =="
          nvcc --version
          echo "== torch =="
          pip show torch

      - name: Checkout Codes
        uses: actions/checkout@v4
        with:
          repository: ${{ env.repo }}
          ref: ${{ env.ref }}

      - name: Install requirements
        run: pip install -r requirements.txt -i http://${{ needs.check-vm.outputs.ip }}/simple/ --trusted-host ${{ needs.check-vm.outputs.ip }}

      - name: Compile
        run: python setup.py bdist_wheel

      - name: Test install
        run: |
          ls -ahl dist
          whl=$(ls -t dist/*.whl | head -n 1 | xargs basename)
          echo "WHL_NAME=$whl" >> $GITHUB_ENV

          twine check dist/$whl
          pip install dist/$whl

      - name: Upload wheel
        continue-on-error: true
        run: |
          sha256=$(sha256sum dist/${{ env.WHL_NAME }})
          response=$(curl -s -F "runid=${{ github.run_id }}" -F "repo=${{ env.repo }}" -F "ref=${{ env.ref }}" -F "sha256=$sha256" -F "file=@dist/${{ env.WHL_NAME }}" http://${{ needs.check-vm.outputs.ip }}/gpu/whl/upload)
          if [ "$response" -eq 0 ]; then
            echo "UPLOADED=1" >> $GITHUB_ENV
          fi

      - name: Upload artifact
        uses: actions/upload-artifact@v4
        continue-on-error: ${{ env.UPLOADED }} == '1'
        with:
          name: ${{ env.WHL_NAME }}
          path: dist/${{ env.WHL_NAME }}

      - name: Upload binaries to release
        uses: svenstaro/upload-release-action@v2
        if: github.event_name == 'release'
        with:
          repo_name: ${{ env.repo }}
          tag: ${{ env.ref }}
          file: dist/${{ env.WHL_NAME }}
          file_glob: true
          overwrite: true

  release-intel:
    strategy:
      fail-fast: false
      matrix:
        tag: ${{ fromJSON(needs.check-vm.outputs.intel_list) }}
      max-parallel: 3
    runs-on: [ self-hosted, intel ]
    needs: 
      - check-vm
      - release-source
    if: (needs.check-vm.outputs.tag == '0' || needs.check-vm.outputs.tag == '2') || github.event_name == 'release'
    container:
      image: ${{ needs.check-vm.outputs.ip }}:5000/modelcloud/gptqmodel:${{ matrix.tag }}
    timeout-minutes: 50
    steps:
      - name: Print Env
        run: |
          echo "== pyenv =="
          pyenv versions
          echo "== python =="
          python --version
          echo "== nvcc =="
          nvcc --version
          echo "== torch =="
          pip show torch

      - name: Checkout Codes
        uses: actions/checkout@v4
        with:
          repository: ${{ env.repo }}
          ref: ${{ env.ref }}

      - name: Install requirements
        run: pip install -r requirements.txt -i http://${{ needs.check-vm.outputs.ip }}/simple/ --trusted-host ${{ needs.check-vm.outputs.ip }}

      - name: Compile
        run: python setup.py bdist_wheel

      - name: Test install
        run: |
          ls -ahl dist
          whl=$(ls -t dist/*.whl | head -n 1 | xargs basename)
          echo "WHL_NAME=$whl" >> $GITHUB_ENV

          twine check dist/$whl
          pip install dist/$whl

      - name: Upload wheel
        continue-on-error: true
        run: |
          sha256=$(sha256sum dist/${{ env.WHL_NAME }})
          response=$(curl -s -F "runid=${{ github.run_id }}" -F "repo=${{ env.repo }}" -F "ref=${{ env.ref }}" -F "sha256=$sha256" -F "file=@dist/${{ env.WHL_NAME }}" http://${{ needs.check-vm.outputs.ip }}/gpu/whl/upload)
          if [ "$response" -eq 0 ]; then
            echo "UPLOADED=1" >> $GITHUB_ENV
          fi

      - name: Upload artifact
        uses: actions/upload-artifact@v4
        continue-on-error: ${{ env.UPLOADED }} == '1'
        with:
          name: ${{ env.WHL_NAME }}
          path: dist/${{ env.WHL_NAME }}

      - name: Upload binaries to release
        uses: svenstaro/upload-release-action@v2
        if: github.event_name == 'release'
        with:
          repo_name: ${{ env.repo }}
          tag: ${{ env.ref }}
          file: dist/${{ env.WHL_NAME }}
          file_glob: true
          overwrite: true

  release-source:
    strategy:
      fail-fast: false
    runs-on: self-hosted
    needs: check-vm
    container:
      image: ${{ needs.check-vm.outputs.ip }}:5000/modelcloud/gptqmodel:compiler_cuda12.4-torch2.4-python3.11
    env:
      CUDA_RELEASE: 0
    steps:
      - name: Print Env
        run: |
          echo "== pyenv =="
          pyenv versions
          echo "== python =="
          python --version
          echo "== nvcc =="
          nvcc --version
          echo "== torch =="
          pip show torch

      - name: Checkout Codes
        uses: actions/checkout@v4

      - name: Install requirements
        run: pip install build -i http://${{ needs.check-vm.outputs.ip }}/simple/ --trusted-host ${{ needs.check-vm.outputs.ip }}

      - name: Compile
        run: python -m build --no-isolation --sdist

      - name: Check dist
        run: |
          ls -ahl dist
          whl=$(ls -t dist/*.gz | head -n 1 | xargs basename)
          echo "WHL_NAME=$whl" >> $GITHUB_ENV

          twine check dist/$whl

      - name: Upload package
        continue-on-error: true
        run: |
          sha256=$(sha256sum dist/${{ env.WHL_NAME }})
          response=$(curl -s -F "runid=${{ github.run_id }}" -F "repo=${{ env.repo }}" -F "ref=${{ env.ref }}" -F "sha256=$sha256" -F "file=@dist/${{ env.WHL_NAME }}" http://${{ needs.check-vm.outputs.ip }}/gpu/whl/upload)
          if [ "$response" -eq 0 ]; then
            echo "UPLOADED=1" >> $GITHUB_ENV
          fi

      - name: Upload to artifact
        uses: actions/upload-artifact@v4
        continue-on-error: ${{ env.UPLOADED }} == '1'
        with:
          name: ${{ env.WHL_NAME }}
          path: dist/${{ env.WHL_NAME }}

      - name: Upload package to release
        uses: svenstaro/upload-release-action@v2
        if: github.event_name == 'release'
        with:
          file: dist/${{ env.WHL_NAME }}
          tag: ${{ env.ref }}
          file_glob: true

      - name: Upload sdist to pypi
        if: github.event_name == 'release'
        env:
          TWINE_USERNAME: "__token__"
          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
        run: |
          python -m twine upload dist/*gz