From 11258f5287a3d760131279dc51df800d9cc39ed6 Mon Sep 17 00:00:00 2001 From: Pavel Chekin Date: Fri, 4 Oct 2024 09:28:55 -0700 Subject: [PATCH] Option to install upstream PyTorch from nightly wheels (#2386) Adding a new workflow input to select how to install upstream PyTorch: build from sources (with the corresponding patches applied) or from the latest nightly wheels from https://download.pytorch.org/whl/nightly/xpu. The default is to build from source. Fixes #1913. --- .github/actions/setup-pytorch/action.yml | 27 ++++++++++++++++++----- .github/workflows/build-test-gpu.yml | 8 +++++++ .github/workflows/build-test-reusable.yml | 5 +++++ .github/workflows/build-test.yml | 8 +++++++ 4 files changed, 43 insertions(+), 5 deletions(-) diff --git a/.github/actions/setup-pytorch/action.yml b/.github/actions/setup-pytorch/action.yml index 726df36f1..16fba7afa 100644 --- a/.github/actions/setup-pytorch/action.yml +++ b/.github/actions/setup-pytorch/action.yml @@ -16,6 +16,9 @@ inputs: ref: description: Branch, tag, commit id default: "" + mode: + description: Source or wheels + default: source runs: using: "composite" steps: @@ -71,7 +74,7 @@ runs: - name: Generate PyTorch cache key shell: bash run: | - PYTORCH_CACHE_KEY=$(echo $PYTHON_VERSION $PYTORCH_COMMIT_ID ${{ hashFiles('scripts/patch-pytorch.sh') }} | sha256sum - | cut -d\ -f1) + PYTORCH_CACHE_KEY=$(echo $PYTHON_VERSION $PYTORCH_COMMIT_ID ${{ hashFiles('scripts/patch-pytorch.sh') }} ${{ inputs.mode }} | sha256sum - | cut -d\ -f1) echo "PYTORCH_CACHE_KEY=$PYTORCH_CACHE_KEY" | tee -a "$GITHUB_ENV" - name: Load PyTorch from a cache @@ -90,11 +93,12 @@ runs: with: repository: ${{ env.PYTORCH_REPO }} ref: ${{ env.PYTORCH_COMMIT_ID }} - submodules: recursive + # To build PyTorch from source we need all submodules, they are not required for benchmarks + submodules: ${{ inputs.mode == 'source' && 'recursive' || 'false' }} path: pytorch - name: Apply additional PR patches - if: ${{ steps.pytorch-cache.outputs.status == 'miss' && inputs.repository == 'pytorch/pytorch' }} + if: ${{ steps.pytorch-cache.outputs.status == 'miss' && inputs.repository == 'pytorch/pytorch' && inputs.mode == 'source' }} shell: bash run: | cd pytorch @@ -108,7 +112,7 @@ runs: pip install 'numpy<2.0.0' - name: Build PyTorch - if: ${{ steps.pytorch-cache.outputs.status == 'miss' }} + if: ${{ steps.pytorch-cache.outputs.status == 'miss' && inputs.mode == 'source' }} shell: bash run: | source ${{ inputs.oneapi }}/setvars.sh @@ -117,11 +121,24 @@ runs: pip install -r requirements.txt python setup.py bdist_wheel - - name: Install PyTorch + - name: Install PyTorch (built from source) + if: ${{ inputs.mode == 'source' }} shell: bash run: | source ${{ inputs.oneapi }}/setvars.sh pip install pytorch/dist/*.whl + + - name: Install PyTorch (from wheels) + if: ${{ inputs.mode == 'wheels' }} + shell: bash + run: | + source ${{ inputs.oneapi }}/setvars.sh + pip install torch --index-url https://download.pytorch.org/whl/nightly/xpu + + - name: Get PyTorch version + shell: bash + run: | + source ${{ inputs.oneapi }}/setvars.sh PYTORCH_VERSION="$(python -c 'import torch;print(torch.__version__)')" echo "PYTORCH_VERSION=$PYTORCH_VERSION" | tee -a "$GITHUB_ENV" diff --git a/.github/workflows/build-test-gpu.yml b/.github/workflows/build-test-gpu.yml index 84442d028..7fe8511b0 100644 --- a/.github/workflows/build-test-gpu.yml +++ b/.github/workflows/build-test-gpu.yml @@ -12,6 +12,13 @@ on: description: PyTorch ref, keep empty for default type: string default: "" + pytorch_mode: + description: PyTorch mode, source or wheels + type: choice + options: + - source + - wheels + default: source upload_test_reports: description: Upload test reports type: boolean @@ -46,6 +53,7 @@ jobs: device: ${{ inputs.runner_label }} runner_label: ${{ inputs.runner_label }} pytorch_ref: ${{ inputs.pytorch_ref }} + pytorch_mode: ${{ inputs.pytorch_mode || 'source' }} python_version: ${{ matrix.python }} upload_test_reports: ${{ inputs.upload_test_reports }} ignore_errors: ${{ inputs.ignore_errors }} diff --git a/.github/workflows/build-test-reusable.yml b/.github/workflows/build-test-reusable.yml index c5aa02778..cc623a810 100644 --- a/.github/workflows/build-test-reusable.yml +++ b/.github/workflows/build-test-reusable.yml @@ -20,6 +20,10 @@ on: description: PyTorch ref, keep empty for default type: string default: "" + pytorch_mode: + description: PyTorch mode, source or wheels + type: string + default: "source" python_version: description: Python version type: string @@ -96,6 +100,7 @@ jobs: with: repository: pytorch/pytorch ref: ${{ inputs.pytorch_ref }} + mode: ${{ inputs.pytorch_mode }} - name: Install pass_rate dependencies run: | diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index dfad5d713..d3d9b29b5 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -12,6 +12,13 @@ on: description: PyTorch ref, keep empty for default type: string default: "" + pytorch_mode: + description: PyTorch mode, source or wheels + type: choice + options: + - source + - wheels + default: source upload_test_reports: description: Upload test reports type: boolean @@ -120,6 +127,7 @@ jobs: driver_version: ${{ matrix.driver }} runner_label: ${{ inputs.runner_label }} pytorch_ref: ${{ inputs.pytorch_ref }} + pytorch_mode: ${{ inputs.pytorch_mode || 'source' }} python_version: ${{ matrix.python }} upload_test_reports: ${{ inputs.upload_test_reports || false }} ignore_errors: ${{ inputs.ignore_errors || false }}