From 6fdeb9cf4135e01bc285b6a766b2a80ed7d6b2c0 Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Tue, 24 Sep 2024 11:07:21 -0700 Subject: [PATCH 01/23] Android try to upload release artifacts --- .github/workflows/android.yml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 54e9dbb761..39d504d2fe 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -23,8 +23,10 @@ concurrency: cancel-in-progress: true jobs: - build-llm-demo: - name: build-llm-demo + build-aar-and-upload: + name: build-aar-and-upload + env: + RELEASE_JOB: ${{ startsWith(github.ref, 'refs/tags/ciflow/android/release') }} uses: pytorch/test-infra/.github/workflows/linux_job.yml@main with: runner: linux.2xlarge @@ -46,9 +48,18 @@ jobs: # Build LLM Demo for Android bash build/build_android_llm_demo.sh ${ARTIFACTS_DIR_NAME} + # Upload to RC if ciflow/android/release tag is present + if [[ "$RELEASE_JOB" = 'true']] + pip install awscli==1.32.18 + AWS_CMD="aws s3 cp" + VERSION_NAME="$(date -f "%Y%m%d")" + shasum -a 256 "${ARTIFACTS_DIR_NAME}/llm_demo/executorch.aar" + ${AWS_CMD} "${ARTIFACTS_DIR_NAME}/llm_demo/executorch.aar" s3://ossci-android/executorch/release/${VERSION_NAME}/executorch.aar --acl public-read + fi + # Running Android emulator directly on the runner and not using Docker run-emulator: - needs: build-llm-demo + needs: build-aar-and-upload # NB: Use metal install for KVM support to run the emulator faster runs-on: linux.24xl.spr-metal env: From ce0df6d05e832344dbd3c1663c6d828ca2109cc0 Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Tue, 24 Sep 2024 11:08:57 -0700 Subject: [PATCH 02/23] dryrun --- .github/workflows/android.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 39d504d2fe..9b4cb462b9 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -46,12 +46,15 @@ jobs: export ARTIFACTS_DIR_NAME=artifacts-to-be-uploaded # Build LLM Demo for Android - bash build/build_android_llm_demo.sh ${ARTIFACTS_DIR_NAME} + # bash build/build_android_llm_demo.sh ${ARTIFACTS_DIR_NAME} + + mkdir -p "${ARTIFACTS_DIR_NAME}/llm_demo" + touch "${ARTIFACTS_DIR_NAME}/llm_demo/executorch.aar" # Upload to RC if ciflow/android/release tag is present if [[ "$RELEASE_JOB" = 'true']] pip install awscli==1.32.18 - AWS_CMD="aws s3 cp" + AWS_CMD="aws s3 cp --dryrun" VERSION_NAME="$(date -f "%Y%m%d")" shasum -a 256 "${ARTIFACTS_DIR_NAME}/llm_demo/executorch.aar" ${AWS_CMD} "${ARTIFACTS_DIR_NAME}/llm_demo/executorch.aar" s3://ossci-android/executorch/release/${VERSION_NAME}/executorch.aar --acl public-read From 4d9676beeca926e49e923409abfd392a3fe74414 Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Tue, 24 Sep 2024 11:27:51 -0700 Subject: [PATCH 03/23] Test --- .github/workflows/android.yml | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 9b4cb462b9..0dea5f7bdd 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -23,10 +23,8 @@ concurrency: cancel-in-progress: true jobs: - build-aar-and-upload: - name: build-aar-and-upload - env: - RELEASE_JOB: ${{ startsWith(github.ref, 'refs/tags/ciflow/android/release') }} + build-aar: + name: build-aar uses: pytorch/test-infra/.github/workflows/linux_job.yml@main with: runner: linux.2xlarge @@ -51,18 +49,23 @@ jobs: mkdir -p "${ARTIFACTS_DIR_NAME}/llm_demo" touch "${ARTIFACTS_DIR_NAME}/llm_demo/executorch.aar" - # Upload to RC if ciflow/android/release tag is present - if [[ "$RELEASE_JOB" = 'true']] - pip install awscli==1.32.18 - AWS_CMD="aws s3 cp --dryrun" - VERSION_NAME="$(date -f "%Y%m%d")" - shasum -a 256 "${ARTIFACTS_DIR_NAME}/llm_demo/executorch.aar" - ${AWS_CMD} "${ARTIFACTS_DIR_NAME}/llm_demo/executorch.aar" s3://ossci-android/executorch/release/${VERSION_NAME}/executorch.aar --acl public-read - fi + upload-release-aar: + name: Upload AAR to RC if ciflow/android/release tag is present + needs: build-aar-and-upload + if: ${{ startsWith(github.ref, 'refs/tags/ciflow/android/release') }} + runs-on: ubuntu-22.04 + timeout-minutes: 10 + run: | + wget https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/llm_demo/executorch.aar + + pip install awscli==1.32.18 + AWS_CMD="aws s3 cp --dryrun" + VERSION_NAME="$(date -f "%Y%m%d")" + ${AWS_CMD} executorch.aar s3://ossci-android/executorch/release/${VERSION_NAME}/executorch.aar --acl public-read # Running Android emulator directly on the runner and not using Docker run-emulator: - needs: build-aar-and-upload + needs: build-aar # NB: Use metal install for KVM support to run the emulator faster runs-on: linux.24xl.spr-metal env: From 17ae8079d326b15457b7d44d4d6d509055bd758f Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Tue, 24 Sep 2024 11:30:51 -0700 Subject: [PATCH 04/23] Fix name --- .github/workflows/android.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 0dea5f7bdd..f8f55c2b8d 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -51,7 +51,7 @@ jobs: upload-release-aar: name: Upload AAR to RC if ciflow/android/release tag is present - needs: build-aar-and-upload + needs: build-aar if: ${{ startsWith(github.ref, 'refs/tags/ciflow/android/release') }} runs-on: ubuntu-22.04 timeout-minutes: 10 From 85981fc1e055864d59ccd9e40831ff5511aca9b1 Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Tue, 24 Sep 2024 11:32:26 -0700 Subject: [PATCH 05/23] Fix steps --- .github/workflows/android.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index f8f55c2b8d..748b9c692b 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -50,18 +50,21 @@ jobs: touch "${ARTIFACTS_DIR_NAME}/llm_demo/executorch.aar" upload-release-aar: - name: Upload AAR to RC if ciflow/android/release tag is present + name: upload-release-aar needs: build-aar if: ${{ startsWith(github.ref, 'refs/tags/ciflow/android/release') }} runs-on: ubuntu-22.04 timeout-minutes: 10 - run: | - wget https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/llm_demo/executorch.aar + steps: + - name: Upload AAR to RC if ciflow/android/release tag is present + shell: bash + run: | + wget https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/llm_demo/executorch.aar - pip install awscli==1.32.18 - AWS_CMD="aws s3 cp --dryrun" - VERSION_NAME="$(date -f "%Y%m%d")" - ${AWS_CMD} executorch.aar s3://ossci-android/executorch/release/${VERSION_NAME}/executorch.aar --acl public-read + pip install awscli==1.32.18 + AWS_CMD="aws s3 cp --dryrun" + VERSION_NAME="$(date -f "%Y%m%d")" + ${AWS_CMD} executorch.aar s3://ossci-android/executorch/release/${VERSION_NAME}/executorch.aar --acl public-read # Running Android emulator directly on the runner and not using Docker run-emulator: From 64b3f0ff74d336f62a14ce8f971dd8f3cb222e82 Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Tue, 24 Sep 2024 11:45:09 -0700 Subject: [PATCH 06/23] Fix date --- .github/workflows/android.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 748b9c692b..367c4b44a7 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -63,7 +63,7 @@ jobs: pip install awscli==1.32.18 AWS_CMD="aws s3 cp --dryrun" - VERSION_NAME="$(date -f "%Y%m%d")" + VERSION_NAME="$(date +'%Y%m%d')" ${AWS_CMD} executorch.aar s3://ossci-android/executorch/release/${VERSION_NAME}/executorch.aar --acl public-read # Running Android emulator directly on the runner and not using Docker From 71aa6cb866467af9a38f805df18cb238c71147b7 Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Tue, 24 Sep 2024 11:57:23 -0700 Subject: [PATCH 07/23] Nondry run --- .github/workflows/android.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 367c4b44a7..eb2c30991a 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -62,7 +62,7 @@ jobs: wget https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/llm_demo/executorch.aar pip install awscli==1.32.18 - AWS_CMD="aws s3 cp --dryrun" + AWS_CMD="aws s3 cp" VERSION_NAME="$(date +'%Y%m%d')" ${AWS_CMD} executorch.aar s3://ossci-android/executorch/release/${VERSION_NAME}/executorch.aar --acl public-read From c569ea863ea0b05289ef6303526091c2c75c1f13 Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Tue, 24 Sep 2024 12:41:00 -0700 Subject: [PATCH 08/23] Fix AWS and ciflow tag --- .github/pytorch-probot.yml | 1 + .github/workflows/android.yml | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/.github/pytorch-probot.yml b/.github/pytorch-probot.yml index 2b66829ed0..f1d4b7d61a 100644 --- a/.github/pytorch-probot.yml +++ b/.github/pytorch-probot.yml @@ -1,6 +1,7 @@ # The schema is from https://github.com/pytorch/pytorch/blob/main/.github/pytorch-probot.yml ciflow_push_tags: - ciflow/android +- ciflow/android/release - ciflow/apple - ciflow/nightly - ciflow/trunk diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index eb2c30991a..c10b1ff100 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -56,6 +56,11 @@ jobs: runs-on: ubuntu-22.04 timeout-minutes: 10 steps: + - name: configure aws credentials + uses: aws-actions/configure-aws-credentials@v1.7.0 + with: + role-to-assume: arn:aws:iam::308535385114:role/gha_executorch_upload-frameworks-android + aws-region: us-east-1 - name: Upload AAR to RC if ciflow/android/release tag is present shell: bash run: | From fbf66e3e79bcf08eb088e338d1917a140380d76c Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Tue, 24 Sep 2024 12:50:11 -0700 Subject: [PATCH 09/23] Real build and upload sha256sums --- .github/workflows/android.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index c10b1ff100..50b2938cc5 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -44,10 +44,9 @@ jobs: export ARTIFACTS_DIR_NAME=artifacts-to-be-uploaded # Build LLM Demo for Android - # bash build/build_android_llm_demo.sh ${ARTIFACTS_DIR_NAME} + bash build/build_android_llm_demo.sh ${ARTIFACTS_DIR_NAME} - mkdir -p "${ARTIFACTS_DIR_NAME}/llm_demo" - touch "${ARTIFACTS_DIR_NAME}/llm_demo/executorch.aar" + shasum -a 256 "${ARTIFACTS_DIR_NAME}/llm_demo/executorch.aar" upload-release-aar: name: upload-release-aar @@ -65,11 +64,13 @@ jobs: shell: bash run: | wget https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/llm_demo/executorch.aar + shasum -a 256 executorch.aar > executorch.aar.sha256sums pip install awscli==1.32.18 AWS_CMD="aws s3 cp" VERSION_NAME="$(date +'%Y%m%d')" ${AWS_CMD} executorch.aar s3://ossci-android/executorch/release/${VERSION_NAME}/executorch.aar --acl public-read + ${AWS_CMD} executorch.aar.sha256sums s3://ossci-android/executorch/release/${VERSION_NAME}/executorch.aar.sha256sums --acl public-read # Running Android emulator directly on the runner and not using Docker run-emulator: From e1167b0df90eca587f4f3fef97ddf6b50220c532 Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Tue, 24 Sep 2024 12:50:40 -0700 Subject: [PATCH 10/23] more permissions --- .github/workflows/android.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 50b2938cc5..c5d790800f 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -54,6 +54,9 @@ jobs: if: ${{ startsWith(github.ref, 'refs/tags/ciflow/android/release') }} runs-on: ubuntu-22.04 timeout-minutes: 10 + permissions: + id-token: write + contents: read steps: - name: configure aws credentials uses: aws-actions/configure-aws-credentials@v1.7.0 From 354b20f758ea33461d85fe2e57017cd258737e65 Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Tue, 24 Sep 2024 12:59:17 -0700 Subject: [PATCH 11/23] Use a separate yml file --- .github/pytorch-probot.yml | 1 - .github/workflows/android.yml | 35 +++-------------------------------- 2 files changed, 3 insertions(+), 33 deletions(-) diff --git a/.github/pytorch-probot.yml b/.github/pytorch-probot.yml index f1d4b7d61a..2b66829ed0 100644 --- a/.github/pytorch-probot.yml +++ b/.github/pytorch-probot.yml @@ -1,7 +1,6 @@ # The schema is from https://github.com/pytorch/pytorch/blob/main/.github/pytorch-probot.yml ciflow_push_tags: - ciflow/android -- ciflow/android/release - ciflow/apple - ciflow/nightly - ciflow/trunk diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index c5d790800f..54e9dbb761 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -23,8 +23,8 @@ concurrency: cancel-in-progress: true jobs: - build-aar: - name: build-aar + build-llm-demo: + name: build-llm-demo uses: pytorch/test-infra/.github/workflows/linux_job.yml@main with: runner: linux.2xlarge @@ -46,38 +46,9 @@ jobs: # Build LLM Demo for Android bash build/build_android_llm_demo.sh ${ARTIFACTS_DIR_NAME} - shasum -a 256 "${ARTIFACTS_DIR_NAME}/llm_demo/executorch.aar" - - upload-release-aar: - name: upload-release-aar - needs: build-aar - if: ${{ startsWith(github.ref, 'refs/tags/ciflow/android/release') }} - runs-on: ubuntu-22.04 - timeout-minutes: 10 - permissions: - id-token: write - contents: read - steps: - - name: configure aws credentials - uses: aws-actions/configure-aws-credentials@v1.7.0 - with: - role-to-assume: arn:aws:iam::308535385114:role/gha_executorch_upload-frameworks-android - aws-region: us-east-1 - - name: Upload AAR to RC if ciflow/android/release tag is present - shell: bash - run: | - wget https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/llm_demo/executorch.aar - shasum -a 256 executorch.aar > executorch.aar.sha256sums - - pip install awscli==1.32.18 - AWS_CMD="aws s3 cp" - VERSION_NAME="$(date +'%Y%m%d')" - ${AWS_CMD} executorch.aar s3://ossci-android/executorch/release/${VERSION_NAME}/executorch.aar --acl public-read - ${AWS_CMD} executorch.aar.sha256sums s3://ossci-android/executorch/release/${VERSION_NAME}/executorch.aar.sha256sums --acl public-read - # Running Android emulator directly on the runner and not using Docker run-emulator: - needs: build-aar + needs: build-llm-demo # NB: Use metal install for KVM support to run the emulator faster runs-on: linux.24xl.spr-metal env: From fe07b7fee391788877695857caaded5a8d6ecc70 Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Tue, 24 Sep 2024 13:01:36 -0700 Subject: [PATCH 12/23] New workflow --- .../workflows/android-release-artifacts.yml | 60 +++++++++++++++++++ .github/workflows/android.yml | 4 +- 2 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/android-release-artifacts.yml diff --git a/.github/workflows/android-release-artifacts.yml b/.github/workflows/android-release-artifacts.yml new file mode 100644 index 0000000000..2e8578336e --- /dev/null +++ b/.github/workflows/android-release-artifacts.yml @@ -0,0 +1,60 @@ +name: Android Release Artifacts + +on: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-aar: + name: build-aar + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + runner: linux.2xlarge + docker-image: executorch-ubuntu-22.04-clang12-android + submodules: 'true' + ref: ${{ github.sha }} + timeout: 90 + upload-artifact: android-apps + upload-artifact-to-s3: true + script: | + set -eux + + # The generic Linux job chooses to use base env, not the one setup by the image + CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") + conda activate "${CONDA_ENV}" + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh buck2 + export ARTIFACTS_DIR_NAME=artifacts-to-be-uploaded + + # Build LLM Demo for Android + bash build/build_android_llm_demo.sh ${ARTIFACTS_DIR_NAME} + + shasum -a 256 "${ARTIFACTS_DIR_NAME}/llm_demo/executorch.aar" + + upload-release-aar: + name: upload-release-aar + needs: build-aar + runs-on: ubuntu-22.04 + timeout-minutes: 10 + permissions: + id-token: write + contents: read + steps: + - name: configure aws credentials + uses: aws-actions/configure-aws-credentials@v1.7.0 + with: + role-to-assume: arn:aws:iam::308535385114:role/gha_executorch_upload-frameworks-android + aws-region: us-east-1 + - name: Upload AAR to RC + shell: bash + run: | + wget https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/llm_demo/executorch.aar + shasum -a 256 executorch.aar > executorch.aar.sha256sums + + pip install awscli==1.32.18 + AWS_CMD="aws s3 cp" + VERSION_NAME="$(date +'%Y%m%d')" + ${AWS_CMD} executorch.aar s3://ossci-android/executorch/release/${VERSION_NAME}/executorch.aar --acl public-read + ${AWS_CMD} executorch.aar.sha256sums s3://ossci-android/executorch/release/${VERSION_NAME}/executorch.aar.sha256sums --acl public-read diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 54e9dbb761..fb1e1586ea 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -44,7 +44,9 @@ jobs: export ARTIFACTS_DIR_NAME=artifacts-to-be-uploaded # Build LLM Demo for Android - bash build/build_android_llm_demo.sh ${ARTIFACTS_DIR_NAME} + # bash build/build_android_llm_demo.sh ${ARTIFACTS_DIR_NAME} + mkdir -p "${ARTIFACTS_DIR_NAME}/llm_demo" + touch "${ARTIFACTS_DIR_NAME}/llm_demo/executorch.aar" # Running Android emulator directly on the runner and not using Docker run-emulator: From f7e25e9a825dfeee4f2ffc1cbc67acffb8068b99 Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Tue, 24 Sep 2024 13:02:21 -0700 Subject: [PATCH 13/23] test --- .github/workflows/android-release-artifacts.yml | 5 ++++- .github/workflows/android.yml | 4 +--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/android-release-artifacts.yml b/.github/workflows/android-release-artifacts.yml index 2e8578336e..28ba815056 100644 --- a/.github/workflows/android-release-artifacts.yml +++ b/.github/workflows/android-release-artifacts.yml @@ -29,7 +29,10 @@ jobs: export ARTIFACTS_DIR_NAME=artifacts-to-be-uploaded # Build LLM Demo for Android - bash build/build_android_llm_demo.sh ${ARTIFACTS_DIR_NAME} + # bash build/build_android_llm_demo.sh ${ARTIFACTS_DIR_NAME} + mkdir -p "${ARTIFACTS_DIR_NAME}/llm_demo" + touch "${ARTIFACTS_DIR_NAME}/llm_demo/executorch.aar" + shasum -a 256 "${ARTIFACTS_DIR_NAME}/llm_demo/executorch.aar" diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index fb1e1586ea..54e9dbb761 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -44,9 +44,7 @@ jobs: export ARTIFACTS_DIR_NAME=artifacts-to-be-uploaded # Build LLM Demo for Android - # bash build/build_android_llm_demo.sh ${ARTIFACTS_DIR_NAME} - mkdir -p "${ARTIFACTS_DIR_NAME}/llm_demo" - touch "${ARTIFACTS_DIR_NAME}/llm_demo/executorch.aar" + bash build/build_android_llm_demo.sh ${ARTIFACTS_DIR_NAME} # Running Android emulator directly on the runner and not using Docker run-emulator: From 95328cd85b78de72e863772ac6a293a8c04c7de4 Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Tue, 24 Sep 2024 13:09:06 -0700 Subject: [PATCH 14/23] Use workflow dispatch on a android/release tag for now --- .github/workflows/android.yml | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 54e9dbb761..131c248a8c 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -23,8 +23,8 @@ concurrency: cancel-in-progress: true jobs: - build-llm-demo: - name: build-llm-demo + build-aar: + name: build-aar uses: pytorch/test-infra/.github/workflows/linux_job.yml@main with: runner: linux.2xlarge @@ -46,9 +46,38 @@ jobs: # Build LLM Demo for Android bash build/build_android_llm_demo.sh ${ARTIFACTS_DIR_NAME} + shasum -a 256 "${ARTIFACTS_DIR_NAME}/llm_demo/executorch.aar" + + upload-release-aar: + name: upload-release-aar + needs: build-aar + if: ${{ startsWith(github.ref, 'refs/tags/android/release') }} + runs-on: ubuntu-22.04 + timeout-minutes: 10 + permissions: + id-token: write + contents: read + steps: + - name: configure aws credentials + uses: aws-actions/configure-aws-credentials@v1.7.0 + with: + role-to-assume: arn:aws:iam::308535385114:role/gha_executorch_upload-frameworks-android + aws-region: us-east-1 + - name: Upload AAR to RC if ciflow/android/release tag is present + shell: bash + run: | + wget https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/llm_demo/executorch.aar + shasum -a 256 executorch.aar > executorch.aar.sha256sums + + pip install awscli==1.32.18 + AWS_CMD="aws s3 cp" + VERSION_NAME="$(date +'%Y%m%d')" + ${AWS_CMD} executorch.aar s3://ossci-android/executorch/release/${VERSION_NAME}/executorch.aar --acl public-read + ${AWS_CMD} executorch.aar.sha256sums s3://ossci-android/executorch/release/${VERSION_NAME}/executorch.aar.sha256sums --acl public-read + # Running Android emulator directly on the runner and not using Docker run-emulator: - needs: build-llm-demo + needs: build-aar # NB: Use metal install for KVM support to run the emulator faster runs-on: linux.24xl.spr-metal env: From 46f0114be8f9b05414f5704f351f5cee22ecd5b9 Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Tue, 24 Sep 2024 13:41:51 -0700 Subject: [PATCH 15/23] add version_name --- .github/workflows/android.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 131c248a8c..0165fbcbc8 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -17,6 +17,11 @@ on: - extension/android/** - extension/module/** workflow_dispatch: + inputs: + version: + description: Version name to be uploaded for AAR release + required: false + type: string concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }} @@ -44,7 +49,9 @@ jobs: export ARTIFACTS_DIR_NAME=artifacts-to-be-uploaded # Build LLM Demo for Android - bash build/build_android_llm_demo.sh ${ARTIFACTS_DIR_NAME} + # bash build/build_android_llm_demo.sh ${ARTIFACTS_DIR_NAME} + mkdir -p "${ARTIFACTS_DIR_NAME}/llm_demo" + echo Hello > "${ARTIFACTS_DIR_NAME}/llm_demo/executorch.aar" shasum -a 256 "${ARTIFACTS_DIR_NAME}/llm_demo/executorch.aar" @@ -71,7 +78,8 @@ jobs: pip install awscli==1.32.18 AWS_CMD="aws s3 cp" - VERSION_NAME="$(date +'%Y%m%d')" + VERSION="${{ inputs.version }}" + VERSION_NAME="${VERSION:-temp_snapshot}" ${AWS_CMD} executorch.aar s3://ossci-android/executorch/release/${VERSION_NAME}/executorch.aar --acl public-read ${AWS_CMD} executorch.aar.sha256sums s3://ossci-android/executorch/release/${VERSION_NAME}/executorch.aar.sha256sums --acl public-read From 307a6e9c49de7d55b054aea57b4b58ba8c8144ad Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Tue, 24 Sep 2024 13:56:55 -0700 Subject: [PATCH 16/23] Add workflow version field --- .github/workflows/android.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 0165fbcbc8..c34beb1fba 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -49,9 +49,7 @@ jobs: export ARTIFACTS_DIR_NAME=artifacts-to-be-uploaded # Build LLM Demo for Android - # bash build/build_android_llm_demo.sh ${ARTIFACTS_DIR_NAME} - mkdir -p "${ARTIFACTS_DIR_NAME}/llm_demo" - echo Hello > "${ARTIFACTS_DIR_NAME}/llm_demo/executorch.aar" + bash build/build_android_llm_demo.sh ${ARTIFACTS_DIR_NAME} shasum -a 256 "${ARTIFACTS_DIR_NAME}/llm_demo/executorch.aar" From e7f13e75680317c5d97aa8c9d1432d9b045058c3 Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Tue, 24 Sep 2024 14:13:11 -0700 Subject: [PATCH 17/23] Use a separate workflow --- .../workflows/android-release-artifacts.yml | 15 ++++++---- .github/workflows/android.yml | 28 ------------------- 2 files changed, 9 insertions(+), 34 deletions(-) diff --git a/.github/workflows/android-release-artifacts.yml b/.github/workflows/android-release-artifacts.yml index 28ba815056..3983885848 100644 --- a/.github/workflows/android-release-artifacts.yml +++ b/.github/workflows/android-release-artifacts.yml @@ -2,6 +2,11 @@ name: Android Release Artifacts on: workflow_dispatch: + inputs: + version: + description: Version name to be uploaded for AAR release + required: false + type: string concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -29,10 +34,7 @@ jobs: export ARTIFACTS_DIR_NAME=artifacts-to-be-uploaded # Build LLM Demo for Android - # bash build/build_android_llm_demo.sh ${ARTIFACTS_DIR_NAME} - mkdir -p "${ARTIFACTS_DIR_NAME}/llm_demo" - touch "${ARTIFACTS_DIR_NAME}/llm_demo/executorch.aar" - + bash build/build_android_llm_demo.sh ${ARTIFACTS_DIR_NAME} shasum -a 256 "${ARTIFACTS_DIR_NAME}/llm_demo/executorch.aar" @@ -50,7 +52,7 @@ jobs: with: role-to-assume: arn:aws:iam::308535385114:role/gha_executorch_upload-frameworks-android aws-region: us-east-1 - - name: Upload AAR to RC + - name: Upload AAR RC to AWS S3 shell: bash run: | wget https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/llm_demo/executorch.aar @@ -58,6 +60,7 @@ jobs: pip install awscli==1.32.18 AWS_CMD="aws s3 cp" - VERSION_NAME="$(date +'%Y%m%d')" + VERSION="${{ inputs.version }}" + VERSION_NAME="${VERSION:-temp_snapshot}" ${AWS_CMD} executorch.aar s3://ossci-android/executorch/release/${VERSION_NAME}/executorch.aar --acl public-read ${AWS_CMD} executorch.aar.sha256sums s3://ossci-android/executorch/release/${VERSION_NAME}/executorch.aar.sha256sums --acl public-read diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index c34beb1fba..86077a18b2 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -53,34 +53,6 @@ jobs: shasum -a 256 "${ARTIFACTS_DIR_NAME}/llm_demo/executorch.aar" - upload-release-aar: - name: upload-release-aar - needs: build-aar - if: ${{ startsWith(github.ref, 'refs/tags/android/release') }} - runs-on: ubuntu-22.04 - timeout-minutes: 10 - permissions: - id-token: write - contents: read - steps: - - name: configure aws credentials - uses: aws-actions/configure-aws-credentials@v1.7.0 - with: - role-to-assume: arn:aws:iam::308535385114:role/gha_executorch_upload-frameworks-android - aws-region: us-east-1 - - name: Upload AAR to RC if ciflow/android/release tag is present - shell: bash - run: | - wget https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/llm_demo/executorch.aar - shasum -a 256 executorch.aar > executorch.aar.sha256sums - - pip install awscli==1.32.18 - AWS_CMD="aws s3 cp" - VERSION="${{ inputs.version }}" - VERSION_NAME="${VERSION:-temp_snapshot}" - ${AWS_CMD} executorch.aar s3://ossci-android/executorch/release/${VERSION_NAME}/executorch.aar --acl public-read - ${AWS_CMD} executorch.aar.sha256sums s3://ossci-android/executorch/release/${VERSION_NAME}/executorch.aar.sha256sums --acl public-read - # Running Android emulator directly on the runner and not using Docker run-emulator: needs: build-aar From aa5efbe8451c5abafc8488dd7a3e411ade19593d Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Tue, 24 Sep 2024 14:14:10 -0700 Subject: [PATCH 18/23] Remove unused --- .github/workflows/android.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 86077a18b2..5c689cb981 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -17,19 +17,14 @@ on: - extension/android/** - extension/module/** workflow_dispatch: - inputs: - version: - description: Version name to be uploaded for AAR release - required: false - type: string concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }} cancel-in-progress: true jobs: - build-aar: - name: build-aar + build-llm-demo: + name: build-llm-demo uses: pytorch/test-infra/.github/workflows/linux_job.yml@main with: runner: linux.2xlarge @@ -55,7 +50,7 @@ jobs: # Running Android emulator directly on the runner and not using Docker run-emulator: - needs: build-aar + needs: build-llm-demo # NB: Use metal install for KVM support to run the emulator faster runs-on: linux.24xl.spr-metal env: From 3a06d101ab0fb23bbcf889609e42c782e7314760 Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Tue, 24 Sep 2024 14:14:32 -0700 Subject: [PATCH 19/23] Remove unused --- .github/workflows/android.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 5c689cb981..54e9dbb761 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -46,8 +46,6 @@ jobs: # Build LLM Demo for Android bash build/build_android_llm_demo.sh ${ARTIFACTS_DIR_NAME} - shasum -a 256 "${ARTIFACTS_DIR_NAME}/llm_demo/executorch.aar" - # Running Android emulator directly on the runner and not using Docker run-emulator: needs: build-llm-demo From 1db1586f82b26abc6d9140965eb1ac58218ddd52 Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Tue, 24 Sep 2024 14:22:40 -0700 Subject: [PATCH 20/23] temp run --- .github/workflows/android-release-artifacts.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/android-release-artifacts.yml b/.github/workflows/android-release-artifacts.yml index 3983885848..687796d6e6 100644 --- a/.github/workflows/android-release-artifacts.yml +++ b/.github/workflows/android-release-artifacts.yml @@ -1,6 +1,8 @@ name: Android Release Artifacts on: + push: + - android-upload-artifact-2 workflow_dispatch: inputs: version: @@ -59,7 +61,7 @@ jobs: shasum -a 256 executorch.aar > executorch.aar.sha256sums pip install awscli==1.32.18 - AWS_CMD="aws s3 cp" + AWS_CMD="aws s3 cp --dryrun" VERSION="${{ inputs.version }}" VERSION_NAME="${VERSION:-temp_snapshot}" ${AWS_CMD} executorch.aar s3://ossci-android/executorch/release/${VERSION_NAME}/executorch.aar --acl public-read From 79d60c9d7c7571c5e11e54344b4e3a9c0f44342a Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Tue, 24 Sep 2024 14:23:12 -0700 Subject: [PATCH 21/23] go back --- .github/workflows/android-release-artifacts.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/android-release-artifacts.yml b/.github/workflows/android-release-artifacts.yml index 687796d6e6..3983885848 100644 --- a/.github/workflows/android-release-artifacts.yml +++ b/.github/workflows/android-release-artifacts.yml @@ -1,8 +1,6 @@ name: Android Release Artifacts on: - push: - - android-upload-artifact-2 workflow_dispatch: inputs: version: @@ -61,7 +59,7 @@ jobs: shasum -a 256 executorch.aar > executorch.aar.sha256sums pip install awscli==1.32.18 - AWS_CMD="aws s3 cp --dryrun" + AWS_CMD="aws s3 cp" VERSION="${{ inputs.version }}" VERSION_NAME="${VERSION:-temp_snapshot}" ${AWS_CMD} executorch.aar s3://ossci-android/executorch/release/${VERSION_NAME}/executorch.aar --acl public-read From 7e1767183e3883885eaf420ddf0f18cde99e289a Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Tue, 24 Sep 2024 14:24:23 -0700 Subject: [PATCH 22/23] test again --- .github/workflows/android-release-artifacts.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/android-release-artifacts.yml b/.github/workflows/android-release-artifacts.yml index 3983885848..5a28d7d88b 100644 --- a/.github/workflows/android-release-artifacts.yml +++ b/.github/workflows/android-release-artifacts.yml @@ -1,6 +1,9 @@ name: Android Release Artifacts on: + push: + branches: + - android-upload-artifact-2 workflow_dispatch: inputs: version: @@ -59,7 +62,7 @@ jobs: shasum -a 256 executorch.aar > executorch.aar.sha256sums pip install awscli==1.32.18 - AWS_CMD="aws s3 cp" + AWS_CMD="aws s3 cp --dryrun" VERSION="${{ inputs.version }}" VERSION_NAME="${VERSION:-temp_snapshot}" ${AWS_CMD} executorch.aar s3://ossci-android/executorch/release/${VERSION_NAME}/executorch.aar --acl public-read From 2549ec564d30f66d7ac34845b5b55530ef6dec59 Mon Sep 17 00:00:00 2001 From: Hansong Zhang Date: Tue, 24 Sep 2024 14:25:54 -0700 Subject: [PATCH 23/23] recover --- .github/workflows/android-release-artifacts.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/android-release-artifacts.yml b/.github/workflows/android-release-artifacts.yml index 5a28d7d88b..3983885848 100644 --- a/.github/workflows/android-release-artifacts.yml +++ b/.github/workflows/android-release-artifacts.yml @@ -1,9 +1,6 @@ name: Android Release Artifacts on: - push: - branches: - - android-upload-artifact-2 workflow_dispatch: inputs: version: @@ -62,7 +59,7 @@ jobs: shasum -a 256 executorch.aar > executorch.aar.sha256sums pip install awscli==1.32.18 - AWS_CMD="aws s3 cp --dryrun" + AWS_CMD="aws s3 cp" VERSION="${{ inputs.version }}" VERSION_NAME="${VERSION:-temp_snapshot}" ${AWS_CMD} executorch.aar s3://ossci-android/executorch/release/${VERSION_NAME}/executorch.aar --acl public-read