Add updates from ORT android emulator handling (#588)

* Update JDK version to 17 in ci.yml * Update com.diffplug.spotless to 6.22.0. * Copy updated scripts to start/stop the emulator from ORT from microsoft/onnxruntime#17903. Minimize the time the emulator is running as well. * Fix includes * Update to JDK 17 in packaging pipelines. * Fix pool name. --------- Co-authored-by: Edward Chen <[email protected]>
microsoft · Oct 31, 2023 · e951e72 · e951e72
1 parent 5fd6bcf
commit e951e72
Show file tree

Hide file tree

Showing 6 changed files with 207 additions and 90 deletions.
diff --git a/.pipelines/android_packaging.yml b/.pipelines/android_packaging.yml
@@ -32,26 +32,26 @@ jobs:
           python ./tools/gen_selectedops.py ./tools/android/package_ops.config
         displayName: "Generate selected ops CMake file"
 
-      - template: templates/run-with-android-emulator-steps.yml
-        parameters:
-          steps:
+      - bash: |
+          set -e -x
 
-          - bash: |
-              set -e -x
+          python ./tools/android/build_aar.py \
+            --output_dir $(Build.BinariesDirectory)/android_aar \
+            --config $(buildConfig) \
+            -- \
+            --one_cmake_extra_define OCOS_ENABLE_SELECTED_OPLIST=ON
 
-              python ./tools/android/build_aar.py \
-                --output_dir $(Build.BinariesDirectory)/android_aar \
-                --config $(buildConfig) \
-                -- \
-                --one_cmake_extra_define OCOS_ENABLE_SELECTED_OPLIST=ON
+          VERSION=$(cat ./version.txt)
+          AAR_PATH="$(Build.BinariesDirectory)/android_aar/aar_out/$(buildConfig)/com/microsoft/onnxruntime/onnxruntime-extensions-android/${VERSION}/onnxruntime-extensions-android-${VERSION}.aar"
 
-              VERSION=$(cat ./version.txt)
-              AAR_PATH="$(Build.BinariesDirectory)/android_aar/aar_out/$(buildConfig)/com/microsoft/onnxruntime/onnxruntime-extensions-android/${VERSION}/onnxruntime-extensions-android-${VERSION}.aar"
+          # Do not output ##vso[] commands with `set -x` or they may be parsed again and include a trailing quote.
+          set +x
+          echo "##vso[task.setvariable variable=ORT_EXTENSIONS_AAR_PATH]${AAR_PATH}"
+        displayName: Build onnxruntime-extensions AAR package
 
-              # Do not output ##vso[] commands with `set -x` or they may be parsed again and include a trailing quote.
-              set +x
-              echo "##vso[task.setvariable variable=ORT_EXTENSIONS_AAR_PATH]${AAR_PATH}"
-            displayName: Build onnxruntime-extensions AAR package
+      - template: templates/run-with-android-emulator-steps.yml
+        parameters:
+          steps:
 
           - bash: |
               set -e -x

diff --git a/.pipelines/ci.yml b/.pipelines/ci.yml
@@ -95,7 +95,7 @@ stages:
 
     - script: cd test && python -m pytest . --verbose
       displayName: Run python test
-  
+
   ###############
   # Linux PyDebug
   ###############
@@ -452,7 +452,7 @@ stages:
         set OCOS_SCB_DEBUG=1
         python -m pip install -v -e .
       displayName: Build onnxruntime-extensions in editable mode.
-        
+
     - script: |
         python -m pip install -r requirements-dev.txt
         python -m pip install torch torchvision torchaudio
@@ -521,22 +521,22 @@ stages:
     - script: brew install coreutils ninja
       displayName: Install coreutils and ninja
 
-    - template: templates/run-with-android-emulator-steps.yml
-      parameters:
-        steps:
+    - bash: |
+        set -e -x
 
-        - bash: |
-            set -e -x
+        _BUILD_CFG="x86_64 $(Build.BinariesDirectory)/android_aar" ./build.android
 
-            _BUILD_CFG="x86_64 $(Build.BinariesDirectory)/android_aar" ./build.android
+        VERSION=$(cat ./version.txt)
+        AAR_PATH="$(Build.BinariesDirectory)/android_aar/aar_out/com/microsoft/onnxruntime/onnxruntime-extensions-android/${VERSION}/onnxruntime-extensions-android-${VERSION}.aar"
 
-            VERSION=$(cat ./version.txt)
-            AAR_PATH="$(Build.BinariesDirectory)/android_aar/aar_out/com/microsoft/onnxruntime/onnxruntime-extensions-android/${VERSION}/onnxruntime-extensions-android-${VERSION}.aar"
+        # Do not output ##vso[] commands with `set -x` or they may be parsed again and include a trailing quote.
+        set +x
+        echo "##vso[task.setvariable variable=ORT_EXTENSIONS_AAR_PATH]${AAR_PATH}"
+      displayName: Build onnxruntime-extensions AAR package
 
-            # Do not output ##vso[] commands with `set -x` or they may be parsed again and include a trailing quote.
-            set +x
-            echo "##vso[task.setvariable variable=ORT_EXTENSIONS_AAR_PATH]${AAR_PATH}"
-          displayName: Build onnxruntime-extensions AAR package
+    - template: templates/run-with-android-emulator-steps.yml
+      parameters:
+        steps:
 
         - bash: |
             set -e -x
@@ -570,18 +570,26 @@ stages:
       - script: brew install ninja
         displayName: Install ninja
 
+      - bash: |
+          python ./tools/build.py \
+            --config RelWithDebInfo \
+            --android \
+            --android_abi x86_64 \
+            --enable_cxx_tests \
+            --update --build --parallel
+        displayName: Build onnxruntime-extensions for Android
+
       - template: templates/run-with-android-emulator-steps.yml
         parameters:
           steps:
-
           - bash: |
               python ./tools/build.py \
                 --config RelWithDebInfo \
                 --android \
                 --android_abi x86_64 \
                 --enable_cxx_tests \
-                --update --build --test --parallel
-            displayName: Build onnxruntime-extensions for Android and run C++ tests on emulator
+                --test
+            displayName: Run C++ tests on emulator
 
 - stage: IosBuilds
   dependsOn: []

diff --git a/.pipelines/templates/run-with-android-emulator-steps.yml b/.pipelines/templates/run-with-android-emulator-steps.yml
@@ -3,34 +3,13 @@ parameters:
   type: stepList
 
 steps:
-- bash: |
-    set -e -x
-
-    ORT_EXTENSIONS_BUILD_ANDROID_EMULATOR_PID_FILE="$(Build.BinariesDirectory)/android_emulator.pid"
-
-    python ./tools/android/run_android_emulator.py \
-      --android-sdk-root "${ANDROID_SDK_ROOT}" \
-      --create-avd --system-image "system-images;android-31;default;x86_64" \
-      --start --emulator-extra-args="-partition-size 4096" \
-      --emulator-pid-file "${ORT_EXTENSIONS_BUILD_ANDROID_EMULATOR_PID_FILE}"
-
-    # Do not output ##vso[] commands with `set -x` or they may be parsed again and include a trailing quote.
-    set +x
-    echo "##vso[task.setvariable variable=ORT_EXTENSIONS_BUILD_ANDROID_EMULATOR_PID_FILE]${ORT_EXTENSIONS_BUILD_ANDROID_EMULATOR_PID_FILE}"
-  displayName: "Create and start Android emulator"
+- template: use-android-emulator.yml
+  parameters:
+    create: true
+    start: true
 
 - ${{ parameters.steps }}
 
-- bash: |
-    set -e -x
-
-    if [[ -n "${ORT_EXTENSIONS_BUILD_ANDROID_EMULATOR_PID_FILE-}" ]]; then
-      python ./tools/android/run_android_emulator.py \
-        --android-sdk-root "${ANDROID_SDK_ROOT}" \
-        --stop \
-        --emulator-pid-file "${ORT_EXTENSIONS_BUILD_ANDROID_EMULATOR_PID_FILE}"
-
-      rm "${ORT_EXTENSIONS_BUILD_ANDROID_EMULATOR_PID_FILE}"
-    fi
-  displayName: "Stop Android emulator"
-  condition: always()
+- template: use-android-emulator.yml
+  parameters:
+    stop: true
diff --git a/.pipelines/templates/use-android-emulator.yml b/.pipelines/templates/use-android-emulator.yml
@@ -0,0 +1,64 @@
+# Android Emulator helpers
+# Copied from https://github.com/microsoft/onnxruntime/blob/main/tools/ci_build/github/azure-pipelines/templates/use-android-emulator.yml
+parameters:
+- name: create
+  type: boolean
+  default: false
+
+- name: start
+  type: boolean
+  default: false
+
+- name: stop
+  type: boolean
+  default: false
+
+steps:
+- ${{ if eq(parameters.create, true) }}:
+  - script: |
+      set -e -x
+      python3 tools/android/run_android_emulator.py \
+        --android-sdk-root $(ANDROID_SDK_ROOT) \
+        --create-avd --system-image "system-images;android-31;default;x86_64"
+    displayName: Create Android Emulator
+
+- ${{ if eq(parameters.start, true) }}:
+  - script: |
+      if test -f $(Build.BinariesDirectory)/emulator.pid; then
+        echo "Emulator PID file was not expected to exist but does and has pid:" \
+          `cat $(Build.BinariesDirectory)/emulator.pid`
+        exit 1
+      fi
+    displayName: Check emulator.pid does not exist
+
+  # Add -verbose to --emulator-extra-args to enable additional logging.
+  - script: |
+      set -e -x
+      python3 tools/android/run_android_emulator.py \
+        --android-sdk-root $(ANDROID_SDK_ROOT) \
+        --start --emulator-extra-args="-partition-size 2047" \
+        --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid
+      echo "Emulator PID:"`cat $(Build.BinariesDirectory)/emulator.pid`
+    displayName: Start Android Emulator
+
+- ${{ if eq(parameters.stop, true) }}:
+  - script: |
+      set -e -x
+      python3 -m pip install psutil
+    displayName: Install psutil for emulator shutdown by run_android_emulator.py
+    condition: always()
+
+  - script: |
+      set -e -x
+      if test -f $(Build.BinariesDirectory)/emulator.pid; then
+        echo "Emulator PID:"`cat $(Build.BinariesDirectory)/emulator.pid`
+        python3 tools/android/run_android_emulator.py \
+          --android-sdk-root $(ANDROID_SDK_ROOT) \
+          --stop \
+          --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid
+        rm $(Build.BinariesDirectory)/emulator.pid
+      else
+        echo "Emulator PID file was expected to exist but does not."
+      fi
+    displayName: Stop Android Emulator
+    condition: always()
diff --git a/tools/android/run_android_emulator.py b/tools/android/run_android_emulator.py
@@ -22,8 +22,8 @@
 def parse_args():
     parser = argparse.ArgumentParser(
         description="Manages the running of an Android emulator. "
-        "Supported modes are to start and stop (default), only start, or only "
-        "stop the emulator."
+        "Supported modes are to create an AVD, and start or stop the emulator. "
+        "The default is to start the emulator and wait for a keypress to stop it (start and stop)."
     )
 
     parser.add_argument("--create-avd", action="store_true", help="Whether to create the Android virtual device.")
@@ -49,8 +49,8 @@ def parse_args():
 
     args = parser.parse_args()
 
-    if not args.start and not args.stop:
-        # unspecified means start and stop
+    if not args.start and not args.stop and not args.create_avd:
+        # unspecified means start and stop if not creating the AVD
         args.start = args.stop = True
 
     if args.start != args.stop and args.emulator_pid_file is None:
@@ -86,14 +86,14 @@ def main():
         emulator_proc = android.start_emulator(**start_emulator_args)
 
         with open(args.emulator_pid_file, mode="w") as emulator_pid_file:
-            print("{}".format(emulator_proc.pid), file=emulator_pid_file)
+            print(f"{emulator_proc.pid}", file=emulator_pid_file)
 
     elif args.stop:
-        with open(args.emulator_pid_file, mode="r") as emulator_pid_file:
+        with open(args.emulator_pid_file) as emulator_pid_file:
             emulator_pid = int(emulator_pid_file.readline().strip())
 
         android.stop_emulator(emulator_pid)
 
 
 if __name__ == "__main__":
-    main()
+    sys.exit(main())