From c072396a2dc1d05ea138b498e76d17ce2e8dc667 Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Thu, 12 Oct 2023 18:46:04 +1000 Subject: [PATCH 1/6] Attempt to make the usage of the Android emulator in CIs more robust - Change approach to detecting boot has completed - use `-delay-adb` and a simple command (`ls`) with `wait-for-device` as the first step - this ensures enough startup has occurred for adb to be responsive - use secondary loop on the python side to check for sys.boot_completed to be set - doing the check on the python side provides more feedback and seems to work well - make the 'stop' logic a lot more precise by using psutil CI updates - add template for using the emulator - update CIs to use template - reorder React Native CI - minimize the time the Android emulator or iOS simulator is running by moving some build steps around - don't run both at the same time - unnecessary and potentially adds significant memory pressure to the machine - fix QNN Android emulator CI as much as possible - now everything works apart from running onnx_test_runner with the QNN EP --- ...arm64-v8a-QNN-crosscompile-ci-pipeline.yml | 64 +++---- ...ndroid-x86_64-crosscompile-ci-pipeline.yml | 65 +++---- .../templates/android-java-api-aar-test.yml | 23 +-- .../templates/react-native-ci.yml | 159 +++++++++--------- .../templates/use-android-emulator.yml | 63 +++++++ tools/python/run_android_emulator.py | 4 +- tools/python/util/android/android.py | 92 ++++++++-- 7 files changed, 280 insertions(+), 190 deletions(-) create mode 100644 tools/ci_build/github/azure-pipelines/templates/use-android-emulator.yml diff --git a/tools/ci_build/github/azure-pipelines/android-arm64-v8a-QNN-crosscompile-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/android-arm64-v8a-QNN-crosscompile-ci-pipeline.yml index bbab9f3d85abb..4ebc6ea510ed8 100644 --- a/tools/ci_build/github/azure-pipelines/android-arm64-v8a-QNN-crosscompile-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/android-arm64-v8a-QNN-crosscompile-ci-pipeline.yml @@ -74,8 +74,8 @@ jobs: --build_dir build_qnn \ --android_sdk_path $ANDROID_HOME \ --android_ndk_path $ANDROID_NDK_HOME \ - --android_abi=arm64-v8a \ - --android_api=30 \ + --android_abi=x86_64 \ + --android_api=31 \ --parallel \ --use_qnn \ --qnn_home $(QNN_SDK_ROOT) \ @@ -85,48 +85,36 @@ jobs: - script: | mkdir -p build_qnn/Release/testdata/QNN/node_tests - cp -r cmake/external/onnx//onnx/backend/test/data/node/test_basic_conv_with_padding build_qnn/Release/testdata/QNN/node_tests + cp -r cmake/external/onnx/onnx/backend/test/data/node/test_basic_conv_with_padding build_qnn/Release/testdata/QNN/node_tests displayName: Initialize test directories - task: JavaToolInstaller@0 - displayName: Use jdk 8 + displayName: Use jdk 11 inputs: - versionSpec: '8' + versionSpec: '11' jdkArchitectureOption: 'x64' jdkSourceOption: 'PreInstalled' - - script: | - python3 tools/python/run_android_emulator.py \ - --android-sdk-root ${ANDROID_SDK_ROOT} \ - --create-avd --system-image "system-images;android-30;google_apis;arm64-v8a" \ - --start --emulator-extra-args="-partition-size 4096" \ - --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid - displayName: Start Android emulator - enabled: false - continueOnError: true + # This is commented out for now. The emulator runs correctly, onnx_test_runner is executable, and the test passes + # with the CPU EP but returns 139 when attempting to use the QNN EP. Maybe some QNN EP parameters need to be provided? + # + # - template: templates/use-android-emulator.yml + # parameters: + # create: true + # start: true - - script: | - $ANDROID_SDK_ROOT/emulator/emulator -list-avds - displayName: List emulators + # - task: CmdLine@2 + # inputs: + # script: | + # set -e -x + # ${ANDROID_SDK_ROOT}/platform-tools/adb push onnx_test_runner /data/local/tmp/ + # ${ANDROID_SDK_ROOT}/platform-tools/adb push testdata/QNN/node_tests/test_basic_conv_with_padding /data/local/tmp/ + # ${ANDROID_SDK_ROOT}/platform-tools/adb shell "chmod +x /data/local/tmp/onnx_test_runner" + # ${ANDROID_SDK_ROOT}/platform-tools/adb shell "/data/local/tmp/onnx_test_runner -v /data/local/tmp/test_basic_conv_with_padding" + # ${ANDROID_SDK_ROOT}/platform-tools/adb shell "/data/local/tmp/onnx_test_runner -v -e qnn /data/local/tmp/test_basic_conv_with_padding" + # workingDirectory: build_qnn/Release + # displayName: Run test_basic_conv_with_padding on emulator - - task: CmdLine@2 - inputs: - script: | - ${ANDROID_SDK_ROOT}/platform-tools/adb shell "mkdir /data/local/tmp/qnn" - ${ANDROID_SDK_ROOT}/platform-tools/adb push onnx_test_runner /data/local/tmp/qnn - ${ANDROID_SDK_ROOT}/platform-tools/adb push testdata/QNN/node_tests/test_basic_conv_with_padding /data/local/tmp/qnn - ${ANDROID_SDK_ROOT}/platform-tools/adb shell "/data/local/tmp/qnn/onnx_test_runner -e qnn /data/local/tmp/qnn/test_basic_conv_with_padding" - workingDirectory: build_qnn/Release - displayName: Run tests - enabled: false - continueOnError: true - - - script: | - python3 tools/python/run_android_emulator.py \ - --android-sdk-root ${ANDROID_SDK_ROOT} \ - --stop \ - --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid - displayName: Stop Android emulator - condition: always() - enabled: false - continueOnError: true + # - template: templates/use-android-emulator.yml + # parameters: + # stop: true diff --git a/tools/ci_build/github/azure-pipelines/android-x86_64-crosscompile-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/android-x86_64-crosscompile-ci-pipeline.yml index f6f6f52440534..9136b21aec626 100644 --- a/tools/ci_build/github/azure-pipelines/android-x86_64-crosscompile-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/android-x86_64-crosscompile-ci-pipeline.yml @@ -257,13 +257,10 @@ stages: - template: "templates/use-android-ndk.yml" - - script: | - python3 tools/python/run_android_emulator.py \ - --android-sdk-root ${ANDROID_SDK_ROOT} \ - --create-avd --system-image "system-images;android-31;default;x86_64" \ - --start --emulator-extra-args="-partition-size 4096" \ - --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid - displayName: Start Android emulator + - template: templates/use-android-emulator.yml + parameters: + create: true + start: true - script: | python3 tools/ci_build/build.py \ @@ -277,13 +274,9 @@ stages: --test displayName: CPU EP, Test on Android Emulator - - script: | - python3 tools/python/run_android_emulator.py \ - --android-sdk-root ${ANDROID_SDK_ROOT} \ - --stop \ - --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid - displayName: Stop Android emulator - condition: always() + - template: templates/use-android-emulator.yml + parameters: + stop: true - template: templates/clean-agent-build-directory-step.yml @@ -329,13 +322,10 @@ stages: - template: "templates/use-android-ndk.yml" - - script: | - python3 tools/python/run_android_emulator.py \ - --android-sdk-root ${ANDROID_SDK_ROOT} \ - --create-avd --system-image "system-images;android-31;default;x86_64" \ - --start --emulator-extra-args="-partition-size 4096" \ - --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid - displayName: Start Android emulator + - template: templates/use-android-emulator.yml + parameters: + create: true + start: true - script: | python3 tools/ci_build/build.py \ @@ -358,13 +348,10 @@ stages: # Build Minimal ORT with NNAPI and reduced Ops, run unit tests on Android Emulator displayName: Build Minimal ORT with NNAPI and run tests - - script: | - python3 tools/python/run_android_emulator.py \ - --android-sdk-root ${ANDROID_SDK_ROOT} \ - --stop \ - --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid - displayName: Stop Android emulator - condition: always() + - template: templates/use-android-emulator.yml + parameters: + stop: true + - template: templates/clean-agent-build-directory-step.yml - stage: MASTER_BUILD_STAGE @@ -415,13 +402,10 @@ stages: $(Build.SourcesDirectory)/protobuf_install displayName: Build Host Protoc - - script: | - python3 tools/python/run_android_emulator.py \ - --android-sdk-root ${ANDROID_SDK_ROOT} \ - --create-avd --system-image "system-images;android-31;default;x86_64" \ - --start --emulator-extra-args="-partition-size 4096" \ - --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid - displayName: Start Android emulator + - template: templates/use-android-emulator.yml + parameters: + create: true + start: true - script: | python3 tools/ci_build/build.py \ @@ -460,13 +444,10 @@ stages: # Build Minimal ORT with NNAPI and reduced Ops, run unit tests on Android Emulator displayName: Build Minimal ORT with NNAPI and run tests - - script: | - python3 tools/python/run_android_emulator.py \ - --android-sdk-root ${ANDROID_SDK_ROOT} \ - --stop \ - --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid - displayName: Stop Android emulator - condition: always() + - template: templates/use-android-emulator.yml + parameters: + stop: true + - template: templates/clean-agent-build-directory-step.yml - job: Update_Dashboard diff --git a/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar-test.yml b/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar-test.yml index 1263b21d4a03e..41322c6ce3c2b 100644 --- a/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar-test.yml +++ b/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar-test.yml @@ -50,13 +50,10 @@ jobs: - template: install-appcenter.yml - - script: | - python3 $(Build.SourcesDirectory)/tools/python/run_android_emulator.py \ - --android-sdk-root ${ANDROID_SDK_ROOT} \ - --create-avd --system-image "system-images;android-31;default;x86_64" \ - --start --emulator-extra-args="-partition-size 4096" \ - --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid - displayName: Start Android emulator + - template: use-android-emulator.yml + parameters: + create: true + start: true - script: | set -e -x @@ -70,6 +67,10 @@ jobs: displayName: Run E2E test using Emulator workingDirectory: $(Build.BinariesDirectory) + - template: use-android-emulator.yml + parameters: + stop: true + - script: | set -e -x cd android_test/android @@ -84,14 +85,6 @@ jobs: displayName: Run E2E tests using App Center workingDirectory: $(Build.BinariesDirectory) - - script: | - python3 $(Build.SourcesDirectory)/tools/python/run_android_emulator.py \ - --android-sdk-root ${ANDROID_SDK_ROOT} \ - --stop \ - --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid - displayName: Stop Android emulator - condition: always() - - template: component-governance-component-detection-steps.yml parameters : condition : 'succeeded' diff --git a/tools/ci_build/github/azure-pipelines/templates/react-native-ci.yml b/tools/ci_build/github/azure-pipelines/templates/react-native-ci.yml index e63939ae0114c..5107fedaba01b 100644 --- a/tools/ci_build/github/azure-pipelines/templates/react-native-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/react-native-ci.yml @@ -185,69 +185,6 @@ stages: workingDirectory: '$(Build.SourcesDirectory)/js/react_native' displayName: yarn js/react_native - - script: | - python3 tools/python/run_android_emulator.py \ - --android-sdk-root $(ANDROID_SDK_ROOT) \ - --create-avd --system-image "system-images;android-30;default;x86_64" \ - --start --emulator-extra-args="-partition-size 4096 -verbose" \ - --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid - displayName: Start Android Emulator - - - script: | - xcrun simctl create iPhoneRNTest com.apple.CoreSimulator.SimDeviceType.iPhone-13 - workingDirectory: '$(Build.SourcesDirectory)/js/react_native/e2e/ios' - displayName: Start iOS Simulator - - - template: android-dump-logs-from-steps.yml - parameters: - steps: - - task: Gradle@3 - inputs: - gradleWrapperFile: '$(Build.SourcesDirectory)/js/react_native/android/gradlew' - workingDirectory: '$(Build.SourcesDirectory)/js/react_native/android' - options: '--stacktrace' - tasks: 'connectedDebugAndroidTest' - publishJUnitResults: true - testResultsFiles: '**/TEST-*.xml' - testRunTitle: 'React Native Android Instrumented Test results' - javaHomeOption: 'path' - jdkDirectory: '$(JAVA_HOME_11_X64)' - sonarQubeRunAnalysis: false - spotBugsAnalysis: false - displayName: Run React Native Android Instrumented Tests - - - script: | - # Mobile build: - # ORT_MOBILE_C_LOCAL_POD_PATH=$(Build.BinariesDirectory)/staging/onnxruntime-mobile-c \ - ORT_C_LOCAL_POD_PATH=$(Build.BinariesDirectory)/staging/onnxruntime-c \ - pod install - workingDirectory: '$(Build.SourcesDirectory)/js/react_native/ios' - displayName: Pod install for onnxruntime react native ios bridge library - - - task: Xcode@5 - inputs: - actions: 'test' - configuration: 'Debug' - sdk: 'iphonesimulator' - xcWorkspacePath: '$(Build.SourcesDirectory)/js/react_native/ios/OnnxruntimeModule.xcworkspace' - scheme: 'OnnxruntimeModuleTest' - packageApp: false - destinationPlatformOption: 'iOS' - destinationSimulators: 'iPhone 13,OS=latest' - workingDirectory: '$(Build.SourcesDirectory)/js/react_native/ios' - xcprettyArgs: '--output build/reports/test-results.xml' - publishJUnitResults: true - testRunTitle: 'React Native iOS Instrumented Test Results' - displayName: Run React Native iOS Instrumented Tests - - - task: PublishTestResults@2 - inputs: - testResultsFiles: '$(Build.SourcesDirectory)/js/react_native/ios/build/reports/test-results.xml' - failTaskOnFailedTests: true - testRunTitle: 'React Native iOS Instrumented Test results' - condition: succeededOrFailed() - displayName: Publish React Native iOS Instrumented Test Results - - task: PowerShell@2 inputs: filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/js/pack-npm-packages.ps1' @@ -267,6 +204,14 @@ stages: workingDirectory: '$(Build.SourcesDirectory)/js/react_native/e2e' displayName: Bootstrap Android and iOS e2e tests + - script: | + # Mobile build: + # ORT_MOBILE_C_LOCAL_POD_PATH=$(Build.BinariesDirectory)/staging/onnxruntime-mobile-c \ + ORT_C_LOCAL_POD_PATH=$(Build.BinariesDirectory)/staging/onnxruntime-c \ + pod install + workingDirectory: '$(Build.SourcesDirectory)/js/react_native/ios' + displayName: Pod install for onnxruntime react native ios bridge library + - script: | # Mobile build: # ORT_MOBILE_C_LOCAL_POD_PATH=$(Build.BinariesDirectory)/staging/onnxruntime-mobile-c \ @@ -301,12 +246,47 @@ stages: workingDirectory: '$(Build.SourcesDirectory)/js/react_native/e2e' displayName: Build React Native Detox Android e2e Tests + - script: | + detox build --configuration ios.sim.release + workingDirectory: '$(Build.SourcesDirectory)/js/react_native/e2e' + displayName: Build React Native Detox iOS e2e Tests + + # + # Unit tests and E2E tests with Android emulator + # + - template: use-android-emulator.yml + parameters: + create: true + start: true + + - template: android-dump-logs-from-steps.yml + parameters: + steps: + - task: Gradle@3 + inputs: + gradleWrapperFile: '$(Build.SourcesDirectory)/js/react_native/android/gradlew' + workingDirectory: '$(Build.SourcesDirectory)/js/react_native/android' + options: '--stacktrace' + tasks: 'connectedDebugAndroidTest' + publishJUnitResults: true + testResultsFiles: '**/TEST-*.xml' + testRunTitle: 'React Native Android Instrumented Test results' + javaHomeOption: 'path' + jdkDirectory: '$(JAVA_HOME_11_X64)' + sonarQubeRunAnalysis: false + spotBugsAnalysis: false + displayName: Run React Native Android Instrumented Tests + - script: | JEST_JUNIT_OUTPUT_FILE=$(Build.SourcesDirectory)/js/react_native/e2e/android-test-results.xml \ detox test --record-logs all --configuration android.emu.release workingDirectory: '$(Build.SourcesDirectory)/js/react_native/e2e' displayName: Run React Native Detox Android e2e Tests + - template: use-android-emulator.yml + parameters: + stop: true + - task: PublishTestResults@2 inputs: testResultsFiles: '$(Build.SourcesDirectory)/js/react_native/e2e/android-test-results.xml' @@ -315,10 +295,37 @@ stages: condition: succeededOrFailed() displayName: Publish React Native Detox Android e2e Test Results + # + # Unit tests and E2E tests with iOS simulator + # - script: | - detox build --configuration ios.sim.release - workingDirectory: '$(Build.SourcesDirectory)/js/react_native/e2e' - displayName: Build React Native Detox iOS e2e Tests + xcrun simctl create iPhoneRNTest com.apple.CoreSimulator.SimDeviceType.iPhone-13 + workingDirectory: '$(Build.SourcesDirectory)/js/react_native/e2e/ios' + displayName: Start iOS Simulator + + - task: Xcode@5 + inputs: + actions: 'test' + configuration: 'Debug' + sdk: 'iphonesimulator' + xcWorkspacePath: '$(Build.SourcesDirectory)/js/react_native/ios/OnnxruntimeModule.xcworkspace' + scheme: 'OnnxruntimeModuleTest' + packageApp: false + destinationPlatformOption: 'iOS' + destinationSimulators: 'iPhone 13,OS=latest' + workingDirectory: '$(Build.SourcesDirectory)/js/react_native/ios' + xcprettyArgs: '--output build/reports/test-results.xml' + publishJUnitResults: true + testRunTitle: 'React Native iOS Instrumented Test Results' + displayName: Run React Native iOS Instrumented Tests + + - task: PublishTestResults@2 + inputs: + testResultsFiles: '$(Build.SourcesDirectory)/js/react_native/ios/build/reports/test-results.xml' + failTaskOnFailedTests: true + testRunTitle: 'React Native iOS Instrumented Test results' + condition: succeededOrFailed() + displayName: Publish React Native iOS Instrumented Test Results - script: | JEST_JUNIT_OUTPUT_FILE=$(Build.SourcesDirectory)/js/react_native/e2e/ios-test-results.xml \ @@ -326,6 +333,12 @@ stages: workingDirectory: '$(Build.SourcesDirectory)/js/react_native/e2e' displayName: Run React Native Detox iOS e2e Tests + - script: | + xcrun simctl delete iPhoneRNTest + workingDirectory: '$(Build.SourcesDirectory)/js/react_native/e2e/ios' + displayName: Stop iOS Simulator + condition: always() + - task: PublishTestResults@2 inputs: testResultsFiles: '$(Build.SourcesDirectory)/js/react_native/e2e/ios-test-results.xml' @@ -341,20 +354,6 @@ stages: condition: succeededOrFailed() displayName: Publish React Native Detox E2E test logs - - script: | - python3 tools/python/run_android_emulator.py \ - --android-sdk-root $(ANDROID_SDK_ROOT) \ - --stop \ - --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid - displayName: Stop Android Emulator - condition: always() - - - script: | - xcrun simctl delete iPhoneRNTest - workingDirectory: '$(Build.SourcesDirectory)/js/react_native/e2e/ios' - displayName: Stop iOS Simulator - condition: always() - - script: | git restore . workingDirectory: '$(Build.SourcesDirectory)/js' diff --git a/tools/ci_build/github/azure-pipelines/templates/use-android-emulator.yml b/tools/ci_build/github/azure-pipelines/templates/use-android-emulator.yml new file mode 100644 index 0000000000000..2a702558fd589 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/templates/use-android-emulator.yml @@ -0,0 +1,63 @@ +# Android Emulator helpers + +parameters: +- name: create + type: boolean + default: false + +- name: start + type: boolean + default: false + +- name: stop + type: boolean + default: false + +steps: +- ${{ if eq(parameters.create, true) }}: + - script: | + set -e -x + python3 tools/python/run_android_emulator.py \ + --android-sdk-root $(ANDROID_SDK_ROOT) \ + --create-avd --system-image "system-images;android-31;default;x86_64" + displayName: Create Android Emulator + +- ${{ if eq(parameters.start, true) }}: + - script: | + if test -f $(Build.BinariesDirectory)/emulator.pid; then + echo "Emulator PID file was not expected to exist but does and has pid:" \ + `cat $(Build.BinariesDirectory)/emulator.pid` + exit 1 + fi + displayName: Check emulator.pid does not exist + + # Add -verbose to --emulator-extra-args to additional logging. + - script: | + set -e -x + python3 tools/python/run_android_emulator.py \ + --android-sdk-root $(ANDROID_SDK_ROOT) \ + --start --emulator-extra-args="-partition-size 2047" \ + --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid + echo "Emulator PID:"`cat $(Build.BinariesDirectory)/emulator.pid` + displayName: Start Android Emulator + +- ${{ if eq(parameters.stop, true) }}: + - script: | + set -e -x + python3 -m pip install psutil + displayName: Install psutil for emulator shutdown by run_android_emulator.py + condition: always() + + - script: | + if test -f $(Build.BinariesDirectory)/emulator.pid; then + echo "Emulator PID:"`cat $(Build.BinariesDirectory)/emulator.pid` + python3 tools/python/run_android_emulator.py \ + --android-sdk-root $(ANDROID_SDK_ROOT) \ + --stop \ + --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid + rm $(Build.BinariesDirectory)/emulator.pid + else + echo "Emulator PID file was expected to exist but does not." + fi + displayName: Stop Android Emulator + condition: always() diff --git a/tools/python/run_android_emulator.py b/tools/python/run_android_emulator.py index 69fa88bd082dc..3e6c7394367d7 100755 --- a/tools/python/run_android_emulator.py +++ b/tools/python/run_android_emulator.py @@ -43,8 +43,8 @@ def parse_args(): args = parser.parse_args() - if not args.start and not args.stop: - # unspecified means start and stop + if not args.start and not args.stop and not args.create_avd: + # unspecified means start and stop if not creating the AVD args.start = args.stop = True if args.start != args.stop and args.emulator_pid_file is None: diff --git a/tools/python/util/android/android.py b/tools/python/util/android/android.py index 0baa21179d32d..0fe76ecb10e5f 100644 --- a/tools/python/util/android/android.py +++ b/tools/python/util/android/android.py @@ -3,7 +3,7 @@ import collections import contextlib -import logging +import datetime import os import shutil import signal @@ -11,10 +11,11 @@ import time import typing -from ..platform_helpers import is_windows +from ..logger import get_logger +from ..platform_helpers import is_linux, is_windows from ..run import run -_log = logging.getLogger("util.android") +_log = get_logger("util.android") SdkToolPaths = collections.namedtuple("SdkToolPaths", ["emulator", "adb", "sdkmanager", "avdmanager"]) @@ -30,7 +31,7 @@ def filename(name, windows_extension): def resolve_path(dirnames, basename): dirnames.insert(0, "") for dirname in dirnames: - path = shutil.which(os.path.join(dirname, basename)) + path = shutil.which(os.path.join(os.path.expanduser(dirname), basename)) if path is not None: path = os.path.realpath(path) _log.debug(f"Found {basename} at {path}") @@ -90,9 +91,23 @@ def _stop_process(proc: subprocess.Popen): def _stop_process_with_pid(pid: int): - # not attempting anything fancier than just sending _stop_signal for now - _log.debug(f"Stopping process - pid: {pid}") - os.kill(pid, _stop_signal) + # minimize scope of external module usage + import psutil + + if psutil.pid_exists(pid): + process = psutil.Process(pid) + _log.debug(f"Stopping process - pid={pid}") + process.terminate() + try: + process.wait(60) + except psutil.TimeoutExpired: + print("Process did not terminate within 60 seconds. Killing.") + process.kill() + time.sleep(10) + if psutil.pid_exists(pid): + print(f"Process still exists. State:{process.status()}") + else: + _log.debug(f"No process exists with pid={pid}") def start_emulator( @@ -107,29 +122,49 @@ def start_emulator( "4096", "-timezone", "America/Los_Angeles", - "-no-snapshot", + "-no-snapstorage", "-no-audio", "-no-boot-anim", - "-no-window", + "-gpu", + "guest", + "-delay-adb", ] + + # For Linux CIs we must use "-no-window" otherwise you'll get + # Fatal: This application failed to start because no Qt platform plugin could be initialized + # + # For macOS CIs use a window so that we can potentially capture the desktop and the emulator screen + # and publish screenshot.jpg and emulator.png as artifacts to debug issues. + # screencapture screenshot.jpg + # $(ANDROID_SDK_HOME)/platform-tools/adb exec-out screencap -p > emulator.png + # + # On Windows it doesn't matter (AFAIK) so allow a window which is nicer for local debugging. + if is_linux(): + emulator_args.append("-no-window") + if extra_args is not None: emulator_args += extra_args emulator_process = emulator_stack.enter_context(_start_process(*emulator_args)) emulator_stack.callback(_stop_process, emulator_process) + # we're specifying -delay-adb so use a trivial command to check when adb is available. waiter_process = waiter_stack.enter_context( _start_process( sdk_tool_paths.adb, "wait-for-device", "shell", - "while [[ -z $(getprop sys.boot_completed) ]]; do sleep 1; done; input keyevent 82", + "ls /data/local/tmp", ) ) + waiter_stack.callback(_stop_process, waiter_process) - # poll subprocesses - sleep_interval_seconds = 1 + # poll subprocesses. + # allow 20 minutes for startup as some CIs are slow. TODO: Make timeout configurable if needed. + sleep_interval_seconds = 10 + end_time = datetime.datetime.now() + datetime.timedelta(minutes=20) + while True: waiter_ret, emulator_ret = waiter_process.poll(), emulator_process.poll() @@ -142,10 +177,41 @@ def start_emulator( break raise RuntimeError(f"Waiter process exited with return code: {waiter_ret}") + if datetime.datetime.now() > end_time: + raise RuntimeError("Emulator startup timeout") + time.sleep(sleep_interval_seconds) - # emulator is ready now + # emulator is started emulator_stack.pop_all() + + # loop to check for sys.boot_completed being set. + # in theory `-delay-adb` should be enough but this extra check seems to be required to be sure. + while True: + # looping on device with `while` seems to be flaky so loop here and call getprop once + args = [ + sdk_tool_paths.adb, + "shell", + # "while [[ -z $(getprop sys.boot_completed) | tr -d '\r' ]]; do sleep 5; done; input keyevent 82", + "getprop sys.boot_completed", + ] + + _log.debug(f"Starting process - args: {args}") + + getprop_output = subprocess.check_output(args, timeout=10) + getprop_value = bytes.decode(getprop_output).strip() + + if getprop_value == "1": + break + + elif datetime.datetime.now() > end_time: + raise RuntimeError("Emulator startup timeout. sys.boot_completed was not set.") + + _log.debug( + f"sys.boot_completed='{getprop_value}'. Sleeping for {sleep_interval_seconds} before retrying." + ) + time.sleep(sleep_interval_seconds) + return emulator_process From daa8ef57e239ce6d0b3dbe61f48fafe116dd71cb Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Thu, 12 Oct 2023 19:58:53 +1000 Subject: [PATCH 2/6] lint --- tools/python/util/android/android.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/python/util/android/android.py b/tools/python/util/android/android.py index 0fe76ecb10e5f..e6f7af2241005 100644 --- a/tools/python/util/android/android.py +++ b/tools/python/util/android/android.py @@ -207,9 +207,7 @@ def start_emulator( elif datetime.datetime.now() > end_time: raise RuntimeError("Emulator startup timeout. sys.boot_completed was not set.") - _log.debug( - f"sys.boot_completed='{getprop_value}'. Sleeping for {sleep_interval_seconds} before retrying." - ) + _log.debug(f"sys.boot_completed='{getprop_value}'. Sleeping for {sleep_interval_seconds} before retrying.") time.sleep(sleep_interval_seconds) return emulator_process From 43e54b473076eae8d30f0d2a9bec53c8be116bf1 Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Fri, 13 Oct 2023 08:02:27 +1000 Subject: [PATCH 3/6] Update tools/ci_build/github/azure-pipelines/templates/use-android-emulator.yml Co-authored-by: Edward Chen <18449977+edgchen1@users.noreply.github.com> --- .../github/azure-pipelines/templates/use-android-emulator.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/use-android-emulator.yml b/tools/ci_build/github/azure-pipelines/templates/use-android-emulator.yml index 2a702558fd589..72fbf4c53fc9e 100644 --- a/tools/ci_build/github/azure-pipelines/templates/use-android-emulator.yml +++ b/tools/ci_build/github/azure-pipelines/templates/use-android-emulator.yml @@ -31,7 +31,7 @@ steps: fi displayName: Check emulator.pid does not exist - # Add -verbose to --emulator-extra-args to additional logging. + # Add -verbose to --emulator-extra-args to enable additional logging. - script: | set -e -x python3 tools/python/run_android_emulator.py \ From ddce27ee8c32b4766a0ad71c178020db4ac53a64 Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Fri, 13 Oct 2023 08:03:18 +1000 Subject: [PATCH 4/6] Address PR comments --- .../github/azure-pipelines/templates/use-android-emulator.yml | 1 + tools/python/run_android_emulator.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/use-android-emulator.yml b/tools/ci_build/github/azure-pipelines/templates/use-android-emulator.yml index 2a702558fd589..7792e8627cbe5 100644 --- a/tools/ci_build/github/azure-pipelines/templates/use-android-emulator.yml +++ b/tools/ci_build/github/azure-pipelines/templates/use-android-emulator.yml @@ -49,6 +49,7 @@ steps: condition: always() - script: | + set -e -x if test -f $(Build.BinariesDirectory)/emulator.pid; then echo "Emulator PID:"`cat $(Build.BinariesDirectory)/emulator.pid` python3 tools/python/run_android_emulator.py \ diff --git a/tools/python/run_android_emulator.py b/tools/python/run_android_emulator.py index 3e6c7394367d7..2826921726556 100755 --- a/tools/python/run_android_emulator.py +++ b/tools/python/run_android_emulator.py @@ -16,8 +16,8 @@ def parse_args(): parser = argparse.ArgumentParser( description="Manages the running of an Android emulator. " - "Supported modes are to start and stop (default), only start, or only " - "stop the emulator." + "Supported modes are to create an AVD, and start or stop the emulator. " + "The default is to start the emulator and wait for a keypress to stop it (start and stop)." ) parser.add_argument("--create-avd", action="store_true", help="Whether to create the Android virtual device.") From 1c7754bccc86d21014c3eac84f674231ab446a94 Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Fri, 13 Oct 2023 11:24:12 +1000 Subject: [PATCH 5/6] Tweak logging so it's clearer when the wait-for-device process exits. --- .../github/azure-pipelines/templates/react-native-ci.yml | 2 +- tools/python/util/android/android.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/react-native-ci.yml b/tools/ci_build/github/azure-pipelines/templates/react-native-ci.yml index 5107fedaba01b..33f956f931f18 100644 --- a/tools/ci_build/github/azure-pipelines/templates/react-native-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/react-native-ci.yml @@ -69,7 +69,7 @@ stages: inputs: versionSpec: "3.9" addToPath: true - rchitecture: "x64" + architecture: "x64" - task: JavaToolInstaller@0 displayName: Use jdk 11 diff --git a/tools/python/util/android/android.py b/tools/python/util/android/android.py index e6f7af2241005..e431978e01f91 100644 --- a/tools/python/util/android/android.py +++ b/tools/python/util/android/android.py @@ -80,6 +80,10 @@ def _start_process(*args) -> subprocess.Popen: def _stop_process(proc: subprocess.Popen): + if proc.returncode is not None: + # process has exited + return + _log.debug(f"Stopping process - args: {proc.args}") proc.send_signal(_stop_signal) @@ -174,6 +178,7 @@ def start_emulator( if waiter_ret is not None: if waiter_ret == 0: + _log.debug(f"adb wait-for-device process has completed.") break raise RuntimeError(f"Waiter process exited with return code: {waiter_ret}") From 66bbc897be65338e22cb3cd7ec31b306c5cf2781 Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Fri, 13 Oct 2023 11:33:47 +1000 Subject: [PATCH 6/6] lint --- tools/python/util/android/android.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/python/util/android/android.py b/tools/python/util/android/android.py index e431978e01f91..47e251d11a38d 100644 --- a/tools/python/util/android/android.py +++ b/tools/python/util/android/android.py @@ -178,7 +178,7 @@ def start_emulator( if waiter_ret is not None: if waiter_ret == 0: - _log.debug(f"adb wait-for-device process has completed.") + _log.debug("adb wait-for-device process has completed.") break raise RuntimeError(f"Waiter process exited with return code: {waiter_ret}")