Benchmarking #3739
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarking | |
| on: | |
| schedule: | |
| - cron: '0 0 * * *' # every night at midnight UTC | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| paths: | |
| # test changes to Sentry SDK sources | |
| - 'Sources/**' | |
| # test changes to benchmarking implementation | |
| - 'Samples/iOS-Swift/iOS-Swift/**' | |
| - 'Samples/iOS-Swift/PerformanceBenchmarks/**' | |
| - '.github/workflows/benchmarking.yml' | |
| - '.sauce/benchmarking-config.yml' | |
| - 'fastlane/**' | |
| - 'scripts/ci-select-xcode.sh' | |
| - 'scripts/set-device-tests-environment.patch' | |
| jobs: | |
| build-benchmark-test-target: | |
| name: Build app and test runner | |
| runs-on: macos-13 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - run: ./scripts/ci-select-xcode.sh | |
| - name: Install SentryCli | |
| run: brew install getsentry/tools/sentry-cli | |
| - run: git apply ./scripts/set-device-tests-environment.patch | |
| - name: Cache iOS-Swift App and dSYM build products | |
| id: ios-swift-cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| DerivedData/Build/Products/Debug-iphoneos/iOS-Swift.app.dSYM | |
| DerivedData/Build/Products/Debug-iphoneos/iOS-Swift.app | |
| key: ios-swift-for-ui-testing-cache-key-${{ hashFiles('Samples/iOS-Swift/iOS-Swift/**') }}-${{ hashFiles('Sources/Sentry/**') }} | |
| - name: Cache iOS-Swift UI Test Runner App build product | |
| id: ios-swift-benchmark-runner-cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| DerivedData/Build/Products/Debug-iphoneos/PerformanceBenchmarks-Runner.app | |
| key: ios-swift-for-ui-testing-cache-key-${{ hashFiles('Samples/iOS-Swift/PerformanceBenchmarks/**') }} | |
| - run: fastlane build_ios_swift_for_tests | |
| env: | |
| APP_STORE_CONNECT_KEY_ID: ${{ secrets.APP_STORE_CONNECT_KEY_ID }} | |
| APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_ISSUER_ID }} | |
| APP_STORE_CONNECT_KEY: ${{ secrets.APP_STORE_CONNECT_KEY }} | |
| FASTLANE_KEYCHAIN_PASSWORD: ${{ secrets.FASTLANE_KEYCHAIN_PASSWORD }} | |
| MATCH_GIT_PRIVATE_KEY: ${{ secrets.MATCH_GIT_PRIVATE_KEY }} | |
| MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }} | |
| MATCH_USERNAME: ${{ secrets.MATCH_USERNAME }} | |
| - run: fastlane build_ios_benchmark_test | |
| env: | |
| APP_STORE_CONNECT_KEY_ID: ${{ secrets.APP_STORE_CONNECT_KEY_ID }} | |
| APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_ISSUER_ID }} | |
| APP_STORE_CONNECT_KEY: ${{ secrets.APP_STORE_CONNECT_KEY }} | |
| FASTLANE_KEYCHAIN_PASSWORD: ${{ secrets.FASTLANE_KEYCHAIN_PASSWORD }} | |
| MATCH_GIT_PRIVATE_KEY: ${{ secrets.MATCH_GIT_PRIVATE_KEY }} | |
| MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }} | |
| MATCH_USERNAME: ${{ secrets.MATCH_USERNAME }} | |
| - name: Upload dSYMs | |
| run: | | |
| sentry-cli --auth-token ${{ secrets.SENTRY_AUTH_TOKEN }} upload-dif --org sentry-sdks --project sentry-cocoa DerivedData/Build/Products/Debug-iphoneos/iOS-Swift.app.dSYM | |
| - name: Archiving DerivedData | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: DerivedData-Xcode | |
| path: | | |
| **/Debug-iphoneos/iOS-Swift.app | |
| **/Debug-iphoneos/PerformanceBenchmarks-Runner.app | |
| run-ui-tests-with-sauce: | |
| name: Run benchmarks on Sauce Labs | |
| runs-on: ubuntu-latest | |
| needs: build-benchmark-test-target | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| suite: ['High-end device', 'Mid-range device', 'Low-end device'] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: DerivedData-Xcode | |
| - run: npm install -g [email protected] | |
| - name: Run Benchmarks in SauceLab | |
| env: | |
| SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} | |
| SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }} | |
| run: saucectl run --select-suite "${{matrix.suite}}" --config .sauce/benchmarking-config.yml --tags benchmark | |
| app-metrics: | |
| name: Collect app metrics | |
| runs-on: macos-13 | |
| steps: | |
| - name: Git checkout | |
| uses: actions/checkout@v3 | |
| - run: ./scripts/ci-select-xcode.sh | |
| - uses: actions/cache@v3 | |
| id: app-plain-cache | |
| with: | |
| path: Tests/Perf/test-app-plain.ipa | |
| key: ${{ github.workflow }}-${{ github.job }}-appplain-${{ hashFiles('fastlane/Fastfile', 'Tests/Perf/test-app-plain/**') }} | |
| - name: Build test app plain | |
| if: steps.app-plain-cache.outputs['cache-hit'] != 'true' | |
| run: fastlane build_perf_test_app_plain | |
| env: | |
| APP_STORE_CONNECT_KEY_ID: ${{ secrets.APP_STORE_CONNECT_KEY_ID }} | |
| APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_ISSUER_ID }} | |
| APP_STORE_CONNECT_KEY: ${{ secrets.APP_STORE_CONNECT_KEY }} | |
| FASTLANE_KEYCHAIN_PASSWORD: ${{ secrets.FASTLANE_KEYCHAIN_PASSWORD }} | |
| MATCH_GIT_PRIVATE_KEY: ${{ secrets.MATCH_GIT_PRIVATE_KEY }} | |
| MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }} | |
| MATCH_USERNAME: ${{ secrets.MATCH_USERNAME }} | |
| - name: Build test app with sentry | |
| run: fastlane build_perf_test_app_sentry | |
| env: | |
| APP_STORE_CONNECT_KEY_ID: ${{ secrets.APP_STORE_CONNECT_KEY_ID }} | |
| APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_ISSUER_ID }} | |
| APP_STORE_CONNECT_KEY: ${{ secrets.APP_STORE_CONNECT_KEY }} | |
| FASTLANE_KEYCHAIN_PASSWORD: ${{ secrets.FASTLANE_KEYCHAIN_PASSWORD }} | |
| MATCH_GIT_PRIVATE_KEY: ${{ secrets.MATCH_GIT_PRIVATE_KEY }} | |
| MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }} | |
| MATCH_USERNAME: ${{ secrets.MATCH_USERNAME }} | |
| - name: Collect app metrics | |
| uses: getsentry/action-app-sdk-overhead-metrics@v1 | |
| with: | |
| config: Tests/Perf/metrics-test.yml | |
| sauce-user: ${{ secrets.SAUCE_USERNAME }} | |
| sauce-key: ${{ secrets.SAUCE_ACCESS_KEY }} |