test.yml

name: Test
on:
  push:
    branches:
      - main
      - release/**

  pull_request:
    paths:
      - "Sources/**"
      - "Tests/**"
      - "test-server/**"
      - "Samples/**"
      - ".github/workflows/test.yml"
      - "fastlane/**"
      - "scripts/tests-with-thread-sanitizer.sh"
      - "scripts/ci-select-xcode.sh"
      - "scripts/xcode-test.sh"
      - ".codecov.yml"
      - Sentry.xcodeproj

      # run the workflow any time an Xcode scheme changes for a sample app
      - "Samples/tvOS-Swift/tvOS-Swift.xcodeproj/xcshareddata/xcschemes/tvOS-Swift.xcscheme"
      - "Samples/iOS-Swift/iOS-Swift.xcodeproj/xcshareddata/xcschemes/iOS13-Swift.xcscheme"
      - "Samples/iOS-Swift/iOS-Swift.xcodeproj/xcshareddata/xcschemes/iOS-SwiftUITests.xcscheme"
      - "Samples/iOS-Swift/iOS-Swift.xcodeproj/xcshareddata/xcschemes/iOS-Swift.xcscheme"
      - "Samples/macOS-Swift/macOS-Swift.xcodeproj/xcshareddata/xcschemes/macOS-Swift.xcscheme"
      - "Samples/iOS-ObjectiveC/iOS-ObjectiveC.xcodeproj/xcshareddata/xcschemes/iOS-ObjectiveC.xcscheme"

jobs:
  build-test-server:
    name: Build test server
    runs-on: macos-13
    steps:
      - uses: actions/checkout@v3
      - name: Cache for Test Server
        id: cache_test_server
        uses: actions/cache@v3
        with:
          path: ./test-server/.build
          key: test-server-${{ hashFiles('./test-server') }}
          restore-keys: |
            test-server-${{ hashFiles('./test-server') }}
            test-server-

      - name: Build Test Server
        if: steps.cache_test_server.outputs.cache-hit != 'true'
        working-directory: test-server
        run: >-
          swift build -c release 2>&1 | tee test-server-build.log

      - name: Copy exec
        working-directory: test-server
        run: cp $(swift build --show-bin-path -c release)/Run test-server-exec

      - name: Archiving DerivedData
        uses: actions/upload-artifact@v3
        with:
          name: test-server
          path: |
            ./test-server/test-server-exec

      - name: Archiving Raw Test Logs
        uses: actions/upload-artifact@v3
        if: ${{  failure() || cancelled() }}
        with:
          name: test-server-build-log-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}}
          path: |
            raw-test-output.log

  unit-tests:
    name: Unit ${{matrix.platform}} - Xcode ${{matrix.xcode}} - OS ${{matrix.test-destination-os}}
    runs-on: ${{matrix.runs-on}}
    timeout-minutes: 20
    needs: build-test-server

    strategy:
      fail-fast: false
      matrix:
        # Can't run tests on watchOS because XCTest is not available
        include:
          # iOS 13.7
          - runs-on: macos-11
            platform: "iOS"
            xcode: "13.2.1"
            test-destination-os: "13.7"

          # iOS 14
          - runs-on: macos-11
            platform: "iOS"
            xcode: "13.2.1"
            test-destination-os: "14.5"

          # iOS 15
          - runs-on: macos-12
            platform: "iOS"
            xcode: "13.4.1"
            test-destination-os: "latest"

          # iOS 16
          - runs-on: macos-13
            platform: "iOS"
            xcode: "14.3"
            test-destination-os: "latest"

          # macOS 11
          - runs-on: macos-11
            platform: "macOS"
            xcode: "13.2.1"
            test-destination-os: "latest"

          # macOS 12
          - runs-on: macos-12
            platform: "macOS"
            xcode: "13.4.1"
            test-destination-os: "latest"

            # macOS 13
          - runs-on: macos-13
            platform: "macOS"
            xcode: "14.3"
            test-destination-os: "latest"

          # Catalyst. We only test the latest version, as
          # the risk something breaking on Catalyst and not
          # on an older iOS or macOS version is low.
          - runs-on: macos-13
            platform: "Catalyst"
            xcode: "14.3"
            test-destination-os: "latest"

          # tvOS 15
          - runs-on: macos-12
            platform: "tvOS"
            xcode: "13.4.1"
            test-destination-os: "latest"

          # tvOS 16
          - runs-on: macos-13
            platform: "tvOS"
            xcode: "14.3"
            test-destination-os: "latest"

    steps:
      - uses: actions/checkout@v3
      - uses: actions/download-artifact@v3
        with:
          name: test-server

      - name: Print hardware info
        run: system_profiler SPHardwareDataType

      - name: Allow test-server to run
        run: chmod +x ./test-server-exec
      - run: ./test-server-exec &

      - run: ./scripts/ci-select-xcode.sh ${{matrix.xcode}}

      # Workaround with a symlink pointed out in: https://github.com/actions/virtual-environments/issues/551#issuecomment-637344435
      - name: Prepare iOS 13.7 simulator
        if: ${{ matrix.platform == 'iOS' && matrix.test-destination-os == '13.7'}}
        run: |
          sudo mkdir -p /Library/Developer/CoreSimulator/Profiles/Runtimes
          sudo ln -s /Applications/Xcode_11.7.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime /Library/Developer/CoreSimulator/Profiles/Runtimes/iOS\ 13.7.simruntime
          xcrun simctl create custom-test-device "iPhone 8" "com.apple.CoreSimulator.SimRuntime.iOS-13-7"

      # Workaround with a symlink pointed out in: https://github.com/actions/virtual-environments/issues/551#issuecomment-637344435
      - name: Prepare iOS 14.5 simulator
        if: ${{ matrix.platform == 'iOS' && matrix.test-destination-os == '14.5'}}
        run: |
          sudo mkdir -p /Library/Developer/CoreSimulator/Profiles/Runtimes
          sudo ln -s /Applications/Xcode_12.5.1.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime /Library/Developer/CoreSimulator/Profiles/Runtimes/iOS\ 14.5.simruntime
          xcrun simctl create custom-test-device "iPhone 8" "com.apple.CoreSimulator.SimRuntime.iOS-14-5"

      - name: Install Slather
        run: gem install slather

      # We split building and running tests in two steps so we know how long running the tests takes.
      - name: Build tests
        id: build_tests
        run: ./scripts/xcode-test.sh ${{matrix.platform}} ${{matrix.test-destination-os}} $GITHUB_REF_NAME ci build-for-testing

      - name: Run tests
        # We call a script with the platform so the destination
        # passed to xcodebuild doesn't end up in the job name,
        # because GitHub Actions don't provide an easy way of
        # manipulating string in expressions.
        run: ./scripts/xcode-test.sh ${{matrix.platform}} ${{matrix.test-destination-os}} $GITHUB_REF_NAME ci test-without-building

      - name: Slowest Tests
        if: ${{ always() }}
        run: ./scripts/xcode-slowest-tests.sh

      - name: Archiving DerivedData Logs
        uses: actions/upload-artifact@v3
        if: steps.build_tests.outcome == 'failure'
        with:
          name: derived-data-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}}
          path: |
            /Users/runner/Library/Developer/Xcode/DerivedData/**/Logs/**

      - name: Archiving Raw Test Logs
        uses: actions/upload-artifact@v3
        if: ${{  failure() || cancelled() }}
        with:
          name: raw-test-output-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}}
          path: |
            raw-test-output.log

      - name: Archiving Crash Logs
        uses: actions/upload-artifact@v3
        if: ${{  failure() || cancelled() }}
        with:
          name: crash-logs-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}}
          path: |
            ~/Library/Logs/DiagnosticReports/**

      # We can upload all coverage reports, because codecov merges them.
      # See https://docs.codecov.io/docs/merging-reports
      # Checkout .codecov.yml to see the config of Codecov
      # We don't upload codecov for release branches, as we don't want a failing coverage check to block a release.
      # We don't upload codecov for scheduled runs as CodeCov only accepts a limited amount of uploads per commit.
      - name: Push code coverage to codecov
        id: codecov_1
        uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # pin@v3.1.4
        if: ${{ contains(matrix.platform, 'iOS') && !contains(github.ref, 'release') && github.event.schedule == '' }}
        with:
          # Although public repos should not have to specify a token there seems to be a bug with the Codecov GH action, which can
          # be solved by specifying the token, see https://github.com/codecov/codecov-action/issues/557#issuecomment-1224970469
          token: ${{ secrets.CODECOV_TOKEN }}
          fail_ci_if_error: false
          verbose: true

      # Sometimes codecov uploads etc can fail. Retry one time to rule out e.g. intermittent network failures.
      - name: Push code coverage to codecov
        id: codecov_2
        uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # pin@v3.1.4
        if: ${{ steps.codecov_1.outcome == 'failure' && contains(matrix.platform, 'iOS') && !contains(github.ref, 'release') && github.event.schedule == '' }}
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          fail_ci_if_error: true
          verbose: true

  # We don't run all unit tests with Thread Sanitizer enabled because
  # that adds a significant overhead.
  thread-sanitizer:
    # Disabled for now, see https://github.com/getsentry/sentry-cocoa/issues/3200
    if: false 
    name: Unit iOS - Thread Sanitizer
    runs-on: macos-13
    # When there are threading issues the tests sometimes keep hanging
    timeout-minutes: 20
    needs: [build-test-server]

    steps:
      - uses: actions/checkout@v3

      - uses: actions/download-artifact@v3
        with:
          name: test-server

      - name: Allow test-server to run
        run: chmod +x ./test-server-exec

      - run: ./test-server-exec &

      - run: ./scripts/ci-select-xcode.sh

      - name: Running tests with ThreadSanitizer
        run: ./scripts/tests-with-thread-sanitizer.sh

      - name: Archiving Test Logs
        uses: actions/upload-artifact@v3
        if: ${{  failure() || cancelled() }}
        with:
          name: thread-sanitizer
          path: thread-sanitizer.log

  ui-tests:
    name: UI Tests for ${{matrix.target}} on Simulators
    runs-on: macos-13
    strategy:
      matrix:
        target: ["ios_swift", "ios_objc", "tvos_swift"]

    steps:
      - uses: actions/checkout@v3
      - run: ./scripts/ci-select-xcode.sh

      # GitHub Actions sometimes fail to launch the UI tests. Therefore we retry
      - name: Run Fastlane
        run: for i in {1..2}; do fastlane ui_tests_${{matrix.target}} && break ; done
        shell: sh

      - name: Archiving Raw Test Logs
        uses: actions/upload-artifact@v3
        if: ${{  failure() || cancelled() }}
        with:
          name: raw-uitest-output-${{matrix.target}}
          path: |
            ~/Library/Logs/scan/*.log
            ./fastlane/test_output/**

  # SwiftUI only supports iOS 14+ so we run it in a separate matrix here
  ui-tests-swift-ui:
    name: UI Tests for SwiftUI on ${{matrix.device}} Simulator
    runs-on: ${{matrix.runs-on}}
    strategy:
      fail-fast: false
      matrix:
        include:
          - runs-on: macos-12
            xcode: "13.4.1"
            device: "iPhone 8 (15.2)"

          - runs-on: macos-13
            xcode: "14.3"
            device: "iPhone 8 (16.1)"

    steps:
      - uses: actions/checkout@v3
      - run: ./scripts/ci-select-xcode.sh ${{matrix.xcode}}

      # GitHub Actions sometimes fail to launch the UI tests. Therefore we retry
      - name: Run Fastlane
        run: for i in {1..2}; do fastlane ui_tests_ios_swiftui device:"${{matrix.device}}" && break ; done
        shell: sh

      - name: Archiving Raw Test Logs
        uses: actions/upload-artifact@v3
        if: ${{  failure() || cancelled() }}
        with:
          name: raw-swiftui-test-output-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}}
          path: |
            ~/Library/Logs/scan/*.log
            ./fastlane/test_output/**

  ui-tests-address-sanitizer:
    name: UI Tests with Address Sanitizer
    runs-on: macos-13

    steps:
      - uses: actions/checkout@v3
      - run: ./scripts/ci-select-xcode.sh

      # GitHub Actions sometimes fail to launch the UI tests. Therefore we retry
      - name: Run Fastlane
        run: for i in {1..2}; do fastlane ui_tests_ios_swift device:"$iPhone 8 (15.5)" address_sanitizer:true && break ; done
        shell: sh

      - name: Archiving Raw Test Logs
        uses: actions/upload-artifact@v3
        if: ${{  failure() || cancelled() }}
        with:
          name: raw-uitest-output-asan
          path: |
            ~/Library/Logs/scan/*.log
            ./fastlane/test_output/**