From 4988d5bf9e0a5be708853cf53d07b8503e4c77be Mon Sep 17 00:00:00 2001 From: Krystof Woldrich <31292499+krystofwoldrich@users.noreply.github.com> Date: Mon, 21 Oct 2024 16:37:40 +0200 Subject: [PATCH] test(ui): Add critical tests run by Maestro (#3802) --- .../integration-tests-ui-critical.yml | 106 ++++++++++++++++++ .github/workflows/system-tests-backend.yml | 33 +++++- Makefile | 10 +- build.gradle.kts | 1 + scripts/test-ui-critical.sh | 35 ++++++ .../sentry-uitest-android-critical/.gitignore | 2 + .../build.gradle.kts | 69 ++++++++++++ .../maestro/corruptEnvelope.yaml | 11 ++ .../maestro/crash.yaml | 6 + .../proguard-rules.pro | 21 ++++ .../src/main/AndroidManifest.xml | 21 ++++ .../uitest/android/critical/MainActivity.kt | 51 +++++++++ settings.gradle.kts | 1 + 13 files changed, 363 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/integration-tests-ui-critical.yml create mode 100755 scripts/test-ui-critical.sh create mode 100644 sentry-android-integration-tests/sentry-uitest-android-critical/.gitignore create mode 100644 sentry-android-integration-tests/sentry-uitest-android-critical/build.gradle.kts create mode 100644 sentry-android-integration-tests/sentry-uitest-android-critical/maestro/corruptEnvelope.yaml create mode 100644 sentry-android-integration-tests/sentry-uitest-android-critical/maestro/crash.yaml create mode 100644 sentry-android-integration-tests/sentry-uitest-android-critical/proguard-rules.pro create mode 100644 sentry-android-integration-tests/sentry-uitest-android-critical/src/main/AndroidManifest.xml create mode 100644 sentry-android-integration-tests/sentry-uitest-android-critical/src/main/java/io/sentry/uitest/android/critical/MainActivity.kt diff --git a/.github/workflows/integration-tests-ui-critical.yml b/.github/workflows/integration-tests-ui-critical.yml new file mode 100644 index 0000000000..6729e40ca5 --- /dev/null +++ b/.github/workflows/integration-tests-ui-critical.yml @@ -0,0 +1,106 @@ +name: UI Tests Critical + +on: + push: + branches: + - main + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + BASE_PATH: "sentry-android-integration-tests/sentry-uitest-android-critical" + BUILD_PATH: "build/outputs/apk/release" + APK_NAME: "sentry-uitest-android-critical-release.apk" + APK_ARTIFACT_NAME: "sentry-uitest-android-critical-release" + MAESTRO_VERSION: "1.39.0" + +jobs: + build: + name: Build sentry-uitest-android-critical + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '17' + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@bb0c460cbf5354b0cddd15bacdf0d6aaa3e5a32b # pin@v3 + with: + gradle-home-cache-cleanup: true + + - name: Build debug APK + run: make assembleUiTestCriticalRelease + + - name: Upload APK artifact + uses: actions/upload-artifact@v4 + with: + name: ${{env.APK_ARTIFACT_NAME}} + path: "${{env.BASE_PATH}}/${{env.BUILD_PATH}}/${{env.APK_NAME}}" + retention-days: 1 + + run-maestro-tests: + name: Run Maestro Tests + needs: build + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup KVM + shell: bash + run: | + # check if virtualization is supported... + sudo apt install -y --no-install-recommends cpu-checker coreutils && echo "CPUs=$(nproc --all)" && kvm-ok + # allow access to KVM to run the emulator + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ + | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + + - name: Download APK artifact + uses: actions/download-artifact@v4 + with: + name: ${{env.APK_ARTIFACT_NAME}} + + - name: Install Maestro + uses: dniHze/maestro-test-action@bda8a93211c86d0a05b7a4597c5ad134566fbde4 # pin@v1.0.0 + with: + version: ${{env.MAESTRO_VERSION}} + + - name: Run tests + uses: reactivecircus/android-emulator-runner@f0d1ed2dcad93c7479e8b2f2226c83af54494915 # pin@v2.32.0 + with: + api-level: 30 + force-avd-creation: false + disable-animations: true + disable-spellchecker: true + target: 'aosp_atd' + channel: canary # Necessary for ATDs + emulator-options: > + -no-window + -no-snapshot-save + -gpu swiftshader_indirect + -noaudio + -no-boot-anim + -camera-back none + -camera-front none + -timezone US/Pacific + script: | + adb install -r -d "${{env.APK_NAME}}" + maestro test "${{env.BASE_PATH}}/maestro" --debug-output "${{env.BASE_PATH}}/maestro-logs" + + - name: Upload Maestro test results + if: failure() + uses: actions/upload-artifact@v4 + with: + name: maestro-logs + path: "${{env.BASE_PATH}}/maestro-logs" + retention-days: 1 diff --git a/.github/workflows/system-tests-backend.yml b/.github/workflows/system-tests-backend.yml index 0098644d97..2222f910ad 100644 --- a/.github/workflows/system-tests-backend.yml +++ b/.github/workflows/system-tests-backend.yml @@ -46,11 +46,33 @@ jobs: - name: Exclude android modules from build run: | - sed -i -e '/.*"sentry-android-ndk",/d' -e '/.*"sentry-android",/d' -e '/.*"sentry-compose",/d' -e '/.*"sentry-android-core",/d' -e '/.*"sentry-android-fragment",/d' -e '/.*"sentry-android-navigation",/d' -e '/.*"sentry-android-okhttp",/d' -e '/.*"sentry-android-sqlite",/d' -e '/.*"sentry-android-timber",/d' -e '/.*"sentry-android-integration-tests:sentry-uitest-android-benchmark",/d' -e '/.*"sentry-android-integration-tests:sentry-uitest-android",/d' -e '/.*"sentry-android-integration-tests:test-app-sentry",/d' -e '/.*"sentry-samples:sentry-samples-android",/d' -e '/.*"sentry-android-replay",/d' settings.gradle.kts + sed -i \ + -e '/.*"sentry-android-ndk",/d' \ + -e '/.*"sentry-android",/d' \ + -e '/.*"sentry-compose",/d' \ + -e '/.*"sentry-android-core",/d' \ + -e '/.*"sentry-android-fragment",/d' \ + -e '/.*"sentry-android-navigation",/d' \ + -e '/.*"sentry-android-okhttp",/d' \ + -e '/.*"sentry-android-sqlite",/d' \ + -e '/.*"sentry-android-timber",/d' \ + -e '/.*"sentry-android-integration-tests:sentry-uitest-android-benchmark",/d' \ + -e '/.*"sentry-android-integration-tests:sentry-uitest-android",/d' \ + -e '/.*"sentry-android-integration-tests:sentry-uitest-android-critical",/d' \ + -e '/.*"sentry-android-integration-tests:test-app-sentry",/d' \ + -e '/.*"sentry-samples:sentry-samples-android",/d' \ + -e '/.*"sentry-android-replay",/d' \ + settings.gradle.kts - name: Exclude android modules from ignore list run: | - sed -i -e '/.*"sentry-uitest-android",/d' -e '/.*"sentry-uitest-android-benchmark",/d' -e '/.*"test-app-sentry",/d' -e '/.*"sentry-samples-android",/d' build.gradle.kts + sed -i \ + -e '/.*"sentry-uitest-android",/d' \ + -e '/.*"sentry-uitest-android-benchmark",/d' \ + -e '/.*"sentry-uitest-android-critical",/d' \ + -e '/.*"test-app-sentry",/d' \ + -e '/.*"sentry-samples-android",/d' \ + build.gradle.kts - name: Build server jar run: | @@ -58,7 +80,12 @@ jobs: - name: Start server and run integration test for sentry-cli commands run: | - test/system-test-sentry-server-start.sh > sentry-mock-server.txt 2>&1 & test/system-test-spring-server-start.sh "${{ matrix.sample }}" > spring-server.txt 2>&1 & test/wait-for-spring.sh && ./gradlew :sentry-samples:${{ matrix.sample }}:systemTest + test/system-test-sentry-server-start.sh \ + > sentry-mock-server.txt 2>&1 & \ + test/system-test-spring-server-start.sh "${{ matrix.sample }}" \ + > spring-server.txt 2>&1 & \ + test/wait-for-spring.sh && \ + ./gradlew :sentry-samples:${{ matrix.sample }}:systemTest - name: Upload test results if: always() diff --git a/Makefile b/Makefile index 2117e6da21..62e6e258f3 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all clean compile javadocs dryRelease update stop checkFormat format api assembleBenchmarkTestRelease assembleUiTestRelease createCoverageReports check preMerge publish +.PHONY: all clean compile javadocs dryRelease update stop checkFormat format api assembleBenchmarkTestRelease assembleUiTestRelease assembleUiTestCriticalRelease createCoverageReports runUiTestCritical check preMerge publish all: stop clean javadocs compile createCoverageReports assembleBenchmarks: assembleBenchmarkTestRelease @@ -53,6 +53,14 @@ assembleUiTestRelease: ./gradlew :sentry-android-integration-tests:sentry-uitest-android:assembleRelease ./gradlew :sentry-android-integration-tests:sentry-uitest-android:assembleAndroidTest -DtestBuildType=release +# Assemble release of the uitest-android-critical module +assembleUiTestCriticalRelease: + ./gradlew :sentry-android-integration-tests:sentry-uitest-android-critical:assembleRelease + +# Run Maestro tests for the uitest-android-critical module +runUiTestCritical: + ./scripts/test-ui-critical.sh + # Create coverage reports # - Jacoco for Java & Android modules # - Kover for KMP modules e.g sentry-compose diff --git a/build.gradle.kts b/build.gradle.kts index 7985a55486..9d53252562 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -68,6 +68,7 @@ apiValidation { "sentry-samples-spring-boot-webflux-jakarta", "sentry-uitest-android", "sentry-uitest-android-benchmark", + "sentry-uitest-android-critical", "test-app-plain", "test-app-sentry", "sentry-samples-netflix-dgs" diff --git a/scripts/test-ui-critical.sh b/scripts/test-ui-critical.sh new file mode 100755 index 0000000000..7bb36eebec --- /dev/null +++ b/scripts/test-ui-critical.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -e + +echo "Checking if ADB is installed..." +if ! command -v adb &> /dev/null; then + echo "ADB is not installed or not in PATH. Please install Android SDK platform tools and ensure ADB is in your PATH." + exit 1 +fi + +echo "Checking if an Android emulator is running..." +if ! adb devices | grep -q "emulator"; then + echo "No Android emulator is currently running. Please start an emulator before running this script." + exit 1 +fi + +echo "Checking if Maestro is installed..." +if ! command -v maestro &> /dev/null; then + echo "Maestro is not installed. Please install Maestro before running this script." + exit 1 +fi + +echo "Building the UI Test Critical app..." +make assembleUiTestCriticalRelease + +echo "Installing the UI Test Critical app on the emulator..." +baseDir="sentry-android-integration-tests/sentry-uitest-android-critical" +buildDir="build/outputs/apk/release" +apkName="sentry-uitest-android-critical-release.apk" +appPath="${baseDir}/${buildDir}/${apkName}" +adb install -r -d "$appPath" + +echo "Running the Maestro tests..." +maestro test \ + "${baseDir}/maestro" \ + --debug-output "${baseDir}/maestro-logs" diff --git a/sentry-android-integration-tests/sentry-uitest-android-critical/.gitignore b/sentry-android-integration-tests/sentry-uitest-android-critical/.gitignore new file mode 100644 index 0000000000..48fc28dcf5 --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android-critical/.gitignore @@ -0,0 +1,2 @@ +/build +/maestro-logs diff --git a/sentry-android-integration-tests/sentry-uitest-android-critical/build.gradle.kts b/sentry-android-integration-tests/sentry-uitest-android-critical/build.gradle.kts new file mode 100644 index 0000000000..cebf744a24 --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android-critical/build.gradle.kts @@ -0,0 +1,69 @@ +import io.gitlab.arturbosch.detekt.Detekt + +plugins { + id("com.android.application") + kotlin("android") +} + +android { + compileSdk = Config.Android.compileSdkVersion + namespace = "io.sentry.uitest.android.critical" + + signingConfigs { + getByName("debug") { + // Debug config remains unchanged + } + } + + defaultConfig { + applicationId = "io.sentry.uitest.android.critical" + minSdk = Config.Android.minSdkVersionCompose + targetSdk = Config.Android.targetSdkVersion + versionCode = 1 + versionName = "1.0" + } + + buildTypes { + release { + isMinifyEnabled = false + signingConfig = signingConfigs.getByName("debug") + proguardFiles( + getDefaultProguardFile("proguard-android-optimize.txt"), + "proguard-rules.pro" + ) + } + } + kotlinOptions { + jvmTarget = JavaVersion.VERSION_1_8.toString() + } + buildFeatures { + compose = true + } + composeOptions { + kotlinCompilerExtensionVersion = Config.androidComposeCompilerVersion + } + variantFilter { + if (Config.Android.shouldSkipDebugVariant(buildType.name)) { + ignore = true + } + } +} + +dependencies { + implementation(kotlin(Config.kotlinStdLib, org.jetbrains.kotlin.config.KotlinCompilerVersion.VERSION)) + implementation(Config.Libs.androidxCore) + implementation(Config.Libs.composeActivity) + implementation(Config.Libs.composeFoundation) + implementation(Config.Libs.composeMaterial) + implementation(Config.Libs.constraintLayout) + implementation(projects.sentryAndroidCore) +} + +tasks.withType { + // Target version of the generated JVM bytecode. It is used for type resolution. + jvmTarget = JavaVersion.VERSION_1_8.toString() +} + +kotlin { + explicitApi() +} diff --git a/sentry-android-integration-tests/sentry-uitest-android-critical/maestro/corruptEnvelope.yaml b/sentry-android-integration-tests/sentry-uitest-android-critical/maestro/corruptEnvelope.yaml new file mode 100644 index 0000000000..dec889731b --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android-critical/maestro/corruptEnvelope.yaml @@ -0,0 +1,11 @@ +appId: io.sentry.uitest.android.critical +--- +- launchApp +- tapOn: "Write Corrupted Envelope" +# The close here ensures the next corrupted envelope +# will be present on the next app launch +- tapOn: "Close SDK" +- tapOn: "Write Corrupted Envelope" +- stopApp +- launchApp +- assertVisible: "Welcome!" diff --git a/sentry-android-integration-tests/sentry-uitest-android-critical/maestro/crash.yaml b/sentry-android-integration-tests/sentry-uitest-android-critical/maestro/crash.yaml new file mode 100644 index 0000000000..f9543f365c --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android-critical/maestro/crash.yaml @@ -0,0 +1,6 @@ +appId: io.sentry.uitest.android.critical +--- +- launchApp +- tapOn: "Crash" +- launchApp +- assertVisible: "Welcome!" diff --git a/sentry-android-integration-tests/sentry-uitest-android-critical/proguard-rules.pro b/sentry-android-integration-tests/sentry-uitest-android-critical/proguard-rules.pro new file mode 100644 index 0000000000..f1b424510d --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android-critical/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile diff --git a/sentry-android-integration-tests/sentry-uitest-android-critical/src/main/AndroidManifest.xml b/sentry-android-integration-tests/sentry-uitest-android-critical/src/main/AndroidManifest.xml new file mode 100644 index 0000000000..0ab5e6052d --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android-critical/src/main/AndroidManifest.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + diff --git a/sentry-android-integration-tests/sentry-uitest-android-critical/src/main/java/io/sentry/uitest/android/critical/MainActivity.kt b/sentry-android-integration-tests/sentry-uitest-android-critical/src/main/java/io/sentry/uitest/android/critical/MainActivity.kt new file mode 100644 index 0000000000..8802f3dca2 --- /dev/null +++ b/sentry-android-integration-tests/sentry-uitest-android-critical/src/main/java/io/sentry/uitest/android/critical/MainActivity.kt @@ -0,0 +1,51 @@ +package io.sentry.uitest.android.critical + +import android.os.Bundle +import androidx.activity.ComponentActivity +import androidx.activity.compose.setContent +import androidx.compose.foundation.layout.Column +import androidx.compose.material3.Button +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import io.sentry.Sentry +import java.io.File + +class MainActivity : ComponentActivity() { + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + val outboxPath = Sentry.getCurrentHub().options.outboxPath + ?: throw RuntimeException("Outbox path is not set.") + + setContent { + MaterialTheme { + Surface() { + Column() { + Text(text = "Welcome!") + Button(onClick = { + throw RuntimeException("Crash the test app.") + }) { + Text("Crash") + } + Button(onClick = { + Sentry.close() + }) { + Text("Close SDK") + } + Button(onClick = { + val file = File(outboxPath, "corrupted.envelope") + val corruptedEnvelopeContent = """ + {"event_id":"1990b5bc31904b7395fd07feb72daf1c","sdk":{"name":"sentry.java.android","version":"7.21.0"}} + {"type":"test","length":50} + """.trimIndent() + file.writeText(corruptedEnvelopeContent) + println("Wrote corrupted envelope to: ${file.absolutePath}") + }) { + Text("Write Corrupted Envelope") + } + } + } + } + } + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index 760c6e6905..77b3be021d 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -61,6 +61,7 @@ include( "sentry-samples:sentry-samples-spring-boot-webflux", "sentry-samples:sentry-samples-spring-boot-webflux-jakarta", "sentry-samples:sentry-samples-netflix-dgs", + "sentry-android-integration-tests:sentry-uitest-android-critical", "sentry-android-integration-tests:sentry-uitest-android-benchmark", "sentry-android-integration-tests:sentry-uitest-android", "sentry-android-integration-tests:test-app-plain",