Reset capture progress on orientation change (#382)

* Reset capture progress on orientation change

* Lint

* Bump versions

* UI update and task confirmation (#385)

* Parameter updates, clipboard copy, larger UI

* Fail active liveness after timeout (#390)

gradle/libs.versions.toml

@@ -2,15 +2,15 @@
 accompanist-permissions = "0.34.0"
 android-gradle-plugin = "8.2.2"
 androidx-activity = "1.9.0"
-androidx-compose-bom = "2024.05.00"
+androidx-compose-bom = "2024.06.00"
 androidx-core = "1.13.1"
 androidx-core-splashscreen = "1.0.1"
-androidx-fragment = "1.7.1"
+androidx-fragment = "1.8.0"
 androidx-lifecycle = "2.8.2"
 androidx-navigation = "2.7.7"
 androidx-test-core = "1.5.0"
 androidx-test-espresso = "3.5.1"
-androidx-test-fragment = "1.7.1"
+androidx-test-fragment = "1.8.0"
 androidx-test-junit = "1.1.5"
 androidx-test-rules = "1.5.0"
 camposer = "0.4.0"

lib/src/main/java/com/smileidentity/compose/selfie/FaceShapedProgressIndicator.kt

@@ -67,7 +67,7 @@ fun FaceShapedProgressIndicator(
             drawPath(FaceShape.path, color = incompleteProgressStrokeColor, style = stroke)
 
             // To prevent a bug where the progress initially shows up as a full circle
-            if (progress == 0f) return@Canvas
+            if (progress == 0f || strokeWidth == 0.dp) return@Canvas
 
             // Note: Height grows downwards
             val faceShapeSize = faceShapeBounds.size

lib/src/main/java/com/smileidentity/compose/selfie/v2/SelfieCaptureScreenV2.kt

@@ -47,9 +47,13 @@ import androidx.compose.ui.geometry.toRect
 import androidx.compose.ui.graphics.Color
 import androidx.compose.ui.graphics.Path
 import androidx.compose.ui.graphics.drawscope.Stroke
+import androidx.compose.ui.hapticfeedback.HapticFeedbackType
 import androidx.compose.ui.platform.LocalContext
+import androidx.compose.ui.platform.LocalHapticFeedback
 import androidx.compose.ui.res.painterResource
 import androidx.compose.ui.res.stringResource
+import androidx.compose.ui.text.font.FontWeight
+import androidx.compose.ui.text.style.TextAlign
 import androidx.compose.ui.unit.dp
 import androidx.lifecycle.compose.collectAsStateWithLifecycle
 import androidx.lifecycle.viewmodel.compose.viewModel
@@ -71,6 +75,7 @@ import com.smileidentity.compose.components.cameraFrameCornerBorder
 import com.smileidentity.compose.preview.Preview
 import com.smileidentity.compose.preview.SmilePreviews
 import com.smileidentity.compose.selfie.AgentModeSwitch
+import com.smileidentity.compose.selfie.FaceShapedProgressIndicator
 import com.smileidentity.ml.SelfieQualityModel
 import com.smileidentity.results.SmartSelfieResult
 import com.smileidentity.results.SmileIDCallback
@@ -89,6 +94,7 @@ import com.ujizin.camposer.state.rememberCameraState
 import com.ujizin.camposer.state.rememberImageAnalyzer
 import kotlinx.collections.immutable.ImmutableMap
 import kotlinx.collections.immutable.persistentMapOf
+import kotlinx.coroutines.delay
 
 /**
  * Orchestrates the Selfie Capture Flow. Requests permissions, sets brightness, handles back press,
@@ -251,10 +257,12 @@ fun ColumnScope.SmartSelfieV2Screen(
             .background(MaterialTheme.colorScheme.tertiaryContainer)
             .padding(16.dp),
     ) {
+        DirectiveHaptics(selfieState)
+
         // Could be loading indicator, composable animation, animated image, or static image
         DirectiveVisual(
             selfieState = selfieState,
-            modifier = Modifier.size(64.dp),
+            modifier = Modifier.size(80.dp),
         )
         Text(
             text = when (selfieState) {
@@ -268,13 +276,14 @@ fun ColumnScope.SmartSelfieV2Screen(
                     R.string.si_smart_selfie_v2_submission_successful,
                 )
             },
-            style = MaterialTheme.typography.labelLarge,
+            style = MaterialTheme.typography.titleLarge,
+            textAlign = TextAlign.Center,
+            fontWeight = FontWeight.Bold,
             modifier = Modifier.padding(top = 16.dp),
         )
         val roundedCornerShape = RoundedCornerShape(32.dp)
-        // val mainBorderColor = MaterialTheme.colorScheme.onTertiaryContainer
-        val mainBorderColor = Color.Black
-        val accentBorderColor = MaterialTheme.colorScheme.errorContainer
+        val mainBorderColor = MaterialTheme.colorScheme.inverseSurface
+        val accentBorderColor = MaterialTheme.colorScheme.tertiary
         Box(
             contentAlignment = Alignment.Center,
             modifier = Modifier
@@ -315,6 +324,14 @@ fun ColumnScope.SmartSelfieV2Screen(
                         .fillMaxSize()
                         .background(Color.Black.copy(alpha = 0.8f)),
                 )
+            } else {
+                FaceShapedProgressIndicator(
+                    progress = 0f,
+                    faceFillPercent = 0.4f,
+                    strokeWidth = 0.dp,
+                    incompleteProgressStrokeColor = Color.Transparent,
+                    backgroundColor = MaterialTheme.colorScheme.scrim.copy(alpha = 0.4f),
+                )
             }
         }
         if (selfieState is SelfieState.Error) {
@@ -358,6 +375,11 @@ private fun ColumnScope.DirectiveVisual(selfieState: SelfieState, modifier: Modi
                 modifier = modifier,
             )
 
+            SelfieHint.EnsureDeviceUpright -> AnimatedImageFromSelfieHint(
+                hint,
+                modifier = modifier,
+            )
+
             SelfieHint.OnlyOneFace -> Face(modifier = modifier)
             SelfieHint.EnsureEntireFaceVisible -> Face(modifier = modifier)
             SelfieHint.PoorImageQuality -> AnimatedImageFromSelfieHint(
@@ -410,6 +432,25 @@ private fun AnimatedImageFromSelfieHint(selfieHint: SelfieHint, modifier: Modifi
     )
 }
 
+@Composable
+private fun DirectiveHaptics(selfieState: SelfieState) {
+    val haptic = LocalHapticFeedback.current
+    if (selfieState is SelfieState.Analyzing) {
+        if (selfieState.hint == SelfieHint.LookUp ||
+            selfieState.hint == SelfieHint.LookRight ||
+            selfieState.hint == SelfieHint.LookLeft
+        ) {
+            LaunchedEffect(selfieState.hint) {
+                // Custom vibration pattern
+                for (i in 0..2) {
+                    haptic.performHapticFeedback(HapticFeedbackType.LongPress)
+                    delay(100)
+                }
+            }
+        }
+    }
+}
+
 @SmilePreviews
 @Composable
 private fun SmartSelfieV2ScreenPreview() {

lib/src/main/java/com/smileidentity/models/v2/FailureReason.kt

@@ -0,0 +1,12 @@
+package com.smileidentity.models.v2
+
+import android.os.Parcelable
+import com.squareup.moshi.Json
+import com.squareup.moshi.JsonClass
+import kotlinx.parcelize.Parcelize
+
+@Parcelize
+@JsonClass(generateAdapter = true)
+data class FailureReason(
+    @Json(name = "mobile_active_liveness_timed_out") val activeLivenessTimedOut: Boolean? = null,
+) : Parcelable

lib/src/main/java/com/smileidentity/networking/SmileIDService.kt

@@ -28,6 +28,7 @@ import com.smileidentity.models.SubmitBvnTotpRequest
 import com.smileidentity.models.SubmitBvnTotpResponse
 import com.smileidentity.models.UploadRequest
 import com.smileidentity.models.ValidDocumentsResponse
+import com.smileidentity.models.v2.FailureReason
 import com.smileidentity.models.v2.Metadata
 import com.smileidentity.models.v2.SmartSelfieResponse
 import java.io.File
@@ -86,6 +87,7 @@ interface SmileIDService {
         @Part("user_id") userId: String? = null,
         @Part("partner_params")
         partnerParams: Map<@JvmSuppressWildcards String, @JvmSuppressWildcards String>? = null,
+        @Part("failure_reason") failureReason: FailureReason? = null,
         @Part("callback_url") callbackUrl: String? = SmileID.callbackUrl.ifBlank { null },
         @Part("sandbox_result") sandboxResult: Int? = null,
         @Part("allow_new_enroll") allowNewEnroll: Boolean? = null,
@@ -110,6 +112,7 @@ interface SmileIDService {
         @Part livenessImages: List<@JvmSuppressWildcards MultipartBody.Part>,
         @Part("partner_params")
         partnerParams: Map<@JvmSuppressWildcards String, @JvmSuppressWildcards String>? = null,
+        @Part("failure_reason") failureReason: FailureReason? = null,
         @Part("callback_url") callbackUrl: String? = SmileID.callbackUrl.ifBlank { null },
         @Part("sandbox_result") sandboxResult: Int? = null,
         @Part("metadata") metadata: Metadata? = Metadata.default(),
@@ -230,6 +233,7 @@ suspend fun SmileIDService.doSmartSelfieEnrollment(
     livenessImages: List<File>,
     userId: String? = null,
     partnerParams: Map<String, String>? = null,
+    failureReason: FailureReason? = null,
     callbackUrl: String? = SmileID.callbackUrl.ifBlank { null },
     sandboxResult: Int? = null,
     allowNewEnroll: Boolean? = null,
@@ -239,6 +243,7 @@ suspend fun SmileIDService.doSmartSelfieEnrollment(
     livenessImages = livenessImages.asFormDataParts("liveness_images", "image/jpeg"),
     userId = userId,
     partnerParams = partnerParams,
+    failureReason = failureReason,
     callbackUrl = callbackUrl,
     sandboxResult = sandboxResult,
     allowNewEnroll = allowNewEnroll,
@@ -262,6 +267,7 @@ suspend fun SmileIDService.doSmartSelfieAuthentication(
     selfieImage: File,
     livenessImages: List<File>,
     partnerParams: Map<String, String>? = null,
+    failureReason: FailureReason? = null,
     callbackUrl: String? = SmileID.callbackUrl.ifBlank { null },
     sandboxResult: Int? = null,
     metadata: Metadata? = Metadata.default(),
@@ -270,6 +276,7 @@ suspend fun SmileIDService.doSmartSelfieAuthentication(
     selfieImage = selfieImage.asFormDataPart("selfie_image", "image/jpeg"),
     livenessImages = livenessImages.asFormDataParts("liveness_images", "image/jpeg"),
     partnerParams = partnerParams,
+    failureReason = failureReason,
     callbackUrl = callbackUrl,
     sandboxResult = sandboxResult,
     metadata = metadata,

lib/src/main/java/com/smileidentity/viewmodel/ActiveLivenessTask.kt

@@ -42,15 +42,15 @@ internal class ActiveLivenessTask(
     private var currentDirectionInitiallySatisfiedAt = Long.MAX_VALUE
 
     // Parameter tuning
-    var livenessStabilityTimeMs = 300L
+    var livenessStabilityTimeMs = 150L
     var orthogonalAngleBuffer = 90f
-    var midwayLrAngleMin = 10f
+    var midwayLrAngleMin = 9f
     var midwayLrAngleMax = 90f
-    var endLrAngleMin = 30f
+    var endLrAngleMin = 27f
     var endLrAngleMax = 90f
-    var midwayUpAngleMin = 10f
+    var midwayUpAngleMin = 7f
     var midwayUpAngleMax = 90f
-    var endUpAngleMin = 20f
+    var endUpAngleMin = 17f
     var endUpAngleMax = 90f
 
     /**

lib/src/main/java/com/smileidentity/viewmodel/SmartSelfieV2ViewModel.kt

@@ -19,6 +19,7 @@ import com.smileidentity.R
 import com.smileidentity.SmileID
 import com.smileidentity.SmileIDCrashReporting
 import com.smileidentity.ml.SelfieQualityModel
+import com.smileidentity.models.v2.FailureReason
 import com.smileidentity.models.v2.SmartSelfieResponse
 import com.smileidentity.networking.doSmartSelfieAuthentication
 import com.smileidentity.networking.doSmartSelfieEnrollment
@@ -32,6 +33,7 @@ import com.smileidentity.util.createSelfieFile
 import com.smileidentity.util.getExceptionHandler
 import com.smileidentity.util.postProcessImageBitmap
 import com.smileidentity.util.rotated
+import com.smileidentity.viewmodel.SelfieHint.EnsureDeviceUpright
 import com.smileidentity.viewmodel.SelfieHint.EnsureEntireFaceVisible
 import com.smileidentity.viewmodel.SelfieHint.LookStraight
 import com.smileidentity.viewmodel.SelfieHint.MoveBack
@@ -83,6 +85,10 @@ enum class SelfieHint(@DrawableRes val animation: Int, @StringRes val text: Int)
         R.drawable.si_tf_face_search,
         R.string.si_smart_selfie_v2_directive_place_entire_head_in_frame,
     ),
+    EnsureDeviceUpright(
+        R.drawable.si_tf_face_search,
+        R.string.si_smart_selfie_v2_directive_ensure_device_upright,
+    ),
     OnlyOneFace(-1, R.string.si_smart_selfie_v2_directive_ensure_one_face),
     EnsureEntireFaceVisible(-1, R.string.si_smart_selfie_v2_directive_ensure_entire_face_visible),
     NeedLight(R.drawable.si_tf_light_flash, R.string.si_smart_selfie_v2_directive_need_more_light),
@@ -150,7 +156,7 @@ class SmartSelfieV2ViewModel(
     private val onResult: SmileIDCallback<SmartSelfieResult>,
 ) : ViewModel() {
     // PARAMETER DEBUGGING
-    private var selfieQualityHistoryLength = 7
+    private var selfieQualityHistoryLength = 5
     private var intraImageMinDelayMs = 250
     private var noFaceResetDelayMs = 500
     private var faceQualityThreshold = 0.5f
@@ -161,8 +167,8 @@ class SmartSelfieV2ViewModel(
     private var maxFaceYawThreshold = 15
     private var maxFaceRollThreshold = 30
     private var forcedFailureTimeoutMs = 60_000L
-    private var loadingIndicatorDelayMs = 200L
-    private var completedDelayMs = 2000L
+    private var loadingIndicatorDelayMs = 100L
+    private var completedDelayMs = 1500L
     private var ignoreFacesSmallerThan = 0.03f
 
     private val activeLiveness = ActiveLivenessTask()
@@ -199,7 +205,7 @@ class SmartSelfieV2ViewModel(
             selfieQuality = 0f,
         ),
     )
-    val uiState = _uiState.asStateFlow().sample(250).stateIn(
+    val uiState = _uiState.asStateFlow().sample(500).stateIn(
         viewModelScope,
         SharingStarted.WhileSubscribed(),
         _uiState.value,
@@ -209,12 +215,14 @@ class SmartSelfieV2ViewModel(
     private var lastAutoCaptureTimeMs = 0L
     private var lastValidFaceDetectTime = 0L
     private var shouldAnalyzeImages = true
+    private var selfieCameraOrientation: Int? = null
     private val modelInputSize = intArrayOf(1, 120, 120, 3)
     private val selfieQualityHistory = mutableListOf<Float>()
     private var forcedFailureTimerExpired = false
     private val shouldUseActiveLiveness: Boolean get() = useStrictMode && !forcedFailureTimerExpired
 
     init {
+        // TODO: re-enable
         // startStrictModeTimerIfNecessary()
     }
 
@@ -270,6 +278,18 @@ class SmartSelfieV2ViewModel(
             return
         }
 
+        // We want to hold the orientation constant for the duration of the capture
+        val desiredOrientation = selfieCameraOrientation ?: imageProxy.imageInfo.rotationDegrees
+        if (imageProxy.imageInfo.rotationDegrees != desiredOrientation) {
+            val message = "Camera orientation changed. Resetting progress"
+            Timber.d(message)
+            SmileIDCrashReporting.hub.addBreadcrumb(message)
+            resetCaptureProgress(EnsureDeviceUpright)
+            imageProxy.close()
+            selfieCameraOrientation = null
+            return
+        }
+
         // YUV_420_888 is the format produced by CameraX and needed for Luminance calculation
         check(imageProxy.format == YUV_420_888) { "Unsupported format: ${imageProxy.format}" }
         val luminance = calculateLuminance(imageProxy)
@@ -431,6 +451,7 @@ class SmartSelfieV2ViewModel(
                 _uiState.update {
                     it.copy(selfieState = SelfieState.Analyzing(activeLiveness.selfieHint))
                 }
+                selfieCameraOrientation = imageProxy.imageInfo.rotationDegrees
                 lastAutoCaptureTimeMs = System.currentTimeMillis()
                 // local variable is for null type safety purposes
                 val selfieFile = createSelfieFile(userId)
@@ -535,6 +556,7 @@ class SmartSelfieV2ViewModel(
                 livenessImages = livenessFiles,
                 allowNewEnroll = allowNewEnroll,
                 partnerParams = extraPartnerParams,
+                failureReason = FailureReason(activeLivenessTimedOut = forcedFailureTimerExpired),
                 metadata = null,
             )
         } else {

lib/src/main/res/values/strings.xml

@@ -52,6 +52,7 @@
     <string name="si_smart_selfie_v2_auth_product_name">SmartSelfie™ Authentication v2</string>
     <string name="si_smart_selfie_v2_enroll_product_name">SmartSelfie™ Enrollment v2</string>
     <string name="si_smart_selfie_v2_directive_place_entire_head_in_frame">Place entire head in frame</string>
+    <string name="si_smart_selfie_v2_directive_ensure_device_upright">Ensure your device is upright</string>
     <string name="si_smart_selfie_v2_directive_ensure_one_face">Ensure only 1 face is visible</string>
     <string name="si_smart_selfie_v2_directive_ensure_entire_face_visible">Ensure your entire face is visible</string>
     <string name="si_smart_selfie_v2_directive_need_more_light">Need more light</string>

sample/src/main/java/com/smileidentity/sample/viewmodel/MainScreenViewModel.kt

@@ -328,13 +328,15 @@ class MainScreenViewModel : ViewModel() {
                 resultCode = null,
                 resultText = response.message,
             )
-            _uiState.update { it.copy(snackbarMessage = message) }
+            _uiState.update {
+                it.copy(clipboardText = AnnotatedString(response.userId), snackbarMessage = message)
+            }
             viewModelScope.launch {
                 DataStoreRepository.addCompletedJob(
                     partnerId = SmileID.config.partnerId,
                     isProduction = uiState.value.isProduction,
                     job = Job(
-                        jobType = SmartSelfieAuthentication,
+                        jobType = SmartSelfieEnrollment,
                         timestamp = response.createdAt,
                         userId = response.userId,
                         jobId = response.jobId,
@@ -347,7 +349,7 @@ class MainScreenViewModel : ViewModel() {
             }
         } else if (result is SmileIDResult.Error) {
             val th = result.throwable
-            val message = "SmartSelfie Authentication error: ${th.message}"
+            val message = "SmartSelfie Enrollment error: ${th.message}"
             Timber.e(th, message)
             _uiState.update { it.copy(snackbarMessage = message) }
         }