From 45d605f6880e61b10e754c50adb7e669b9a47965 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Nicolas <6371790+pynicolas@users.noreply.github.com> Date: Thu, 22 Jan 2026 06:45:47 +0100 Subject: [PATCH] Live analysis: make the detected quad move smoothly (#28) --- .../app/ui/screens/camera/CameraPreview.kt | 29 +++++-- .../app/ui/screens/camera/CameraUiState.kt | 1 + .../app/ui/screens/camera/CameraViewModel.kt | 23 +++--- .../app/ui/screens/camera/QuadStabilizer.kt | 75 +++++++++++++++++++ 4 files changed, 112 insertions(+), 16 deletions(-) create mode 100644 app/src/main/java/org/fairscan/app/ui/screens/camera/QuadStabilizer.kt diff --git a/app/src/main/java/org/fairscan/app/ui/screens/camera/CameraPreview.kt b/app/src/main/java/org/fairscan/app/ui/screens/camera/CameraPreview.kt index 5363a90..8171360 100644 --- a/app/src/main/java/org/fairscan/app/ui/screens/camera/CameraPreview.kt +++ b/app/src/main/java/org/fairscan/app/ui/screens/camera/CameraPreview.kt @@ -47,6 +47,7 @@ import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.runtime.setValue +import androidx.compose.runtime.withFrameNanos import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.geometry.Offset @@ -64,6 +65,7 @@ import androidx.lifecycle.LifecycleOwner import androidx.lifecycle.compose.LocalLifecycleOwner import org.fairscan.app.ui.components.CameraPermissionState import org.fairscan.imageprocessing.Point +import org.fairscan.imageprocessing.Quad import org.fairscan.imageprocessing.scaledTo import java.util.concurrent.ExecutorService import java.util.concurrent.Executors @@ -207,17 +209,32 @@ fun bindCameraUseCases( @Composable fun AnalysisOverlay(liveAnalysisState: LiveAnalysisState, debugMode: Boolean) { - val binaryMask = liveAnalysisState.binaryMask - if (binaryMask == null) { - return - } + val binaryMask = liveAnalysisState.binaryMask ?: return + val targetQuad = liveAnalysisState.stableQuad + var displayedQuad by remember { mutableStateOf(null) } val quadColor = MaterialTheme.colorScheme.primary + + LaunchedEffect(targetQuad) { + if (targetQuad == null) { + displayedQuad = null + return@LaunchedEffect + } + + while (true) { + displayedQuad = displayedQuad?.let { current -> + lerpQuad(current, targetQuad, 0.15f) + } ?: targetQuad + + withFrameNanos { } + } + } + Canvas(modifier = Modifier.fillMaxSize()) { if (debugMode) { drawMask(this, binaryMask) } - if (liveAnalysisState.documentQuad != null) { - val scaledQuad = liveAnalysisState.documentQuad.scaledTo( + displayedQuad?.let { quad -> + val scaledQuad = quad.scaledTo( fromWidth = binaryMask.width, fromHeight = binaryMask.height, toWidth = size.width.toInt(), diff --git a/app/src/main/java/org/fairscan/app/ui/screens/camera/CameraUiState.kt b/app/src/main/java/org/fairscan/app/ui/screens/camera/CameraUiState.kt index 8bf7d32..2aa09f0 100644 --- a/app/src/main/java/org/fairscan/app/ui/screens/camera/CameraUiState.kt +++ b/app/src/main/java/org/fairscan/app/ui/screens/camera/CameraUiState.kt @@ -23,6 +23,7 @@ data class LiveAnalysisState( val inferenceTime: Long = 0L, val binaryMask: Bitmap? = null, val documentQuad: Quad? = null, + val stableQuad: Quad? = null, ) data class CameraUiState( diff --git a/app/src/main/java/org/fairscan/app/ui/screens/camera/CameraViewModel.kt b/app/src/main/java/org/fairscan/app/ui/screens/camera/CameraViewModel.kt index 2fcc218..9c0829d 100644 --- a/app/src/main/java/org/fairscan/app/ui/screens/camera/CameraViewModel.kt +++ b/app/src/main/java/org/fairscan/app/ui/screens/camera/CameraViewModel.kt @@ -26,7 +26,6 @@ import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asSharedFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.filterNotNull -import kotlinx.coroutines.flow.map import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import org.fairscan.app.AppContainer @@ -59,6 +58,7 @@ class CameraViewModel(appContainer: AppContainer): ViewModel() { private var _liveAnalysisState = MutableStateFlow(LiveAnalysisState()) val liveAnalysisState: StateFlow = _liveAnalysisState.asStateFlow() + private var quadStabilizer = QuadStabilizer() private val _captureState = MutableStateFlow(CaptureState.Idle) val captureState: StateFlow = _captureState @@ -68,17 +68,20 @@ class CameraViewModel(appContainer: AppContainer): ViewModel() { imageSegmentationService.initialize() imageSegmentationService.segmentation .filterNotNull() - .map { + .collect { result -> // TODO Should we really call toBinaryMask if it's used only in debug mode? - val binaryMask = it.segmentation.toBinaryMask() - LiveAnalysisState( - inferenceTime = it.inferenceTime, - binaryMask = binaryMask, - documentQuad = detectDocumentQuad(it.segmentation, isLiveAnalysis = true), + val binaryMask = result.segmentation.toBinaryMask() + val rawQuad = detectDocumentQuad( + result.segmentation, + isLiveAnalysis = true + ) + val stableQuad = quadStabilizer.update(rawQuad) + _liveAnalysisState.value = LiveAnalysisState( + inferenceTime = result.inferenceTime, + binaryMask = binaryMask, + documentQuad = rawQuad, + stableQuad = stableQuad, ) - } - .collect { - _liveAnalysisState.value = it } } } diff --git a/app/src/main/java/org/fairscan/app/ui/screens/camera/QuadStabilizer.kt b/app/src/main/java/org/fairscan/app/ui/screens/camera/QuadStabilizer.kt new file mode 100644 index 0000000..4d92db0 --- /dev/null +++ b/app/src/main/java/org/fairscan/app/ui/screens/camera/QuadStabilizer.kt @@ -0,0 +1,75 @@ +/* + * Copyright 2025 Pierre-Yves Nicolas + * + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) + * any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ +package org.fairscan.app.ui.screens.camera + +import org.fairscan.imageprocessing.Point +import org.fairscan.imageprocessing.Quad +import org.fairscan.imageprocessing.norm + +class QuadStabilizer { + + private var stableCount = 0 + private var lastRawQuad: Quad? = null + + fun update(rawQuad: Quad?): Quad? { + lastRawQuad = rawQuad + + if (rawQuad == null) { + stableCount = 0 + return null + } + + val lastRaw = lastRawQuad + if (lastRaw == null) { + stableCount = 1 + return null + } + + val dist = lastRaw.maxCornerDistanceTo(rawQuad) + // 20f is based on the assumption that the preview has a size of 640×480 + if (dist < 20f) { + stableCount++ + } else { + stableCount = 1 + } + + return if (stableCount >= 3) rawQuad else null + } +} + +private fun Quad.maxCornerDistanceTo(other: Quad): Float { + return listOf( + norm(topLeft, other.topLeft), + norm(topRight, other.topRight), + norm(bottomRight, other.bottomRight), + norm(bottomLeft, other.bottomLeft), + ).max().toFloat() +} + +fun lerp(a: Point, b: Point, alpha: Float): Point { + return Point( + x = a.x + alpha * (b.x - a.x), + y = a.y + alpha * (b.y - a.y) + ) +} + +fun lerpQuad(a: Quad, b: Quad, alpha: Float): Quad { + return Quad( + topLeft = lerp(a.topLeft, b.topLeft, alpha), + topRight = lerp(a.topRight, b.topRight, alpha), + bottomRight = lerp(a.bottomRight, b.bottomRight, alpha), + bottomLeft = lerp(a.bottomLeft, b.bottomLeft, alpha), + ) +}