Jetpack Compose Digital Ink Recognition Accuracy Much Worse Than XML Layout: Why?

Question

I attempted to integrate Digital Ink Recognition into an Android app using Jetpack Compose, but I noticed that the accuracy of recognition is significantly lower compared to when using XML layouts. Despite following the same implementation steps, the recognition results are consistently worse in Compose. Can someone help me understand why this might be happening and how I can improve the accuracy of Digital Ink Recognition in Jetpack Compose?

I followed the usual steps to integrate Digital Ink Recognition into my Android app using Jetpack Compose, expecting similar accuracy to when using XML layouts. However, the recognition results in Compose are noticeably worse, with more errors and inaccuracies. I tried adjusting various parameters and configurations, but the accuracy did not improve significantly.

Now Will Share my Compose Code

Main Activity ->

val TAG = "Image Recognition"


public class MainActivity : ComponentActivity() {

    companion object {
        var inkBuilder = StrokeManager.inkBuilder
    }

    override fun onCreate(savedInstanceState: Bundle?) {
        super.onCreate(savedInstanceState)

        lifecycleScope.launch {
            StrokeManager.downloadModel()
        }

        setContent {
            DemoTheme {
                App()
            }
        }
    }
}

@OptIn(ExperimentalComposeUiApi::class)
@Composable
fun App() {

    val context = LocalContext.current

    var bitmap: ImageBitmap? by remember {
        mutableStateOf(null)
    }

    var textState by remember {
        mutableStateOf("")
    }

    var imageUri: Uri? by remember {
        mutableStateOf(null)
    }

    val lines = remember {
        mutableStateListOf<Line>()
    }

    val imageLauncher = rememberLauncherForActivityResult(
        ActivityResultContracts.GetContent()
    ) { result ->
        if (result != null) {
            imageUri = result
        }
    }


    Column(
        verticalArrangement = Arrangement.Center,
        modifier = Modifier.fillMaxWidth()
    ) {
        Canvas(
            modifier = Modifier
                .fillMaxWidth()
                .height(300.dp)
                .pointerInput(true) {
                    detectDragGestures { change, dragAmount ->
                        change.consume()

                        val line = Line(
                            start = change.position - dragAmount,
                            end = change.position
                        )

                        lines.add(line)

                    }
                }.pointerInteropFilter {
                    addNewTouchEvent(it)
                    true
                }
        ) {
            lines.forEach { line ->
                drawLine(
                    color = line.color,
                    start = line.start,
                    end = line.end,
                    strokeWidth = line.strokeWidth.toPx(),
                    cap = StrokeCap.Round
                )
            }

        }

        Button(
            modifier = Modifier
                .height(50.dp)
                .fillMaxWidth(),
            onClick = {

                    StrokeManager.recognize(){
                        textState = it
                    }

            }
        ) {
            Text("Recognise Character")
        }

        Spacer(modifier = Modifier.height(10.dp))

        Button(
            modifier = Modifier
                .height(50.dp)
                .fillMaxWidth(),
            onClick = {
                lines.clear()
                inkBuilder = Ink.builder()
                textState = ""
            }
        ) {
            Text("Clear")
        }



        if (textState.isNotEmpty()) {
            Text(textState)
        }

    }

}

data class Line(
    val start: Offset,
    val end: Offset,
    val color: Color = Color.Black,
    val strokeWidth: Dp = 8.dp
)

StrokeManager.kt ->

object StrokeManager {

    private val TAG = "Image Recognition"

    var inkBuilder: Ink.Builder = Ink.builder()
    private lateinit var strokeBuilder: Ink.Stroke.Builder
    private lateinit var digitalInkRecognitionModel: DigitalInkRecognitionModel

    fun addNewTouchEvent(event: MotionEvent) {

        val action = event.actionMasked
        val x = event.x
        val y = event.y

        when (action) {

            MotionEvent.ACTION_DOWN -> {
                strokeBuilder = Ink.Stroke.builder()
                strokeBuilder.addPoint(Ink.Point.create(x, y))
            }
            MotionEvent.ACTION_MOVE -> strokeBuilder.addPoint(Ink.Point.create(x, y))
            MotionEvent.ACTION_UP -> {
                strokeBuilder.addPoint(Ink.Point.create(x, y))
                inkBuilder.addStroke(strokeBuilder.build())
            }
            else -> {
                // its seems only this part of code is working
                // not the above cases as told by doc. maybe due its Compose code Using XML
                if(!this::strokeBuilder.isInitialized){
                    strokeBuilder = Ink.Stroke.builder()
                }

                strokeBuilder.addPoint(Ink.Point.create(x, y))
                inkBuilder.addStroke(strokeBuilder.build())

                // Action not relevant for ink construction
                Log.d(TAG, "Action not relevant for ink construction")
            }
        }

    }


    // Define a suspend function to download the model
    suspend fun downloadModel() {

        val modelIdentifier = DigitalInkRecognitionModelIdentifier.fromLanguageTag("en-IN")

        digitalInkRecognitionModel = DigitalInkRecognitionModel.builder(modelIdentifier!!).build()

        val remoteModelManager = RemoteModelManager.getInstance()

        remoteModelManager.isModelDownloaded(digitalInkRecognitionModel)
            .addOnSuccessListener { isDownloaded ->
                if (isDownloaded) {
                    Log.i(TAG, "Model is already downloaded")
                } else {
                    Log.i(TAG, "Model is not downloaded")
                    remoteModelManager.download(digitalInkRecognitionModel,DownloadConditions.Builder().build())
                        .addOnSuccessListener {
                            Log.i( TAG,"Model Downloaded!")
                        }
                        .addOnFailureListener { e ->
                            Log.e(TAG,"Error while downloading a model : $e"
                            )
                        }

                }
            }

    }

    fun recognize(
        recognised : (String) -> Unit
    ) {

            if(!this::digitalInkRecognitionModel.isInitialized){
                Log.e(TAG, "Model not initialized")
                return
            }

            // Get a recognizer for the language
            val recognizer: DigitalInkRecognizer =
                DigitalInkRecognition.getClient(
                    DigitalInkRecognizerOptions.builder(digitalInkRecognitionModel).build()
                )

            val ink = inkBuilder.build()

            recognizer
                .recognize(ink)
                .addOnSuccessListener { result ->
                    var textState = ""
                    result.candidates.forEach{
                        textState += "-> \n $it.text"
                    }
                    recognised(textState)
                    Log.d(TAG, "Recognised Text: ${textState}")
                }
                .addOnFailureListener { e ->
                    Log.e(TAG, "Exception $e")
                    recognised("Error")
                }
    }
}

Now The XML code I used was from GitHub and I have thoroughly gone through it

link to his repo -> https://github.com/icanerdogan/DigitalInkRecognition-MLKit/tree/main

XML code MainActivity ->

class MainActivity : AppCompatActivity() {

    private val binding by lazy { ActivityMainBinding.inflate(layoutInflater) }

    override fun onCreate(savedInstanceState: Bundle?) {
        super.onCreate(savedInstanceState)
        setContentView(binding.root)



        binding.apply {
            textView.movementMethod = ScrollingMovementMethod()
            StrokeManager.downloadInkRecognition()

            buttonRecognize.setOnClickListener {
                textView.visibility = View.VISIBLE
                buttonRecognize.visibility = View.INVISIBLE
                StrokeManager.drawRecognizer(textView)
            }

            buttonClear.setOnClickListener {
                drawView.clear()
                StrokeManager.clear()
                textView.visibility = View.INVISIBLE
                buttonRecognize.visibility = View.VISIBLE
            }
        }
    }
}

DrawView.kt ->

class DrawView(context: Context, attributeSet: AttributeSet?) : View(context, attributeSet) {

    private var currentStrokePaint : Paint = Paint()
    private val canvasPaint : Paint = Paint(Paint.DITHER_FLAG)
    private val currentStroke : Path = Path()

    private var drawCanvas : Canvas? = null
    private lateinit var canvasBitmap : Bitmap

    init {
        currentStrokePaint.color = Color.BLACK
        currentStrokePaint.isAntiAlias = true
        currentStrokePaint.strokeWidth = STROKE_WIDTH_DP
        currentStrokePaint.style = Paint.Style.STROKE
        currentStrokePaint.strokeJoin = Paint.Join.ROUND
        currentStrokePaint.strokeCap = Paint.Cap.ROUND
    }

    override fun onTouchEvent(event: MotionEvent): Boolean {
        val action = event.actionMasked
        val x = event.x
        val y = event.y

        when(action) {
            MotionEvent.ACTION_DOWN -> currentStroke.moveTo(x, y)
            MotionEvent.ACTION_MOVE -> currentStroke.lineTo(x, y)
            MotionEvent.ACTION_UP -> {
                currentStroke.lineTo(x, y)
                drawCanvas?.drawPath(currentStroke, currentStrokePaint)
                currentStroke.reset()
            }
            else -> {}
        }
        StrokeManager.addNewTouchEvent(event)
        invalidate()
        return true
    }

    override fun onDraw(canvas: Canvas) {
        canvas.drawBitmap(canvasBitmap, 0f, 0f, canvasPaint)
        canvas.drawPath(currentStroke, currentStrokePaint)
    }

    fun clear() {
        onSizeChanged(canvasBitmap.width, canvasBitmap.height, canvasBitmap.width, canvasBitmap.height)
    }

    override fun onSizeChanged(w: Int, h: Int, oldw: Int, oldh: Int) {
        canvasBitmap = Bitmap.createBitmap(w, h, Bitmap.Config.ARGB_8888)
        drawCanvas = Canvas(canvasBitmap)
        invalidate()
    }
    companion object {
        private const val STROKE_WIDTH_DP = 6.0f
    }
}

XML code

<?xml version="1.0" encoding="utf-8"?>
<FrameLayout xmlns:android="http://schemas.android.com/apk/res/android"
    xmlns:app="http://schemas.android.com/apk/res-auto"
    xmlns:tools="http://schemas.android.com/tools"
    android:layout_width="match_parent"
    android:layout_height="match_parent"
    android:orientation="vertical"
    tools:context=".MainActivity">

    <androidx.appcompat.widget.AppCompatButton
        android:id="@+id/buttonClear"
        android:layout_width="40dp"
        android:layout_height="40dp"
        android:layout_gravity="top|end"
        android:background="@drawable/icon_delete"/>

    <FrameLayout
        android:layout_width="match_parent"
        android:layout_height="match_parent">

        <com.ibrahimcanerdogan.digitalinkrecognition.view.DrawView
            android:id="@+id/drawView"
            android:layout_width="match_parent"
            android:layout_height="match_parent"
            android:background="@color/white"/>

    </FrameLayout>

    <FrameLayout
        android:layout_width="match_parent"
        android:layout_height="match_parent">

        <androidx.appcompat.widget.AppCompatTextView
            android:id="@+id/textView"
            android:layout_width="match_parent"
            android:layout_height="200dp"
            android:background="@drawable/background_textview"
            android:layout_gravity="bottom"
            android:paddingStart="20dp"
            android:paddingEnd="20dp"
            android:paddingTop="10dp"
            android:textSize="17sp"
            android:textColor="@color/black"
            android:scrollbars="vertical"
            android:visibility="invisible"/>

        <com.google.android.material.floatingactionbutton.ExtendedFloatingActionButton
            android:id="@+id/buttonRecognize"
            android:layout_width="wrap_content"
            android:layout_height="wrap_content"
            android:layout_gravity="center|bottom"
            android:layout_margin="15dp"
            app:icon="@drawable/icon_start"
            android:text="@string/recognize"
            android:textAllCaps="true"/>
    </FrameLayout>
</FrameLayout>

Answer 1

I have tried your code and found the issue is with pointerInteropFilter which does not capture all the gestures correctly. I have replaced it with using detectTapGestures for capturing gestures and detectTapGestures for capturing when a user writes a dot.

Below is the improved code.

@Composable
fun HandwritingCanvas(
    modifier: Modifier = Modifier,
) {
    val lineColor = colorScheme.onSurface
    val lines = remember { mutableStateListOf<Line>() }
    val recognizedText = remember { mutableStateOf("") }

    Column {
        Canvas(
            modifier = modifier
                .fillMaxWidth()
                .height(300.dp)
                .pointerInput(true) {
                    detectDragGestures(
                        onDragStart = { offset ->
                            val event = MotionEvent.obtain(
                                System.currentTimeMillis(),
                                System.currentTimeMillis(),
                                MotionEvent.ACTION_DOWN,
                                offset.x,
                                offset.y,
                                0
                            )
                            addNewTouchEvent(event)
                        },
                        onDragEnd = {
                            val event = MotionEvent.obtain(
                                System.currentTimeMillis(),
                                System.currentTimeMillis(),
                                MotionEvent.ACTION_UP,
                                0f,
                                0f,
                                0
                            )
                            addNewTouchEvent(event)
                        },
                        onDrag = { change, dragAmount ->
                            change.consume()
                            val line = Line(
                                start = change.position - dragAmount,
                                end = change.position,
                            )
                            lines.add(line)
                            val event = MotionEvent.obtain(
                                System.currentTimeMillis(),
                                System.currentTimeMillis(),
                                MotionEvent.ACTION_MOVE,
                                change.position.x,
                                change.position.y,
                                0
                            )
                            addNewTouchEvent(event)
                        }
                    )
                }
                .pointerInput(true) {
                    detectTapGestures(
                        onTap = { offset: Offset ->
                            val line = Line(
                                start = offset,
                                end = offset,
                            )
                            lines.add(line)
                            val event = MotionEvent.obtain(
                                System.currentTimeMillis(),
                                System.currentTimeMillis(),
                                MotionEvent.ACTION_DOWN,
                                offset.x,
                                offset.y,
                                0
                            )
                            addNewTouchEvent(event)
                            val event2 = MotionEvent.obtain(
                                System.currentTimeMillis(),
                                System.currentTimeMillis(),
                                MotionEvent.ACTION_UP,
                                offset.x,
                                offset.y,
                                0
                            )
                            addNewTouchEvent(event2)
                        }
                    )
                }
        ) {
            lines.forEach { line ->
                drawLine(
                    color = lineColor,
                    start = line.start,
                    end = line.end,
                    strokeWidth = line.strokeWidth.toPx(),
                    cap = StrokeCap.Round
                )
            }
        }
        Text("Detected text:\n${recognizedText.value}")
        Button(onClick = { StrokeManager.recognize {
            text -> recognizedText.value = text
        } }) {
            Text("Recognize")
        }
        Button(onClick = {
            lines.clear()
            StrokeManager.inkBuilder = Ink.builder()
        }) {
            Text("Clear")
        }
    }
}
data class Line(
    val start: Offset,
    val end: Offset,
    val strokeWidth: Dp = 4.dp,
)

object StrokeManager {

    private val TAG = "Image Recognition"

    var inkBuilder: Ink.Builder = Ink.builder()
    private lateinit var strokeBuilder: Ink.Stroke.Builder
    private lateinit var digitalInkRecognitionModel: DigitalInkRecognitionModel

    init {
        this.downloadModel()
    }

    fun addNewTouchEvent(event: MotionEvent) {

        val action = event.actionMasked
        val x = event.x
        val y = event.y

        when (action) {

            MotionEvent.ACTION_DOWN -> {
                strokeBuilder = Ink.Stroke.builder()
                strokeBuilder.addPoint(Ink.Point.create(x, y))
            }
            MotionEvent.ACTION_MOVE ->
                strokeBuilder.addPoint(Ink.Point.create(x, y))
            MotionEvent.ACTION_UP ->
                inkBuilder.addStroke(strokeBuilder.build())
        }
    }

    fun downloadModel() {
        val modelIdentifier = DigitalInkRecognitionModelIdentifier.fromLanguageTag("en")

        digitalInkRecognitionModel = DigitalInkRecognitionModel.builder(modelIdentifier!!).build()

        val remoteModelManager = RemoteModelManager.getInstance()

        remoteModelManager.isModelDownloaded(digitalInkRecognitionModel)
            .addOnSuccessListener { isDownloaded ->
                if (isDownloaded) {
                    Log.i(TAG, "Model is already downloaded")
                } else {
                    Log.i(TAG, "Model is not downloaded")
                    remoteModelManager.download(digitalInkRecognitionModel,
                        DownloadConditions.Builder().build())
                        .addOnSuccessListener {
                            Log.i( TAG,"Model Downloaded!")
                        }
                        .addOnFailureListener { e ->
                            Log.e(TAG,"Error while downloading a model : $e"
                            )
                        }

                }
            }

    }

    fun recognize(
        recognised : (String) -> Unit
    ) {

        if(!this::digitalInkRecognitionModel.isInitialized){
            Log.e(TAG, "Model not initialized")
            return
        }

        // Get a recognizer for the language
        val recognizer: DigitalInkRecognizer =
            DigitalInkRecognition.getClient(
                DigitalInkRecognizerOptions.builder(digitalInkRecognitionModel).build()
            )

        val ink = inkBuilder.build()

        recognizer
            .recognize(ink)
            .addOnSuccessListener { result ->
                var textState = result.candidates[0].text
                for (i in 1 until result.candidates.size) {
                    textState += "\n " + result.candidates[i].text
                }
                recognised(textState)
                Log.d(TAG, "Recognised Text: ${textState}")
            }
            .addOnFailureListener { e ->
                Log.e(TAG, "Exception $e")
                recognised("Error")
            }
    }
}