[Mobile] NNAPI EP giving wrong outputs for certain devices.

[Yash-Vardhan-Sharma]

Describe the issue

Enabling NNAPI EP for mobilenet_v2 inference results in model output to get wrong on some devices like -

Oneplus 7, Samsung Galaxy S20+ 5G
Featuring a Snapdragon 865 5G Mobile Platform, POCO X3 Pro(Android 13), Realme 8 Pro (Android 13), OnePlus 8 (Android 13), OnePlus 9R (Android 13), POCO F1 (Android 10)

To reproduce

Place "mobilenetv2_fp32.onnx", "goldfish.jpeg" to app/src/main/assets/ directory.

Download these from here and here.

Below is the kotlin file.

package com.example.ort_sample

import android.os.Bundle
import androidx.activity.ComponentActivity
import ai.onnxruntime.*
import ai.onnxruntime.OrtSession.SessionOptions
import android.content.Context
import android.graphics.Bitmap
import android.graphics.Canvas
import android.graphics.Color
import android.os.SystemClock
import java.nio.FloatBuffer
import java.util.Collections
import android.util.Log

import android.graphics.BitmapFactory

class ImageUtils {
    companion object {
        fun loadAndResizeImage(
            context: Context,
            fileName: String,
            targetWidth: Int,
            targetHeight: Int
        ): Bitmap {
            // Load the image from assets
            val inputStream = context.assets.open(fileName)
            val originalBitmap = BitmapFactory.decodeStream(inputStream)

            // Resize the image
            val resizedBitmap =
                Bitmap.createScaledBitmap(originalBitmap, targetWidth, targetHeight, true)

            // Close the input stream
            inputStream.close()

            return resizedBitmap

        }
    }
}



const val DIM_BATCH_SIZE = 1;
const val DIM_PIXEL_SIZE = 3;
const val IMAGE_SIZE_X = 224;
const val IMAGE_SIZE_Y = 224;


//Converts BITMAP TO FLOATBUFFER
fun preProcess(bitmap: Bitmap): FloatBuffer {
    val imgData = FloatBuffer.allocate(
        DIM_BATCH_SIZE
                * DIM_PIXEL_SIZE
                * IMAGE_SIZE_X
                * IMAGE_SIZE_Y
    )
    imgData.rewind()
    val stride = IMAGE_SIZE_X * IMAGE_SIZE_Y
    val bmpData = IntArray(stride)
    bitmap.getPixels(bmpData, 0, bitmap.width, 0, 0, bitmap.width, bitmap.height)
    for (i in 0..IMAGE_SIZE_X - 1) {
        for (j in 0..IMAGE_SIZE_Y - 1) {
            val idx = IMAGE_SIZE_Y * i + j
            val pixelValue = bmpData[idx]
            imgData.put(idx, (((pixelValue shr 16 and 0xFF) / 255f - 0.485f) / 0.229f))
            imgData.put(idx + stride, (((pixelValue shr 8 and 0xFF) / 255f - 0.456f) / 0.224f))
            imgData.put(idx + stride * 2, (((pixelValue and 0xFF) / 255f - 0.406f) / 0.225f))
        }
    }

    imgData.rewind()
    return imgData
}




// Function to create a black bitmap
fun createBlackBitmap(width: Int, height: Int): Bitmap {

    val bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888)
    val canvas = Canvas(bitmap)
    canvas.drawColor(Color.BLACK)

    return bitmap
}



fun runInference(session: OrtSession, imgData: FloatBuffer ): String{

    val inputName = session.inputNames?.iterator()?.next()
    val shape = longArrayOf(1, 3, 224, 224)
    val env = OrtEnvironment.getEnvironment()

    env.use {
        val tensor = OnnxTensor.createTensor(env, imgData, shape)
        val startTime = SystemClock.uptimeMillis()
        tensor.use {
            val output = session.run(Collections.singletonMap(inputName, tensor))
            output.use {
                @Suppress("UNCHECKED_CAST")
                val rawOutput = ((output?.get(0)?.value) as Array<FloatArray>)[0]
                val arrayString = rawOutput.joinToString(separator = ", ", prefix = "[", postfix = "]") { it.toString() }

                return arrayString
            }
        }
    }
}



class MainActivity : ComponentActivity() {
    override fun onCreate(savedInstanceState: Bundle?) {
        super.onCreate(savedInstanceState)
        val context: Context = this

        var sessionOptionsNNAPI: SessionOptions = SessionOptions()
        sessionOptionsNNAPI.addNnapi()
        var sessionOptionsNormal: SessionOptions = SessionOptions()
        val environment = OrtEnvironment.getEnvironment()

        var sessionWithNNAPI = environment.createSession(
            context.assets.open("mobilenetv2_fp32.onnx").readBytes(),
            sessionOptionsNNAPI
        )
        var sessionWithoutNNAPI = environment.createSession(
            context.assets.open("mobilenetv2_fp32.onnx").readBytes(),
            sessionOptionsNormal
        )

        val blackImgBitmap = createBlackBitmap(224, 224)

        val blackImgData = preProcess(blackImgBitmap)

        val nnapiOutputString1 = runInference(sessionWithNNAPI, blackImgData)
        val cpuOutputString1 = runInference(sessionWithoutNNAPI, blackImgData)


        Log.d("OUT IMG1 NNAPI", nnapiOutputString1)
        Log.d("OUT IMG1 CPU", cpuOutputString1)

        val goldfishImgBitmap = ImageUtils.loadAndResizeImage(this, "goldfish.jpeg", 224, 224)

        val goldfishImgData = preProcess(goldfishImgBitmap)

        val nnapiOutputString2 = runInference(sessionWithNNAPI, goldfishImgData)
        val cpuOutputString2 = runInference(sessionWithoutNNAPI, goldfishImgData)


        Log.d("OUT IMG2 NNAPI", nnapiOutputString2)
        Log.d("OUT IMG2 CPU", cpuOutputString2)



        }
    }

Above is the screenshot for the outputs on OnePlus 7 device described below.
OUT IMG1/2 CPU are the correct outputs expected .
OUT IMG1/2 NNAPI are the incorrect outputs provided by NNAPI on the device

Urgency

High

Platform

Android

OS Version

OxygenOS, Android 12

ONNX Runtime Installation

Released Package

Compiler Version (if 'Built from Source')

No response

Package Name (if 'Released Package')

onnxruntime-android

ONNX Runtime Version or Commit ID

1.16.3

ONNX Runtime API

Java/Kotlin

Architecture

ARM64

Execution Provider

Default CPU, NNAPI

Execution Provider Library Version

No response

[skottmckay]

Closing as duplicate. Further discussion in #19518