Android CameraX Analyzer Image with format YUV_420_888 to OpenCV Mat

Question

With Android CameraX Analyzer ImageProxy uses ImageReader under the hood with a default YUV_420_888 image format.

I'd like to convert it in OpenCV Mat in order to use OpenCV inside my analyzer:

override fun analyze(imageProxy: ImageProxy, rotationDegrees: Int) {
    try {
      imageProxy.image?.let {
        // ImageProxy uses an ImageReader under the hood:
        // https://developer.android.com/reference/androidx/camera/core/ImageProxy.html
        // That has a default format of YUV_420_888 if not changed that's the default
        // Android camera format.
        // https://developer.android.com/reference/android/graphics/ImageFormat.html#YUV_420_888
        // https://developer.android.com/reference/android/media/ImageReader.html

        // Sanity check
        if (it.format == ImageFormat.YUV_420_888
            && it.planes.size == 3
        ) {
           // TODO - convert ImageProxy.image to Mat
        } else {
          // Manage other image formats
          // TODO - https://developer.android.com/reference/android/media/Image.html
        }
      }
    } catch (ise: IllegalStateException) {
      ise.printStackTrace()
    }
  }

How can I do that?

Answer 1

Looking at OpenCV JavaCamera2Frame class in its GitHub repo you can write an Image extension function like that:

(ported to Kotlin)

// Ported from opencv private class JavaCamera2Frame
fun Image.yuvToRgba(): Mat {
  val rgbaMat = Mat()

  if (format == ImageFormat.YUV_420_888
      && planes.size == 3) {

    val chromaPixelStride = planes[1].pixelStride

    if (chromaPixelStride == 2) { // Chroma channels are interleaved
      assert(planes[0].pixelStride == 1)
      assert(planes[2].pixelStride == 2)
      val yPlane = planes[0].buffer
      val uvPlane1 = planes[1].buffer
      val uvPlane2 = planes[2].buffer
      val yMat = Mat(height, width, CvType.CV_8UC1, yPlane)
      val uvMat1 = Mat(height / 2, width / 2, CvType.CV_8UC2, uvPlane1)
      val uvMat2 = Mat(height / 2, width / 2, CvType.CV_8UC2, uvPlane2)
      val addrDiff = uvMat2.dataAddr() - uvMat1.dataAddr()
      if (addrDiff > 0) {
        assert(addrDiff == 1L)
        Imgproc.cvtColorTwoPlane(yMat, uvMat1, rgbaMat, Imgproc.COLOR_YUV2RGBA_NV12)
      } else {
        assert(addrDiff == -1L)
        Imgproc.cvtColorTwoPlane(yMat, uvMat2, rgbaMat, Imgproc.COLOR_YUV2RGBA_NV21)
      }
    } else { // Chroma channels are not interleaved
      val yuvBytes = ByteArray(width * (height + height / 2))
      val yPlane = planes[0].buffer
      val uPlane = planes[1].buffer
      val vPlane = planes[2].buffer

      yPlane.get(yuvBytes, 0, width * height)

      val chromaRowStride = planes[1].rowStride
      val chromaRowPadding = chromaRowStride - width / 2

      var offset = width * height
      if (chromaRowPadding == 0) {
        // When the row stride of the chroma channels equals their width, we can copy
        // the entire channels in one go
        uPlane.get(yuvBytes, offset, width * height / 4)
        offset += width * height / 4
        vPlane.get(yuvBytes, offset, width * height / 4)
      } else {
        // When not equal, we need to copy the channels row by row
        for (i in 0 until height / 2) {
          uPlane.get(yuvBytes, offset, width / 2)
          offset += width / 2
          if (i < height / 2 - 1) {
            uPlane.position(uPlane.position() + chromaRowPadding)
          }
        }
        for (i in 0 until height / 2) {
          vPlane.get(yuvBytes, offset, width / 2)
          offset += width / 2
          if (i < height / 2 - 1) {
            vPlane.position(vPlane.position() + chromaRowPadding)
          }
        }
      }

      val yuvMat = Mat(height + height / 2, width, CvType.CV_8UC1)
      yuvMat.put(0, 0, yuvBytes)
      Imgproc.cvtColor(yuvMat, rgbaMat, Imgproc.COLOR_YUV2RGBA_I420, 4)
    }
  }

  return rgbaMat
}

And so you can write:

override fun analyze(imageProxy: ImageProxy, rotationDegrees: Int) {
    try {
      imageProxy.image?.let {
        // ImageProxy uses an ImageReader under the hood:
        // https://developer.android.com/reference/androidx/camera/core/ImageProxy.html
        // That has a default format of YUV_420_888 if not changed that's the default
        // Android camera format.
        // https://developer.android.com/reference/android/graphics/ImageFormat.html#YUV_420_888
        // https://developer.android.com/reference/android/media/ImageReader.html

        // Sanity check
        if (it.format == ImageFormat.YUV_420_888
            && it.planes.size == 3
        ) {        
          val rgbaMat = it.yuvToRgba()
        } else {
          // Manage other image formats
          // TODO - https://developer.android.com/reference/android/media/Image.html
        }
      }
    } catch (ise: IllegalStateException) {
      ise.printStackTrace()
    }
  }

Answer 2

 private Mat convertYUVtoMat(@NonNull Image img) {
    byte[] nv21;

    ByteBuffer yBuffer = img.getPlanes()[0].getBuffer();
    ByteBuffer uBuffer = img.getPlanes()[1].getBuffer();
    ByteBuffer vBuffer = img.getPlanes()[2].getBuffer();

    int ySize = yBuffer.remaining();
    int uSize = uBuffer.remaining();
    int vSize = vBuffer.remaining();

    nv21 = new byte[ySize + uSize + vSize];

    yBuffer.get(nv21, 0, ySize);
    vBuffer.get(nv21, ySize, vSize);
    uBuffer.get(nv21, ySize + vSize, uSize);

    Mat yuv = new Mat(img.getHeight() + img.getHeight()/2, img.getWidth(), CvType.CV_8UC1);
    yuv.put(0, 0, nv21);
    Mat rgb = new Mat();
    Imgproc.cvtColor(yuv, rgb, Imgproc.COLOR_YUV2RGB_NV21, 3);
    Core.rotate(rgb, rgb, Core.ROTATE_90_CLOCKWISE);
    return  rgb;
}

This method converts the Camerax API YUV_420_888 image to OpenCV's Mat (RGB) object. (Working 2021)