kivy camera application with opencv in android shows black screen

Question

I'm trying to build an camera Android app based on OpenCv in Kivy:

main.py

import kivy
from kivy.app import App
from kivy.uix.label import Label
from kivy.uix.camera import Camera
import cv2
from kivy.uix.image import Image
from kivy.clock import Clock
from kivy.graphics.texture import Texture
import numpy as np

class KivyCamera(Image):
    def __init__(self, capture, fps, **kwargs):
        super(KivyCamera, self).__init__(**kwargs)
        self.capture = capture
        Clock.schedule_interval(self.update, 1.0 / fps)

    def update(self, dt):
        ret, frame = self.capture.read()

        if ret:
            # convert it to texture
            buf1 = cv2.flip(frame, 0)
            buf = buf1.tostring()
            image_texture = Texture.create(size=(frame.shape[1], frame.shape[0]), colorfmt='bgr')
            image_texture.blit_buffer(buf, colorfmt='bgr', bufferfmt='ubyte')
            # display image from the texture
            self.texture = image_texture

class MainApp(App):
    def build(self):
        self.capture = cv2.VideoCapture(0)
        self.camera = KivyCamera(capture=self.capture, fps=30)
        return self.camera

if __name__== "__main__":
    MainApp().run()

buildozer.spec

[app]
title = Test_app
package.name = myapp
package.domain = org.test
source.dir = .
source.include_exts = py,png,jpg,kv,atlas,xml
version = 0.1
requirements = python3,kivy,numpy,opencv
orientation = portrait

# Android specific
fullscreen = 0
android.permissions = INTERNET, ACCESS_FINE_LOCATION, WRITE_EXTERNAL_STORAGE, CAMERA
android.arch = armeabi-v7a
[buildozer]
log_level = 2
warn_on_root = 1

Code works successfully in windows. Then i have build the the code with buildozer for android, when I open the Android App it shows a black screen with a small square in the left corner of the screen. I think the cv2.VideoCapture() is not working properly.So I change cv2.VideoCapture(0) to cv2.VideoCapture(-1) and to cv2.VideoCapture(1). But both doesn't work.

Can anyone help me out with this ?

Answer 1

I have 2 solutions to get it to work on Android.

---

Solution 1:

I was inspired by kivy-for-android-opencv-demo, found on GitHub: link! Because Kivy no longer supports Python2, here is my solution for Python3.

main.py

from kivy.app import App
from kivy.uix.boxlayout import BoxLayout
from kivy.graphics.texture import Texture
from kivy.uix.camera import Camera
from kivy.lang import Builder
import numpy as np
import cv2

Builder.load_file("myapplayout.kv")

class AndroidCamera(Camera):
    camera_resolution = (640, 480)
    counter = 0

    def _camera_loaded(self, *largs):
        self.texture = Texture.create(size=np.flip(self.camera_resolution), colorfmt='rgb')
        self.texture_size = list(self.texture.size)

    def on_tex(self, *l):
        if self._camera._buffer is None:
            return None
        frame = self.frame_from_buf()
        self.frame_to_screen(frame)
        super(AndroidCamera, self).on_tex(*l)

    def frame_from_buf(self):
        w, h = self.resolution
        frame = np.frombuffer(self._camera._buffer.tostring(), 'uint8').reshape((h + h // 2, w))
        frame_bgr = cv2.cvtColor(frame, 93)
        return np.rot90(frame_bgr, 3)

    def frame_to_screen(self, frame):
        frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
        cv2.putText(frame_rgb, str(self.counter), (20, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2, cv2.LINE_AA)
        self.counter += 1
        flipped = np.flip(frame_rgb, 0)
        buf = flipped.tostring()
        self.texture.blit_buffer(buf, colorfmt='rgb', bufferfmt='ubyte')

class MyLayout(BoxLayout):
    pass

class MyApp(App):
    def build(self):
        return MyLayout()

if __name__ == '__main__':
    MyApp().run()

myapplayout.kv

<MyLayout>
    orientation: 'vertical'
    size: root.width, root.height

    AndroidCamera:
        index: 0
        resolution: self.camera_resolution
        allow_stretch: True
        play: True

in buildozer.spec:

requirements = python3,kivy==2.0.0,opencv==4.5.2,numpy
android.permissions = CAMERA

---

Solution 2:

I get the frame from the widget of the displayed camera image, 4 times a second. If you don't need every single frame, and it's not necessary to draw things like text or boxes on top of the frames, then it's an easy solution.

main.py

from kivy.app import App
from kivy.uix.camera import Camera
from kivy.uix.boxlayout import BoxLayout
from kivy.lang import Builder
from kivy.clock import Clock
import numpy as np
import cv2

Builder.load_file('myapplayout.kv')

class AndroidCamera(Camera):
    camera_resolution = (640, 480)
    cam_ratio = camera_resolution[0] / camera_resolution[1]

class MyLayout(BoxLayout):
    pass


class MyApp(App):
    counter = 0

    def build(self):
        return MyLayout()

    def on_start(self):
        Clock.schedule_once(self.get_frame, 5)

    def get_frame(self, dt):
        cam = self.root.ids.a_cam
        image_object = cam.export_as_image(scale=round((400 / int(cam.height)), 2))
        w, h = image_object._texture.size
        frame = np.frombuffer(image_object._texture.pixels, 'uint8').reshape(h, w, 4)
        gray = cv2.cvtColor(frame, cv2.COLOR_RGBA2GRAY)
        self.root.ids.frame_counter.text = f'frame: {self.counter}'
        self.counter += 1
        Clock.schedule_once(self.get_frame, 0.25)

if __name__ == "__main__":
    MyApp().run()

myapplayout.kv

<MyLayout>:
    orientation: 'vertical'
    size: root.width, root.height

    GridLayout:
        rows: 2

        RelativeLayout:
            size_hint: 1, 0.8

            AndroidCamera:
                index: 0
                id: a_cam
                resolution: self.camera_resolution
                allow_stretch: True
                play: True
                canvas.before:
                    PushMatrix
                    Rotate:
                        angle: -90
                        origin: self.center
                    Scale:
                        x: self.cam_ratio
                        y: self.cam_ratio
                        origin: self.center
                canvas.after:
                    PopMatrix

        Label:
            size_hint: 1, 0.2
            id: frame_counter
            font_size: self.height * 0.4
            text: ''

The buildozer.spec is the same as in solution 1.

---

Finally, don't forget to add permission to camera after installation. (No permission request is included in my code.)