Using video stream as open GL ES 2.0 texture

Question

I'm trying to capture video and display it to the screen by setting an Open GL ES texture to an android surfaceTexture. I can't use a TextureView and implement SurfaceTextureListener as per this tutorial since I am using Google Cardboard.

I have followed the Android documentation on how to initialise Open GL ES 2.0 and use it, and also this tutorial on texturing.

Putting the 2 together I get a blank screen and occasionally get <core_glBindTexture:572>: GL_INVALID_OPERATION in the console window.

Overwhelmed by so many new concepts that I don't know, I'm not able to debug or just understand if the two approach can be used like this. Here is my drawing code, it is initialised in the onSurfaceCreated() of the MainActivity class, and drawn from onEyeDraw() which is Cardboard's draw function.

package com.example.rich.test3;

import android.hardware.Camera;
import android.opengl.GLES20;
import android.view.TextureView;

import java.nio.ShortBuffer;
import java.nio.FloatBuffer;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;

/**
 * Created by rich on 03/05/2015.
 */
public class Square {

private java.nio.FloatBuffer vertexBuffer;
private java.nio.ShortBuffer drawListBuffer;
private final java.nio.FloatBuffer mCubeTextureCoordinates;

float color[] = { 1.f, 1.f, 1.f, 1.0f };

private final String vertexShaderCode =
        "attribute vec4 vPosition;" +
        "attribute vec2 a_TexCoordinate;" +
        "varying vec2 v_TexCoordinate;" +
                "void main() {" +
                " gl_Position = vPosition;" +
                " v_TexCoordinate = a_TexCoordinate;" +
                "}";

private final String fragmentShaderCode =
        "precision mediump float;" +
                "uniform vec4 vColor;" +
                "uniform sampler2D u_Texture;" +
                "varying vec2 v_TexCoordinate;" +
                "void main() {" +
                "gl_FragColor = (texture2D(u_Texture, v_TexCoordinate));" +
                "}";

// number of coordinates per vertex in this array
static final int COORDS_PER_VERTEX = 3;
static float squareCoords[] = {
        -0.5f, -0.5f, 0.0f,   // bottom left
        0.5f, -0.5f, 0.0f,   // bottom right
        -0.5f,  0.5f, 0.0f,   // top left
        0.5f,  0.5f, 0.0f}; // top right

private short drawOrder[] = { 0, 1, 2, 0, 2, 3 }; // order to draw vertices

private int mProgram;

private int mPositionHandle;
private int mColorHandle;
private int mTextureUniformHandle;
private int mTextureCoordinateHandle;
private final int mTextureCoordinateDataSize = 2;

private final int vertexCount = squareCoords.length / COORDS_PER_VERTEX;
private final int vertexStride = COORDS_PER_VERTEX * 4; // 4 bytes per vertex

private int mTextureDataHandle;

float textureCoordinates[] =
        {0.0f, 1.0f,
        1.0f, 1.0f,
        0.0f, 0.0f,
        1.0f, 0.0f };

Camera _camera;
TextureView _textureView;
int[] textures;
android.graphics.SurfaceTexture _surface;

public Square()
{
    ByteBuffer bb = ByteBuffer.allocateDirect(
            // (# of coordinate values * 4 bytes per float)
            squareCoords.length * 4);
    bb.order(ByteOrder.nativeOrder());
    vertexBuffer = bb.asFloatBuffer();
    vertexBuffer.put(squareCoords);
    vertexBuffer.position(0);

    // initialize byte buffer for the draw list
    ByteBuffer dlb = ByteBuffer.allocateDirect(
            // (# of coordinate values * 2 bytes per short)
            drawOrder.length * 2);
    dlb.order(ByteOrder.nativeOrder());
    drawListBuffer = dlb.asShortBuffer();
    drawListBuffer.put(drawOrder);
    drawListBuffer.position(0);

    mCubeTextureCoordinates = ByteBuffer.allocateDirect(textureCoordinates.length * 4)
            .order(ByteOrder.nativeOrder()).asFloatBuffer();
    mCubeTextureCoordinates.put(textureCoordinates).position(0);

    // create empty OpenGL ES Program
    mProgram = GLES20.glCreateProgram();

    textures = new int[1];
    GLES20.glGenTextures(1, textures, 0);

    _surface = new android.graphics.SurfaceTexture(textures[0]);
    _camera = Camera.open();
    Camera.Size previewSize = _camera.getParameters().getPreviewSize();

    try
    {
        _camera.setPreviewTexture(_surface);
    }
    catch (java.io.IOException ex)
    {
        // Console.writeLine (ex.Message);
    }

    final int vertexShaderHandle = GLES20.glCreateShader(GLES20.GL_VERTEX_SHADER);
    GLES20.glShaderSource(vertexShaderHandle, vertexShaderCode);
    GLES20.glCompileShader(vertexShaderHandle);
    final int[] compileStatus = new int[1];
    GLES20.glGetShaderiv(vertexShaderHandle, GLES20.GL_COMPILE_STATUS, compileStatus, 0);
    if (compileStatus[0] == 0)
    {
        //do check here
    }

    final int fragmentShaderHandle = GLES20.glCreateShader(GLES20.GL_FRAGMENT_SHADER);
    GLES20.glShaderSource(fragmentShaderHandle, fragmentShaderCode);
    GLES20.glCompileShader(fragmentShaderHandle);
    GLES20.glGetShaderiv(fragmentShaderHandle, GLES20.GL_COMPILE_STATUS, compileStatus, 0);
    if (compileStatus[0] == 0)
    {
        //do check here
    }

    GLES20.glAttachShader(mProgram, vertexShaderHandle);
    GLES20.glAttachShader(mProgram, fragmentShaderHandle);
    GLES20.glBindAttribLocation(mProgram, 0, "a_Position");
    GLES20.glBindAttribLocation(mProgram, 0, "a_TexCoordinate");

    GLES20.glLinkProgram(mProgram);
    final int[] linkStatus = new int[1];
    GLES20.glGetProgramiv(mProgram, GLES20.GL_LINK_STATUS, linkStatus, 0);
    if (linkStatus[0] == 0)
    {
        //do check here
    }

    GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textures[0]);
    mTextureDataHandle = textures[0];

    // Set filtering
    GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST);
    GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_NEAREST);
}

public void draw()
{
    _surface.updateTexImage();
    GLES20.glUseProgram(mProgram);

    mTextureUniformHandle = GLES20.glGetUniformLocation(mProgram, "u_Texture");
    mPositionHandle = GLES20.glGetAttribLocation(mProgram, "a_Position");
    mColorHandle = GLES20.glGetAttribLocation(mProgram, "a_Color");
    mTextureCoordinateHandle = GLES20.glGetAttribLocation(mProgram, "a_TexCoordinate");

    GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
    GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureDataHandle);

    GLES20.glVertexAttribPointer(mPositionHandle, COORDS_PER_VERTEX,
            GLES20.GL_FLOAT, false,
            vertexStride, vertexBuffer);
    GLES20.glVertexAttribPointer(mTextureCoordinateHandle, mTextureCoordinateDataSize, GLES20.GL_FLOAT, false,
            0, mCubeTextureCoordinates);

    GLES20.glEnableVertexAttribArray(mTextureCoordinateHandle);
    GLES20.glEnableVertexAttribArray(mPositionHandle);
    GLES20.glUniform1i(mTextureUniformHandle, 0);

    GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, vertexCount);
    GLES20.glDisableVertexAttribArray(mPositionHandle);
}

}

Answer 1

When rendering a SurfaceTexture texture object, you need to use the GL_TEXTURE_EXTERNAL_OES texture target:

The texture object uses the GL_TEXTURE_EXTERNAL_OES texture target, which is defined by the GL_OES_EGL_image_external OpenGL ES extension. This limits how the texture may be used. Each time the texture is bound it must be bound to the GL_TEXTURE_EXTERNAL_OES target rather than the GL_TEXTURE_2D target. Additionally, any OpenGL ES 2.0 shader that samples from the texture must declare its use of this extension using, for example, an "#extension GL_OES_EGL_image_external : require" directive. Such shaders must also access the texture using the samplerExternalOES GLSL sampler type.

So you need to change your fragment shader like this, adding the #extension declaration and declaring your texture uniform as samplerExternalOES:

private final String fragmentShaderCode =
    "#extension GL_OES_EGL_image_external : require\n" +
    "precision mediump float;" +
    "uniform vec4 vColor;" +
    "uniform samplerExternalOES u_Texture;" +
    "varying vec2 v_TexCoordinate;" +
    "void main() {" +
            "gl_FragColor = (texture2D(u_Texture, v_TexCoordinate));" +
    "}";

Also in your draw() function, bind the texture like this:

GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureDataHandle);