Convert dual fisheye video to equirectangular

Question

I used open framework to convert dual fisheye video (received from ricoh theta S camera) to equirectangular format (for an iOS application).

dual fisheye image sample:

equirectangular image sample:

I use below shader:

equirectangular.frag

  // based on ThetaShaderPack_20150926 (http://stereoarts.jp/) written by Nora.
#ifdef GL_ES
// define default precision for float, vec, mat.
precision highp float;
#endif

#define PI 3.14159265358979
#define _THETA_S_Y_SCALE    (640.0 / 720.0)

uniform sampler2D mainTex;
uniform float radius;
uniform vec4 uvOffset;

varying vec2 texCoordVarying;

void main (void) {
    vec2 revUV = texCoordVarying.st;
    if (texCoordVarying.x <= 0.5) {
        revUV.x = revUV.x * 2.0;
    } else {
        revUV.x = (revUV.x - 0.5) * 2.0;
    }

    revUV *= PI;

    vec3 p = vec3(cos(revUV.x), cos(revUV.y), sin(revUV.x));
    p.xz *= sqrt(1.0 - p.y * p.y);

    float r = 1.0 - asin(p.z) / (PI / 2.0);
    vec2 st = vec2(p.y, p.x);

    st *= r / sqrt(1.0 - p.z * p.z);
    st *= radius;
    st += 0.5;

    if (texCoordVarying.x <= 0.5) {
        st.x *= 0.5;
        st.x += 0.5;
        st.y = 1.0 - st.y;
        st.xy += uvOffset.wz;
    } else {
        st.x = 1.0 - st.x;
        st.x *= 0.5;
        st.xy += uvOffset.yx;
    }

    st.y = st.y * _THETA_S_Y_SCALE;

    gl_FragColor = texture2D(mainTex, st);
}

equirectanguler.vert

uniform mat4 projectionMatrix;
uniform mat4 modelViewMatrix;
uniform mat4 textureMatrix;
uniform mat4 modelViewProjectionMatrix;

attribute vec4  position;
attribute vec4  color;
attribute vec3  normal;
attribute vec2  texcoord;

varying vec2 texCoordVarying;

void main() {
    texCoordVarying = texcoord;
    gl_Position = modelViewProjectionMatrix * position;
}

with below code: main.mm

#include "ofApp.h"

int main() {

    //  here are the most commonly used iOS window settings.
    //------------------------------------------------------
    ofiOSWindowSettings settings;
    settings.enableRetina = false; // enables retina resolution if the device supports it.
    settings.enableDepth = false; // enables depth buffer for 3d drawing.
    settings.enableAntiAliasing = false; // enables anti-aliasing which smooths out graphics on the screen.
    settings.numOfAntiAliasingSamples = 0; // number of samples used for anti-aliasing.
    settings.enableHardwareOrientation = false; // enables native view orientation.
    settings.enableHardwareOrientationAnimation = false; // enables native orientation changes to be animated.
    settings.glesVersion = OFXIOS_RENDERER_ES2; // type of renderer to use, ES1, ES2, ES3
    settings.windowMode = OF_FULLSCREEN;
    ofCreateWindow(settings);

    return ofRunApp(new ofApp);
}

```

`offApp.mm`

```
#include "ofApp.h"

//--------------------------------------------------------------
void ofApp::setup(){    
    ofDisableArbTex();

    devices = theta.listDevices();
    bool isDeviceConnected = false;
    for(int i = 0; i < devices.size(); i++){
        if(devices[i].deviceName == "RICOH THETA S"){
            theta.setDeviceID(devices[i].id);
            isDeviceConnected = true;
        }
    }
    if(!isDeviceConnected){
        ofLog(OF_LOG_ERROR, "RICOH THETA S is not found.");
    }
    theta.initGrabber(360, 568);

    shader.load("shaders/equirectanguler");

    fbo.allocate(320, 568);

    sphere = ofSpherePrimitive(568, 64).getMesh();
    for(int i=0;i<sphere.getNumTexCoords();i++){
        sphere.setTexCoord(i, ofVec2f(1.0) - sphere.getTexCoord(i));
    }
    for(int i=0;i<sphere.getNumNormals();i++){
        sphere.setNormal(i, sphere.getNormal(i) * ofVec3f(-1));
    }

    offset.set("uvOffset", ofVec4f(0,0.0,0,0.0), ofVec4f(-0.1), ofVec4f(0.1));
    radius.set("radius", 0.445, 0.0, 1.0);
    showSphere.set("showSphere", false);
    thetaParams.add(offset);
    thetaParams.add(radius);
    gui.setup(thetaParams);
    gui.add(showSphere);

    cam.setAutoDistance(false);
    cam.setDistance(0);
}

//--------------------------------------------------------------
void ofApp::update(){
    theta.update();
}

//--------------------------------------------------------------
void ofApp::draw(){
    if(theta.isFrameNew()){

        fbo.begin();
        ofClear(0);
        shader.begin();
        shader.setUniformTexture("mainTex", theta.getTexture(), 0);
        shader.setUniforms(thetaParams);
        theta.draw(0, 0, 320, 568);
        shader.end();
        fbo.end();

    }

    if(!showSphere){

        fbo.draw(0, 0, 320, 568);

    }else{

        ofEnableDepthTest();
        cam.begin();
        fbo.getTexture().bind();
        sphere.draw();
        fbo.getTexture().unbind();
        cam.end();

    }

    ofDisableDepthTest();
    gui.draw();
}

//--------------------------------------------------------------
void ofApp::exit(){

}

//--------------------------------------------------------------
void ofApp::touchDown(ofTouchEventArgs & touch){

}

//--------------------------------------------------------------
void ofApp::touchMoved(ofTouchEventArgs & touch){

}

//--------------------------------------------------------------
void ofApp::touchUp(ofTouchEventArgs & touch){

}

//--------------------------------------------------------------
void ofApp::touchDoubleTap(ofTouchEventArgs & touch){

}

//--------------------------------------------------------------
void ofApp::touchCancelled(ofTouchEventArgs & touch){

}

//--------------------------------------------------------------
void ofApp::lostFocus(){

}

//--------------------------------------------------------------
void ofApp::gotFocus(){

}

//--------------------------------------------------------------
void ofApp::gotMemoryWarning(){

}

//--------------------------------------------------------------
void ofApp::deviceOrientationChanged(int newOrientation){

}

offApp.h

#pragma once

#include "ofxiOS.h"
#include "ofxGui.h"

class ofApp : public ofxiOSApp {

    public:
        void setup();
        void update();
        void draw();
        void exit();

        void touchDown(ofTouchEventArgs & touch);
        void touchMoved(ofTouchEventArgs & touch);
        void touchUp(ofTouchEventArgs & touch);
        void touchDoubleTap(ofTouchEventArgs & touch);
        void touchCancelled(ofTouchEventArgs & touch);

        void lostFocus();
        void gotFocus();
        void gotMemoryWarning();
        void deviceOrientationChanged(int newOrientation);

    ofVideoGrabber theta;
    vector<ofVideoDevice> devices;
    ofShader shader;
    ofFbo fbo;
    ofEasyCam cam;
    ofVboMesh sphere;

    ofParameter<ofVec4f> offset;
    ofParameter<float> radius;
    ofParameter<bool> showSphere;
    ofParameterGroup thetaParams;
    ofxPanel gui;
};

And here is the result: on iphone5s

pls point me out what i missed.

Answer 1

I switch from OpenFrameworks to GPUImage and be able to convert dual-fisheye video to equirectangular format successfully on iOS devices.

Used same shader program. check my repository