Access framebuffer in Qt3D

Question

My task: Calculate the pixel coordinates (e.g. make a snapshot) of a 3D mesh to find the 2D shape of this mesh from a specific camera angle.

I'm currently using Qt3D with a QGeometryRenderer to render a scene containing a mesh to a QWidget which works fine. I tried to render the content of the QWidget into a Pixmap with QWidget::render() as proposed by this post How to create screenshot of QWidget?. Saving the pixmap to a .jpg results in a blank image with a default background color which makes sense because the QWidget is not holding the mesh object itself.

Here is how the scene is set in my mainwindow.cpp

// sets the scene objects, camera, lights,...
void MainWindow::setScene() {
    scene = custommesh->createScene(mesh->getVertices(), 
            mesh->getVerticesNormals(), 
            mesh->getFaceNormals(), 
            mesh->getVerticesIndex(), 
            mesh->getFacesIndex());              // QEntity*                         
    custommesh->setMaterial(scene);              // CustomMeshRenderer object
    camera = custommesh->setCamera(view);
    custommesh->setLight(scene, camera);
    custommesh->setCamController(scene, camera);

    view->setRootEntity(scene);                  // Qt3DExtras::Qt3DWindow object

    // Setting up a QWiget working as a container for the view
    QWidget *container = QWidget::createWindowContainer(view);
    container->setMinimumSize(QSize(500, 500));
    QSizePolicy policy = QSizePolicy(QSizePolicy::Policy(5), QSizePolicy::Policy(5));
    policy.setHorizontalStretch(1);
    policy.setVerticalStretch(1);
    container->setSizePolicy(policy);
    container->setObjectName("meshWidget");

    this->ui->meshLayout->insertWidget(0, container);
}

As for the rendering here is the custommeshrenderer class where the QGeometryRenderer is defined and a QEntity* is returned when initializing the mesh.

#include "custommeshrenderer.h"
#include <Qt3DRender/QAttribute>
#include <Qt3DExtras>
#include <Qt3DRender/QGeometryRenderer>


CustommeshRenderer::CustommeshRenderer()
{
    rootEntity = new Qt3DCore::QEntity;
    customMeshEntity = new Qt3DCore::QEntity(rootEntity);
    transform = new Qt3DCore::QTransform;

    customMeshRenderer = new Qt3DRender::QGeometryRenderer;
    customGeometry = new Qt3DRender::QGeometry(customMeshRenderer);

    m_pVertexDataBuffer = new Qt3DRender::QBuffer(Qt3DRender::QBuffer::VertexBuffer, customGeometry);
    m_pNormalDataBuffer = new Qt3DRender::QBuffer(Qt3DRender::QBuffer::VertexBuffer, customGeometry);
    m_pColorDataBuffer = new Qt3DRender::QBuffer(Qt3DRender::QBuffer::VertexBuffer, customGeometry);
    m_pIndexDataBuffer = new Qt3DRender::QBuffer(Qt3DRender::QBuffer::IndexBuffer, customGeometry);

}

/**
    Set vertices and their normals for the scene

    @param vertices List with all vertices of the mesh
    @param vertices_normals List with all vertice normals
    @param face_normals List with all face normals
    @param vertice_idx List with the indices for the vertices
    @param face_idx List with all indices for the faces
    @return Entity where some components were added
*/
Qt3DCore::QEntity *CustommeshRenderer::createScene(QList<QVector3D> vertices, QList<QVector3D> vertices_normals, QList<QVector3D> face_normals, QList<int> vertices_idx, QList<QVector3D> faces_idx) {

    // Setting scale to 8.0
    transform->setScale(8.0f);

    // Setting all the colors to (200, 0, 0)
    QList<QVector3D> color_list;
    for(int i = 0; i < vertices.length(); i++) {
        color_list.append(QVector3D(200.0f, 0.0f, 0.0f));
    }

    // Fill vertexBuffer with data which hold the vertices, normals and colors
    // Build structure: Size of Verticles List * 3 (x,y,z) * 4 (since x,y,z are floats, which needs 4 bytes each) 
    vertexBufferData.resize(vertices.length() * 3 * (int)sizeof(float));
    float *rawVertexArray = reinterpret_cast<float *>(vertexBufferData.data());

    normalBufferData.resize(vertices_normals.length() * 3 * (int)sizeof(float));
    float *rawNormalArray = reinterpret_cast<float *>(normalBufferData.data());

    colorBufferData.resize(color_list.length() * 3 * (int)sizeof(float));
    float *rawColorArray = reinterpret_cast<float *>(colorBufferData.data());


    setRawVertexArray(rawVertexArray, vertices);
    setRawNormalArray(rawNormalArray, vertices_normals);
    setRawColorArray(rawColorArray, color_list);

    //Fill indexBufferData with data which holds the triangulation information (patches/tris/lines)
    indexBufferData.resize(faces_idx.length() * 3 * (int)sizeof(uint));
    uint *rawIndexArray = reinterpret_cast<uint *>(indexBufferData.data());

    setRawIndexArray(rawIndexArray, faces_idx);

    //Set data to buffers
    m_pVertexDataBuffer->setData(vertexBufferData);
    m_pNormalDataBuffer->setData(normalBufferData);
    m_pColorDataBuffer->setData(colorBufferData);
    m_pIndexDataBuffer->setData(indexBufferData);

    // Attributes
    Qt3DRender::QAttribute *positionAttribute = new Qt3DRender::QAttribute();
    positionAttribute->setAttributeType(Qt3DRender::QAttribute::VertexAttribute);
    positionAttribute->setBuffer(m_pVertexDataBuffer);
   // positionAttribute->setBuffer(m_pVertexDataBuffer.data());
    positionAttribute->setDataType(Qt3DRender::QAttribute::Float);
    positionAttribute->setDataSize(3);
    positionAttribute->setByteOffset(0);
    positionAttribute->setByteStride(3 * sizeof(float));
    positionAttribute->setCount(vertices.length());
    positionAttribute->setName(Qt3DRender::QAttribute::defaultPositionAttributeName());

    Qt3DRender::QAttribute *normalAttribute = new Qt3DRender::QAttribute();
    normalAttribute->setAttributeType(Qt3DRender::QAttribute::VertexAttribute);
    normalAttribute->setBuffer(m_pNormalDataBuffer);
    //normalAttribute->setBuffer(m_pNormalDataBuffer.data());
    normalAttribute->setDataType(Qt3DRender::QAttribute::Float);
    normalAttribute->setDataSize(3);
    normalAttribute->setByteOffset(0);
    normalAttribute->setByteStride(3 * sizeof(float));
    normalAttribute->setCount(vertices.length());
    normalAttribute->setName(Qt3DRender::QAttribute::defaultNormalAttributeName());

    Qt3DRender::QAttribute* colorAttribute = new Qt3DRender::QAttribute();
    colorAttribute->setAttributeType(Qt3DRender::QAttribute::VertexAttribute);
    colorAttribute->setBuffer(m_pColorDataBuffer);
    //colorAttribute->setBuffer(m_pColorDataBuffer.data());
    colorAttribute->setDataType(Qt3DRender::QAttribute::Float);
    colorAttribute->setDataSize(3);
    colorAttribute->setByteOffset(0);
    colorAttribute->setByteStride(3 * sizeof(float));
    colorAttribute->setCount(vertices.length());
    colorAttribute->setName(Qt3DRender::QAttribute::defaultColorAttributeName());

    Qt3DRender::QAttribute *indexAttribute = new Qt3DRender::QAttribute();
    indexAttribute->setAttributeType(Qt3DRender::QAttribute::IndexAttribute);
    indexAttribute->setBuffer(m_pIndexDataBuffer);
    //indexAttribute->setBuffer(m_pIndexDataBuffer.data());
    indexAttribute->setDataType(Qt3DRender::QAttribute::UnsignedInt);
    indexAttribute->setDataSize(3);
    indexAttribute->setByteOffset(0);
    indexAttribute->setByteStride(3 * sizeof(uint));
    indexAttribute->setCount(face_normals.length());

    customGeometry->addAttribute(positionAttribute);
    customGeometry->addAttribute(normalAttribute);
    /*customGeometry->addAttribute(colorAttribute);*/
    customGeometry->addAttribute(indexAttribute);

    //Set the final geometry and primitive type
    customMeshRenderer->setPrimitiveType(Qt3DRender::QGeometryRenderer::Triangles);
    customMeshRenderer->setVerticesPerPatch(3);
    customMeshRenderer->setGeometry(customGeometry);


    customMeshRenderer->setVertexCount(faces_idx.length()*3);

    customMeshEntity->addComponent(customMeshRenderer);
    customMeshEntity->addComponent(transform);

    setMaterial(customMeshEntity);

    return rootEntity;
}

What is the best way access the framebuffer or is there any other method to take a snapshot of the mesh?

My last hope would be to implement the rendering pipeline (at least from projected coords to pixel coords) myself, but i would prefer another solution. Unfortunately I have to rely on Qt3D and can't switch to other classes like QOpenGLWidget. At least I haven't found a possibility to integrate it yet.

I'm pretty new to Qt3D and the lack of detailed documentation doesn't make it easier.

Answer 1

You can use QRenderCapture for this. This essentially does a glReadPixels for you. The documentation is a bit sparse on this one, but there is an example online.

Alternatively, I implemented an offline renderer, which could help you in case that you don't want a whole 3D window.

I'm not sure what you mean by

Calculate the pixel coordinates (e.g. make a snapshot) of a 3D mesh to find the 2D shape of this mesh from a specific camera angle

but if you e.g. want to render the whole mesh in only one color (without highlights), you could try QPerVertexColorMaterial, which gave me exactly that result.