OpenGL - object outline

Question

I'm trying to implement a selection-outline feature. This is what I get up to now.

As you can see, the objects are selected correctly when the mouse hovers and a contour is drawn around the selected object.

What I would like to do now is to outline the visible edges of the object in this way

In the image on the left is what I have now, and in the right image is what I want to achieve.

This is the procedure I use now.

void paintGL()
{
    /* ... */

    int w = geometry().width();
    int h = geometry().height();

    glEnable(GL_DEPTH_TEST);
    glDepthFunc(GL_LESS);

    glEnable(GL_STENCIL_TEST);
    glStencilFunc(GL_NOTEQUAL, 1, 0xFF);
    glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
    glStencilMask(0xFF);

    setClearColor(Qt::GlobalColor::darkGray);

    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);

    glStencilMask(0x00);
    DrawGeometry();

    if (HoveredSphere != RgbFromColorToString(Qt::GlobalColor::black))
    {
        glBindFramebuffer(GL_FRAMEBUFFER, addFBO(FBOIndex::OUTLINE));
        {
            glStencilFunc(GL_ALWAYS, 1, 0xFF);
            glStencilMask(0xFF);

            glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);

            DrawOutline(HoveredSphere, 1.0f - 0.025f);
        }
        glBindFramebuffer(GL_FRAMEBUFFER, defaultFramebufferObject());

        glBindFramebuffer(GL_READ_FRAMEBUFFER, addFBO(FBOIndex::OUTLINE));
        glBindFramebuffer(GL_DRAW_FRAMEBUFFER, defaultFramebufferObject());
        {
            // copy stencil buffer
            GLbitfield mask = GL_STENCIL_BUFFER_BIT;
            glBlitFramebuffer(0, 0, w, h, 0, 0, w, h, mask, GL_NEAREST);

            glStencilFunc(GL_NOTEQUAL, 1, 0xFF);
            glStencilMask(0x00);

            glDepthFunc(GL_LEQUAL);
            DrawOutline(HoveredSphere, 1.0f);
        }
        glBindFramebuffer(GL_FRAMEBUFFER, defaultFramebufferObject());
    }

    update();
}

Where DrawGeometry draws all the objects, and DrawOutline draws the selected object scaled by the factor passed as the second parameter.

Thanks for any suggestions.

Answer 1

By following the tips of @MichaelMahn, I found a solution.

First of all, I draw the silhouette of the visible parts of the selected object in a texture.

And then I use this texture to calculate the outline by checking the neighboring pixels to figure out whether or not I stand on the edge of the silhouette.

outline fragment shader

#version 450

uniform sampler2D silhouette;

in FragData
{
    smooth vec2 coords;
} frag;

out vec4 PixelColor;

void main()
{
    // if the pixel is black (we are on the silhouette)
    if (texture(silhouette, frag.coords).xyz == vec3(0.0f))
    {
        vec2 size = 1.0f / textureSize(silhouette, 0);

        for (int i = -1; i <= +1; i++)
        {
            for (int j = -1; j <= +1; j++)
            {
                if (i == 0 && j == 0)
                {
                    continue;
                }

                vec2 offset = vec2(i, j) * size;

                // and if one of the neighboring pixels is white (we are on the border)
                if (texture(silhouette, frag.coords + offset).xyz == vec3(1.0f))
                {
                    PixelColor = vec4(vec3(1.0f), 1.0f);
                    return;
                }
            }
        }
    }

    discard;
}

paintgl

void paintGL()
{
    int w = geometry().width();
    int h = geometry().height();

    setClearColor(Qt::GlobalColor::darkGray);

    glEnable(GL_DEPTH_TEST);
    glDepthFunc(GL_LESS);

    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);

    DrawGeometry();

    // if we hover a sphere
    if (HoveredSphere != RgbFromColorToString(Qt::GlobalColor::black))
    {
        glBindFramebuffer(GL_READ_FRAMEBUFFER, defaultFramebufferObject());
        glBindFramebuffer(GL_DRAW_FRAMEBUFFER, addFBO(FBOIndex::SILHOUETTE));
        {
            // copy depth buffer
            GLbitfield mask = GL_DEPTH_BUFFER_BIT;
            glBlitFramebuffer(0, 0, w, h, 0, 0, w, h, mask, GL_NEAREST);

            // set clear color
            setClearColor(Qt::GlobalColor::white);
            // enable depth test
            glEnable(GL_DEPTH_TEST);
            glDepthFunc(GL_LEQUAL);
            // clear color buffer
            glClear(GL_COLOR_BUFFER_BIT);

            // draw silhouette
            DrawSilhouette(HoveredSphere);
        }
        glBindFramebuffer(GL_FRAMEBUFFER, defaultFramebufferObject());

        // clear depth buffer
        glClear(GL_DEPTH_BUFFER_BIT);

        // draw outline
        DrawOutline();
    }
}

PROBLEM :: Now I'd like to parameterize the width of the contour, whose thickness is currently fixed at 1 pixel.

Thank you so much for any suggestion!