Rotating a Open GL camera correctly using GLM

Question

I have a camera class, which is initialized like so:

CameraFP::CameraFP()  {
    this->aspect_ratio = 800.0f / 600.0f;
    this->fov = 45.0f;
    this->near_plane = 0.1f;
    this->far_plane = 1000.0f;
    this->position = glm::vec3(0, 0, 0);
    this->target = position + glm::vec3(0, 0, -1);
    this->up = glm::vec3(0, 1, 0);
    this->m_rotation = glm::mat4(1.0);

    m_view = glm::lookAt(position, target, up);
    m_projection = glm::perspective(fov, aspect_ratio, near_plane, far_plane);
}

And here are other functions of import:

void CameraFP::update(sf::Window *app)  {
    process_keyboard(app);
    process_mouse(app);

    calculate_view();
}

void CameraFP::process_keyboard(sf::Window *app)  {
    const sf::Input *input = &app->GetInput();

    up = m_rotation * glm::vec3(0, 1, 0);

    glm::vec3 forward = glm::vec3(0, 0, -1);
    glm::vec3 forward_rotated = m_rotation * forward;

    glm::vec3 right = glm::vec3(1, 0, 0);
    glm::vec3 right_rotated = m_rotation * right;

    if (input->IsKeyDown(sf::Key::W))  {
        position += forward_rotated;
    }
    if (input->IsKeyDown(sf::Key::S))  {
        position -= forward_rotated;
    }
    if (input->IsKeyDown(sf::Key::A))  {
        position -= right_rotated;
    }
    if (input->IsKeyDown(sf::Key::D))  {
        position += right_rotated;
    }
}

void CameraFP::process_mouse(sf::Window *app)  {
    // TODO: Make the below constants, and take framerate into account
    GLfloat SPEED_X = 0.000001f;
    GLfloat SPEED_Y = 0.000001f;

    GLfloat mouse_x = app->GetInput().GetMouseX();
    GLfloat mouse_y = app->GetInput().GetMouseY();

    GLfloat mouse_x_delta = old_mouse_x - mouse_x;
    GLfloat mouse_y_delta = old_mouse_y - mouse_y;

    if (mouse_x_delta != 0 ||
        mouse_y_delta != 0)  {
        if (mouse_x_delta != 0)  {
            y_rot += mouse_x_delta * SPEED_X;

            m_rotation = glm::rotate(m_rotation, y_rot, glm::vec3(0, 1, 0));
        }
        if (mouse_y_delta != 0)  {
            x_rot += mouse_y_delta * SPEED_Y;

            m_rotation = glm::rotate(m_rotation, x_rot, glm::vec3(1, 0, 0));;
        }
    }

    this->old_mouse_x = mouse_x;
    this->old_mouse_y = mouse_y;

    app->SetCursorPosition(app->GetWidth() / 2, app->GetHeight() / 2);
}


void CameraFP::calculate_view()  {
    glm::vec3 forward = glm::vec3(0, 0, -1);
    glm::vec3 forward_rotated = m_rotation * forward;

    target = position += glm::normalize(forward_rotated);

    m_view = glm::lookAt(position, target, up);
}

My problem is that when I compile the project, the compiler outputs an error saying:

\CameraFP.cpp|59|error: no match for 'operator*' in '((CameraFP*)this)->CameraFP::m_rotation * glm::detail::tvec3<float>(((const int&)((const int*)(&0))), ((const int&)((const int*)(&1))), ((const int&)((const int*)(&0))))'|

From what I understand vec = mat4 * vec should yield a rotated vector? Since I haven't been able to test this code, I don't know if the function work correctly.

Edit

Updated code according to the comments and awnsers. My problem is now that I get a BSOD, somewhere in the render function...

void CameraFP::process_keyboard(sf::Window *app)  {
    const sf::Input *input = &app->GetInput();

    up = m_rotation * glm::vec4(0.0f, 1.0f, 0.0f, 0.0f);

    glm::vec4 forward = glm::vec4(0.0f, 0.0f, -1.0f, 0.0f);
    glm::vec4 forward_rotated = m_rotation * forward;

    glm::vec4 right = glm::vec4(1.0f, 0.0f, 0.0f, 0.0f);
    glm::vec4 right_rotated = m_rotation * right;

    if (input->IsKeyDown(sf::Key::W))  {
        position += forward_rotated;
    }
    if (input->IsKeyDown(sf::Key::S))  {
        position -= forward_rotated;
    }
    if (input->IsKeyDown(sf::Key::A))  {
        position -= right_rotated;
    }
    if (input->IsKeyDown(sf::Key::D))  {
        position += right_rotated;
    }
}

void CameraFP::process_mouse(sf::Window *app)  {
    // TODO: Make the below constants, and take framerate into account
    GLfloat SPEED_X = 0.000001f;
    GLfloat SPEED_Y = 0.000001f;

    GLfloat mouse_x = app->GetInput().GetMouseX();
    GLfloat mouse_y = app->GetInput().GetMouseY();

    GLfloat mouse_x_delta = old_mouse_x - mouse_x;
    GLfloat mouse_y_delta = old_mouse_y - mouse_y;

    if (mouse_x_delta != 0 ||
        mouse_y_delta != 0)  {
        if (mouse_x_delta != 0)  {
            y_rot += mouse_x_delta * SPEED_X;

            m_rotation = glm::rotate(m_rotation, y_rot, glm::vec3(0.0f, 1.0f, 0.0f));
        }
        if (mouse_y_delta != 0)  {
            x_rot += mouse_y_delta * SPEED_Y;

            m_rotation = glm::rotate(m_rotation, x_rot, glm::vec3(1.0f, 0.0f, 0.0f));;
        }
    }

    this->old_mouse_x = mouse_x;
    this->old_mouse_y = mouse_y;

    app->SetCursorPosition(app->GetWidth() / 2, app->GetHeight() / 2);
}

void CameraFP::calculate_view()  {
    glm::vec4 forward = glm::vec4(0.0f, 0.0f, -1.0f, 0.0f);
    glm::vec4 forward_rotated = m_rotation * forward;

    target = position += forward_rotated;

    m_view = glm::lookAt(v4tov3(position), v4tov3(target), v4tov3(up));
}

glm::vec3 v4tov3(glm::vec4 v1)  {
    return glm::vec3(v1.x, v1.y, v1.z);
}

Edit 2

Problem now is with the camera rotation with the mouse, it just doesn't work, for some reason changes on the x axis oft times effect change on the y and vice versa. In addition, if I move the mouse right or left on the x axis (y rotation) the camera rotates left...

void CameraFP::process_mouse(sf::Clock *clock, sf::Window *app)  {
    // TODO: Make the below constants, and take framerate into account
    GLfloat SPEED_X = 0.25f;
    GLfloat SPEED_Y = 0.25f;

    GLfloat screen_x = app->GetWidth();
    GLfloat screen_y = app->GetHeight();

    GLfloat mouse_x = float(screen_x / 2 - app->GetInput().GetMouseX());
    GLfloat mouse_y = float(screen_y / 2 - app->GetInput().GetMouseY());

    GLfloat mouse_x_delta = old_mouse_x - mouse_x;
    GLfloat mouse_y_delta = old_mouse_y - mouse_y;

    GLfloat current_time = clock->GetElapsedTime();
    GLfloat delta_time = current_time - last_time;

    this->last_time = current_time;

    if (mouse_x_delta != 0 ||
        mouse_y_delta != 0)  {
        if (mouse_x_delta != 0)  {
            y_rot += glm::radians(delta_time * SPEED_X * mouse_x);

            m_rotation = glm::rotate(m_rotation, y_rot, glm::vec3(0.0f, 1.0f, 0.0f));

            std::cout << "Y Rotation: " << y_rot << "\n";
        }
        if (mouse_y_delta != 0)  {
            x_rot += glm::radians(delta_time * SPEED_Y * mouse_y);

            m_rotation = glm::rotate(m_rotation, x_rot, glm::vec3(1.0f, 0.0f, 0.0f));

            std::cout << "X rotation: " << x_rot << "\n";
        }
    }

    app->SetCursorPosition(screen_x / 2, screen_y / 2);

    this->old_mouse_x = float(screen_x / 2 - app->GetInput().GetMouseX());
    this->old_mouse_y = float(screen_y / 2 - app->GetInput().GetMouseY());
}

Answer 1

Replace all glm::vec3(0, 1, 0); by glm::vec3(0.0f, 1.0f, 0.0f);

As for the vec-mac multiplication, AquilaRapax is right in that you can only multiply a mat4 with a vec4. But since you're multiplying directions, the 4rth coordinate should be 0.0f, not 1.0f. This will have the effect to ignore the translations (1.0 will teke them into account, which you don't want)

See http://www.opengl-tutorial.org/beginners-tutorials/tutorial-3-matrices/ for details on matrices.

However, it's often a good idea to keep vec3 instead of vec4's, mostly for clarity purposes (i.e., glm::vec3 mPosition instead of glm::vec4 mPosition). It is thus handy to have 2 functions like these (untested) :

glm::vec3 TransformDirection(glm::vec3 pDirection, glm::mat4 pMatrix){
    return pMatrix * glm::vec4(pDirection, 0.0f);
}

glm::vec3 TransformPosition(glm::vec3 pDirection, glm::mat4 pMatrix){
    return pMatrix * glm::vec4(pDirection, 1.0f);
}