How to get world coordinates from screen coordinates in Vispy

Question

I am not sure how to get from screen coordinates to world coordinates. I am using VisPy and I would like to implement ray tracing and picking ability in 3D.

I prepared some code based on a cube example. The code below sends a crude ray through the screen by changing z value and prints 3D coordinates (in ''on_mouse_press '' method). However the results are not correct. If i click top right corner of the cube somewhere along the ray should be printed (3,3,3), but it's not. Can anybody help me with this?

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vispy: gallery 50
"""
This example shows how to display 3D objects.
You should see a colored outlined spinning cube.
"""

import numpy as np
from vispy import app, gloo
from vispy.util.transforms import perspective, translate, rotate

vert = """
// Uniforms
// ------------------------------------
uniform   mat4 u_model;
uniform   mat4 u_view;
uniform   mat4 u_projection;
uniform   vec4 u_color;

// Attributes
// ------------------------------------
attribute vec3 a_position;
attribute vec4 a_color;
attribute vec3 a_normal;

// Varying
// ------------------------------------
varying vec4 v_color;

void main()
{
    v_color = a_color * u_color;
    gl_Position = u_projection * u_view * u_model * vec4(a_position,1.0);
}
"""


frag = """
uniform mat4 u_model;
uniform mat4 u_view;
uniform mat4 u_normal;

uniform vec3 u_light_intensity;
uniform vec3 u_light_position;

varying vec3 v_position;
varying vec3 v_normal;
varying vec4 v_color;

void main()
{
    gl_FragColor = v_color;
}
"""


# -----------------------------------------------------------------------------
def cube(num_of_cubes):
    """
    Build vertices for a colored cube.

    V  is the vertices
    I1 is the indices for a filled cube (use with GL_TRIANGLES)
    I2 is the indices for an outline cube (use with GL_LINES)
    """

    for i in range(0,num_of_cubes):
        # Vertices positions
        v = np.array([[1, 1, 1], [-1, 1, 1], [-1, -1, 1], [1, -1, 1],
             [1, -1, -1], [1, 1, -1], [-1, 1, -1], [-1, -1, -1]],dtype=np.float32)

        v[:,0]=v[:,0]+2.
        v[:,1]=v[:,1]+2.
        v[:,2]=v[:,2]+2.

        # Face Normals
        n =np.array([[0, 0, 1], [1, 0, 0], [0, 1, 0],
             [-1, 0, 1], [0, -1, 0], [0, 0, -1]],dtype=np.float32)
        # Vertice colors
        c = np.array([[0, 0, 1, 1], [0, 0, 1, 1], [0, 0, 1, 1], [0, 0, 1, 1],
             [0, 0, 1, 1], [0, 0, 1, 1], [0, 0, 1, 1], [0, 0, 1, 1]],dtype=np.float32)

        V_aux = np.array([(v[0], n[0], c[0]), (v[1], n[0], c[1]),
                      (v[2], n[0], c[2]), (v[3], n[0], c[3]),
                      (v[0], n[1], c[0]), (v[3], n[1], c[3]),
                      (v[4], n[1], c[4]), (v[5], n[1], c[5]),
                      (v[0], n[2], c[0]), (v[5], n[2], c[5]),
                      (v[6], n[2], c[6]), (v[1], n[2], c[1]),
                      (v[1], n[3], c[1]), (v[6], n[3], c[6]),
                      (v[7], n[3], c[7]), (v[2], n[3], c[2]),
                      (v[7], n[4], c[7]), (v[4], n[4], c[4]),
                      (v[3], n[4], c[3]), (v[2], n[4], c[2]),
                      (v[4], n[5], c[4]), (v[7], n[5], c[7]),
                      (v[6], n[5], c[6]), (v[5], n[5], c[5])]
        )
        I1_aux = np.resize(np.array([0, 1, 2, 0, 2, 3], dtype=np.uint32), 6 * (2 * 3))
        I1_aux += np.repeat(4 * np.arange(2 * 3, dtype=np.uint32), 6)

        I2_aux = np.resize(
            np.array([0, 1, 1, 2, 2, 3, 3, 0], dtype=np.uint32), 6 * (2 * 4))
        I2_aux += np.repeat(4 * np.arange(6, dtype=np.uint32), 8)


        if i==0:
            V=V_aux
            I1=I1_aux
            I2=I2_aux
        else:
            V=np.vstack((V,V_aux))
            I1=np.vstack((I1,I1_aux+i*24))
            I2=np.vstack((I2,I2_aux+i*24))




    return V, I1, I2


# -----------------------------------------------------------------------------
class Canvas(app.Canvas):

    def __init__(self):
        app.Canvas.__init__(self, keys='interactive', size=(800, 600))

        num_of_cubes=1 #number of cubes to draw
        self.V, self.filled, self.outline = cube(num_of_cubes)


        self.store_pos=np.array((0,0)) #for mouse interaction

        self.vert_data=np.vstack(self.V[:,0])
        self.V_buf=np.vstack(self.V[:,0])
        self.V_buf.dtype=[('a_position',np.float32,3)]
        self.vert_buf=gloo.VertexBuffer(self.V_buf)

        self.N_buf=np.vstack(self.V[:,1])
        self.N_buf.dtype=[('a_normal',np.float32,3)]
        self.norm_buf=gloo.VertexBuffer(self.N_buf)

        self.C_buf=np.vstack(self.V[:,2])
        self.C_buf.dtype=[('a_color',np.float32,4)]
        self.colo_buf=gloo.VertexBuffer(self.C_buf)

        self.filled_buf=gloo.IndexBuffer(self.filled.flatten())
        self.outline_buf=gloo.IndexBuffer(self.outline.flatten())

        self.program = gloo.Program(vert, frag)
        self.translate = 1

        #self.vert_buf=gloo.VertexBuffer(self.vertices.flatten())
        self.program.bind(self.vert_buf)
        self.program.bind(self.norm_buf)
        self.program.bind(self.colo_buf)


        self.view = translate((0, 0, -10))
        self.model = np.eye(4, dtype=np.float32)

        gloo.set_viewport(0, 0, self.physical_size[0], self.physical_size[1])
        self.projection = perspective(45.0, self.size[0] /
                                      float(self.size[1]), 2.0, 10.0)

        self.program['u_projection'] = self.projection

        self.program['u_model'] = self.model
        self.program['u_view'] = self.view

        self.theta = 0
        self.phi = 0

        gloo.set_clear_color('white')
        gloo.set_state('opaque')
        gloo.set_polygon_offset(1, 1)

        self._timer = app.Timer('auto', connect=self.on_timer, start=True)

        self.show()
        self.t=0


    # ---------------------------------
    def on_timer(self, event):
        self.update()

    # ---------------------------------
    def print_mouse_event(self, event, what):
        modifiers = ', '.join([key.name for key in event.modifiers])
        print('%s - pos: %r, button: %s, modifiers: %s, delta: %r' %
              (what, event.pos, event.button, modifiers, event.delta))

    def on_mouse_press(self, event):
        self.print_mouse_event(event, 'Mouse press')

        #convert to NDC
        left=event.pos[0]*2/self.size[0]-1
        bottom=(self.size[1]-event.pos[1])*2/self.size[1]-1


        z_clip=np.linspace(-1.,1.,100)
        for val in z_clip:
            aux=np.dot(np.dot(np.linalg.inv(self.view),np.linalg.inv(self.projection)),np.array((left,bottom,val,1.)))
            pos3d=aux/aux[3]
            print(pos3d)

    def on_mouse_wheel(self, event):

        self.translate -= event.delta[1]
        self.translate = max(-1, self.translate)
        self.view[3,2]=-self.translate

        self.program['u_view'] = self.view
        self.update()



    def on_draw(self, event):
        gloo.clear()

        # Filled cube

        gloo.set_state(blend=False, depth_test=True, polygon_offset_fill=True)
        self.program['u_color'] = 1, 0, 1, 1


        self.program.draw('triangles', self.filled_buf)

        # Outline
        gloo.set_state(polygon_offset_fill=False, blend=True, depth_mask=False)
        gloo.set_depth_mask(False)
        self.program['u_color'] = 0, 0, 0, 1

        self.program.draw('lines', self.outline_buf)
        gloo.set_depth_mask(True)


# -----------------------------------------------------------------------------
if __name__ == '__main__':
    c = Canvas()
    app.run()

Answer 1

A clicked point on the screen maps to a line in your scene.

The object in view.scene.transform represents the mapping between scene and screen coordinates. .map(points) will transform points from scene to screen. .imap(points) maps from screen coordinates back to world coordinates.

To get the line your screen point corresponds to. You can imap a point on the screen, and another point offset from the screen in z:

def get_view_axis_in_scene_coordinates(view):
    import numpy
    tform=view.scene.transform
    w,h = view.canvas.size
    screen_center = numpy.array([w/2,h/2,0,1]) # in homogeneous screen coordinates
    d1 = numpy.array([0,0,1,0]) # in homogeneous screen coordinates
    point_in_front_of_screen_center = screen_center + d1 # in homogeneous screen coordinates
    p1 = tform.imap(point_in_front_of_screen_center) # in homogeneous scene coordinates
    p0 = tform.imap(screen_center) # in homogeneous screen coordinates
    assert(abs(p1[3]-1.0) < 1e-5) # normalization necessary before subtraction
    assert(abs(p0[3]-1.0) < 1e-5)
    return p0[0:3],p1[0:3] # 2 point representation of view axis in 3d scene coordinates

I adapted it be a bit closer to what you want; you need to replace screen_center with the clicked point. Note, I did this for orthogonal projection; think it works for perspective too but haven't tested it.

Related: Get view direction relative to scene in vispy?