openGL MSAA does not work on linux with nouveau drivers

Question

Recently I have tried using MSAA (multisample anti-aliasing) under Linux. The result was surprising: everything worked like a charm with Nvidia proprietary drivers, however with nouveau drivers it does not work. The strange thing about this is that glxingo says that GLX_ARB_multisample is supported and I am able to get the appropriate FBConfig with GLX_SAMPLE_BUFFERS set to 1 and GLX_SAMPLES set to 4 (or 8). But when it comes to rendering, the picture is as no anti-aliasing is applied. The problem exists in both my project and freeglut, so I will provide a small glut-based program that demonstarates the problem:

/*
 * Test multisampling and polygon smoothing.
 *
 * Brian Paul
 * 4 November 2002
 */

#include <stdio.h>
#include <stdlib.h>
#include <math.h>
#include <GL/glut.h>


static GLfloat Zrot = 0;
static GLboolean Anim = GL_TRUE;
static GLboolean HaveMultisample = GL_TRUE;
static GLboolean DoMultisample = GL_TRUE;


static void
PrintString(const char *s)
{
   while (*s) {
      glutBitmapCharacter(GLUT_BITMAP_8_BY_13, (int) *s);
      s++;
   }
}


static void
Polygon( GLint verts, GLfloat radius, GLfloat z )
{
   int i;
   for (i = 0; i < verts; i++) {
      float a = (i * 2.0 * 3.14159) / verts;
      float x = radius * cos(a);
      float y = radius * sin(a);
      glVertex3f(x, y, z);
   }
}


static void
DrawObject( void )
{
   glLineWidth(3.0);
   glColor3f(1, 1, 1);
   glBegin(GL_LINE_LOOP);
   Polygon(12, 1.2, 0);
   glEnd();

   glLineWidth(1.0);
   glColor3f(1, 1, 1);
   glBegin(GL_LINE_LOOP);
   Polygon(12, 1.1, 0);
   glEnd();

   glColor3f(1, 0, 0);
   glBegin(GL_POLYGON);
   Polygon(12, 0.4, 0.3);
   glEnd();

   glColor3f(0, 1, 0);
   glBegin(GL_POLYGON);
   Polygon(12, 0.6, 0.2);
   glEnd();

   glColor3f(0, 0, 1);
   glBegin(GL_POLYGON);
   Polygon(12, 0.8, 0.1);
   glEnd();

   glColor3f(1, 1, 1);
   glBegin(GL_POLYGON);
   Polygon(12, 1.0, 0);
   glEnd();
}


static void
Display( void )
{
   glClear( GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT );

   glColor3f(1, 1, 1);
   if (HaveMultisample) {
      glRasterPos2f(-3.1, -1.6);
      if (DoMultisample)
         PrintString("MULTISAMPLE");
      else
         PrintString("MULTISAMPLE (off)");
   }
   glRasterPos2f(-0.8, -1.6);
   PrintString("No antialiasing");
   glRasterPos2f(1.6, -1.6);
   PrintString("GL_POLYGON_SMOOTH");

   /* multisample */
   if (HaveMultisample) {
      glEnable(GL_DEPTH_TEST);
      if (DoMultisample)
         glEnable(GL_MULTISAMPLE_ARB);
      glPushMatrix();
      glTranslatef(-2.5, 0, 0);
      glPushMatrix();
      glRotatef(Zrot, 0, 0, 1);
      DrawObject();
      glPopMatrix();
      glPopMatrix();
      glDisable(GL_MULTISAMPLE_ARB);
      glDisable(GL_DEPTH_TEST);
   }

   /* non-aa */
   glEnable(GL_DEPTH_TEST);
   glPushMatrix();
   glTranslatef(0, 0, 0);
   glPushMatrix();
   glRotatef(Zrot, 0, 0, 1);
   DrawObject();
   glPopMatrix();
   glPopMatrix();
   glDisable(GL_DEPTH_TEST);

   /* polygon smooth */
   glEnable(GL_POLYGON_SMOOTH);
   glEnable(GL_LINE_SMOOTH);
   glEnable(GL_BLEND);
   glPushMatrix();
   glTranslatef(2.5, 0, 0);
   glPushMatrix();
   glRotatef(Zrot, 0, 0, 1);
   DrawObject();
   glPopMatrix();
   glPopMatrix();
   glDisable(GL_LINE_SMOOTH);
   glDisable(GL_POLYGON_SMOOTH);
   glDisable(GL_BLEND);

   glutSwapBuffers();
}


static void
Reshape( int width, int height )
{
   GLfloat ar = (float) width / (float) height;
   glViewport( 0, 0, width, height );
   glMatrixMode( GL_PROJECTION );
   glLoadIdentity();
   glOrtho(-2.0*ar, 2.0*ar, -2.0, 2.0, -1.0, 1.0);
   glMatrixMode( GL_MODELVIEW );
   glLoadIdentity();
}


static void
Idle( void )
{
   Zrot = 0.01 * glutGet(GLUT_ELAPSED_TIME);
   glutPostRedisplay();
}


static void
Key( unsigned char key, int x, int y )
{
   const GLfloat step = 1.0;
   (void) x;
   (void) y;
   switch (key) {
      case 'a':
         Anim = !Anim;
         if (Anim)
            glutIdleFunc(Idle);
         else
            glutIdleFunc(NULL);
         break;
      case 'm':
         DoMultisample = !DoMultisample;
         break;
      case 'z':
         Zrot = (int) (Zrot - step);
         break;
      case 'Z':
         Zrot = (int) (Zrot + step);
         break;
      case 27:
         exit(0);
         break;
   }
   glutPostRedisplay();
}


static void
Init( void )
{
   /* GLUT imposes the four samples/pixel requirement */
   int s;
   glGetIntegerv(GL_SAMPLES_ARB, &s);
   if (!glutExtensionSupported("GL_ARB_multisample") || s < 1) {
      printf("Warning: multisample antialiasing not supported.\n");
      HaveMultisample = GL_FALSE;
   }
   printf("GL_RENDERER = %s\n", (char *) glGetString(GL_RENDERER));
   printf("GL_SAMPLES_ARB = %d\n", s);

   glBlendFunc(GL_SRC_ALPHA, GL_ONE);
   glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
   glBlendFunc(GL_SRC_ALPHA_SATURATE, GL_ONE);

   glGetIntegerv(GL_MULTISAMPLE_ARB, &s);
   printf("GL_MULTISAMPLE_ARB = %d\n", s);
}


int
main( int argc, char *argv[] )
{
   glutInit( &argc, argv );
   glutInitWindowPosition( 0, 0 );
   glutInitWindowSize( 600, 300 );
   glutInitDisplayMode( GLUT_RGB | GLUT_ALPHA | GLUT_DOUBLE |
                        GLUT_DEPTH | GLUT_MULTISAMPLE );
   glutCreateWindow(argv[0]);
   glutReshapeFunc( Reshape );
   glutKeyboardFunc( Key );
   glutDisplayFunc( Display );
   if (Anim)
      glutIdleFunc( Idle );
   Init();
   glutMainLoop();
   return 0;
}

Here is glxinfo output (excluding the visuals info):

name of display: :0.0
display: :0  screen: 0
direct rendering: Yes
server glx vendor string: SGI
server glx version string: 1.4
server glx extensions:
    GLX_ARB_multisample, GLX_EXT_import_context, GLX_EXT_texture_from_pixmap, 
    GLX_EXT_visual_info, GLX_EXT_visual_rating, GLX_MESA_copy_sub_buffer, 
    GLX_OML_swap_method, GLX_SGI_swap_control, GLX_SGIS_multisample, 
    GLX_SGIX_fbconfig, GLX_SGIX_pbuffer, GLX_SGIX_visual_select_group, 
    GLX_INTEL_swap_event
client glx vendor string: Mesa Project and SGI
client glx version string: 1.4
client glx extensions:
    GLX_ARB_create_context, GLX_ARB_create_context_profile, 
    GLX_ARB_get_proc_address, GLX_ARB_multisample, GLX_EXT_import_context, 
    GLX_EXT_visual_info, GLX_EXT_visual_rating, GLX_EXT_framebuffer_sRGB, 
    GLX_EXT_create_context_es2_profile, GLX_MESA_copy_sub_buffer, 
    GLX_MESA_multithread_makecurrent, GLX_MESA_swap_control, 
    GLX_OML_swap_method, GLX_OML_sync_control, GLX_SGI_make_current_read, 
    GLX_SGI_swap_control, GLX_SGI_video_sync, GLX_SGIS_multisample, 
    GLX_SGIX_fbconfig, GLX_SGIX_pbuffer, GLX_SGIX_visual_select_group, 
    GLX_EXT_texture_from_pixmap, GLX_INTEL_swap_event
GLX version: 1.4
GLX extensions:
    GLX_ARB_get_proc_address, GLX_ARB_multisample, GLX_EXT_import_context, 
    GLX_EXT_visual_info, GLX_EXT_visual_rating, GLX_MESA_copy_sub_buffer, 
    GLX_MESA_multithread_makecurrent, GLX_MESA_swap_control, 
    GLX_OML_swap_method, GLX_OML_sync_control, GLX_SGI_make_current_read, 
    GLX_SGI_swap_control, GLX_SGI_video_sync, GLX_SGIS_multisample, 
    GLX_SGIX_fbconfig, GLX_SGIX_pbuffer, GLX_SGIX_visual_select_group, 
    GLX_EXT_texture_from_pixmap
OpenGL vendor string: nouveau
OpenGL renderer string: Gallium 0.4 on NVC1
OpenGL version string: 2.1 Mesa 8.0.4
OpenGL shading language version string: 1.30
OpenGL extensions:
    GL_ARB_multisample, GL_EXT_abgr, GL_EXT_bgra, GL_EXT_blend_color, 
    GL_EXT_blend_minmax, GL_EXT_blend_subtract, GL_EXT_copy_texture, 
    GL_EXT_polygon_offset, GL_EXT_subtexture, GL_EXT_texture_object, 
    GL_EXT_vertex_array, GL_EXT_compiled_vertex_array, GL_EXT_texture, 
    GL_EXT_texture3D, GL_IBM_rasterpos_clip, GL_ARB_point_parameters, 
    GL_EXT_draw_range_elements, GL_EXT_packed_pixels, GL_EXT_point_parameters, 
    GL_EXT_rescale_normal, GL_EXT_separate_specular_color, 
    GL_EXT_texture_edge_clamp, GL_SGIS_generate_mipmap, 
    GL_SGIS_texture_border_clamp, GL_SGIS_texture_edge_clamp, 
    GL_SGIS_texture_lod, GL_ARB_framebuffer_sRGB, GL_ARB_multitexture, 
    GL_EXT_framebuffer_sRGB, GL_IBM_multimode_draw_arrays, 
    GL_IBM_texture_mirrored_repeat, GL_ARB_texture_cube_map, 
    GL_ARB_texture_env_add, GL_ARB_transpose_matrix, 
    GL_EXT_blend_func_separate, GL_EXT_fog_coord, GL_EXT_multi_draw_arrays, 
    GL_EXT_secondary_color, GL_EXT_texture_env_add, 
    GL_EXT_texture_filter_anisotropic, GL_EXT_texture_lod_bias, 
    GL_INGR_blend_func_separate, GL_NV_blend_square, GL_NV_light_max_exponent, 
    GL_NV_texgen_reflection, GL_NV_texture_env_combine4, 
    GL_SUN_multi_draw_arrays, GL_ARB_texture_border_clamp, 
    GL_ARB_texture_compression, GL_EXT_framebuffer_object, 
    GL_EXT_texture_env_combine, GL_EXT_texture_env_dot3, GL_MESA_window_pos, 
    GL_NV_packed_depth_stencil, GL_NV_texture_rectangle, GL_ARB_depth_texture, 
    GL_ARB_occlusion_query, GL_ARB_shadow, GL_ARB_texture_env_combine, 
    GL_ARB_texture_env_crossbar, GL_ARB_texture_env_dot3, 
    GL_ARB_texture_mirrored_repeat, GL_ARB_window_pos, 
    GL_EXT_stencil_two_side, GL_EXT_texture_cube_map, GL_NV_depth_clamp, 
    GL_NV_fog_distance, GL_APPLE_packed_pixels, GL_APPLE_vertex_array_object, 
    GL_ARB_draw_buffers, GL_ARB_fragment_program, GL_ARB_fragment_shader, 
    GL_ARB_shader_objects, GL_ARB_vertex_program, GL_ARB_vertex_shader, 
    GL_ATI_draw_buffers, GL_ATI_texture_env_combine3, GL_EXT_shadow_funcs, 
    GL_EXT_stencil_wrap, GL_MESA_pack_invert, GL_NV_primitive_restart, 
    GL_ARB_depth_clamp, GL_ARB_fragment_program_shadow, 
    GL_ARB_half_float_pixel, GL_ARB_occlusion_query2, GL_ARB_point_sprite, 
    GL_ARB_shading_language_100, GL_ARB_sync, GL_ARB_texture_non_power_of_two, 
    GL_ARB_vertex_buffer_object, GL_ATI_blend_equation_separate, 
    GL_EXT_blend_equation_separate, GL_OES_read_format, 
    GL_ARB_color_buffer_float, GL_ARB_pixel_buffer_object, 
    GL_ARB_texture_compression_rgtc, GL_ARB_texture_rectangle, 
    GL_ATI_texture_compression_3dc, GL_EXT_packed_float, 
    GL_EXT_pixel_buffer_object, GL_EXT_texture_compression_rgtc, 
    GL_EXT_texture_mirror_clamp, GL_EXT_texture_rectangle, 
    GL_EXT_texture_sRGB, GL_EXT_texture_shared_exponent, 
    GL_ARB_framebuffer_object, GL_EXT_framebuffer_blit, 
    GL_EXT_framebuffer_multisample, GL_EXT_packed_depth_stencil, 
    GL_ARB_vertex_array_object, GL_ATI_separate_stencil, 
    GL_ATI_texture_mirror_once, GL_EXT_draw_buffers2, GL_EXT_draw_instanced, 
    GL_EXT_gpu_program_parameters, GL_EXT_texture_array, 
    GL_EXT_texture_compression_latc, GL_EXT_texture_integer, 
    GL_EXT_texture_sRGB_decode, GL_EXT_timer_query, GL_OES_EGL_image, 
    GL_MESA_texture_array, GL_ARB_copy_buffer, GL_ARB_depth_buffer_float, 
    GL_ARB_draw_instanced, GL_ARB_half_float_vertex, GL_ARB_instanced_arrays, 
    GL_ARB_map_buffer_range, GL_ARB_texture_rg, GL_ARB_texture_swizzle, 
    GL_ARB_vertex_array_bgra, GL_EXT_separate_shader_objects, 
    GL_EXT_texture_swizzle, GL_EXT_vertex_array_bgra, 
    GL_NV_conditional_render, GL_AMD_conservative_depth, 
    GL_AMD_draw_buffers_blend, GL_ARB_ES2_compatibility, 
    GL_ARB_draw_buffers_blend, GL_ARB_draw_elements_base_vertex, 
    GL_ARB_explicit_attrib_location, GL_ARB_fragment_coord_conventions, 
    GL_ARB_provoking_vertex, GL_ARB_sampler_objects, GL_ARB_seamless_cube_map, 
    GL_ARB_shader_texture_lod, GL_EXT_provoking_vertex, GL_EXT_texture_snorm, 
    GL_MESA_texture_signed_rgba, GL_NV_texture_barrier, GL_ARB_robustness, 
    GL_ARB_transform_feedback2, GL_ARB_conservative_depth, 
    GL_ARB_texture_storage, GL_EXT_transform_feedback

glewIsSupported says the extension is available, glGetIntegerv says the number of samples is 4, as specified when getting the FBConfig. I have no idea of what is going on.

Edit: Here is the output of glXQueryExtensionsString: GLX_ARB_get_proc_address GLX_ARB_multisample GLX_EXT_import_context GLX_EXT_visual_info GLX_EXT_visual_rating GLX_MESA_copy_sub_buffer GLX_MESA_multithread_makecurrent GLX_MESA_swap_control GLX_OML_swap_method GLX_OML_sync_control GLX_SGI_make_current_read GLX_SGI_swap_control GLX_SGI_video_sync GLX_SGIS_multisample GLX_SGIX_fbconfig GLX_SGIX_pbuffer GLX_SGIX_visual_select_group GLX_EXT_texture_from_pixmap

Answer 1

FORGET ABOUT MSAA. On nouveau UNLESS you have very fresh MESA + kernel + X.org.

MSAA is requirement for OpenGL 3.0 and up. So it was not high on priority list, up until recently. You need Mesa 9.0 for that.

(If you want tips on what to do to get those needed software, pls post your hardware, and software - distro, kernel, mesa, xorg - versions).