vo_gl: fix 10 bit with Mesa drivers (Intel/Nouveau on Linux)

The GL_LUMINANCE16 texture format had only 8 bit precision on Mesa
based drivers. This caused heavy degradation of the image when playing
formats with more than 8 bits per pixel, such as 10 bit h264. Use
GL_R16 instead, which at least Mesa and Nvidia drivers actually
implement as 16 bit textures. Since sampling from this texture format
doesn't return anything meaningful in the other color components
(unlike luminance textures), the shader code has to be slightly
changed.

GL_R16 requires the GL_ARB_texture_rg extension. Check for it, and fall
back to the old texture format if it's not available.

The low precision of the GL_LUMINANCE16 format has just been fixed in
upstream Mesa, but it'll take a while before that fix is available in
distros.

1 files changed, 5 insertions, 2 deletions

diff --git a/libvo/vo_gl.c b/libvo/vo_gl.c
index e99ff4432f..b38a8d75e4 100644
--- a/libvo/vo_gl.c
+++ b/libvo/vo_gl.c
@@ -103,6 +103,7 @@ struct gl_priv {
     uint32_t image_d_width;
     uint32_t image_d_height;
     int many_fmts;
+    int have_texture_rg;
     int ati_hack;
     int force_pbo;
     int use_glFinish;
@@ -503,6 +504,7 @@ static void autodetectGlExtensions(struct vo *vo)
         if (extensions && strstr(extensions, "_pixel_buffer_object"))
             p->force_pbo = is_ati;
     }
+    p->have_texture_rg = extensions && strstr(extensions, "GL_ARB_texture_rg");
     if (p->use_rectangle == -1) {
         p->use_rectangle = 0;
         if (extensions) {
@@ -669,7 +671,8 @@ static int config(struct vo *vo, uint32_t width, uint32_t height,
     p->image_d_height = d_height;
     p->is_yuv = mp_get_chroma_shift(p->image_format, &xs, &ys, NULL) > 0;
     p->is_yuv |= (xs << 8) | (ys << 16);
-    glFindFormat(format, NULL, &p->texfmt, &p->gl_format, &p->gl_type);
+    glFindFormat(format, p->have_texture_rg, NULL, &p->texfmt, &p->gl_format,
+                 &p->gl_type);
 
     p->vo_flipped = !!(flags & VOFLAG_FLIPPING);
 
@@ -1207,7 +1210,7 @@ static int query_format(struct vo *vo, uint32_t format)
     if (!p->use_ycbcr && (format == IMGFMT_UYVY || format == IMGFMT_YVYU))
         return 0;
     if (p->many_fmts &&
-        glFindFormat(format, NULL, NULL, NULL, NULL))
+        glFindFormat(format, p->have_texture_rg, NULL, NULL, NULL, NULL))
         return caps;
     return 0;
 }