vo_gl: fix 10 bit with Mesa drivers (Intel/Nouveau on Linux)

The GL_LUMINANCE16 texture format had only 8 bit precision on Mesa
based drivers. This caused heavy degradation of the image when playing
formats with more than 8 bits per pixel, such as 10 bit h264. Use
GL_R16 instead, which at least Mesa and Nvidia drivers actually
implement as 16 bit textures. Since sampling from this texture format
doesn't return anything meaningful in the other color components
(unlike luminance textures), the shader code has to be slightly
changed.

GL_R16 requires the GL_ARB_texture_rg extension. Check for it, and fall
back to the old texture format if it's not available.

The low precision of the GL_LUMINANCE16 format has just been fixed in
upstream Mesa, but it'll take a while before that fix is available in
distros.

3 files changed, 18 insertions, 8 deletions

diff --git a/libvo/gl_common.c b/libvo/gl_common.c
index 47b0429ccf..626f7e4643 100644
--- a/libvo/gl_common.c
+++ b/libvo/gl_common.c
@@ -88,6 +88,7 @@ static const struct gl_name_map_struct gl_name_map[] = {
     MAP(GL_RGB10), MAP(GL_RGB12), MAP(GL_RGB16), MAP(GL_RGBA2),
     MAP(GL_RGBA4), MAP(GL_RGB5_A1), MAP(GL_RGBA8), MAP(GL_RGB10_A2),
     MAP(GL_RGBA12), MAP(GL_RGBA16), MAP(GL_LUMINANCE8), MAP(GL_LUMINANCE16),
+    MAP(GL_R16),
 
     // format
     MAP(GL_RGB), MAP(GL_RGBA), MAP(GL_RED), MAP(GL_GREEN), MAP(GL_BLUE),
@@ -146,7 +147,7 @@ const char *glValName(GLint value)
  * \return 1 if format is supported by OpenGL, 0 if not.
  * \ingroup gltexture
  */
-int glFindFormat(uint32_t fmt, int *bpp, GLint *gl_texfmt,
+int glFindFormat(uint32_t fmt, int have_texture_rg, int *bpp, GLint *gl_texfmt,
                  GLenum *gl_format, GLenum *gl_type)
 {
     int supported = 1;
@@ -190,9 +191,9 @@ int glFindFormat(uint32_t fmt, int *bpp, GLint *gl_texfmt,
         break;
     case IMGFMT_420P16:
         supported = 0; // no native YUV support
-        *gl_texfmt = GL_LUMINANCE16;
+        *gl_texfmt = have_texture_rg ? GL_R16 : GL_LUMINANCE16;
         *bpp = 16;
-        *gl_format = GL_LUMINANCE;
+        *gl_format = have_texture_rg ? GL_RED : GL_LUMINANCE;
         *gl_type = GL_UNSIGNED_SHORT;
         break;
     case IMGFMT_YV12:
@@ -592,6 +593,8 @@ int glFmt2bpp(GLenum format, GLenum type)
     case GL_RGBA:
     case GL_BGRA:
         return 4 * component_size;
+    case GL_RED:
+        return component_size;
     }
     return 0; // unknown
 }
@@ -853,7 +856,8 @@ static void gen_spline_lookup_tex(GL *gl, GLenum unit)
 }
 
 #define SAMPLE(dest, coord, texture) \
-    "TEX dest, " coord ", " texture ", $tex_type;\n"
+    "TEX textemp, " coord ", " texture ", $tex_type;\n" \
+    "MOV " dest ", textemp.r;\n"
 
 static const char *bilin_filt_template =
     SAMPLE("yuv.$out_comp","fragment.texcoord[$in_tex]","texture[$in_tex]");
diff --git a/libvo/gl_common.h b/libvo/gl_common.h
index d1d034b99a..f2e07e7e8b 100644
--- a/libvo/gl_common.h
+++ b/libvo/gl_common.h
@@ -218,6 +218,9 @@
 #ifndef GL_LUMINANCE16
 #define GL_LUMINANCE16 0x8042
 #endif
+#ifndef GL_R16
+#define GL_R16 0x822A
+#endif
 #ifndef GL_UNPACK_CLIENT_STORAGE_APPLE
 #define GL_UNPACK_CLIENT_STORAGE_APPLE 0x85B2
 #endif
@@ -245,8 +248,8 @@ void glAdjustAlignment(GL *gl, int stride);
 
 const char *glValName(GLint value);
 
-int glFindFormat(uint32_t format, int *bpp, GLint *gl_texfmt,
-                 GLenum *gl_format, GLenum *gl_type);
+int glFindFormat(uint32_t format, int have_texture_rg, int *bpp,
+                 GLint *gl_texfmt, GLenum *gl_format, GLenum *gl_type);
 int glFmt2bpp(GLenum format, GLenum type);
 void glCreateClearTex(GL *gl, GLenum target, GLenum fmt, GLenum format,
                       GLenum type, GLint filter, int w, int h,
diff --git a/libvo/vo_gl.c b/libvo/vo_gl.c
index e99ff4432f..b38a8d75e4 100644
--- a/libvo/vo_gl.c
+++ b/libvo/vo_gl.c
@@ -103,6 +103,7 @@ struct gl_priv {
     uint32_t image_d_width;
     uint32_t image_d_height;
     int many_fmts;
+    int have_texture_rg;
     int ati_hack;
     int force_pbo;
     int use_glFinish;
@@ -503,6 +504,7 @@ static void autodetectGlExtensions(struct vo *vo)
         if (extensions && strstr(extensions, "_pixel_buffer_object"))
             p->force_pbo = is_ati;
     }
+    p->have_texture_rg = extensions && strstr(extensions, "GL_ARB_texture_rg");
     if (p->use_rectangle == -1) {
         p->use_rectangle = 0;
         if (extensions) {
@@ -669,7 +671,8 @@ static int config(struct vo *vo, uint32_t width, uint32_t height,
     p->image_d_height = d_height;
     p->is_yuv = mp_get_chroma_shift(p->image_format, &xs, &ys, NULL) > 0;
     p->is_yuv |= (xs << 8) | (ys << 16);
-    glFindFormat(format, NULL, &p->texfmt, &p->gl_format, &p->gl_type);
+    glFindFormat(format, p->have_texture_rg, NULL, &p->texfmt, &p->gl_format,
+                 &p->gl_type);
 
     p->vo_flipped = !!(flags & VOFLAG_FLIPPING);
 
@@ -1207,7 +1210,7 @@ static int query_format(struct vo *vo, uint32_t format)
     if (!p->use_ycbcr && (format == IMGFMT_UYVY || format == IMGFMT_YVYU))
         return 0;
     if (p->many_fmts &&
-        glFindFormat(format, NULL, NULL, NULL, NULL))
+        glFindFormat(format, p->have_texture_rg, NULL, NULL, NULL, NULL))
         return caps;
     return 0;
 }