vo_direct3d: add hack for using 2 channel textures for 10 bit pixel formats

This actually applies to YUV formats with 9-16 bit depths. This hack is
disabled by default, and the VO will use 16 bit textures normally.

It can be enabled by passing the no16bit-textures option is passed to
vo_direct3d. Then the VO will use D3DFMT_A8L8 as texture formats for the
YUV plane (instead of D3DFMT_L16), and compute the sampled two color
values back into one.

In some cases it might be undesireable to use 16 bit texture formats. At
least some OpenGL drivers on Linux (Mesa + Intel) round values sampled
from 16 bit textures back into 8 bit, which loses 8 from 10 bit color
information when playing 10 bit formats. It is unknown whether there are
D3D9 drivers which do this, so this commit might be removed later.

1 files changed, 75 insertions, 5 deletions

diff --git a/libvo/vo_direct3d.c b/libvo/vo_direct3d.c
index 95bff26454..fdecee8c94 100644
--- a/libvo/vo_direct3d.c
+++ b/libvo/vo_direct3d.c
@@ -48,13 +48,15 @@
 
 // shaders generated by fxc.exe from d3d_shader_yuv.hlsl
 #include "d3d_shader_yuv.h"
+#include "d3d_shader_yuv_2ch.h"
 
 
 // TODO: beg someone to add this (there is already IMGFMT_Y8)
 // equals MAKEFOURCC('Y', '1', '6', ' ')
 #define IMGFMT_Y16 0x20363159
+#define IMGFMT_A8Y8 MAKEFOURCC('A', '8', 'Y', '8')
 
-#define IMGFMT_IS_Y(x) ((x) == IMGFMT_Y8 || (x) == IMGFMT_Y16)
+#define IMGFMT_IS_Y(x) ((x) == IMGFMT_Y8 || (x) == IMGFMT_Y16 || (x) == IMGFMT_A8Y8)
 #define IMGFMT_Y_DEPTH(x) ((x) == IMGFMT_Y8 ? 8 : 16)
 
 #define DEVTYPE D3DDEVTYPE_HAL
@@ -132,6 +134,7 @@ typedef struct d3d_priv {
     int opt_texture_memory;
     int opt_swap_discard;
     int opt_exact_backbuffer;
+    int opt_16bit_textures;
 
     struct vo *vo;
 
@@ -151,6 +154,7 @@ typedef struct d3d_priv {
                                 StretchRect */
     bool use_shaders;           /**< use shader for YUV color conversion
                                 (or possibly for RGB video equalizers) */
+    bool use_2ch_hack;          /**< 2 byte YUV formats use 2 channel hack */
 
     int plane_count;
     struct texplane planes[3];
@@ -190,6 +194,7 @@ typedef struct d3d_priv {
     int max_texture_height;         /**< from the device capabilities */
 
     D3DMATRIX d3d_colormatrix;
+    float d3d_depth_vector[4];
     struct mp_csp_details colorspace;
     struct mp_csp_equalizer video_eq;
 
@@ -224,6 +229,7 @@ static const struct fmt_entry fmt_table[] = {
     // grayscale (can be considered both packed and planar)
     {IMGFMT_Y8,    D3DFMT_L8},
     {IMGFMT_Y16,   D3DFMT_L16},
+    {IMGFMT_A8Y8,  D3DFMT_A8L8},
     {0},
 };
 
@@ -1013,6 +1019,9 @@ static uint32_t d3d_draw_frame(d3d_priv *priv)
             IDirect3DDevice9_SetPixelShaderConstantF(priv->d3d_device, 0,
                                                      &priv->d3d_colormatrix._11,
                                                      4);
+            IDirect3DDevice9_SetPixelShaderConstantF(priv->d3d_device, 5,
+                                                     priv->d3d_depth_vector,
+                                                     1);
         }
 
         IDirect3DDevice9_SetFVF(priv->d3d_device, D3DFVF_VIDEO_VERTEX);
@@ -1130,7 +1139,14 @@ static D3DFORMAT check_shader_conversion(d3d_priv *priv, uint32_t fmt)
     bool is_8bit = component_bits == 8;
     if (!is_8bit && priv->opt_only_8bit)
         return 0;
-    return check_format(priv, is_8bit ? IMGFMT_Y8 : IMGFMT_Y16, true);
+    int texfmt = IMGFMT_Y8;
+    if (!is_8bit) {
+        if (priv->opt_16bit_textures)
+            texfmt = IMGFMT_Y16;
+        else
+            texfmt = IMGFMT_A8Y8;
+    }
+    return check_format(priv, texfmt, true);
 }
 
 // Return if the image format can be used. If it can, decide which rendering
@@ -1165,6 +1181,7 @@ static bool init_rendering_mode(d3d_priv *priv, uint32_t fmt, bool initialize)
 
     priv->use_shaders = false;
     priv->use_textures = false;
+    priv->use_2ch_hack = false;
     priv->movie_src_fmt = 0;
     priv->pixel_shader_data = NULL;
     priv->plane_count = 0;
@@ -1209,7 +1226,14 @@ static bool init_rendering_mode(d3d_priv *priv, uint32_t fmt, bool initialize)
                     planes[n].clearval = get_chroma_clear_val(component_bits);
                 }
             }
-            priv->pixel_shader_data = d3d_shader_yuv;
+            if (shader_d3dfmt != D3DFMT_A8L8) {
+                priv->pixel_shader_data = d3d_shader_yuv;
+            } else {
+                mp_msg(MSGT_VO, MSGL_WARN, "<vo_direct3d>Using YUV 2ch hack.\n");
+
+                priv->pixel_shader_data = d3d_shader_yuv_2ch;
+                priv->use_2ch_hack = true;
+            }
         }
 
         for (n = 0; n < priv->plane_count; n++) {
@@ -1253,6 +1277,30 @@ static int query_format(d3d_priv *priv, uint32_t movie_fmt)
  *                                                                          *
  ****************************************************************************/
 
+static void get_2ch_depth_multiplier(int depth, float *out_f1, float *out_f2) {
+    // How to get these values:
+    //  The suffix i8 and i16 is for values with 8/16 bit fixed point numbers.
+    //  The suffix f is for float, ideally in the range 0.0-1.0.
+    //  c_i8 is a two component vector, sampled from a two channel texture.
+    //  (c_i8.x is the low byte, c_i8.y is the high byte)
+    //  r_f is the resulting color scalar value.
+    //
+    //  c_i8 = c_f * (2^8-1)
+    //  r_i16 = c_i8.x + c_i8.y * 2^8
+    //  r_f = r_i16 / (2^16-1)
+    //      = c_f.x * (2^8-1) / (2^16-1) + c_f.y * (2^8-1) * 2^8 / (2^16-1)
+    //      = c_f.x * ((2^8-1) / (2^16-1)) + c_f.y * (2^8 * ((2^8-1) / (2^16-1)))
+    // out = ((2^8-1) / (2^16-1),  2^8 * ((2^8-1) / (2^16-1)))
+    // The result color is r_f = dot(c_f, out).
+    // Same goes for other bit depth, such as 10 bit. Assuming (2^depth-1) is
+    // the maximum possible value at that depth, you have to scale the value
+    // r_i16 with it, the factor (2^16-1) in the formula above has to be
+    // replaced with (2^depth-1).
+    float factor = (float)((1 << 8) - 1) / (float)((1 << depth) - 1);
+    *out_f1 = factor;
+    *out_f2 = 256.0 * factor;
+}
+
 static void update_colorspace(d3d_priv *priv)
 {
     float coeff[3][4];
@@ -1260,8 +1308,19 @@ static void update_colorspace(d3d_priv *priv)
     mp_csp_copy_equalizer_values(&csp, &priv->video_eq);
 
     if (priv->use_shaders) {
-        csp.input_bits = priv->planes[0].bits_per_pixel;
-        csp.texture_bits = (csp.input_bits + 7) & ~7;
+        if (!priv->use_2ch_hack) {
+            csp.input_bits = priv->planes[0].bits_per_pixel;
+            csp.texture_bits = (csp.input_bits + 7) & ~7;
+        } else {
+            float f1, f2;
+            get_2ch_depth_multiplier(priv->planes[0].bits_per_pixel, &f1, &f2);
+            priv->d3d_depth_vector[0] = f1;
+            priv->d3d_depth_vector[1] = f2;
+            priv->d3d_depth_vector[2] = priv->d3d_depth_vector[3] = 0;
+            // no change
+            csp.input_bits = 8;
+            csp.texture_bits = 8;
+        }
     }
 
     mp_get_yuv2rgb_coeffs(&csp, coeff);
@@ -1314,6 +1373,14 @@ const char *options_help_text = "-vo direct3d command line help:\n"
 "        Might be slower too, as it must (?) clear every frame.\n"
 "    exact-backbuffer\n"
 "        Always resize the backbuffer to window size.\n"
+"    no16bit-textures\n"
+"        Don't use textures with a 16 bit color channel for YUV formats that\n"
+"        use more than 8 bits per component. Instead, use D3DFMT_A8L8 textures\n"
+"        and compute the values sampled from the 2 channels back into one.\n"
+"        Might be slower, since the shader becomes slightly more complicated.\n"
+"        Might work better, if your drivers either don't support D3DFMT_L16,\n"
+"        or if either the texture unit or the shaders don't operate in at least\n"
+"        16 bit precision.\n"
 "";
 
 /** @brief libvo Callback: Preinitialize the video card.
@@ -1331,6 +1398,8 @@ static int preinit_internal(struct vo *vo, const char *arg, bool allow_shaders)
     *priv = (d3d_priv) {
         .vo = vo,
 
+        .opt_16bit_textures = true,
+
         .colorspace = MP_CSP_DETAILS_DEFAULTS,
         .video_eq = { MP_CSP_EQ_CAPS_COLORMATRIX },
     };
@@ -1351,6 +1420,7 @@ static int preinit_internal(struct vo *vo, const char *arg, bool allow_shaders)
         {"texture-memory", OPT_ARG_INT, &priv->opt_texture_memory},
         {"swap-discard", OPT_ARG_BOOL, &priv->opt_swap_discard},
         {"exact-backbuffer", OPT_ARG_BOOL, &priv->opt_exact_backbuffer},
+        {"16bit-textures", OPT_ARG_BOOL, &priv->opt_16bit_textures},
         {NULL}
     };
     if (subopt_parse(arg, subopts) != 0) {