vo_direct3d: add hack for using 2 channel textures for 10 bit pixel formats

This actually applies to YUV formats with 9-16 bit depths. This hack is
disabled by default, and the VO will use 16 bit textures normally.

It can be enabled by passing the no16bit-textures option is passed to
vo_direct3d. Then the VO will use D3DFMT_A8L8 as texture formats for the
YUV plane (instead of D3DFMT_L16), and compute the sampled two color
values back into one.

In some cases it might be undesireable to use 16 bit texture formats. At
least some OpenGL drivers on Linux (Mesa + Intel) round values sampled
from 16 bit textures back into 8 bit, which loses 8 from 10 bit color
information when playing 10 bit formats. It is unknown whether there are
D3D9 drivers which do this, so this commit might be removed later.

1 files changed, 27 insertions, 3 deletions

diff --git a/libvo/d3d_shader_yuv.hlsl b/libvo/d3d_shader_yuv.hlsl
index 9d46e536fc..b17e257210 100644
--- a/libvo/d3d_shader_yuv.hlsl
+++ b/libvo/d3d_shader_yuv.hlsl
@@ -1,20 +1,44 @@
 // Compile with:
 // fxc.exe /Tps_2_0 /Fhd3d_shader_yuv.h d3d_shader_yuv.hlsl /Vnd3d_shader_yuv
+// fxc.exe /Tps_2_0 /Fhd3d_shader_yuv_2ch.h d3d_shader_yuv.hlsl /Vnd3d_shader_yuv_2ch /DUSE_2CH=1
+
+// Be careful with this shader. You can't use constant slots, since we don't
+// load the shader with D3DX. All uniform variables are mapped to hardcoded
+// constant slots.
 
 sampler2D tex0 : register(s0);
 sampler2D tex1 : register(s1);
 sampler2D tex2 : register(s2);
 
 uniform float4x4 colormatrix : register(c0);
+uniform float2 depth : register(c5);
+
+#ifdef USE_2CH
+
+float1 sample(sampler2D tex, float2 t)
+{
+    // Sample from A8L8 format as if we sampled a single value from L16.
+    // We compute the 2 channel values back into one.
+    return dot(tex2D(tex, t).xw, depth);
+}
+
+#else
+
+float1 sample(sampler2D tex, float2 t)
+{
+    return tex2D(tex, t).x;
+}
+
+#endif
 
 float4 main(float2 t0 : TEXCOORD0,
             float2 t1 : TEXCOORD1,
             float2 t2 : TEXCOORD2)
             : COLOR
 {
-    float4 c = float4(tex2D(tex0, t0).x,
-                      tex2D(tex1, t1).x,
-                      tex2D(tex2, t2).x,
+    float4 c = float4(sample(tex0, t0),
+                      sample(tex1, t1),
+                      sample(tex2, t2),
                       1);
     return mul(c, colormatrix);
 }