vo_direct3d: support NV12 with shaders

Semi-important, because --hwdec=dxva2 outputs NV12, and we really don't
want people to end up with the "old" StretchRect method.

Unfortunately, I couldn't actually get it to work. It seems most D3D
drivers (including the wine D3D implementation) reject D3DFMT_A8L8,
and I could not find any other 2-channel 8 bit Direct3D 9 format. It
seems newer D3D APIs have DXGI_FORMAT_R8G8_UNORM, but there's no way
to get it in D3D9.

Still pushing this; maybe it actually works on some drivers.

1 files changed, 12 insertions, 9 deletions

diff --git a/video/out/d3d_shader_yuv.hlsl b/video/out/d3d_shader_yuv.hlsl
index 0e78554254..b74f42e6dc 100644
--- a/video/out/d3d_shader_yuv.hlsl
+++ b/video/out/d3d_shader_yuv.hlsl
@@ -1,5 +1,6 @@
 // Compile with:
-// fxc.exe /Tps_2_0 /Fhd3d_shader_yuv.h d3d_shader_yuv.hlsl /Vnd3d_shader_yuv
+// fxc.exe /Tps_2_0 -DUSE_420P=1 /Fhd3d_shader_420p.h d3d_shader_yuv.hlsl /Vnd3d_shader_420p
+// fxc.exe /Tps_2_0 -DUSE_NV12=1 /Fhd3d_shader_nv12.h d3d_shader_yuv.hlsl /Vnd3d_shader_nv12
 
 // Be careful with this shader. You can't use constant slots, since we don't
 // load the shader with D3DX. All uniform variables are mapped to hardcoded
@@ -11,19 +12,21 @@ sampler2D tex2 : register(s2);
 
 uniform float4x4 colormatrix : register(c0);
 
-float1 sample(sampler2D tex, float2 t)
-{
-    return tex2D(tex, t).x;
-}
-
 float4 main(float2 t0 : TEXCOORD0,
             float2 t1 : TEXCOORD1,
             float2 t2 : TEXCOORD2)
             : COLOR
 {
-    float4 c = float4(sample(tex0, t0),
-                      sample(tex1, t1),
-                      sample(tex2, t2),
+#ifdef USE_420P
+    float4 c = float4(tex2D(tex0, t0).x,
+                      tex2D(tex1, t1).x,
+                      tex2D(tex2, t2).x,
+                      1);
+#endif
+#ifdef USE_NV12
+    float4 c = float4(tex2D(tex0, t0).x,
+                      tex2D(tex1, t1).xz,
                       1);
+#endif
     return mul(c, colormatrix);
 }