vo_direct3d: support NV12 with shaders

Semi-important, because --hwdec=dxva2 outputs NV12, and we really don't
want people to end up with the "old" StretchRect method.

Unfortunately, I couldn't actually get it to work. It seems most D3D
drivers (including the wine D3D implementation) reject D3DFMT_A8L8,
and I could not find any other 2-channel 8 bit Direct3D 9 format. It
seems newer D3D APIs have DXGI_FORMAT_R8G8_UNORM, but there's no way
to get it in D3D9.

Still pushing this; maybe it actually works on some drivers.

4 files changed, 359 insertions, 196 deletions

diff --git a/video/out/d3d_shader_yuv.h b/video/out/d3d_shader_420p.h
index 49ef753b4c..8f19825458 100644
--- a/video/out/d3d_shader_yuv.h
+++ b/video/out/d3d_shader_420p.h
@@ -1,142 +1,142 @@
-#if 0
-//
-// Generated by Microsoft (R) HLSL Shader Compiler 9.27.952.3022
-//
-//   fxc /Tps_2_0 /Fhz:\tmp\mplayer\libvo\d3d_shader_yuv.h
-//    z:\tmp\mplayer\libvo\d3d_shader_yuv.hlsl /Vnd3d_shader_yuv
-//
-//
-// Parameters:
-//
-//   float4x4 colormatrix;
-//   sampler2D tex0;
-//   sampler2D tex1;
-//   sampler2D tex2;
-//
-//
-// Registers:
-//
-//   Name         Reg   Size
-//   ------------ ----- ----
-//   colormatrix  c0       4
-//   tex0         s0       1
-//   tex1         s1       1
-//   tex2         s2       1
-//
-
-    ps_2_0
-    def c4, 1, 0, 0, 0
-    dcl t0.xy
-    dcl t1.xy
-    dcl t2.xy
-    dcl_2d s0
-    dcl_2d s1
-    dcl_2d s2
-    texld r0, t0, s0
-    texld r1, t1, s1
-    texld r2, t2, s2
-    mov r0.y, r1.x
-    mov r0.z, r2.x
-    mov r0.w, c4.x
-    dp4 r1.x, r0, c0
-    dp4 r1.y, r0, c1
-    dp4 r1.z, r0, c2
-    dp4 r1.w, r0, c3
-    mov oC0, r1
-
-// approximately 11 instruction slots used (3 texture, 8 arithmetic)
-#endif
-
-const BYTE d3d_shader_yuv[] =
-{
-      0,   2, 255, 255, 254, 255, 
-     67,   0,  67,  84,  65,  66, 
-     28,   0,   0,   0, 215,   0, 
-      0,   0,   0,   2, 255, 255, 
-      4,   0,   0,   0,  28,   0, 
-      0,   0,   0,   1,   0,   0, 
-    208,   0,   0,   0, 108,   0, 
-      0,   0,   2,   0,   0,   0, 
-      4,   0,   2,   0, 120,   0, 
-      0,   0,   0,   0,   0,   0, 
-    136,   0,   0,   0,   3,   0, 
-      0,   0,   1,   0,   2,   0, 
-    144,   0,   0,   0,   0,   0, 
-      0,   0, 160,   0,   0,   0, 
-      3,   0,   1,   0,   1,   0, 
-      6,   0, 168,   0,   0,   0, 
-      0,   0,   0,   0, 184,   0, 
-      0,   0,   3,   0,   2,   0, 
-      1,   0,  10,   0, 192,   0, 
-      0,   0,   0,   0,   0,   0, 
-     99, 111, 108, 111, 114, 109, 
-     97, 116, 114, 105, 120,   0, 
-      3,   0,   3,   0,   4,   0, 
-      4,   0,   1,   0,   0,   0, 
-      0,   0,   0,   0, 116, 101, 
-    120,  48,   0, 171, 171, 171, 
-      4,   0,  12,   0,   1,   0, 
-      1,   0,   1,   0,   0,   0, 
-      0,   0,   0,   0, 116, 101, 
-    120,  49,   0, 171, 171, 171, 
-      4,   0,  12,   0,   1,   0, 
-      1,   0,   1,   0,   0,   0, 
-      0,   0,   0,   0, 116, 101, 
-    120,  50,   0, 171, 171, 171, 
-      4,   0,  12,   0,   1,   0, 
-      1,   0,   1,   0,   0,   0, 
-      0,   0,   0,   0, 112, 115, 
-     95,  50,  95,  48,   0,  77, 
-    105,  99, 114, 111, 115, 111, 
-    102, 116,  32,  40,  82,  41, 
-     32,  72,  76,  83,  76,  32, 
-     83, 104,  97, 100, 101, 114, 
-     32,  67, 111, 109, 112, 105, 
-    108, 101, 114,  32,  57,  46, 
-     50,  55,  46,  57,  53,  50, 
-     46,  51,  48,  50,  50,   0, 
-     81,   0,   0,   5,   4,   0, 
-     15, 160,   0,   0, 128,  63, 
-      0,   0,   0,   0,   0,   0, 
-      0,   0,   0,   0,   0,   0, 
-     31,   0,   0,   2,   0,   0, 
-      0, 128,   0,   0,   3, 176, 
-     31,   0,   0,   2,   0,   0, 
-      0, 128,   1,   0,   3, 176, 
-     31,   0,   0,   2,   0,   0, 
-      0, 128,   2,   0,   3, 176, 
-     31,   0,   0,   2,   0,   0, 
-      0, 144,   0,   8,  15, 160, 
-     31,   0,   0,   2,   0,   0, 
-      0, 144,   1,   8,  15, 160, 
-     31,   0,   0,   2,   0,   0, 
-      0, 144,   2,   8,  15, 160, 
-     66,   0,   0,   3,   0,   0, 
-     15, 128,   0,   0, 228, 176, 
-      0,   8, 228, 160,  66,   0, 
-      0,   3,   1,   0,  15, 128, 
-      1,   0, 228, 176,   1,   8, 
-    228, 160,  66,   0,   0,   3, 
-      2,   0,  15, 128,   2,   0, 
-    228, 176,   2,   8, 228, 160, 
-      1,   0,   0,   2,   0,   0, 
-      2, 128,   1,   0,   0, 128, 
-      1,   0,   0,   2,   0,   0, 
-      4, 128,   2,   0,   0, 128, 
-      1,   0,   0,   2,   0,   0, 
-      8, 128,   4,   0,   0, 160, 
-      9,   0,   0,   3,   1,   0, 
-      1, 128,   0,   0, 228, 128, 
-      0,   0, 228, 160,   9,   0, 
-      0,   3,   1,   0,   2, 128, 
-      0,   0, 228, 128,   1,   0, 
-    228, 160,   9,   0,   0,   3, 
-      1,   0,   4, 128,   0,   0, 
-    228, 128,   2,   0, 228, 160, 
-      9,   0,   0,   3,   1,   0, 
-      8, 128,   0,   0, 228, 128, 
-      3,   0, 228, 160,   1,   0, 
-      0,   2,   0,   8,  15, 128, 
-      1,   0, 228, 128, 255, 255, 
-      0,   0
-};
+#if 0

+//

+// Generated by Microsoft (R) HLSL Shader Compiler 9.29.952.3111

+//

+//   fxc /Tps_2_0 -DUSE_420P=1 /Fhd3d_shader_420p.h d3d_shader_yuv.hlsl

+//    /Vnd3d_shader_420p

+//

+//

+// Parameters:

+//

+//   float4x4 colormatrix;

+//   sampler2D tex0;

+//   sampler2D tex1;

+//   sampler2D tex2;

+//

+//

+// Registers:

+//

+//   Name         Reg   Size

+//   ------------ ----- ----

+//   colormatrix  c0       4

+//   tex0         s0       1

+//   tex1         s1       1

+//   tex2         s2       1

+//

+

+    ps_2_0

+    def c4, 1, 0, 0, 0

+    dcl t0.xy

+    dcl t1.xy

+    dcl t2.xy

+    dcl_2d s0

+    dcl_2d s1

+    dcl_2d s2

+    texld r0, t0, s0

+    texld r1, t1, s1

+    texld r2, t2, s2

+    mov r0.y, r1.x

+    mov r0.z, r2.x

+    mov r0.w, c4.x

+    dp4 r1.x, r0, c0

+    dp4 r1.y, r0, c1

+    dp4 r1.z, r0, c2

+    dp4 r1.w, r0, c3

+    mov oC0, r1

+

+// approximately 11 instruction slots used (3 texture, 8 arithmetic)

+#endif

+

+const BYTE d3d_shader_420p[] =

+{

+      0,   2, 255, 255, 254, 255, 

+     67,   0,  67,  84,  65,  66, 

+     28,   0,   0,   0, 215,   0, 

+      0,   0,   0,   2, 255, 255, 

+      4,   0,   0,   0,  28,   0, 

+      0,   0,   0,   1,   0,   0, 

+    208,   0,   0,   0, 108,   0, 

+      0,   0,   2,   0,   0,   0, 

+      4,   0,   2,   0, 120,   0, 

+      0,   0,   0,   0,   0,   0, 

+    136,   0,   0,   0,   3,   0, 

+      0,   0,   1,   0,   2,   0, 

+    144,   0,   0,   0,   0,   0, 

+      0,   0, 160,   0,   0,   0, 

+      3,   0,   1,   0,   1,   0, 

+      6,   0, 168,   0,   0,   0, 

+      0,   0,   0,   0, 184,   0, 

+      0,   0,   3,   0,   2,   0, 

+      1,   0,  10,   0, 192,   0, 

+      0,   0,   0,   0,   0,   0, 

+     99, 111, 108, 111, 114, 109, 

+     97, 116, 114, 105, 120,   0, 

+      3,   0,   3,   0,   4,   0, 

+      4,   0,   1,   0,   0,   0, 

+      0,   0,   0,   0, 116, 101, 

+    120,  48,   0, 171, 171, 171, 

+      4,   0,  12,   0,   1,   0, 

+      1,   0,   1,   0,   0,   0, 

+      0,   0,   0,   0, 116, 101, 

+    120,  49,   0, 171, 171, 171, 

+      4,   0,  12,   0,   1,   0, 

+      1,   0,   1,   0,   0,   0, 

+      0,   0,   0,   0, 116, 101, 

+    120,  50,   0, 171, 171, 171, 

+      4,   0,  12,   0,   1,   0, 

+      1,   0,   1,   0,   0,   0, 

+      0,   0,   0,   0, 112, 115, 

+     95,  50,  95,  48,   0,  77, 

+    105,  99, 114, 111, 115, 111, 

+    102, 116,  32,  40,  82,  41, 

+     32,  72,  76,  83,  76,  32, 

+     83, 104,  97, 100, 101, 114, 

+     32,  67, 111, 109, 112, 105, 

+    108, 101, 114,  32,  57,  46, 

+     50,  57,  46,  57,  53,  50, 

+     46,  51,  49,  49,  49,   0, 

+     81,   0,   0,   5,   4,   0, 

+     15, 160,   0,   0, 128,  63, 

+      0,   0,   0,   0,   0,   0, 

+      0,   0,   0,   0,   0,   0, 

+     31,   0,   0,   2,   0,   0, 

+      0, 128,   0,   0,   3, 176, 

+     31,   0,   0,   2,   0,   0, 

+      0, 128,   1,   0,   3, 176, 

+     31,   0,   0,   2,   0,   0, 

+      0, 128,   2,   0,   3, 176, 

+     31,   0,   0,   2,   0,   0, 

+      0, 144,   0,   8,  15, 160, 

+     31,   0,   0,   2,   0,   0, 

+      0, 144,   1,   8,  15, 160, 

+     31,   0,   0,   2,   0,   0, 

+      0, 144,   2,   8,  15, 160, 

+     66,   0,   0,   3,   0,   0, 

+     15, 128,   0,   0, 228, 176, 

+      0,   8, 228, 160,  66,   0, 

+      0,   3,   1,   0,  15, 128, 

+      1,   0, 228, 176,   1,   8, 

+    228, 160,  66,   0,   0,   3, 

+      2,   0,  15, 128,   2,   0, 

+    228, 176,   2,   8, 228, 160, 

+      1,   0,   0,   2,   0,   0, 

+      2, 128,   1,   0,   0, 128, 

+      1,   0,   0,   2,   0,   0, 

+      4, 128,   2,   0,   0, 128, 

+      1,   0,   0,   2,   0,   0, 

+      8, 128,   4,   0,   0, 160, 

+      9,   0,   0,   3,   1,   0, 

+      1, 128,   0,   0, 228, 128, 

+      0,   0, 228, 160,   9,   0, 

+      0,   3,   1,   0,   2, 128, 

+      0,   0, 228, 128,   1,   0, 

+    228, 160,   9,   0,   0,   3, 

+      1,   0,   4, 128,   0,   0, 

+    228, 128,   2,   0, 228, 160, 

+      9,   0,   0,   3,   1,   0, 

+      8, 128,   0,   0, 228, 128, 

+      3,   0, 228, 160,   1,   0, 

+      0,   2,   0,   8,  15, 128, 

+      1,   0, 228, 128, 255, 255, 

+      0,   0

+};

diff --git a/video/out/d3d_shader_nv12.h b/video/out/d3d_shader_nv12.h
new file mode 100644
index 0000000000..74b592fa04
--- /dev/null
+++ b/video/out/d3d_shader_nv12.h
@@ -0,0 +1,123 @@
+#if 0

+//

+// Generated by Microsoft (R) HLSL Shader Compiler 9.29.952.3111

+//

+//   fxc /Tps_2_0 -DUSE_NV12=1 /Fhd3d_shader_nv12.h d3d_shader_yuv.hlsl

+//    /Vnd3d_shader_nv12

+//

+//

+// Parameters:

+//

+//   float4x4 colormatrix;

+//   sampler2D tex0;

+//   sampler2D tex1;

+//

+//

+// Registers:

+//

+//   Name         Reg   Size

+//   ------------ ----- ----

+//   colormatrix  c0       4

+//   tex0         s0       1

+//   tex1         s1       1

+//

+

+    ps_2_0

+    def c4, 1, 0, 0, 0

+    dcl t0.xy

+    dcl t1.xy

+    dcl_2d s0

+    dcl_2d s1

+    texld r0, t0, s0

+    texld r1, t1, s1

+    mov r0.y, r1.x

+    mov r0.z, r1.z

+    mov r0.w, c4.x

+    dp4 r1.x, r0, c0

+    dp4 r1.y, r0, c1

+    dp4 r1.z, r0, c2

+    dp4 r1.w, r0, c3

+    mov oC0, r1

+

+// approximately 10 instruction slots used (2 texture, 8 arithmetic)

+#endif

+

+const BYTE d3d_shader_nv12[] =

+{

+      0,   2, 255, 255, 254, 255, 

+     56,   0,  67,  84,  65,  66, 

+     28,   0,   0,   0, 171,   0, 

+      0,   0,   0,   2, 255, 255, 

+      3,   0,   0,   0,  28,   0, 

+      0,   0,   0,   1,   0,   0, 

+    164,   0,   0,   0,  88,   0, 

+      0,   0,   2,   0,   0,   0, 

+      4,   0,   2,   0, 100,   0, 

+      0,   0,   0,   0,   0,   0, 

+    116,   0,   0,   0,   3,   0, 

+      0,   0,   1,   0,   2,   0, 

+    124,   0,   0,   0,   0,   0, 

+      0,   0, 140,   0,   0,   0, 

+      3,   0,   1,   0,   1,   0, 

+      6,   0, 148,   0,   0,   0, 

+      0,   0,   0,   0,  99, 111, 

+    108, 111, 114, 109,  97, 116, 

+    114, 105, 120,   0,   3,   0, 

+      3,   0,   4,   0,   4,   0, 

+      1,   0,   0,   0,   0,   0, 

+      0,   0, 116, 101, 120,  48, 

+      0, 171, 171, 171,   4,   0, 

+     12,   0,   1,   0,   1,   0, 

+      1,   0,   0,   0,   0,   0, 

+      0,   0, 116, 101, 120,  49, 

+      0, 171, 171, 171,   4,   0, 

+     12,   0,   1,   0,   1,   0, 

+      1,   0,   0,   0,   0,   0, 

+      0,   0, 112, 115,  95,  50, 

+     95,  48,   0,  77, 105,  99, 

+    114, 111, 115, 111, 102, 116, 

+     32,  40,  82,  41,  32,  72, 

+     76,  83,  76,  32,  83, 104, 

+     97, 100, 101, 114,  32,  67, 

+    111, 109, 112, 105, 108, 101, 

+    114,  32,  57,  46,  50,  57, 

+     46,  57,  53,  50,  46,  51, 

+     49,  49,  49,   0,  81,   0, 

+      0,   5,   4,   0,  15, 160, 

+      0,   0, 128,  63,   0,   0, 

+      0,   0,   0,   0,   0,   0, 

+      0,   0,   0,   0,  31,   0, 

+      0,   2,   0,   0,   0, 128, 

+      0,   0,   3, 176,  31,   0, 

+      0,   2,   0,   0,   0, 128, 

+      1,   0,   3, 176,  31,   0, 

+      0,   2,   0,   0,   0, 144, 

+      0,   8,  15, 160,  31,   0, 

+      0,   2,   0,   0,   0, 144, 

+      1,   8,  15, 160,  66,   0, 

+      0,   3,   0,   0,  15, 128, 

+      0,   0, 228, 176,   0,   8, 

+    228, 160,  66,   0,   0,   3, 

+      1,   0,  15, 128,   1,   0, 

+    228, 176,   1,   8, 228, 160, 

+      1,   0,   0,   2,   0,   0, 

+      2, 128,   1,   0,   0, 128, 

+      1,   0,   0,   2,   0,   0, 

+      4, 128,   1,   0, 170, 128, 

+      1,   0,   0,   2,   0,   0, 

+      8, 128,   4,   0,   0, 160, 

+      9,   0,   0,   3,   1,   0, 

+      1, 128,   0,   0, 228, 128, 

+      0,   0, 228, 160,   9,   0, 

+      0,   3,   1,   0,   2, 128, 

+      0,   0, 228, 128,   1,   0, 

+    228, 160,   9,   0,   0,   3, 

+      1,   0,   4, 128,   0,   0, 

+    228, 128,   2,   0, 228, 160, 

+      9,   0,   0,   3,   1,   0, 

+      8, 128,   0,   0, 228, 128, 

+      3,   0, 228, 160,   1,   0, 

+      0,   2,   0,   8,  15, 128, 

+      1,   0, 228, 128, 255, 255, 

+      0,   0

+};

diff --git a/video/out/d3d_shader_yuv.hlsl b/video/out/d3d_shader_yuv.hlsl
index 0e78554254..b74f42e6dc 100644
--- a/video/out/d3d_shader_yuv.hlsl
+++ b/video/out/d3d_shader_yuv.hlsl
@@ -1,5 +1,6 @@
 // Compile with:
-// fxc.exe /Tps_2_0 /Fhd3d_shader_yuv.h d3d_shader_yuv.hlsl /Vnd3d_shader_yuv
+// fxc.exe /Tps_2_0 -DUSE_420P=1 /Fhd3d_shader_420p.h d3d_shader_yuv.hlsl /Vnd3d_shader_420p
+// fxc.exe /Tps_2_0 -DUSE_NV12=1 /Fhd3d_shader_nv12.h d3d_shader_yuv.hlsl /Vnd3d_shader_nv12
 
 // Be careful with this shader. You can't use constant slots, since we don't
 // load the shader with D3DX. All uniform variables are mapped to hardcoded
@@ -11,19 +12,21 @@ sampler2D tex2 : register(s2);
 
 uniform float4x4 colormatrix : register(c0);
 
-float1 sample(sampler2D tex, float2 t)
-{
-    return tex2D(tex, t).x;
-}
-
 float4 main(float2 t0 : TEXCOORD0,
             float2 t1 : TEXCOORD1,
             float2 t2 : TEXCOORD2)
             : COLOR
 {
-    float4 c = float4(sample(tex0, t0),
-                      sample(tex1, t1),
-                      sample(tex2, t2),
+#ifdef USE_420P
+    float4 c = float4(tex2D(tex0, t0).x,
+                      tex2D(tex1, t1).x,
+                      tex2D(tex2, t2).x,
+                      1);
+#endif
+#ifdef USE_NV12
+    float4 c = float4(tex2D(tex0, t0).x,
+                      tex2D(tex1, t1).xz,
                       1);
+#endif
     return mul(c, colormatrix);
 }
diff --git a/video/out/vo_direct3d.c b/video/out/vo_direct3d.c
index 88eb619b8a..bf24d164c8 100644
--- a/video/out/vo_direct3d.c
+++ b/video/out/vo_direct3d.c
@@ -43,8 +43,8 @@
 #include "bitmap_packer.h"
 
 // shaders generated by fxc.exe from d3d_shader_yuv.hlsl
-#include "d3d_shader_yuv.h"
-
+#include "d3d_shader_420p.h"
+#include "d3d_shader_nv12.h"
 
 #define IMGFMT_IS_Y(x) ((x) == IMGFMT_Y8 || (x) == IMGFMT_Y16)
 #define IMGFMT_Y_DEPTH(x) ((x) == IMGFMT_Y8 ? 8 : 16)
@@ -110,6 +110,13 @@ struct osdpart {
     struct bitmap_packer *packer;
 };
 
+enum shaders {
+    SHADER_NONE = 0,
+    SHADER_420P,
+    SHADER_NV12,
+    NUM_SHADERS
+};
+
 /* Global variables "priv" structure. I try to keep their count low.
  */
 typedef struct d3d_priv {
@@ -144,13 +151,14 @@ typedef struct d3d_priv {
     struct mp_image_params params;
     bool use_textures;          /**< use 3D texture rendering, instead of
                                 StretchRect */
-    bool use_shaders;           /**< use shader for YUV color conversion
+    int use_shaders;            /**< use shader for YUV color conversion, and
+                                the SHADER_ id (0 is SHADER_NONE)
                                 (or possibly for RGB video equalizers) */
 
     int plane_count;
     struct texplane planes[3];
 
-    IDirect3DPixelShader9 *pixel_shader;
+    IDirect3DPixelShader9 *pixel_shaders[NUM_SHADERS];
 
     D3DFORMAT movie_src_fmt;        /**< Movie colorspace format (depends on
                                     the movie's codec) */
@@ -216,6 +224,8 @@ static const struct fmt_entry fmt_table[] = {
     // grayscale (can be considered both packed and planar)
     {IMGFMT_Y8,    D3DFMT_L8},
     {IMGFMT_Y16,   D3DFMT_L16},
+    // 2 channels (needed for NV12 shader)
+    {IMGFMT_YA8,   D3DFMT_A8L8},
     {0},
 };
 
@@ -693,9 +703,11 @@ static void fill_d3d_presentparams(d3d_priv *priv,
 // Create a new backbuffer. Create or Reset the D3D device.
 static bool change_d3d_backbuffer(d3d_priv *priv)
 {
-    if (priv->pixel_shader)
-        IDirect3DPixelShader9_Release(priv->pixel_shader);
-    priv->pixel_shader = NULL;
+    for (int n = 0; n < NUM_SHADERS; n++) {
+        if (priv->pixel_shaders[n])
+            IDirect3DPixelShader9_Release(priv->pixel_shaders[n]);
+        priv->pixel_shaders[n] = NULL;
+    }
 
     int window_w = priv->vo->dwidth;
     int window_h = priv->vo->dheight;
@@ -740,12 +752,21 @@ static bool change_d3d_backbuffer(d3d_priv *priv)
                present_params.BackBufferWidth, present_params.BackBufferHeight,
                window_w, window_h);
 
-    if (FAILED(IDirect3DDevice9_CreatePixelShader(priv->d3d_device,
-               (DWORD *)d3d_shader_yuv, &priv->pixel_shader)))
-    {
-        priv->pixel_shader = NULL;
-        if (!priv->opt_disable_shaders)
-            MP_WARN(priv, "Shader could not be created - disabling shaders.\n");
+    const DWORD* shaders[NUM_SHADERS] = {
+        [SHADER_420P] = (DWORD *)d3d_shader_420p,
+        [SHADER_NV12] = (DWORD *)d3d_shader_nv12,
+    };
+
+    for (int n = 0; n < NUM_SHADERS; n++) {
+        if (!shaders[n])
+            continue;
+        if (FAILED(IDirect3DDevice9_CreatePixelShader(priv->d3d_device,
+                   shaders[n], &priv->pixel_shaders[n])))
+        {
+            priv->pixel_shaders[n] = NULL;
+            if (!priv->opt_disable_shaders)
+                MP_WARN(priv, "Shader could not be created - disabling shaders.\n");
+        }
     }
 
     return 1;
@@ -755,9 +776,11 @@ static void destroy_d3d(d3d_priv *priv)
 {
     destroy_d3d_surfaces(priv);
 
-    if (priv->pixel_shader)
-        IDirect3DPixelShader9_Release(priv->pixel_shader);
-    priv->pixel_shader = NULL;
+    for (int n = 0; n < NUM_SHADERS; n++) {
+        if (priv->pixel_shaders[n])
+            IDirect3DPixelShader9_Release(priv->pixel_shaders[n]);
+        priv->pixel_shaders[n] = NULL;
+    }
 
     if (priv->d3d_device)
         IDirect3DDevice9_Release(priv->d3d_device);
@@ -901,8 +924,9 @@ static uint32_t d3d_draw_frame(d3d_priv *priv)
             }
         }
 
-        if (priv->pixel_shader) {
-            IDirect3DDevice9_SetPixelShader(priv->d3d_device, priv->pixel_shader);
+        if (priv->use_shaders) {
+            IDirect3DDevice9_SetPixelShader(priv->d3d_device,
+                                    priv->pixel_shaders[priv->use_shaders]);
             IDirect3DDevice9_SetPixelShaderConstantF(priv->d3d_device, 0,
                                                      &priv->d3d_colormatrix._11,
                                                      4);
@@ -1011,24 +1035,36 @@ static D3DFORMAT check_format(d3d_priv *priv, uint32_t movie_fmt,
 }
 
 // Check whether YUV conversion with shaders can be done.
-// Returns the format the individual planes should use (or 0 on failure)
-static D3DFORMAT check_shader_conversion(d3d_priv *priv, uint32_t fmt)
+static int check_shader_conversion(d3d_priv *priv, uint32_t fmt,
+                                   D3DFORMAT shader_d3dfmts[MP_MAX_PLANES])
 {
     if (priv->opt_disable_shaders)
         return 0;
     struct mp_imgfmt_desc desc = mp_imgfmt_get_desc(fmt);
-    if (!(desc.flags & MP_IMGFLAG_YUV_P) || !(desc.flags & MP_IMGFLAG_NE))
-        return 0;
-    if (desc.num_planes != 3)
-        return 0;
-    int component_bits = desc.plane_bits;
-    if (component_bits < 8 || component_bits > 16)
-        return 0;
-    bool is_8bit = component_bits == 8;
-    if (!is_8bit && priv->opt_only_8bit)
-        return 0;
-    int texfmt = is_8bit ? IMGFMT_Y8 : IMGFMT_Y16;
-    return check_format(priv, texfmt, true);
+    if ((desc.flags & MP_IMGFLAG_YUV_P) && (desc.flags & MP_IMGFLAG_NE)) {
+        if (desc.num_planes > MP_MAX_PLANES)
+            return 0;
+        int component_bits = desc.plane_bits;
+        if (component_bits < 8 || component_bits > 16)
+            return 0;
+        bool is_8bit = component_bits == 8;
+        if (!is_8bit && priv->opt_only_8bit)
+            return 0;
+        int texfmt = is_8bit ? IMGFMT_Y8 : IMGFMT_Y16;
+        D3DFORMAT d3dfmt = check_format(priv, texfmt, true);
+        if (d3dfmt) {
+            for (int n = 0; n < desc.num_planes; n++)
+                shader_d3dfmts[n] = d3dfmt;
+            return SHADER_420P;
+        }
+    }
+    if (fmt == IMGFMT_NV12) {
+        shader_d3dfmts[0] = check_format(priv, IMGFMT_Y8, true);
+        shader_d3dfmts[1] = check_format(priv, IMGFMT_YA8, true);
+        if (shader_d3dfmts[0] && shader_d3dfmts[1])
+            return SHADER_NV12;
+    }
+    return 0;
 }
 
 // Return if the image format can be used. If it can, decide which rendering
@@ -1040,41 +1076,42 @@ static bool init_rendering_mode(d3d_priv *priv, uint32_t fmt, bool initialize)
     int n;
     int blit_d3dfmt = check_format(priv, fmt, false);
     int texture_d3dfmt = check_format(priv, fmt, true);
-    int shader_d3dfmt = check_shader_conversion(priv, fmt);
+    D3DFORMAT shader_d3dfmts[MP_MAX_PLANES] = {0};
+    int shader = check_shader_conversion(priv, fmt, shader_d3dfmts);
 
     if (priv->opt_disable_textures)
         texture_d3dfmt = 0;
-    if (priv->opt_disable_shaders || !priv->pixel_shader)
-        shader_d3dfmt = 0;
+    if (priv->opt_disable_shaders || !priv->pixel_shaders[shader])
+        shader = 0;
     if (priv->opt_disable_stretchrect)
         blit_d3dfmt = 0;
 
-    if (!(blit_d3dfmt || shader_d3dfmt || texture_d3dfmt))
+    if (!(blit_d3dfmt || shader || texture_d3dfmt))
         return false;
 
     MP_VERBOSE(priv, "Accepted rendering methods for "
-           "format='%s': StretchRect=%#x, Texture=%#x, Texture+Shader=%#x.\n",
-           vo_format_name(fmt), blit_d3dfmt, texture_d3dfmt, shader_d3dfmt);
+           "format='%s': StretchRect=%#x, Texture=%#x, Texture+Shader=%d.\n",
+           vo_format_name(fmt), blit_d3dfmt, texture_d3dfmt, shader);
 
     if (!initialize)
         return true;
 
     // initialization doesn't fail beyond this point
 
-    priv->use_shaders = false;
+    priv->use_shaders = 0;
     priv->use_textures = false;
     priv->movie_src_fmt = 0;
     priv->plane_count = 0;
     priv->image_format = fmt;
 
     if (blit_d3dfmt && priv->opt_prefer_stretchrect)
-        texture_d3dfmt = shader_d3dfmt = 0;
+        texture_d3dfmt = shader = 0;
 
     if (texture_d3dfmt) {
         priv->use_textures = true;
-    } else if (shader_d3dfmt) {
+    } else if (shader) {
         priv->use_textures = true;
-        priv->use_shaders = true;
+        priv->use_shaders = shader;
     } else {
         assert(!!blit_d3dfmt);
     }
@@ -1095,9 +1132,9 @@ static bool init_rendering_mode(d3d_priv *priv, uint32_t fmt, bool initialize)
             MP_VERBOSE(priv, "Using YUV shaders.\n");
 
             struct mp_imgfmt_desc desc = mp_imgfmt_get_desc(priv->image_format);
-            priv->plane_count = 3;
-            for (n = 0; n < 3; n++) {
-                planes[n].d3d_format = shader_d3dfmt;
+            priv->plane_count = desc.num_planes;
+            for (n = 0; n < priv->plane_count; n++) {
+                planes[n].d3d_format = shader_d3dfmts[n];
                 planes[n].bits_per_pixel = desc.plane_bits;
                 planes[n].shift_x = desc.xs[n];
                 planes[n].shift_y = desc.ys[n];