vo_direct3d: implement YUV conversion using shaders

Use the 3D rendering functions and shaders to render the video frame. This
is very similar to vo_gl. Most planar formats with varying chroma shifts
and bit depths are supported (including 10 bit), as well as some RGB
formats. The old code that used D3D's StretchRect function is still
available, but will by default be used with the least priority.

Also implement video equalizers and colorspace selection. These use the
same code as vo_gl and vo_vdpau, and are available only if shader YUV
conversion is active.

The rendering is extremely simple and naive, there are no filters etc.

Since compiling shaders seems to require the 500 MB DirectX SDK, all
shaders come in pre-compiled form, and there is no make rule to compile
them. mplayer should be compilable without 500 MB of garbage.
The main problem is that compiling shaders within an application seems to
require d3dx9_*.dll, which isn't installed by default.

Add an option ("disable-texture-align") that allows making the video
textures exactly the same size as the source video. The code used to align
the OSD texture size on 16 for unknown reasons, but since this was perhaps
a good reason, this behavior is kept for video textures as well. (There is
a chance that the alignment improves performance and stability with some
drivers.) Passing this option disables this behavior. It is useful for
reducing texture filtering artifacts at the bottom/right borders.

3 files changed, 842 insertions, 192 deletions

diff --git a/libvo/d3d_shader_yuv.h b/libvo/d3d_shader_yuv.h
new file mode 100644
index 0000000000..49ef753b4c
--- /dev/null
+++ b/libvo/d3d_shader_yuv.h
@@ -0,0 +1,142 @@
+#if 0
+//
+// Generated by Microsoft (R) HLSL Shader Compiler 9.27.952.3022
+//
+//   fxc /Tps_2_0 /Fhz:\tmp\mplayer\libvo\d3d_shader_yuv.h
+//    z:\tmp\mplayer\libvo\d3d_shader_yuv.hlsl /Vnd3d_shader_yuv
+//
+//
+// Parameters:
+//
+//   float4x4 colormatrix;
+//   sampler2D tex0;
+//   sampler2D tex1;
+//   sampler2D tex2;
+//
+//
+// Registers:
+//
+//   Name         Reg   Size
+//   ------------ ----- ----
+//   colormatrix  c0       4
+//   tex0         s0       1
+//   tex1         s1       1
+//   tex2         s2       1
+//
+
+    ps_2_0
+    def c4, 1, 0, 0, 0
+    dcl t0.xy
+    dcl t1.xy
+    dcl t2.xy
+    dcl_2d s0
+    dcl_2d s1
+    dcl_2d s2
+    texld r0, t0, s0
+    texld r1, t1, s1
+    texld r2, t2, s2
+    mov r0.y, r1.x
+    mov r0.z, r2.x
+    mov r0.w, c4.x
+    dp4 r1.x, r0, c0
+    dp4 r1.y, r0, c1
+    dp4 r1.z, r0, c2
+    dp4 r1.w, r0, c3
+    mov oC0, r1
+
+// approximately 11 instruction slots used (3 texture, 8 arithmetic)
+#endif
+
+const BYTE d3d_shader_yuv[] =
+{
+      0,   2, 255, 255, 254, 255, 
+     67,   0,  67,  84,  65,  66, 
+     28,   0,   0,   0, 215,   0, 
+      0,   0,   0,   2, 255, 255, 
+      4,   0,   0,   0,  28,   0, 
+      0,   0,   0,   1,   0,   0, 
+    208,   0,   0,   0, 108,   0, 
+      0,   0,   2,   0,   0,   0, 
+      4,   0,   2,   0, 120,   0, 
+      0,   0,   0,   0,   0,   0, 
+    136,   0,   0,   0,   3,   0, 
+      0,   0,   1,   0,   2,   0, 
+    144,   0,   0,   0,   0,   0, 
+      0,   0, 160,   0,   0,   0, 
+      3,   0,   1,   0,   1,   0, 
+      6,   0, 168,   0,   0,   0, 
+      0,   0,   0,   0, 184,   0, 
+      0,   0,   3,   0,   2,   0, 
+      1,   0,  10,   0, 192,   0, 
+      0,   0,   0,   0,   0,   0, 
+     99, 111, 108, 111, 114, 109, 
+     97, 116, 114, 105, 120,   0, 
+      3,   0,   3,   0,   4,   0, 
+      4,   0,   1,   0,   0,   0, 
+      0,   0,   0,   0, 116, 101, 
+    120,  48,   0, 171, 171, 171, 
+      4,   0,  12,   0,   1,   0, 
+      1,   0,   1,   0,   0,   0, 
+      0,   0,   0,   0, 116, 101, 
+    120,  49,   0, 171, 171, 171, 
+      4,   0,  12,   0,   1,   0, 
+      1,   0,   1,   0,   0,   0, 
+      0,   0,   0,   0, 116, 101, 
+    120,  50,   0, 171, 171, 171, 
+      4,   0,  12,   0,   1,   0, 
+      1,   0,   1,   0,   0,   0, 
+      0,   0,   0,   0, 112, 115, 
+     95,  50,  95,  48,   0,  77, 
+    105,  99, 114, 111, 115, 111, 
+    102, 116,  32,  40,  82,  41, 
+     32,  72,  76,  83,  76,  32, 
+     83, 104,  97, 100, 101, 114, 
+     32,  67, 111, 109, 112, 105, 
+    108, 101, 114,  32,  57,  46, 
+     50,  55,  46,  57,  53,  50, 
+     46,  51,  48,  50,  50,   0, 
+     81,   0,   0,   5,   4,   0, 
+     15, 160,   0,   0, 128,  63, 
+      0,   0,   0,   0,   0,   0, 
+      0,   0,   0,   0,   0,   0, 
+     31,   0,   0,   2,   0,   0, 
+      0, 128,   0,   0,   3, 176, 
+     31,   0,   0,   2,   0,   0, 
+      0, 128,   1,   0,   3, 176, 
+     31,   0,   0,   2,   0,   0, 
+      0, 128,   2,   0,   3, 176, 
+     31,   0,   0,   2,   0,   0, 
+      0, 144,   0,   8,  15, 160, 
+     31,   0,   0,   2,   0,   0, 
+      0, 144,   1,   8,  15, 160, 
+     31,   0,   0,   2,   0,   0, 
+      0, 144,   2,   8,  15, 160, 
+     66,   0,   0,   3,   0,   0, 
+     15, 128,   0,   0, 228, 176, 
+      0,   8, 228, 160,  66,   0, 
+      0,   3,   1,   0,  15, 128, 
+      1,   0, 228, 176,   1,   8, 
+    228, 160,  66,   0,   0,   3, 
+      2,   0,  15, 128,   2,   0, 
+    228, 176,   2,   8, 228, 160, 
+      1,   0,   0,   2,   0,   0, 
+      2, 128,   1,   0,   0, 128, 
+      1,   0,   0,   2,   0,   0, 
+      4, 128,   2,   0,   0, 128, 
+      1,   0,   0,   2,   0,   0, 
+      8, 128,   4,   0,   0, 160, 
+      9,   0,   0,   3,   1,   0, 
+      1, 128,   0,   0, 228, 128, 
+      0,   0, 228, 160,   9,   0, 
+      0,   3,   1,   0,   2, 128, 
+      0,   0, 228, 128,   1,   0, 
+    228, 160,   9,   0,   0,   3, 
+      1,   0,   4, 128,   0,   0, 
+    228, 128,   2,   0, 228, 160, 
+      9,   0,   0,   3,   1,   0, 
+      8, 128,   0,   0, 228, 128, 
+      3,   0, 228, 160,   1,   0, 
+      0,   2,   0,   8,  15, 128, 
+      1,   0, 228, 128, 255, 255, 
+      0,   0
+};
diff --git a/libvo/d3d_shader_yuv.hlsl b/libvo/d3d_shader_yuv.hlsl
new file mode 100644
index 0000000000..9d46e536fc
--- /dev/null
+++ b/libvo/d3d_shader_yuv.hlsl
@@ -0,0 +1,20 @@
+// Compile with:
+// fxc.exe /Tps_2_0 /Fhd3d_shader_yuv.h d3d_shader_yuv.hlsl /Vnd3d_shader_yuv
+
+sampler2D tex0 : register(s0);
+sampler2D tex1 : register(s1);
+sampler2D tex2 : register(s2);
+
+uniform float4x4 colormatrix : register(c0);
+
+float4 main(float2 t0 : TEXCOORD0,
+            float2 t1 : TEXCOORD1,
+            float2 t2 : TEXCOORD2)
+            : COLOR
+{
+    float4 c = float4(tex2D(tex0, t0).x,
+                      tex2D(tex1, t1).x,
+                      tex2D(tex2, t2).x,
+                      1);
+    return mul(c, colormatrix);
+}
diff --git a/libvo/vo_direct3d.c b/libvo/vo_direct3d.c
index a1124f5412..79c91cbea3 100644
--- a/libvo/vo_direct3d.c
+++ b/libvo/vo_direct3d.c
@@ -27,9 +27,13 @@
 #include <assert.h>
 #include <d3d9.h>
 #include "config.h"
+#include "options.h"
+#include "subopt-helper.h"
 #include "talloc.h"
 #include "video_out.h"
 #include "video_out_internal.h"
+#include "csputils.h"
+#include "libmpcodecs/img_format.h"
 #include "fastmemcpy.h"
 #include "mp_msg.h"
 #include "aspect.h"
@@ -39,18 +43,26 @@
 #include "sub/sub.h"
 #include "eosd_packer.h"
 
+// shaders generated by fxc.exe from d3d_shader_yuv.hlsl
+#include "d3d_shader_yuv.h"
+
 static const vo_info_t info =
 {
     "Direct3D 9 Renderer",
     "direct3d",
-    "Georgi Petrov (gogothebee) <gogothebee@gmail.com>",
+    "Georgi Petrov (gogothebee) <gogothebee@gmail.com> and others",
     ""
 };
 
-// texture format for EOSD
-// 0: use D3DFMT_A8L8
-// 1: use D3DFMT_A8 (doesn't work with wine)
-#define USE_A8 0
+// TODO: beg someone to add this (there is already IMGFMT_Y8)
+// equals MAKEFOURCC('Y', '1', '6', ' ')
+#define IMGFMT_Y16 0x20363159
+
+#define IMGFMT_IS_Y(x) ((x) == IMGFMT_Y8 || (x) == IMGFMT_Y16)
+#define IMGFMT_Y_DEPTH(x) ((x) == IMGFMT_Y8 ? 8 : 16)
+
+#define DEVTYPE D3DDEVTYPE_HAL
+//#define DEVTYPE D3DDEVTYPE_REF
 
 /*
  * Link essential libvo functions: preinit, config, control, draw_frame,
@@ -74,6 +86,14 @@ typedef struct {
     float tu, tv;
 } vertex_eosd;
 
+#define D3DFVF_VIDEO_VERTEX (D3DFVF_XYZ | D3DFVF_TEX3)
+
+typedef struct {
+    float x, y, z;
+    // pairs of texture coordinates for up to 3 planes
+    float t[3][2];
+} vertex_video;
+
 struct d3dtex {
     // user-requested size
     int w, h;
@@ -91,9 +111,32 @@ struct d3dtex {
     IDirect3DTexture9 *device;
 };
 
+struct texplane {
+    int bytes_per_pixel;
+    int bits_per_pixel;
+    // chroma shifts
+    // e.g. get the plane's width in pixels with (priv->src_width >> shift_x)
+    int shift_x, shift_y;
+    D3DFORMAT d3d_format;
+    struct d3dtex texture;
+    // temporary locking during uploading the frame (e.g. for draw_slice)
+    D3DLOCKED_RECT locked_rect;
+    // value used to clear the image with memset (YUV chroma planes do not use
+    // the value 0 for this)
+    uint8_t clearval;
+};
+
 /* Global variables "priv" structure. I try to keep their count low.
  */
 static struct global_priv {
+    int opt_prefer_stretchrect;
+    int opt_disable_textures;
+    int opt_disable_stretchrect;
+    int opt_disable_shaders;
+    int opt_only_8bit;
+    int opt_disable_eosd;
+    int opt_disable_texture_align;
+
     int is_paused;              /**< 1 = Movie is paused,
                                 0 = Movie is not paused */
     int is_clear_needed;        /**< 1 = Clear the backbuffer before StretchRect
@@ -107,6 +150,17 @@ static struct global_priv {
     int src_height;             /**< Source (movie) heigth */
     int border_x;               /**< horizontal border value for OSD */
     int border_y;               /**< vertical border value for OSD */
+    int image_format;           /**< mplayer image format */
+    bool use_textures;          /**< use 3D texture rendering, instead of
+                                StretchRect */
+    bool use_shaders;           /**< use shader for YUV color conversion
+                                (or possibly for RGB video equalizers) */
+
+    int plane_count;
+    struct texplane planes[3];
+
+    IDirect3DPixelShader9 *pixel_shader;
+    const BYTE *pixel_shader_data;
 
     D3DFORMAT movie_src_fmt;        /**< Movie colorspace format (depends on
                                     the movie's codec) */
@@ -118,13 +172,14 @@ static struct global_priv {
 
     LPDIRECT3D9        d3d_handle;  /**< Direct3D Handle */
     LPDIRECT3DDEVICE9  d3d_device;  /**< The Direct3D Adapter */
+    bool d3d_in_scene;              /**< BeginScene was called, EndScene not */
     IDirect3DSurface9 *d3d_surface; /**< Offscreen Direct3D Surface. MPlayer
                                     renders inside it. Uses colorspace
                                     priv->movie_src_fmt */
     struct d3dtex texture_osd;      /**< RGBA */
     IDirect3DSurface9 *d3d_backbuf; /**< Video card's back buffer (used to
                                     display next frame) */
-    struct d3dtex texture_eosd;     /**< A8 or A8L8 (see USE_A8) */
+    struct d3dtex texture_eosd;     /**< A8 */
     int cur_backbuf_width;          /**< Current backbuffer width */
     int cur_backbuf_height;         /**< Current backbuffer height */
     int is_osd_populated;           /**< 1 = OSD texture has something to display,
@@ -138,34 +193,44 @@ static struct global_priv {
     int max_texture_width;          /**< from the device capabilities */
     int max_texture_height;         /**< from the device capabilities */
 
+    D3DMATRIX d3d_colormatrix;
+    struct mp_csp_details colorspace;
+    struct mp_csp_equalizer video_eq;
+
     struct eosd_packer *eosd;       /**< EOSD packer (image positions etc.) */
     vertex_eosd *eosd_vb;           /**< temporary memory for D3D when rendering EOSD */
 } *priv;
 
-typedef struct {
+struct fmt_entry {
     const unsigned int  mplayer_fmt;   /**< Given by MPlayer */
     const D3DFORMAT     fourcc;        /**< Required by D3D's test function */
-} struct_fmt_table;
+};
 
 /* Map table from reported MPlayer format to the required
    fourcc. This is needed to perform the format query. */
 
-static const struct_fmt_table fmt_table[] = {
+static const struct fmt_entry fmt_table[] = {
+    // planar YUV
     {IMGFMT_YV12,  MAKEFOURCC('Y','V','1','2')},
     {IMGFMT_I420,  MAKEFOURCC('I','4','2','0')},
     {IMGFMT_IYUV,  MAKEFOURCC('I','Y','U','V')},
     {IMGFMT_YVU9,  MAKEFOURCC('Y','V','U','9')},
+    // packed YUV
     {IMGFMT_YUY2,  D3DFMT_YUY2},
     {IMGFMT_UYVY,  D3DFMT_UYVY},
+    // packed RGB
     {IMGFMT_BGR32, D3DFMT_X8R8G8B8},
     {IMGFMT_RGB32, D3DFMT_X8B8G8R8},
     {IMGFMT_BGR24, D3DFMT_R8G8B8}, //untested
     {IMGFMT_BGR16, D3DFMT_R5G6B5},
     {IMGFMT_BGR15, D3DFMT_X1R5G5B5},
     {IMGFMT_BGR8 , D3DFMT_R3G3B2}, //untested
+    // grayscale (can be considered both packed and planar)
+    {IMGFMT_Y8,    D3DFMT_L8},
+    {IMGFMT_Y16,   D3DFMT_L16},
+    {0},
 };
 
-#define DISPLAY_FORMAT_TABLE_ENTRIES (sizeof(fmt_table) / sizeof(fmt_table[0]))
 
 typedef enum back_buffer_action {
     BACKBUFFER_CREATE,
@@ -174,7 +239,28 @@ typedef enum back_buffer_action {
 
 static void generate_eosd(mp_eosd_images_t *);
 static void draw_eosd(void);
+static void update_colorspace(void);
+static void d3d_clear_video_textures(void);
+static int resize_d3d(void);
 
+static void d3d_matrix_identity(D3DMATRIX *m)
+{
+    memset(m, 0, sizeof(D3DMATRIX));
+    m->_11 = m->_22 = m->_33 = m->_44 = 1.0f;
+}
+
+static void d3d_matrix_ortho(D3DMATRIX *m, float left, float right,
+                             float bottom, float top)
+{
+    d3d_matrix_identity(m);
+    m->_11 = 2.0f / (right - left);
+    m->_22 = 2.0f / (top - bottom);
+    m->_33 = 1.0f;
+    m->_41 = -(right + left) / (right - left);
+    m->_42 = -(top + bottom) / (top - bottom);
+    m->_43 = 0;
+    m->_44 = 1.0f;
+}
 
 /****************************************************************************
  *                                                                          *
@@ -186,6 +272,18 @@ static void draw_eosd(void);
  *                                                                          *
  ****************************************************************************/
 
+static bool d3d_begin_scene(void)
+{
+    if (!priv->d3d_in_scene) {
+        if (FAILED(IDirect3DDevice9_BeginScene(priv->d3d_device))) {
+            mp_msg(MSGT_VO, MSGL_ERR, "<vo_direct3d>BeginScene failed.\n");
+            return false;
+        }
+        priv->d3d_in_scene = true;
+    }
+    return true;
+}
+
 /** @brief Calculate scaled fullscreen movie rectangle with
  *  preserved aspect ratio.
  */
@@ -241,9 +339,10 @@ static void d3d_fix_texture_size(int *width, int *height)
         /* device only supports square textures */
         tex_width = tex_height = tex_width > tex_height ? tex_width : tex_height;
     // better round up to a multiple of 16
-    // (xxx: why???)
-    tex_width  = (tex_width  + 15) & ~15;
-    tex_height = (tex_height + 15) & ~15;
+    if (!priv->opt_disable_texture_align) {
+        tex_width  = (tex_width  + 15) & ~15;
+        tex_height = (tex_height + 15) & ~15;
+    }
 
     *width = tex_width;
     *height = tex_height;
@@ -304,35 +403,69 @@ error_exit:
 
 static IDirect3DBaseTexture9 *d3dtex_get_render_texture(struct d3dtex *tex)
 {
-    return (IDirect3DBaseTexture9 *)
-                (priv->device_texture_sys ? tex->system : tex->device);
+    return (IDirect3DBaseTexture9 *)(tex->device ? tex->device : tex->system);
 }
 
 // Copy system texture contents to device texture.
 static bool d3dtex_update(struct d3dtex *tex)
 {
-    if (priv->device_texture_sys)
+    if (!tex->device)
         return true;
     return !FAILED(IDirect3DDevice9_UpdateTexture(priv->d3d_device,
                    (IDirect3DBaseTexture9 *)tex->system,
                    (IDirect3DBaseTexture9 *)tex->device));
 }
 
-/** @brief Destroy D3D Offscreen and Backbuffer surfaces.
- */
-static void destroy_d3d_surfaces(void)
+static void d3d_unlock_video_objects(void)
 {
-    mp_msg(MSGT_VO, MSGL_V, "<vo_direct3d>destroy_d3d_surfaces called.\n");
-    /* Let's destroy the old (if any) D3D Surfaces */
+    bool any_failed = false;
 
-    if (priv->locked_rect.pBits)
-        IDirect3DSurface9_UnlockRect(priv->d3d_surface);
+    if (priv->locked_rect.pBits) {
+        if (FAILED(IDirect3DSurface9_UnlockRect(priv->d3d_surface)))
+            any_failed = true;
+    }
     priv->locked_rect.pBits = NULL;
 
+    for (int n = 0; n < priv->plane_count; n++) {
+        struct texplane *plane = &priv->planes[n];
+        if (plane->locked_rect.pBits) {
+            if (FAILED(IDirect3DTexture9_UnlockRect(plane->texture.system, 0)))
+                any_failed = true;
+        }
+        plane->locked_rect.pBits = NULL;
+    }
+
+    if (any_failed)
+        mp_msg(MSGT_VO, MSGL_V, "<vo_direct3d>Unlocking video objects failed.\n");
+}
+
+// Free video surface/textures, shaders, etc.
+static void d3d_destroy_video_objects(void)
+{
+    d3d_unlock_video_objects();
+
     if (priv->d3d_surface)
         IDirect3DSurface9_Release(priv->d3d_surface);
     priv->d3d_surface = NULL;
 
+    for (int n = 0; n < priv->plane_count; n++) {
+        d3dtex_release(&priv->planes[n].texture);
+    }
+
+    if (priv->pixel_shader)
+        IDirect3DPixelShader9_Release(priv->pixel_shader);
+    priv->pixel_shader = NULL;
+}
+
+/** @brief Destroy D3D Offscreen and Backbuffer surfaces.
+ */
+static void destroy_d3d_surfaces(void)
+{
+    mp_msg(MSGT_VO, MSGL_V, "<vo_direct3d>destroy_d3d_surfaces called.\n");
+    /* Let's destroy the old (if any) D3D Surfaces */
+
+    d3d_destroy_video_objects();
+
     d3dtex_release(&priv->texture_osd);
 
     if (priv->d3d_backbuf)
@@ -345,6 +478,102 @@ static void destroy_d3d_surfaces(void)
         eosd_packer_reinit(priv->eosd, 0, 0);
 }
 
+// Allocate video surface or textures, and create shaders if needed.
+static bool d3d_configure_video_objects(void)
+{
+    int n;
+    bool need_clear = false;
+
+    assert(priv->image_format != 0);
+
+    if (priv->use_textures) {
+        for (n = 0; n < priv->plane_count; n++) {
+            struct texplane *plane = &priv->planes[n];
+
+            if (!plane->texture.system) {
+                if (!d3dtex_allocate(&plane->texture,
+                                     plane->d3d_format,
+                                     priv->src_width >> plane->shift_x,
+                                     priv->src_height >> plane->shift_y))
+                {
+                    mp_msg(MSGT_VO, MSGL_ERR, "<vo_direct3d>Allocating plane %d"
+                           " failed.\n", n);
+                    return false;
+                }
+
+                mp_msg(MSGT_VO, MSGL_V, "<vo_direct3d>Allocated plane %d:"
+                       " %d bit, shift=%d/%d size=%d/%d (%d/%d).\n", n,
+                       plane->bits_per_pixel,
+                       plane->shift_x, plane->shift_y,
+                       plane->texture.w, plane->texture.h,
+                       plane->texture.tex_w, plane->texture.tex_h);
+
+                need_clear = true;
+            }
+        }
+
+        if (need_clear)
+            d3d_clear_video_textures();
+
+        if (priv->pixel_shader_data) {
+            if (!priv->pixel_shader &&
+                FAILED(IDirect3DDevice9_CreatePixelShader(priv->d3d_device,
+                    (DWORD *)priv->pixel_shader_data, &priv->pixel_shader)))
+            {
+                mp_msg(MSGT_VO, MSGL_ERR, "<vo_direct3d>Failed to create "
+                       "YUV conversion pixel shader.\n");
+                return false;
+            }
+        }
+
+    } else {
+
+        if (!priv->d3d_surface &&
+            FAILED(IDirect3DDevice9_CreateOffscreenPlainSurface(
+                priv->d3d_device, priv->src_width, priv->src_height,
+                priv->movie_src_fmt, D3DPOOL_DEFAULT, &priv->d3d_surface, NULL))) {
+            mp_msg(MSGT_VO, MSGL_ERR,
+                "<vo_direct3d>Allocating offscreen surface failed.\n");
+            return false;
+        }
+    }
+
+    return true;
+}
+
+static bool d3d_lock_video_textures(void)
+{
+    for (int n = 0; n < priv->plane_count; n++) {
+        struct texplane *plane = &priv->planes[n];
+
+        if (!plane->locked_rect.pBits) {
+            if (FAILED(IDirect3DTexture9_LockRect(plane->texture.system, 0,
+                                                  &plane->locked_rect, NULL, 0)))
+            {
+                mp_msg(MSGT_VO, MSGL_V, "<vo_direct3d>Texture lock failure.\n");
+                d3d_unlock_video_objects();
+                return false;
+            }
+        }
+    }
+
+    return true;
+}
+
+static void d3d_clear_video_textures(void)
+{
+    if (!d3d_lock_video_textures())
+        return;
+
+    for (int n = 0; n < priv->plane_count; n++) {
+        struct texplane *plane = &priv->planes[n];
+        memset(plane->locked_rect.pBits, plane->clearval,
+               plane->locked_rect.Pitch * plane->texture.tex_h);
+    }
+
+    d3d_unlock_video_objects();
+}
+
 /** @brief Create D3D Offscreen and Backbuffer surfaces. Each
  *         surface is created only if it's not already present.
  *  @return 1 on success, 0 on failure
@@ -355,15 +584,6 @@ static int create_d3d_surfaces(void)
     int tex_width = osd_width, tex_height = osd_height;
     mp_msg(MSGT_VO, MSGL_V, "<vo_direct3d>create_d3d_surfaces called.\n");
 
-    if (!priv->d3d_surface &&
-        FAILED(IDirect3DDevice9_CreateOffscreenPlainSurface(
-               priv->d3d_device, priv->src_width, priv->src_height,
-               priv->movie_src_fmt, D3DPOOL_DEFAULT, &priv->d3d_surface, NULL))) {
-        mp_msg(MSGT_VO, MSGL_ERR,
-               "<vo_direct3d>Allocating offscreen surface failed.\n");
-        return 0;
-    }
-
     if (!priv->d3d_backbuf &&
         FAILED(IDirect3DDevice9_GetBackBuffer(priv->d3d_device, 0, 0,
                                               D3DBACKBUFFER_TYPE_MONO,
@@ -372,6 +592,9 @@ static int create_d3d_surfaces(void)
         return 0;
     }
 
+    if (!d3d_configure_video_objects())
+        return 0;
+
     /* create OSD */
 
     d3d_fix_texture_size(&tex_width, &tex_height);
@@ -405,13 +628,23 @@ static int create_d3d_surfaces(void)
     priv->texture_osd.h = osd_height;
 
     /* setup default renderstate */
-    IDirect3DDevice9_SetRenderState(priv->d3d_device, D3DRS_SRCBLEND, D3DBLEND_ONE);
+    IDirect3DDevice9_SetRenderState(priv->d3d_device, D3DRS_SRCBLEND, D3DBLEND_SRCALPHA);
     IDirect3DDevice9_SetRenderState(priv->d3d_device, D3DRS_DESTBLEND, D3DBLEND_INVSRCALPHA);
     IDirect3DDevice9_SetRenderState(priv->d3d_device, D3DRS_ALPHAFUNC, D3DCMP_GREATER);
     IDirect3DDevice9_SetRenderState(priv->d3d_device, D3DRS_ALPHAREF, (DWORD)0x0);
     IDirect3DDevice9_SetRenderState(priv->d3d_device, D3DRS_LIGHTING, FALSE);
-    IDirect3DDevice9_SetSamplerState(priv->d3d_device, 0, D3DSAMP_MINFILTER, D3DTEXF_LINEAR);
-    IDirect3DDevice9_SetSamplerState(priv->d3d_device, 0, D3DSAMP_MAGFILTER, D3DTEXF_LINEAR);
+
+    // we use up to 3 samplers for up to 3 YUV planes
+    for (int n = 0; n < 3; n++) {
+        IDirect3DDevice9_SetSamplerState(priv->d3d_device, n, D3DSAMP_MINFILTER,
+                                         D3DTEXF_LINEAR);
+        IDirect3DDevice9_SetSamplerState(priv->d3d_device, n, D3DSAMP_MAGFILTER,
+                                         D3DTEXF_LINEAR);
+        IDirect3DDevice9_SetSamplerState(priv->d3d_device, n, D3DSAMP_ADDRESSU,
+                                         D3DTADDRESS_CLAMP);
+        IDirect3DDevice9_SetSamplerState(priv->d3d_device, n, D3DSAMP_ADDRESSV,
+                                         D3DTADDRESS_CLAMP);
+    }
 
     if (priv->eosd && !priv->texture_eosd.system)
         eosd_packer_reinit(priv->eosd, priv->max_texture_width,
@@ -466,7 +699,7 @@ static int change_d3d_backbuffer(back_buffer_action_e action)
     if (action == BACKBUFFER_CREATE &&
         FAILED(IDirect3D9_CreateDevice(priv->d3d_handle,
                                        D3DADAPTER_DEFAULT,
-                                       D3DDEVTYPE_HAL, vo_w32_window,
+                                       DEVTYPE, vo_w32_window,
                                        D3DCREATE_SOFTWARE_VERTEXPROCESSING,
                                        &present_params, &priv->d3d_device))) {
             mp_msg(MSGT_VO, MSGL_V,
@@ -496,7 +729,6 @@ static int change_d3d_backbuffer(back_buffer_action_e action)
 static int configure_d3d(void)
 {
     D3DDISPLAYMODE disp_mode;
-    D3DVIEWPORT9 vp = {0, 0, vo_dwidth, vo_dheight, 0, 1};
 
     mp_msg(MSGT_VO, MSGL_V, "<vo_direct3d>configure_d3d called.\n");
 
@@ -521,13 +753,7 @@ static int configure_d3d(void)
     if (!create_d3d_surfaces())
         return 0;
 
-    if (FAILED(IDirect3DDevice9_SetViewport(priv->d3d_device,
-                                            &vp))) {
-        mp_msg(MSGT_VO, MSGL_ERR, "<vo_direct3d>Setting viewport failed.\n");
-        return 0;
-    }
-
-    calc_fs_rect();
+    resize_d3d();
 
     return 1;
 }
@@ -597,6 +823,11 @@ static int resize_d3d(void)
         return 0;
     }
 
+    // so that screen coordinates map to D3D ones
+    D3DMATRIX view;
+    d3d_matrix_ortho(&view, 0.5f, vp.Width + 0.5f, vp.Height + 0.5f, 0.5f);
+    IDirect3DDevice9_SetTransform(priv->d3d_device, D3DTS_VIEW, &view);
+
     calc_fs_rect();
 
 #ifdef CONFIG_FREETYPE
@@ -632,6 +863,20 @@ static void uninit_d3d(void)
 
 static uint32_t d3d_draw_frame(void);
 
+static uint32_t d3d_upload_and_render_frame_texture(mp_image_t *mpi)
+{
+    if (!(mpi->flags & MP_IMGFLAG_DRAW_CALLBACK))
+        draw_slice(mpi->planes, mpi->stride, mpi->w, mpi->h, 0, 0);
+
+    d3d_unlock_video_objects();
+
+    for (int n = 0; n < priv->plane_count; n++) {
+        d3dtex_update(&priv->planes[n].texture);
+    }
+
+    return d3d_draw_frame();
+}
+
 /** @brief Render a frame on the screen.
  *  @param mpi mpi structure with the decoded frame inside
  *  @return VO_TRUE on success, VO_ERROR on failure
@@ -647,6 +892,9 @@ static uint32_t d3d_upload_and_render_frame(mp_image_t *mpi)
     if (!priv->d3d_device)
         return VO_TRUE;
 
+    if (priv->use_textures)
+        return d3d_upload_and_render_frame_texture(mpi);
+
     if (mpi->flags & MP_IMGFLAG_DRAW_CALLBACK)
         goto skip_upload;
 
@@ -668,22 +916,17 @@ static uint32_t d3d_upload_and_render_frame(mp_image_t *mpi)
                mpi->height, priv->locked_rect.Pitch, mpi->stride[0]);
 
 skip_upload:
-    /* This unlock is used for both slice_draw path and DRAW_IMAGE path. */
-    if (FAILED(IDirect3DSurface9_UnlockRect(priv->d3d_surface))) {
-        mp_msg(MSGT_VO, MSGL_V, "<vo_direct3d>Surface unlock failed.\n");
-        return VO_ERROR;
-    }
-    priv->locked_rect.pBits = NULL;
+    d3d_unlock_video_objects();
 
     return d3d_draw_frame();
 }
 
 static uint32_t d3d_draw_frame(void)
 {
-    if (FAILED(IDirect3DDevice9_BeginScene(priv->d3d_device))) {
-        mp_msg(MSGT_VO, MSGL_ERR, "<vo_direct3d>BeginScene failed.\n");
+    int n;
+
+    if (!d3d_begin_scene())
         return VO_ERROR;
-    }
 
     if (priv->is_clear_needed) {
         IDirect3DDevice9_Clear(priv->d3d_device, 0, NULL,
@@ -691,52 +934,257 @@ static uint32_t d3d_draw_frame(void)
         priv->is_clear_needed = 0;
     }
 
-    if (FAILED(IDirect3DDevice9_StretchRect(priv->d3d_device,
-                                            priv->d3d_surface,
-                                            &priv->fs_panscan_rect,
-                                            priv->d3d_backbuf,
-                                            &priv->fs_movie_rect,
-                                            D3DTEXF_LINEAR))) {
-        mp_msg(MSGT_VO, MSGL_ERR,
-               "<vo_direct3d>Copying frame to the backbuffer failed.\n");
-        return VO_ERROR;
-    }
+    if (priv->use_textures) {
 
-    if (FAILED(IDirect3DDevice9_EndScene(priv->d3d_device))) {
-        mp_msg(MSGT_VO, MSGL_ERR, "<vo_direct3d>EndScene failed.\n");
-        return VO_ERROR;
+        for (n = 0; n < priv->plane_count; n++) {
+            IDirect3DDevice9_SetTexture(priv->d3d_device, n,
+                d3dtex_get_render_texture(&priv->planes[n].texture));
+        }
+
+        RECT rm = priv->fs_movie_rect;
+        RECT rs = priv->fs_panscan_rect;
+
+        vertex_video vb[] = {
+            { rm.left,  rm.top,    0.0f},
+            { rm.right, rm.top,    0.0f},
+            { rm.left,  rm.bottom, 0.0f},
+            { rm.right, rm.bottom, 0.0f}
+        };
+
+        float texc[4][2] = {
+            { rs.left,  rs.top},
+            { rs.right, rs.top},
+            { rs.left,  rs.bottom},
+            { rs.right, rs.bottom}
+        };
+
+        for (n = 0; n < priv->plane_count; n++) {
+            float s_x = (1.0f / (1 << priv->planes[n].shift_x))
+                        / priv->planes[n].texture.tex_w;
+            float s_y = (1.0f / (1 << priv->planes[n].shift_y))
+                        / priv->planes[n].texture.tex_h;
+            for (int i = 0; i < 4; i++) {
+                vb[i].t[n][0] = texc[i][0] * s_x;
+                vb[i].t[n][1] = texc[i][1] * s_y;
+            }
+        }
+
+        if (priv->pixel_shader) {
+            IDirect3DDevice9_SetPixelShader(priv->d3d_device, priv->pixel_shader);
+            IDirect3DDevice9_SetPixelShaderConstantF(priv->d3d_device, 0,
+                                                     &priv->d3d_colormatrix._11,
+                                                     4);
+        }
+
+        IDirect3DDevice9_SetFVF(priv->d3d_device, D3DFVF_VIDEO_VERTEX);
+        IDirect3DDevice9_DrawPrimitiveUP(priv->d3d_device, D3DPT_TRIANGLESTRIP,
+                                         2, &vb[0], sizeof(vertex_video));
+
+        IDirect3DDevice9_SetPixelShader(priv->d3d_device, NULL);
+
+        for (n = 0; n < priv->plane_count; n++) {
+            IDirect3DDevice9_SetTexture(priv->d3d_device, n, NULL);
+        }
+
+    } else {
+        if (FAILED(IDirect3DDevice9_StretchRect(priv->d3d_device,
+                                                priv->d3d_surface,
+                                                &priv->fs_panscan_rect,
+                                                priv->d3d_backbuf,
+                                                &priv->fs_movie_rect,
+                                                D3DTEXF_LINEAR))) {
+            mp_msg(MSGT_VO, MSGL_ERR,
+                "<vo_direct3d>Copying frame to the backbuffer failed.\n");
+            return VO_ERROR;
+        }
     }
 
     return VO_TRUE;
 }
 
-static const struct_fmt_table *check_format(uint32_t movie_fmt)
+// Return the high byte of the value that represents white in chroma (U/V)
+static int get_chroma_clear_val(int bit_depth)
 {
-    int i;
-    for (i = 0; i < DISPLAY_FORMAT_TABLE_ENTRIES; i++) {
-        if (fmt_table[i].mplayer_fmt == movie_fmt) {
-            /* Test conversion from Movie colorspace to
-             * display's target colorspace. */
-            if (FAILED(IDirect3D9_CheckDeviceFormatConversion(priv->d3d_handle,
-                                                              D3DADAPTER_DEFAULT,
-                                                              D3DDEVTYPE_HAL,
-                                                              fmt_table[i].fourcc,
-                                                              priv->desktop_fmt))) {
-                mp_msg(MSGT_VO, MSGL_V, "<vo_direct3d>Rejected image format: %s\n",
-                       vo_format_name(fmt_table[i].mplayer_fmt));
-                return NULL;
+    return 1 << (bit_depth - 1 & 7);
+}
+
+// this macro is supposed to work on all formats supported by 3D rendering, and
+// that produce "reasonable" output (i.e. no mixed up colors)
+#define IMGFMT_IS_ANY_RND(x) (IMGFMT_IS_BGR(x) || IMGFMT_IS_RGB(x) || IMGFMT_IS_Y(x))
+
+// pixel size in bit for any IMGFMT_IS_ANY_RND(x)==true
+// we assume that the actual pixel strides are always aligned on bytes
+static int imgfmt_any_rnd_depth(int fmt)
+{
+    if (IMGFMT_IS_BGR(fmt))
+        return IMGFMT_BGR_DEPTH(fmt);
+    if (IMGFMT_IS_RGB(fmt))
+        return IMGFMT_RGB_DEPTH(fmt);
+    if (IMGFMT_IS_Y(fmt))
+        return IMGFMT_Y_DEPTH(fmt);
+    assert(false);
+    return 0;
+}
+
+static D3DFORMAT check_format(uint32_t movie_fmt, bool as_texture)
+{
+    const char *type = as_texture ? "texture rendering" : "StretchRect";
+    const struct fmt_entry *cur = &fmt_table[0];
+
+    // Don't try to handle weird packed texture formats (although I don't know
+    // if D3D9 would even accept any such format for 3D rendering; and we
+    // certainly don't try any tricks like matching it to RGB formats and
+    // applying a YUV conversion matrix)
+    if (as_texture && !IMGFMT_IS_ANY_RND(movie_fmt))
+        return 0;
+
+    while (cur->mplayer_fmt) {
+        if (cur->mplayer_fmt == movie_fmt) {
+            HRESULT res;
+            if (as_texture) {
+                res = IDirect3D9_CheckDeviceFormat(priv->d3d_handle,
+                            D3DADAPTER_DEFAULT,
+                            DEVTYPE,
+                            priv->desktop_fmt,
+                            D3DUSAGE_DYNAMIC | D3DUSAGE_QUERY_FILTER,
+                            D3DRTYPE_TEXTURE,
+                            cur->fourcc);
+            } else {
+                /* Test conversion from Movie colorspace to
+                * display's target colorspace. */
+                res = IDirect3D9_CheckDeviceFormatConversion(priv->d3d_handle,
+                            D3DADAPTER_DEFAULT,
+                            DEVTYPE,
+                            cur->fourcc,
+                            priv->desktop_fmt);
+            }
+            if (FAILED(res)) {
+                mp_msg(MSGT_VO, MSGL_V, "<vo_direct3d>Rejected image format "
+                       "(%s): %s\n", type, vo_format_name(cur->mplayer_fmt));
+                return 0;
             }
 
-            mp_msg(MSGT_VO, MSGL_V, "<vo_direct3d>Accepted image format: %s\n",
-                   vo_format_name(fmt_table[i].mplayer_fmt));
+            mp_msg(MSGT_VO, MSGL_DBG2, "<vo_direct3d>Accepted image format "
+                   "(%s): %s\n", type, vo_format_name(cur->mplayer_fmt));
 
-            return &fmt_table[i];
+            return cur->fourcc;
         }
+        cur++;
     }
 
     return 0;
 }
 
+// Check whether YUV conversion with shaders can be done.
+// Returns the format the individual planes should use (or 0 on failure)
+static D3DFORMAT check_shader_conversion(uint32_t fmt)
+{
+    if (priv->opt_disable_shaders)
+        return 0;
+    int component_bits;
+    if (!mp_get_chroma_shift(fmt, NULL, NULL, &component_bits))
+        return 0;
+    if (component_bits < 8 || component_bits > 16)
+        return 0;
+    bool is_8bit = component_bits == 8;
+    if (!is_8bit && priv->opt_only_8bit)
+        return