vo_opengl: refactor how hwdec interop exports textures

Rename gl_hwdec_driver.map_image to map_frame, and let it fill out a
struct gl_hwdec_frame describing the exact texture layout. This gives
more flexibility to what the hwdec interop can export. In particular, it
can export strange component orders/permutations and textures with
padded size. (The latter originating from cropped video.)

The way gl_hwdec_frame works is in the spirit of the rest of the
vo_opengl video processing code, which tends to put as much information
in immediate state (as part of the dataflow), instead of declaring it
globally. To some degree this duplicates the texplane and img_tex
structs, but until we somehow unify those, it's better to give the hwdec
state its own struct. The fact that changing the hwdec struct would
require changes and testing on at least 4 platform/GPU combinations
makes duplicating it almost a requirement to avoid pain later.

Make gl_hwdec_driver.reinit set the new image format and remove the
gl_hwdec.converted_imgfmt field.

Likewise, gl_hwdec.gl_texture_target is replaced with
gl_hwdec_plane.gl_target.

Split out a init_image_desc function from init_format. The latter is not
called in the hwdec case at all anymore. Setting up most of struct
texplane is also completely separate in the hwdec and normal cases.

video.c does not check whether the hwdec "mapped" image format is
supported. This should not really happen anyway, and if it does, the
hwdec interop backend must fail at creation time, so this is not an
issue.

11 files changed, 274 insertions, 184 deletions

diff --git a/video/out/opengl/hwdec.c b/video/out/opengl/hwdec.c
index b4b5c23580..fee4a16059 100644
--- a/video/out/opengl/hwdec.c
+++ b/video/out/opengl/hwdec.c
@@ -72,7 +72,6 @@ static struct gl_hwdec *load_hwdec_driver(struct mp_log *log, GL *gl,
         .global = global,
         .gl = gl,
         .devs = devs,
-        .gl_texture_target = GL_TEXTURE_2D,
         .probing = is_auto,
     };
     mp_verbose(log, "Loading hwdec driver '%s'\n", drv->name);
diff --git a/video/out/opengl/hwdec.h b/video/out/opengl/hwdec.h
index fcc6d3c11e..ae48d485cb 100644
--- a/video/out/opengl/hwdec.h
+++ b/video/out/opengl/hwdec.h
@@ -14,14 +14,17 @@ struct gl_hwdec {
     void *priv;
     // For working around the vdpau vs. vaapi mess.
     bool probing;
-    // hwdec backends must set this to an IMGFMT_ that has an equivalent
-    // internal representation in gl_video.c as the hardware texture.
-    // It's used to build the rendering chain. For example, setting it to
-    // IMGFMT_RGB0 indicates that the video texture is RGB.
-    int converted_imgfmt;
-    // Normally this is GL_TEXTURE_2D, but the hwdec driver can set it to
-    // GL_TEXTURE_RECTANGLE. This is needed because VideoToolbox is shit.
-    GLenum gl_texture_target;
+};
+
+struct gl_hwdec_plane {
+    GLuint gl_texture;
+    GLenum gl_target;
+    int tex_w, tex_h; // allocated texture size
+    char swizzle[5]; // component order (if length is 0, use defaults)
+};
+
+struct gl_hwdec_frame {
+    struct gl_hwdec_plane planes[4];
 };
 
 struct gl_hwdec_driver {
@@ -32,16 +35,19 @@ struct gl_hwdec_driver {
     // The hardware surface IMGFMT_ that must be passed to map_image later.
     int imgfmt;
     // Create the hwdec device. It must add it to hw->devs, if applicable.
-    // This also must set hw->converted_imgfmt.
     int (*create)(struct gl_hwdec *hw);
     // Prepare for rendering video. (E.g. create textures.)
     // Called on initialization, and every time the video size changes.
     // *params must be set to the format the hw textures return.
-    // This also can update hw->converted_imgfmt.
     int (*reinit)(struct gl_hwdec *hw, struct mp_image_params *params);
     // Return textures that contain a copy or reference of the given hw_image.
-    int (*map_image)(struct gl_hwdec *hw, struct mp_image *hw_image,
-                     GLuint *out_textures);
+    // The textures mirror the format returned by the reinit params argument.
+    // The textures must remain valid until unmap is called.
+    // hw_image remains referenced by the caller until unmap is called.
+    int (*map_frame)(struct gl_hwdec *hw, struct mp_image *hw_image,
+                     struct gl_hwdec_frame *out_frame);
+    // Must be idempotent.
+    void (*unmap)(struct gl_hwdec *hw);
 
     void (*destroy)(struct gl_hwdec *hw);
 };
diff --git a/video/out/opengl/hwdec_d3d11egl.c b/video/out/opengl/hwdec_d3d11egl.c
index caf27e1dc4..ae7e35f953 100644
--- a/video/out/opengl/hwdec_d3d11egl.c
+++ b/video/out/opengl/hwdec_d3d11egl.c
@@ -195,8 +195,6 @@ static int create(struct gl_hwdec *hw)
         goto fail;
     }
 
-    hw->converted_imgfmt = IMGFMT_RGB0;
-
     p->hwctx = (struct mp_hwdec_ctx){
         .type = HWDEC_D3D11VA,
         .driver_name = hw->driver->name,
@@ -218,8 +216,6 @@ static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
 
     destroy_objects(hw);
 
-    assert(params->imgfmt == hw->driver->imgfmt);
-
     D3D11_TEXTURE2D_DESC texdesc = {
         .Width = params->w,
         .Height = params->h,
@@ -277,6 +273,7 @@ static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
     eglBindTexImage(p->egl_display, p->egl_surface, EGL_BACK_BUFFER);
     gl->BindTexture(GL_TEXTURE_2D, 0);
 
+    params->imgfmt = IMGFMT_RGB0;
     return 0;
 fail:
     destroy_objects(hw);
@@ -358,8 +355,8 @@ fail:
     return -1;
 }
 
-static int map_image(struct gl_hwdec *hw, struct mp_image *hw_image,
-                     GLuint *out_textures)
+static int map_frame(struct gl_hwdec *hw, struct mp_image *hw_image,
+                     struct gl_hwdec_frame *out_frame)
 {
     struct priv *p = hw->priv;
     HRESULT hr;
@@ -410,7 +407,16 @@ static int map_image(struct gl_hwdec *hw, struct mp_image *hw_image,
         return -1;
     }
 
-    out_textures[0] = p->gl_texture;
+    *out_frame = (struct gl_hwdec_frame){
+        .planes = {
+            {
+                .gl_texture = p->gl_texture,
+                .gl_target = GL_TEXTURE_2D,
+                .tex_w = hw_image->w,
+                .tex_h = hw_image->h,
+            },
+        },
+    };
     return 0;
 }
 
@@ -420,6 +426,6 @@ const struct gl_hwdec_driver gl_hwdec_d3d11egl = {
     .imgfmt = IMGFMT_D3D11VA,
     .create = create,
     .reinit = reinit,
-    .map_image = map_image,
+    .map_frame = map_frame,
     .destroy = destroy,
 };
diff --git a/video/out/opengl/hwdec_dxva2.c b/video/out/opengl/hwdec_dxva2.c
index 35d091f14f..d832bb4c68 100644
--- a/video/out/opengl/hwdec_dxva2.c
+++ b/video/out/opengl/hwdec_dxva2.c
@@ -43,8 +43,6 @@ static int create(struct gl_hwdec *hw)
         .ctx = d3d,
     };
     hwdec_devices_add(hw->devs, &p->hwctx);
-
-    hw->converted_imgfmt = 0;
     return 0;
 }
 
@@ -53,8 +51,8 @@ static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
     return -1;
 }
 
-static int map_image(struct gl_hwdec *hw, struct mp_image *hw_image,
-                     GLuint *out_textures)
+static int map_frame(struct gl_hwdec *hw, struct mp_image *hw_image,
+                     struct gl_hwdec_frame *out_frame)
 {
     return -1;
 }
@@ -65,6 +63,6 @@ const struct gl_hwdec_driver gl_hwdec_dxva2 = {
     .imgfmt = -1,
     .create = create,
     .reinit = reinit,
-    .map_image = map_image,
+    .map_frame = map_frame,
     .destroy = destroy,
 };
diff --git a/video/out/opengl/hwdec_dxva2egl.c b/video/out/opengl/hwdec_dxva2egl.c
index 1384e2bb47..3a4ee6089f 100644
--- a/video/out/opengl/hwdec_dxva2egl.c
+++ b/video/out/opengl/hwdec_dxva2egl.c
@@ -205,8 +205,6 @@ static int create(struct gl_hwdec *hw)
         goto fail;
     }
 
-    hw->converted_imgfmt = IMGFMT_RGB0;
-
     p->hwctx = (struct mp_hwdec_ctx){
         .type = HWDEC_DXVA2,
         .driver_name = hw->driver->name,
@@ -228,8 +226,6 @@ static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
 
     destroy_textures(hw);
 
-    assert(params->imgfmt == hw->driver->imgfmt);
-
     HANDLE share_handle = NULL;
     hr = IDirect3DDevice9Ex_CreateTexture(p->device9ex,
                                           params->w, params->h,
@@ -274,14 +270,15 @@ static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
     gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
     gl->BindTexture(GL_TEXTURE_2D, 0);
 
+    params->imgfmt = IMGFMT_RGB0;
     return 0;
 fail:
     destroy_textures(hw);
     return -1;
 }
 
-static int map_image(struct gl_hwdec *hw, struct mp_image *hw_image,
-                     GLuint *out_textures)
+static int map_frame(struct gl_hwdec *hw, struct mp_image *hw_image,
+                     struct gl_hwdec_frame *out_frame)
 {
     struct priv *p = hw->priv;
     GL *gl = hw->gl;
@@ -334,7 +331,16 @@ static int map_image(struct gl_hwdec *hw, struct mp_image *hw_image,
     eglBindTexImage(p->egl_display, p->egl_surface, EGL_BACK_BUFFER);
     gl->BindTexture(GL_TEXTURE_2D, 0);
 
-    out_textures[0] = p->gl_texture;
+    *out_frame = (struct gl_hwdec_frame){
+        .planes = {
+            {
+                .gl_texture = p->gl_texture,
+                .gl_target = GL_TEXTURE_2D,
+                .tex_w = hw_image->w,
+                .tex_h = hw_image->h,
+            },
+        },
+    };
     return 0;
 }
 
@@ -344,6 +350,6 @@ const struct gl_hwdec_driver gl_hwdec_dxva2egl = {
     .imgfmt = IMGFMT_DXVA2,
     .create = create,
     .reinit = reinit,
-    .map_image = map_image,
+    .map_frame = map_frame,
     .destroy = destroy,
 };
diff --git a/video/out/opengl/hwdec_dxva2gldx.c b/video/out/opengl/hwdec_dxva2gldx.c
index 97f1918a2c..cf65748474 100644
--- a/video/out/opengl/hwdec_dxva2gldx.c
+++ b/video/out/opengl/hwdec_dxva2gldx.c
@@ -107,8 +107,6 @@ static int create(struct gl_hwdec *hw)
         .ctx = (IDirect3DDevice9 *)p->device,
     };
     hwdec_devices_add(hw->devs, &p->hwctx);
-
-    hw->converted_imgfmt = SHARED_SURFACE_MPFMT;
     return 0;
 }
 
@@ -120,8 +118,6 @@ static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
 
     destroy_objects(hw);
 
-    assert(params->imgfmt == hw->driver->imgfmt);
-
     HANDLE share_handle = NULL;
     hr = IDirect3DDevice9Ex_CreateRenderTarget(
         p->device,
@@ -164,14 +160,16 @@ static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
         goto fail;
     }
 
+    params->imgfmt = SHARED_SURFACE_MPFMT;
+
     return 0;
 fail:
     destroy_objects(hw);
     return -1;
 }
 
-static int map_image(struct gl_hwdec *hw, struct mp_image *hw_image,
-                     GLuint *out_textures)
+static int map_frame(struct gl_hwdec *hw, struct mp_image *hw_image,
+                     struct gl_hwdec_frame *out_frame)
 {
     assert(hw_image && hw_image->imgfmt == hw->driver->imgfmt);
     GL *gl = hw->gl;
@@ -201,7 +199,16 @@ static int map_image(struct gl_hwdec *hw, struct mp_image *hw_image,
         return -1;
     }
 
-    out_textures[0] = p->texture;
+    *out_frame = (struct gl_hwdec_frame){
+        .planes = {
+            {
+                .gl_texture = p->texture,
+                .gl_target = GL_TEXTURE_2D,
+                .tex_w = hw_image->w,
+                .tex_h = hw_image->h,
+            },
+        },
+    };
     return 0;
 }
 
@@ -211,6 +218,6 @@ const struct gl_hwdec_driver gl_hwdec_dxva2gldx = {
     .imgfmt = IMGFMT_DXVA2,
     .create = create,
     .reinit = reinit,
-    .map_image = map_image,
+    .map_frame = map_frame,
     .destroy = destroy,
 };
diff --git a/video/out/opengl/hwdec_osx.c b/video/out/opengl/hwdec_osx.c
index 5aa4d3dcd4..2c1ffc71ae 100644
--- a/video/out/opengl/hwdec_osx.c
+++ b/video/out/opengl/hwdec_osx.c
@@ -33,6 +33,7 @@ struct vt_gl_plane_format {
     GLenum gl_format;
     GLenum gl_type;
     GLenum gl_internal_format;
+    char swizzle[5];
 };
 
 struct vt_format {
@@ -65,7 +66,7 @@ static struct vt_format vt_formats[] = {
         .imgfmt = IMGFMT_UYVY,
         .planes = 1,
         .gl = {
-            { GL_RGB_422_APPLE, GL_UNSIGNED_SHORT_8_8_APPLE, GL_RGB }
+            { GL_RGB_422_APPLE, GL_UNSIGNED_SHORT_8_8_APPLE, GL_RGB, "gbra" }
         }
     },
     {
@@ -163,14 +164,8 @@ static int create(struct gl_hwdec *hw)
         return -1;
 
     struct priv *p = talloc_zero(hw, struct priv);
-    struct vt_format *f = vt_get_gl_format_from_imgfmt(IMGFMT_NV12);
-    if (!f)
-        return -1;
-
     hw->priv = p;
-    hw->converted_imgfmt = f->imgfmt;
 
-    hw->gl_texture_target = GL_TEXTURE_RECTANGLE;
     hw->gl->GenTextures(MP_MAX_PLANES, p->gl_planes);
 
     p->vtctx = (struct mp_vt_ctx){
@@ -197,16 +192,13 @@ static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
         return -1;
     }
 
-    hw->converted_imgfmt = f->imgfmt;
+    params->imgfmt = f->imgfmt;
     return 0;
 }
 
-static int map_image(struct gl_hwdec *hw, struct mp_image *hw_image,
-                     GLuint *out_textures)
+static int map_frame(struct gl_hwdec *hw, struct mp_image *hw_image,
+                     struct gl_hwdec_frame *out_frame)
 {
-    if (!check_hwdec(hw))
-        return -1;
-
     struct priv *p = hw->priv;
     GL *gl = hw->gl;
 
@@ -230,11 +222,13 @@ static int map_image(struct gl_hwdec *hw, struct mp_image *hw_image,
     const int planes  = CVPixelBufferGetPlaneCount(p->pbuf);
     assert(planar && planes == f->planes || f->planes == 1);
 
+    GLenum gl_target = GL_TEXTURE_RECTANGLE;
+
     for (int i = 0; i < f->planes; i++) {
-        gl->BindTexture(hw->gl_texture_target, p->gl_planes[i]);
+        gl->BindTexture(gl_target, p->gl_planes[i]);
 
         CGLError err = CGLTexImageIOSurface2D(
-            CGLGetCurrentContext(), hw->gl_texture_target,
+            CGLGetCurrentContext(), gl_target,
             f->gl[i].gl_internal_format,
             IOSurfaceGetWidthOfPlane(surface, i),
             IOSurfaceGetHeightOfPlane(surface, i),
@@ -244,9 +238,14 @@ static int map_image(struct gl_hwdec *hw, struct mp_image *hw_image,
             MP_ERR(hw, "error creating IOSurface texture for plane %d: %s (%x)\n",
                    i, CGLErrorString(err), gl->GetError());
 
-        gl->BindTexture(hw->gl_texture_target, 0);
+        gl->BindTexture(gl_target, 0);
 
-        out_textures[i] = p->gl_planes[i];
+        out_frame->planes[i] = (struct gl_hwdec_plane){
+            .gl_texture = p->gl_planes[i],
+            .gl_target = gl_target,
+            .tex_w = IOSurfaceGetWidthOfPlane(surface, i),
+            .tex_h = IOSurfaceGetHeightOfPlane(surface, i),
+        };
     }
 
     return 0;
@@ -269,6 +268,6 @@ const struct gl_hwdec_driver gl_hwdec_videotoolbox = {
     .imgfmt = IMGFMT_VIDEOTOOLBOX,
     .create = create,
     .reinit = reinit,
-    .map_image = map_image,
+    .map_frame = map_frame,
     .destroy = destroy,
 };
diff --git a/video/out/opengl/hwdec_vaegl.c b/video/out/opengl/hwdec_vaegl.c
index 84e6e83a73..6c52cdde11 100644
--- a/video/out/opengl/hwdec_vaegl.c
+++ b/video/out/opengl/hwdec_vaegl.c
@@ -114,7 +114,7 @@ struct priv {
     EGLImageKHR images[4];
     VAImage current_image;
     bool buffer_acquired;
-    struct mp_image *current_ref;
+    int current_mpfmt;
 
     EGLImageKHR (EGLAPIENTRY *CreateImageKHR)(EGLDisplay, EGLContext,
                                               EGLenum, EGLClientBuffer,
@@ -125,7 +125,7 @@ struct priv {
 
 static bool test_format(struct gl_hwdec *hw);
 
-static void unref_image(struct gl_hwdec *hw)
+static void unmap_frame(struct gl_hwdec *hw)
 {
     struct priv *p = hw->priv;
     VAStatus status;
@@ -149,8 +149,6 @@ static void unref_image(struct gl_hwdec *hw)
         p->current_image.image_id = VA_INVALID_ID;
     }
 
-    mp_image_unrefp(&p->current_ref);
-
     va_unlock(p->ctx);
 }
 
@@ -167,7 +165,7 @@ static void destroy_textures(struct gl_hwdec *hw)
 static void destroy(struct gl_hwdec *hw)
 {
     struct priv *p = hw->priv;
-    unref_image(hw);
+    unmap_frame(hw);
     destroy_textures(hw);
     if (p->ctx)
         hwdec_devices_remove(hw->devs, &p->ctx->hwctx);
@@ -242,8 +240,6 @@ static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
     // Recreate them to get rid of all previous image data (possibly).
     destroy_textures(hw);
 
-    assert(params->imgfmt == hw->driver->imgfmt);
-
     gl->GenTextures(4, p->gl_textures);
     for (int n = 0; n < 4; n++) {
         gl->BindTexture(GL_TEXTURE_2D, p->gl_textures[n]);
@@ -254,9 +250,9 @@ static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
     }
     gl->BindTexture(GL_TEXTURE_2D, 0);
 
-    hw->converted_imgfmt = va_fourcc_to_imgfmt(params->hw_subfmt);
-    if (hw->converted_imgfmt != IMGFMT_NV12 &&
-        hw->converted_imgfmt != IMGFMT_420P)
+    p->current_mpfmt = va_fourcc_to_imgfmt(params->hw_subfmt);
+    if (p->current_mpfmt != IMGFMT_NV12 &&
+        p->current_mpfmt != IMGFMT_420P)
     {
         MP_FATAL(p, "unsupported VA image format %s\n",
                  mp_tag_str(params->hw_subfmt));
@@ -264,7 +260,9 @@ static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
     }
 
     MP_VERBOSE(p, "format: %s %s\n", mp_tag_str(params->hw_subfmt),
-               mp_imgfmt_to_name(hw->converted_imgfmt));
+               mp_imgfmt_to_name(p->current_mpfmt));
+
+    params->imgfmt = p->current_mpfmt;
 
     return 0;
 }
@@ -277,17 +275,15 @@ static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
     attribs[num_attribs] = EGL_NONE;                    \
     } while(0)
 
-static int map_image(struct gl_hwdec *hw, struct mp_image *hw_image,
-                     GLuint *out_textures)
+static int map_frame(struct gl_hwdec *hw, struct mp_image *hw_image,
+                     struct gl_hwdec_frame *out_frame)
 {
     struct priv *p = hw->priv;
     GL *gl = hw->gl;
     VAStatus status;
     VAImage *va_image = &p->current_image;
 
-    unref_image(hw);
-
-    mp_image_setrefp(&p->current_ref, hw_image);
+    unmap_frame(hw);
 
     va_lock(p->ctx);
 
@@ -296,9 +292,9 @@ static int map_image(struct gl_hwdec *hw, struct mp_image *hw_image,
         goto err;
 
     int mpfmt = va_fourcc_to_imgfmt(va_image->format.fourcc);
-    if (hw->converted_imgfmt != mpfmt) {
+    if (p->current_mpfmt != mpfmt) {
         MP_FATAL(p, "mid-stream hwdec format change (%s -> %s) not supported\n",
-                 mp_imgfmt_to_name(hw->converted_imgfmt), mp_imgfmt_to_name(mpfmt));
+                 mp_imgfmt_to_name(p->current_mpfmt), mp_imgfmt_to_name(mpfmt));
         goto err;
     }
 
@@ -337,12 +333,17 @@ static int map_image(struct gl_hwdec *hw, struct mp_image *hw_image,
         gl->BindTexture(GL_TEXTURE_2D, p->gl_textures[n]);
         p->EGLImageTargetTexture2DOES(GL_TEXTURE_2D, p->images[n]);
 
-        out_textures[n] = p->gl_textures[n];
+        out_frame->planes[n] = (struct gl_hwdec_plane){
+            .gl_texture = p->gl_textures[n],
+            .gl_target = GL_TEXTURE_2D,
+            .tex_w = mp_image_plane_w(&layout, n),
+            .tex_h = mp_image_plane_h(&layout, n),
+        };
     }
     gl->BindTexture(GL_TEXTURE_2D, 0);
 
     if (va_image->format.fourcc == VA_FOURCC_YV12)
-        MPSWAP(GLuint, out_textures[1], out_textures[2]);
+        MPSWAP(struct gl_hwdec_plane, out_frame->planes[1], out_frame->planes[2]);
 
     va_unlock(p->ctx);
     return 0;
@@ -350,7 +351,7 @@ static int map_image(struct gl_hwdec *hw, struct mp_image *hw_image,
 err:
     va_unlock(p->ctx);
     MP_FATAL(p, "mapping VAAPI EGL image failed\n");
-    unref_image(hw);
+    unmap_frame(hw);
     return -1;
 }
 
@@ -366,10 +367,10 @@ static bool test_format(struct gl_hwdec *hw)
         va_surface_init_subformat(surface);
         struct mp_image_params params = surface->params;
         if (reinit(hw, &params) >= 0) {
-            GLuint textures[4];
-            ok = map_image(hw, surface, textures) >= 0;
+            struct gl_hwdec_frame frame = {0};
+            ok = map_frame(hw, surface, &frame) >= 0;
         }
-        unref_image(hw);
+        unmap_frame(hw);
     }
     talloc_free(surface);
     talloc_free(alloc);
@@ -383,6 +384,7 @@ const struct gl_hwdec_driver gl_hwdec_vaegl = {
     .imgfmt = IMGFMT_VAAPI,
     .create = create,
     .reinit = reinit,
-    .map_image = map_image,
+    .map_frame = map_frame,
+    .unmap = unmap_frame,
     .destroy = destroy,
 };
diff --git a/video/out/opengl/hwdec_vaglx.c b/video/out/opengl/hwdec_vaglx.c
index a9c5d03af4..2e3017c193 100644
--- a/video/out/opengl/hwdec_vaglx.c
+++ b/video/out/opengl/hwdec_vaglx.c
@@ -128,7 +128,6 @@ static int create(struct gl_hwdec *hw)
 
     p->ctx->hwctx.driver_name = hw->driver->name;
     hwdec_devices_add(hw->devs, &p->ctx->hwctx);
-    hw->converted_imgfmt = IMGFMT_RGB0;
     return 0;
 }
 
@@ -139,8 +138,6 @@ static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
 
     destroy_texture(hw);
 
-    assert(params->imgfmt == hw->driver->imgfmt);
-
     gl->GenTextures(1, &p->gl_texture);
     gl->BindTexture(GL_TEXTURE_2D, p->gl_texture);
     gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
@@ -169,11 +166,13 @@ static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
     p->glXBindTexImage(p->xdisplay, p->glxpixmap, GLX_FRONT_EXT, NULL);
     gl->BindTexture(GL_TEXTURE_2D, 0);
 
+    params->imgfmt = IMGFMT_RGB0;
+
     return 0;
 }
 
-static int map_image(struct gl_hwdec *hw, struct mp_image *hw_image,
-                     GLuint *out_textures)
+static int map_frame(struct gl_hwdec *hw, struct mp_image *hw_image,
+                     struct gl_hwdec_frame *out_frame)
 {
     struct priv *p = hw->priv;
     VAStatus status;
@@ -190,7 +189,16 @@ static int map_image(struct gl_hwdec *hw, struct mp_image *hw_image,
     CHECK_VA_STATUS(p, "vaPutSurface()");
     va_unlock(p->ctx);
 
-    out_textures[0] = p->gl_texture;
+    *out_frame = (struct gl_hwdec_frame){
+        .planes = {
+            {
+                .gl_texture = p->gl_texture,
+                .gl_target = GL_TEXTURE_2D,
+                .tex_w = hw_image->w,
+                .tex_h = hw_image->h,
+            },
+        },
+    };
     return 0;
 }
 
@@ -200,6 +208,6 @@ const struct gl_hwdec_driver gl_hwdec_vaglx = {
     .imgfmt = IMGFMT_VAAPI,
     .create = create,
     .reinit = reinit,
-    .map_image = map_image,
+    .map_frame = map_frame,
     .destroy = destroy,
 };
diff --git a/video/out/opengl/hwdec_vdpau.c b/video/out/opengl/hwdec_vdpau.c
index e3b69941bf..7f61134376 100644
--- a/video/out/opengl/hwdec_vdpau.c
+++ b/video/out/opengl/hwdec_vdpau.c
@@ -121,7 +121,6 @@ static int create(struct gl_hwdec *hw)
     }
     p->ctx->hwctx.driver_name = hw->driver->name;
     hwdec_devices_add(hw->devs, &p->ctx->hwctx);
-    hw->converted_imgfmt = IMGFMT_RGB0;
     return 0;
 }
 
@@ -167,17 +166,17 @@ static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
 
     glCheckError(gl, hw->log, "After initializing vdpau OpenGL interop");
 
+    params->imgfmt = IMGFMT_RGB0;
+
     return 0;
 }
 
-static int map_image(struct gl_hwdec *hw, struct mp_image *hw_image,
-                     GLuint *out_textures)
+static int map_frame(struct gl_hwdec *hw, struct mp_image *hw_image,
+                     struct gl_hwdec_frame *out_frame)
 {
     struct priv *p = hw->priv;
     GL *gl = hw->gl;
 
-    assert(hw_image && hw_image->imgfmt == IMGFMT_VDPAU);
-
     int pe = mp_vdpau_handle_preemption(p->ctx, &p->preemption_counter);
     if (pe < 1) {
         mark_vdpau_objects_uninitialized(hw);
@@ -190,23 +189,40 @@ static int map_image(struct gl_hwdec *hw, struct mp_image *hw_image,
     if (!p->vdpgl_surface)
         return -1;
 
-    if (p->mapped)
-        gl->VDPAUUnmapSurfacesNV(1, &p->vdpgl_surface);
-
     mp_vdpau_mixer_render(p->mixer, NULL, p->vdp_surface, NULL, hw_image, NULL);
 
     gl->VDPAUMapSurfacesNV(1, &p->vdpgl_surface);
     p->mapped = true;
-    out_textures[0] = p->gl_texture;
+    *out_frame = (struct gl_hwdec_frame){
+        .planes = {
+            {
+                .gl_texture = p->gl_texture,
+                .gl_target = GL_TEXTURE_2D,
+                .tex_w = p->image_params.w,
+                .tex_h = p->image_params.h,
+            },
+        },
+    };
     return 0;
 }
 
+static void unmap(struct gl_hwdec *hw)
+{
+    struct priv *p = hw->priv;
+    GL *gl = hw->gl;
+
+    if (p->mapped)
+        gl->VDPAUUnmapSurfacesNV(1, &p->vdpgl_surface);
+    p->mapped = false;
+}
+
 const struct gl_hwdec_driver gl_hwdec_vdpau = {
     .name = "vdpau-glx",
     .api = HWDEC_VDPAU,
     .imgfmt = IMGFMT_VDPAU,
     .create = create,
     .reinit = reinit,
-    .map_image = map_image,
+    .map_frame = map_frame,
+    .unmap = unmap,
     .destroy = destroy,
 };
diff --git a/video/out/opengl/video.c b/video/out/opengl/video.c
index 79807d072b..b78295b368 100644
--- a/video/out/opengl/video.c
+++ b/video/out/opengl/video.c
@@ -91,6 +91,7 @@ static const struct gl_vao_entry vertex_vao[] = {
 
 struct texplane {
     int w, h;
+    int tex_w, tex_h;
     GLint gl_internal_format;
     GLenum gl_target;
     bool use_integer;
@@ -98,12 +99,14 @@ struct texplane {
     GLenum gl_type;
     GLuint gl_texture;
     int gl_buffer;
+    char swizzle[5];
 };
 
 struct video_image {
     struct texplane planes[4];
     bool image_flipped;
     struct mp_image *mpi;       // original input image
+    bool hwdec_mapped;
 };
 
 enum plane_type {
@@ -128,7 +131,7 @@ struct img_tex {
     int w, h; // logical size (with pre_transform applied)
     struct gl_transform pre_transform; // source texture space
     struct gl_transform transform; // rendering transformation
-    bool texture_la; // it's a GL_LUMINANCE_ALPHA texture (access with .ra not .rg)
+    char swizzle[5];
 };
 
 struct fbosurface {
@@ -156,8 +159,6 @@ struct gl_video {
 
     struct gl_shader_cache *sc;
 
-    GLenum gl_target; // texture target (GL_TEXTURE_2D, ...) for video and FBOs
-
     struct gl_vao vao;
 
     struct osd_state *osd_state;
@@ -536,7 +537,8 @@ const struct m_sub_options gl_video_conf = {
 static void uninit_rendering(struct gl_video *p);
 static void uninit_scaler(struct gl_video *p, struct scaler *scaler);
 static void check_gl_features(struct gl_video *p);
-static bool init_format(int fmt, struct gl_video *init);
+static bool init_format(struct gl_video *p, int fmt, bool test_only);
+static void init_image_desc(struct gl_video *p, int fmt);
 static void gl_video_upload_image(struct gl_video *p, struct mp_image *mpi);
 static void assign_options(struct gl_video_opts *dst, struct gl_video_opts *src);
 static void get_scale_factors(struct gl_video *p, bool transpose_rot, double xy[2]);
@@ -855,14 +857,14 @@ static void pass_get_img_tex(struct gl_video *p, struct video_image *vimg,
             .gl_target = t->gl_target,
             .multiplier = tex_mul,
             .use_integer = t->use_integer,
-            .tex_w = t->w,
-            .tex_h = t->h,
+            .tex_w = t->tex_w,
+            .tex_h = t->tex_h,
             .w = t->w,
             .h = t->h,
             .transform = type == PLANE_CHROMA ? chroma : identity_trans,
             .components = p->image_desc.components[n],
-            .texture_la = t->gl_format == GL_LUMINANCE_ALPHA,
         };
+        snprintf(tex[n].swizzle, sizeof(tex[n].swizzle), "%s", t->swizzle);
         get_plane_source_transform(p, t->w, t->h, &tex[n].pre_transform);
         if (p->image_params.rotate % 180 == 90)
             MPSWAP(int, tex[n].w, tex[n].h);
@@ -873,17 +875,15 @@ static void init_video(struct gl_video *p)
 {
     GL *gl = p->gl;
 
-    init_format(p->image_params.imgfmt, p);
-    p->gl_target = p->opts.use_rectangle ? GL_TEXTURE_RECTANGLE : GL_TEXTURE_2D;
-
     check_gl_features(p);
 
-    if (p->hwdec_active) {
+    if (p->hwdec && p->hwdec->driver->imgfmt == p->image_params.imgfmt) {
         if (p->hwdec->driver->reinit(p->hwdec, &p->image_params) < 0)
             MP_ERR(p, "Initializing texture for hardware decoding failed.\n");
-        init_format(p->image_params.imgfmt, p);
-        p->image_params.imgfmt = p->image_desc.id;
-        p->gl_target = p->hwdec->gl_texture_target;
+        init_image_desc(p, p->image_params.imgfmt);
+        p->hwdec_active = true;
+    } else {
+        init_format(p, p->image_params.imgfmt, false);
     }
 
     mp_image_params_guess_csp(&p->image_params);
@@ -899,44 +899,61 @@ static void init_video(struct gl_video *p)
 
     debug_check_gl(p, "before video texture creation");
 
-    struct video_image *vimg = &p->image;
+    if (!p->hwdec_active) {
+        struct video_image *vimg = &p->image;
 
-    struct mp_image layout = {0};
-    mp_image_set_params(&layout, &p->image_params);
+        GLenum gl_target =
+            p->opts.use_rectangle ? GL_TEXTURE_RECTANGLE : GL_TEXTURE_2D;
 
-    for (int n = 0; n < p->plane_count; n++) {
-        struct texplane *plane = &vimg->planes[n];
+        struct mp_image layout = {0};
+        mp_image_set_params(&layout, &p->image_params);
+
+        for (int n = 0; n < p->plane_count; n++) {
+            struct texplane *plane = &vimg->planes[n];
 
-        plane->gl_target = p->gl_target;
+            plane->gl_target = gl_target;
 
-        plane->w = mp_image_plane_w(&layout, n);
-        plane->h = mp_image_plane_h(&layout, n);
+            plane->w = plane->tex_w = mp_image_plane_w(&layout, n);
+            plane->h = plane->tex_h = mp_image_plane_h(&layout, n);
 
-        if (!p->hwdec_active) {
             gl->ActiveTexture(GL_TEXTURE0 + n);
             gl->GenTextures(1, &plane->gl_texture);
-            gl->BindTexture(p->gl_target, plane->gl_texture);
+            gl->BindTexture(gl_target, plane->gl_texture);
 
-            gl->TexImage2D(p->gl_target, 0, plane->gl_internal_format,
+            gl->TexImage2D(gl_target, 0, plane->gl_internal_format,
                            plane->w, plane->h, 0,
                            plane->gl_format, plane->gl_type, NULL);
 
             int filter = plane->use_integer ? GL_NEAREST : GL_LINEAR;
-            gl->TexParameteri(p->gl_target, GL_TEXTURE_MIN_FILTER, filter);
-            gl->TexParameteri(p->gl_target, GL_TEXTURE_MAG_FILTER, filter);
-            gl->TexParameteri(p->gl_target, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
-            gl->TexParameteri(p->gl_target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
-        }
+            gl->TexParameteri(gl_target, GL_TEXTURE_MIN_FILTER, filter);
+            gl->TexParameteri(gl_target, GL_TEXTURE_MAG_FILTER, filter);
+            gl->TexParameteri(gl_target, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+            gl->TexParameteri(gl_target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
 
-        MP_VERBOSE(p, "Texture for plane %d: %dx%d\n", n, plane->w, plane->h);
+            MP_VERBOSE(p, "Texture for plane %d: %dx%d\n", n, plane->w, plane->h);
+        }
+        gl->ActiveTexture(GL_TEXTURE0);
     }
-    gl->ActiveTexture(GL_TEXTURE0);
 
     debug_check_gl(p, "after video texture creation");
 
     reinit_rendering(p);
 }
 
+static void unref_current_image(struct gl_video *p)
+{
+    struct video_image *vimg = &p->image;
+
+    if (vimg->hwdec_mapped) {
+        assert(p->hwdec_active);