vo_opengl: refactor how hwdec interop exports textures

Rename gl_hwdec_driver.map_image to map_frame, and let it fill out a
struct gl_hwdec_frame describing the exact texture layout. This gives
more flexibility to what the hwdec interop can export. In particular, it
can export strange component orders/permutations and textures with
padded size. (The latter originating from cropped video.)

The way gl_hwdec_frame works is in the spirit of the rest of the
vo_opengl video processing code, which tends to put as much information
in immediate state (as part of the dataflow), instead of declaring it
globally. To some degree this duplicates the texplane and img_tex
structs, but until we somehow unify those, it's better to give the hwdec
state its own struct. The fact that changing the hwdec struct would
require changes and testing on at least 4 platform/GPU combinations
makes duplicating it almost a requirement to avoid pain later.

Make gl_hwdec_driver.reinit set the new image format and remove the
gl_hwdec.converted_imgfmt field.

Likewise, gl_hwdec.gl_texture_target is replaced with
gl_hwdec_plane.gl_target.

Split out a init_image_desc function from init_format. The latter is not
called in the hwdec case at all anymore. Setting up most of struct
texplane is also completely separate in the hwdec and normal cases.

video.c does not check whether the hwdec "mapped" image format is
supported. This should not really happen anyway, and if it does, the
hwdec interop backend must fail at creation time, so this is not an
issue.

1 files changed, 18 insertions, 12 deletions

diff --git a/video/out/opengl/hwdec.h b/video/out/opengl/hwdec.h
index fcc6d3c11e..ae48d485cb 100644
--- a/video/out/opengl/hwdec.h
+++ b/video/out/opengl/hwdec.h
@@ -14,14 +14,17 @@ struct gl_hwdec {
     void *priv;
     // For working around the vdpau vs. vaapi mess.
     bool probing;
-    // hwdec backends must set this to an IMGFMT_ that has an equivalent
-    // internal representation in gl_video.c as the hardware texture.
-    // It's used to build the rendering chain. For example, setting it to
-    // IMGFMT_RGB0 indicates that the video texture is RGB.
-    int converted_imgfmt;
-    // Normally this is GL_TEXTURE_2D, but the hwdec driver can set it to
-    // GL_TEXTURE_RECTANGLE. This is needed because VideoToolbox is shit.
-    GLenum gl_texture_target;
+};
+
+struct gl_hwdec_plane {
+    GLuint gl_texture;
+    GLenum gl_target;
+    int tex_w, tex_h; // allocated texture size
+    char swizzle[5]; // component order (if length is 0, use defaults)
+};
+
+struct gl_hwdec_frame {
+    struct gl_hwdec_plane planes[4];
 };
 
 struct gl_hwdec_driver {
@@ -32,16 +35,19 @@ struct gl_hwdec_driver {
     // The hardware surface IMGFMT_ that must be passed to map_image later.
     int imgfmt;
     // Create the hwdec device. It must add it to hw->devs, if applicable.
-    // This also must set hw->converted_imgfmt.
     int (*create)(struct gl_hwdec *hw);
     // Prepare for rendering video. (E.g. create textures.)
     // Called on initialization, and every time the video size changes.
     // *params must be set to the format the hw textures return.
-    // This also can update hw->converted_imgfmt.
     int (*reinit)(struct gl_hwdec *hw, struct mp_image_params *params);
     // Return textures that contain a copy or reference of the given hw_image.
-    int (*map_image)(struct gl_hwdec *hw, struct mp_image *hw_image,
-                     GLuint *out_textures);
+    // The textures mirror the format returned by the reinit params argument.
+    // The textures must remain valid until unmap is called.
+    // hw_image remains referenced by the caller until unmap is called.
+    int (*map_frame)(struct gl_hwdec *hw, struct mp_image *hw_image,
+                     struct gl_hwdec_frame *out_frame);
+    // Must be idempotent.
+    void (*unmap)(struct gl_hwdec *hw);
 
     void (*destroy)(struct gl_hwdec *hw);
 };