vo_opengl: separate hwdec context and mapping, port it to use ra

This does two separate rather intrusive things:

 1. Make the hwdec context (which does initialization, provides the
    device to the decoder, and other basic state) and frame mapping
    (getting textures from a mp_image) separate. This is more
    flexible, and you could map multiple images at once. It will
    help removing some hwdec special-casing from video.c.
 2. Switch all hwdec API use to ra. Of course all code is still
    GL specific, but in theory it would be possible to support other
    backends. The most important change is that the hwdec interop
    returns ra objects, instead of anything GL specific. This removes
    the last dependency on GL-specific header files from video.c.

I'm mixing these separate changes because both requires essentially
rewriting all the glue code, so better do them at once. For the same
reason, this change isn't done incrementally.

hwdec_ios.m is untested, since I can't test it. Apart from superficial
mistakes, this also requires dealing with Apple's texture format
fuckups: they force you to use GL_LUMINANCE[_ALPHA] instead of GL_RED
and GL_RG. We also need to report the correct format via ra_tex to
the renderer, which is done by find_la_variant(). It's unknown whether
this works correctly.

hwdec_rpi.c as well as vo_rpi.c are still broken. (I need to pull my
RPI out of a dusty pile of devices and cables, so, later.)

1 files changed, 148 insertions, 116 deletions

diff --git a/video/out/opengl/hwdec_vdpau.c b/video/out/opengl/hwdec_vdpau.c
index 9ddec18e06..8f09d549e7 100644
--- a/video/out/opengl/hwdec_vdpau.c
+++ b/video/out/opengl/hwdec_vdpau.c
@@ -21,7 +21,7 @@
 #include <GL/glx.h>
 
 #include "hwdec.h"
-#include "gl_utils.h"
+#include "ra_gl.h"
 #include "video/vdpau.h"
 #include "video/vdpau_mixer.h"
 
@@ -29,13 +29,14 @@
 // follow it. I'm not sure about the original nvidia headers.
 #define BRAINDEATH(x) ((void *)(uintptr_t)(x))
 
-static int reinit(struct gl_hwdec *hw, struct mp_image_params *params);
+struct priv_owner {
+    struct mp_vdpau_ctx *ctx;
+};
 
 struct priv {
-    struct mp_log *log;
     struct mp_vdpau_ctx *ctx;
+    GL *gl;
     uint64_t preemption_counter;
-    struct mp_image_params image_params;
     GLuint gl_textures[4];
     bool vdpgl_initialized;
     GLvdpauSurfaceNV vdpgl_surface;
@@ -45,10 +46,40 @@ struct priv {
     bool mapped;
 };
 
-static void unmap(struct gl_hwdec *hw)
+static int init(struct ra_hwdec *hw)
+{
+    Display *x11disp = glXGetCurrentDisplay();
+    if (!x11disp || !ra_is_gl(hw->ra))
+        return -1;
+    GL *gl = ra_gl_get(hw->ra);
+    if (!(gl->mpgl_caps & MPGL_CAP_VDPAU))
+        return -1;
+    struct priv_owner *p = hw->priv;
+    p->ctx = mp_vdpau_create_device_x11(hw->log, x11disp, true);
+    if (!p->ctx)
+        return -1;
+    if (mp_vdpau_handle_preemption(p->ctx, NULL) < 1)
+        return -1;
+    if (hw->probing && mp_vdpau_guess_if_emulated(p->ctx))
+        return -1;
+    p->ctx->hwctx.driver_name = hw->driver->name;
+    hwdec_devices_add(hw->devs, &p->ctx->hwctx);
+    return 0;
+}
+
+static void uninit(struct ra_hwdec *hw)
 {
-    struct priv *p = hw->priv;
-    GL *gl = hw->gl;
+    struct priv_owner *p = hw->priv;
+
+    if (p->ctx)
+        hwdec_devices_remove(hw->devs, &p->ctx->hwctx);
+    mp_vdpau_destroy(p->ctx);
+}
+
+static void mapper_unmap(struct ra_hwdec_mapper *mapper)
+{
+    struct priv *p = mapper->priv;
+    GL *gl = p->gl;
 
     if (p->mapped) {
         gl->VDPAUUnmapSurfacesNV(1, &p->vdpgl_surface);
@@ -60,96 +91,69 @@ static void unmap(struct gl_hwdec *hw)
     p->mapped = false;
 }
 
-static void mark_vdpau_objects_uninitialized(struct gl_hwdec *hw)
+static void mark_vdpau_objects_uninitialized(struct ra_hwdec_mapper *mapper)
 {
-    struct priv *p = hw->priv;
+    struct priv *p = mapper->priv;
 
     p->vdp_surface = VDP_INVALID_HANDLE;
     p->mapped = false;
 }
 
-static void destroy_objects(struct gl_hwdec *hw)
+static void mapper_uninit(struct ra_hwdec_mapper *mapper)
 {
-    struct priv *p = hw->priv;
-    GL *gl = hw->gl;
+    struct priv *p = mapper->priv;
+    GL *gl = p->gl;
     struct vdp_functions *vdp = &p->ctx->vdp;
     VdpStatus vdp_st;
 
-    unmap(hw);
+    assert(!p->mapped);
 
     if (p->vdpgl_surface)
         gl->VDPAUUnregisterSurfaceNV(p->vdpgl_surface);
     p->vdpgl_surface = 0;
 
-    glDeleteTextures(4, p->gl_textures);
-    for (int n = 0; n < 4; n++)
+    gl->DeleteTextures(4, p->gl_textures);
+    for (int n = 0; n < 4; n++) {
         p->gl_textures[n] = 0;
+        ra_tex_free(mapper->ra, &mapper->tex[n]);
+    }
 
     if (p->vdp_surface != VDP_INVALID_HANDLE) {
         vdp_st = vdp->output_surface_destroy(p->vdp_surface);
-        CHECK_VDP_WARNING(p, "Error when calling vdp_output_surface_destroy");
+        CHECK_VDP_WARNING(mapper, "Error when calling vdp_output_surface_destroy");
     }
     p->vdp_surface = VDP_INVALID_HANDLE;
 
-    gl_check_error(gl, hw->log, "Before uninitializing OpenGL interop");
+    gl_check_error(gl, mapper->log, "Before uninitializing OpenGL interop");
 
     if (p->vdpgl_initialized)
         gl->VDPAUFiniNV();
 
     p->vdpgl_initialized = false;
 
-    gl_check_error(gl, hw->log, "After uninitializing OpenGL interop");
-}
+    gl_check_error(gl, mapper->log, "After uninitializing OpenGL interop");
 
-static void destroy(struct gl_hwdec *hw)
-{
-    struct priv *p = hw->priv;
-
-    destroy_objects(hw);
     mp_vdpau_mixer_destroy(p->mixer);
-    if (p->ctx)
-        hwdec_devices_remove(hw->devs, &p->ctx->hwctx);
-    mp_vdpau_destroy(p->ctx);
 }
 
-static int create(struct gl_hwdec *hw)
+static int mapper_init(struct ra_hwdec_mapper *mapper)
 {
-    GL *gl = hw->gl;
-    Display *x11disp = glXGetCurrentDisplay();
-    if (!x11disp)
-        return -1;
-    if (!(gl->mpgl_caps & MPGL_CAP_VDPAU))
-        return -1;
-    struct priv *p = talloc_zero(hw, struct priv);
-    hw->priv = p;
-    p->log = hw->log;
-    p->ctx = mp_vdpau_create_device_x11(hw->log, x11disp, true);
-    if (!p->ctx)
-        return -1;
-    if (mp_vdpau_handle_preemption(p->ctx, &p->preemption_counter) < 1)
-        return -1;
-    p->vdp_surface = VDP_INVALID_HANDLE;
-    p->mixer = mp_vdpau_mixer_create(p->ctx, hw->log);
-    if (hw->probing && mp_vdpau_guess_if_emulated(p->ctx)) {
-        destroy(hw);
-        return -1;
-    }
-    p->ctx->hwctx.driver_name = hw->driver->name;
-    hwdec_devices_add(hw->devs, &p->ctx->hwctx);
-    return 0;
-}
+    struct priv_owner *p_owner = mapper->owner->priv;
+    struct priv *p = mapper->priv;
 
-static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
-{
-    struct priv *p = hw->priv;
-    GL *gl = hw->gl;
+    p->gl = ra_gl_get(mapper->ra);
+    p->ctx = p_owner->ctx;
+
+    GL *gl = p->gl;
     struct vdp_functions *vdp = &p->ctx->vdp;
     VdpStatus vdp_st;
 
-    destroy_objects(hw);
+    p->vdp_surface = VDP_INVALID_HANDLE;
+    p->mixer = mp_vdpau_mixer_create(p->ctx, mapper->log);
+    if (!p->mixer)
+        return -1;
 
-    assert(params->imgfmt == hw->driver->imgfmt);
-    p->image_params = *params;
+    mapper->dst_params = mapper->src_params;
 
     if (mp_vdpau_handle_preemption(p->ctx, &p->preemption_counter) < 0)
         return -1;
@@ -158,28 +162,73 @@ static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
 
     p->vdpgl_initialized = true;
 
-    p->direct_mode = params->hw_subfmt == IMGFMT_NV12 ||
-                     params->hw_subfmt == IMGFMT_420P;
+    p->direct_mode = mapper->dst_params.hw_subfmt == IMGFMT_NV12 ||
+                     mapper->dst_params.hw_subfmt == IMGFMT_420P;
+    mapper->vdpau_fields = p->direct_mode;
 
     gl->GenTextures(4, p->gl_textures);
-    for (int n = 0; n < 4; n++) {
-        gl->BindTexture(GL_TEXTURE_2D, p->gl_textures[n]);
-        GLenum filter = p->direct_mode ? GL_NEAREST : GL_LINEAR;
-        gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, filter);
-        gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, filter);
-        gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
-        gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
-    }
-    gl->BindTexture(GL_TEXTURE_2D, 0);
 
     if (p->direct_mode) {
-        params->imgfmt = IMGFMT_NV12;
-        params->hw_subfmt = 0;
+        mapper->dst_params.imgfmt = IMGFMT_NV12;
+        mapper->dst_params.hw_subfmt = 0;
+
+        for (int n = 0; n < 4; n++) {
+            gl->BindTexture(GL_TEXTURE_2D, p->gl_textures[n]);
+            gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
+            gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+            gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+            gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+            gl->BindTexture(GL_TEXTURE_2D, 0);
+
+            bool chroma = n >= 2;
+
+            struct ra_tex_params params = {
+                .dimensions = 2,
+                .w = mapper->src_params.w / (chroma ? 2 : 1),
+                .h = mapper->src_params.h / (chroma ? 4 : 2),
+                .d = 1,
+                .format = ra_find_unorm_format(mapper->ra, 1, chroma ? 2 : 1),
+                .render_src = true,
+            };
+
+            if (!params.format)
+                return -1;
+
+            mapper->tex[n] =
+                ra_create_wrapped_tex(mapper->ra, &params, p->gl_textures[n]);
+            if (!mapper->tex[n])
+                return -1;
+        }
     } else {
+        gl->BindTexture(GL_TEXTURE_2D, p->gl_textures[0]);
+        gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+        gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+        gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+        gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+        gl->BindTexture(GL_TEXTURE_2D, 0);
+
+        struct ra_tex_params params = {
+            .dimensions = 2,
+            .w = mapper->src_params.w,
+            .h = mapper->src_params.h,
+            .d = 1,
+            .format = ra_find_unorm_format(mapper->ra, 1, 4),
+            .render_src = true,
+            .src_linear = true,
+        };
+
+        if (!params.format)
+            return -1;
+
+        mapper->tex[0] =
+            ra_create_wrapped_tex(mapper->ra, &params, p->gl_textures[0]);
+        if (!mapper->tex[0])
+            return -1;
+
         vdp_st = vdp->output_surface_create(p->ctx->vdp_device,
                                             VDP_RGBA_FORMAT_B8G8R8A8,
-                                            params->w, params->h, &p->vdp_surface);
-        CHECK_VDP_ERROR(p, "Error when calling vdp_output_surface_create");
+                                            params.w, params.h, &p->vdp_surface);
+        CHECK_VDP_ERROR(mapper, "Error when calling vdp_output_surface_create");
 
         p->vdpgl_surface = gl->VDPAURegisterOutputSurfaceNV(BRAINDEATH(p->vdp_surface),
                                                             GL_TEXTURE_2D,
@@ -189,41 +238,40 @@ static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
 
         gl->VDPAUSurfaceAccessNV(p->vdpgl_surface, GL_READ_ONLY);
 
-        params->imgfmt = IMGFMT_RGB0;
-        params->hw_subfmt = 0;
+        mapper->dst_params.imgfmt = IMGFMT_RGB0;
+        mapper->dst_params.hw_subfmt = 0;
     }
 
-    gl_check_error(gl, hw->log, "After initializing vdpau OpenGL interop");
+    gl_check_error(gl, mapper->log, "After initializing vdpau OpenGL interop");
 
     return 0;
 }
 
-static int map_frame(struct gl_hwdec *hw, struct mp_image *hw_image,
-                     struct gl_hwdec_frame *out_frame)
+static int mapper_map(struct ra_hwdec_mapper *mapper)
 {
-    struct priv *p = hw->priv;
-    GL *gl = hw->gl;
+    struct priv *p = mapper->priv;
+    GL *gl = p->gl;
     struct vdp_functions *vdp = &p->ctx->vdp;
     VdpStatus vdp_st;
 
     int pe = mp_vdpau_handle_preemption(p->ctx, &p->preemption_counter);
     if (pe < 1) {
-        mark_vdpau_objects_uninitialized(hw);
+        mark_vdpau_objects_uninitialized(mapper);
         if (pe < 0)
             return -1;
-        struct mp_image_params params = p->image_params;
-        if (reinit(hw, &params) < 0)
+        mapper_uninit(mapper);
+        if (mapper_init(mapper) < 0)
             return -1;
     }
 
     if (p->direct_mode) {
-        VdpVideoSurface surface = (intptr_t)hw_image->planes[3];
+        VdpVideoSurface surface = (intptr_t)mapper->src->planes[3];
 
         // We need the uncropped size.
         VdpChromaType s_chroma_type;
         uint32_t s_w, s_h;
         vdp_st = vdp->video_surface_get_parameters(surface, &s_chroma_type, &s_w, &s_h);
-        CHECK_VDP_ERROR(hw, "Error when calling vdp_video_surface_get_parameters");
+        CHECK_VDP_ERROR(mapper, "Error when calling vdp_video_surface_get_parameters");
 
         p->vdpgl_surface = gl->VDPAURegisterVideoSurfaceNV(BRAINDEATH(surface),
                                                            GL_TEXTURE_2D,
@@ -235,49 +283,33 @@ static int map_frame(struct gl_hwdec *hw, struct mp_image *hw_image,
         gl->VDPAUMapSurfacesNV(1, &p->vdpgl_surface);
 
         p->mapped = true;
-        *out_frame = (struct gl_hwdec_frame){
-            .vdpau_fields = true,
-        };
-        for (int n = 0; n < 4; n++) {
-            bool chroma = n >= 2;
-            out_frame->planes[n] = (struct gl_hwdec_plane){
-                .gl_texture = p->gl_textures[n],
-                .gl_target = GL_TEXTURE_2D,
-                .tex_w = s_w / (chroma ? 2 : 1),
-                .tex_h = s_h / (chroma ? 4 : 2),
-            };
-        };
     } else {
         if (!p->vdpgl_surface)
             return -1;
 
-        mp_vdpau_mixer_render(p->mixer, NULL, p->vdp_surface, NULL, hw_image, NULL);
+        mp_vdpau_mixer_render(p->mixer, NULL, p->vdp_surface, NULL, mapper->src,
+                              NULL);
 
         gl->VDPAUMapSurfacesNV(1, &p->vdpgl_surface);
 
         p->mapped = true;
-        *out_frame = (struct gl_hwdec_frame){
-            .planes = {
-                {
-                    .gl_texture = p->gl_textures[0],
-                    .gl_target = GL_TEXTURE_2D,
-                    .tex_w = p->image_params.w,
-                    .tex_h = p->image_params.h,
-                },
-            },
-        };
     }
 
     return 0;
 }
 
-const struct gl_hwdec_driver gl_hwdec_vdpau = {
+const struct ra_hwdec_driver ra_hwdec_vdpau = {
     .name = "vdpau-glx",
+    .priv_size = sizeof(struct priv_owner),
     .api = HWDEC_VDPAU,
-    .imgfmt = IMGFMT_VDPAU,
-    .create = create,
-    .reinit = reinit,
-    .map_frame = map_frame,
-    .unmap = unmap,
-    .destroy = destroy,
+    .imgfmts = {IMGFMT_VDPAU, 0},
+    .init = init,
+    .uninit = uninit,
+    .mapper = &(const struct ra_hwdec_mapper_driver){
+        .priv_size = sizeof(struct priv),
+        .init = mapper_init,
+        .uninit = mapper_uninit,
+        .map = mapper_map,
+        .unmap = mapper_unmap,
+    },
 };