vo_opengl: separate hwdec context and mapping, port it to use ra

This does two separate rather intrusive things:

 1. Make the hwdec context (which does initialization, provides the
    device to the decoder, and other basic state) and frame mapping
    (getting textures from a mp_image) separate. This is more
    flexible, and you could map multiple images at once. It will
    help removing some hwdec special-casing from video.c.
 2. Switch all hwdec API use to ra. Of course all code is still
    GL specific, but in theory it would be possible to support other
    backends. The most important change is that the hwdec interop
    returns ra objects, instead of anything GL specific. This removes
    the last dependency on GL-specific header files from video.c.

I'm mixing these separate changes because both requires essentially
rewriting all the glue code, so better do them at once. For the same
reason, this change isn't done incrementally.

hwdec_ios.m is untested, since I can't test it. Apart from superficial
mistakes, this also requires dealing with Apple's texture format
fuckups: they force you to use GL_LUMINANCE[_ALPHA] instead of GL_RED
and GL_RG. We also need to report the correct format via ra_tex to
the renderer, which is done by find_la_variant(). It's unknown whether
this works correctly.

hwdec_rpi.c as well as vo_rpi.c are still broken. (I need to pull my
RPI out of a dusty pile of devices and cables, so, later.)

1 files changed, 132 insertions, 77 deletions

diff --git a/video/out/opengl/hwdec_ios.m b/video/out/opengl/hwdec_ios.m
index 9461f3d980..ec6eb422d6 100644
--- a/video/out/opengl/hwdec_ios.m
+++ b/video/out/opengl/hwdec_ios.m
@@ -29,21 +29,27 @@
 
 #include "video/mp_image_pool.h"
 #include "video/vt.h"
-#include "formats.h"
+#include "ra_gl.h"
 #include "hwdec.h"
 
-struct priv {
+struct priv_owner {
     struct mp_hwdec_ctx hwctx;
+};
 
+struct priv {
     CVPixelBufferRef pbuf;
     CVOpenGLESTextureCacheRef gl_texture_cache;
     CVOpenGLESTextureRef gl_planes[MP_MAX_PLANES];
-    struct gl_imgfmt_desc desc;
+    struct ra_imgfmt_desc desc;
 };
 
-static bool check_hwdec(struct gl_hwdec *hw)
+static bool check_hwdec(struct ra_hwdec *hw)
 {
-    if (hw->gl->es < 200) {
+    if (!ra_is_gl(hw->ra))
+        return false;
+
+    GL *gl = ra_gl_get(hw->ra);
+    if (gl->es < 200) {
         MP_ERR(hw, "need OpenGLES 2.0 for CVOpenGLESTextureCacheCreateTextureFromImage()\n");
         return false;
     }
@@ -56,25 +62,12 @@ static bool check_hwdec(struct gl_hwdec *hw)
     return true;
 }
 
-static int create_hwdec(struct gl_hwdec *hw)
+static int init(struct ra_hwdec *hw)
 {
-    if (!check_hwdec(hw))
-        return -1;
-
-    struct priv *p = talloc_zero(hw, struct priv);
-    hw->priv = p;
-
-    CVReturn err = CVOpenGLESTextureCacheCreate(
-        kCFAllocatorDefault,
-        NULL,
-        [EAGLContext currentContext],
-        NULL,
-        &p->gl_texture_cache);
+    struct priv_owner *p = hw->priv;
 
-    if (err != noErr) {
-        MP_ERR(hw, "Failure in CVOpenGLESTextureCacheCreate: %d\n", err);
+    if (!check_hwdec(hw))
         return -1;
-    }
 
     p->hwctx = (struct mp_hwdec_ctx){
         .type = HWDEC_VIDEOTOOLBOX,
@@ -92,32 +85,98 @@ static int create_hwdec(struct gl_hwdec *hw)
     return 0;
 }
 
-static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
+static void uninit(struct ra_hwdec *hw)
+{
+    struct priv_owner *p = hw->priv;
+
+    if (p->hwctx.ctx)
+        hwdec_devices_remove(hw->devs, &p->hwctx);
+    av_buffer_unref(&p->hwctx.av_device_ref);
+}
+
+// In GLES3 mode, CVOpenGLESTextureCacheCreateTextureFromImage()
+// will return error -6683 unless invoked with GL_LUMINANCE and
+// GL_LUMINANCE_ALPHA (http://stackoverflow.com/q/36213994/332798)
+// If a format trues to use GL_RED/GL_RG instead, try to find a format
+// that uses GL_LUMINANCE[_ALPHA] instead.
+static const struct ra_format *find_la_variant(struct ra *ra,
+                                               const struct ra_format *fmt)
+{
+    GLint internal_format;
+    GLenum format;
+    GLenum type;
+    ra_gl_get_format(fmt, &internal_format, &format, &type);
+
+    if (format == GL_RED) {
+        format = internal_format = GL_LUMINANCE;
+    } else if (format == GL_RG) {
+        format = internal_format = GL_LUMINANCE_ALPHA;
+    } else {
+        return fmt;
+    }
+
+    for (int n = 0; n < ra->num_formats; n++) {
+        const struct ra_format *fmt2 = ra->formats[n];
+        GLint internal_format2;
+        GLenum format2;
+        GLenum type2;
+        ra_gl_get_format(fmt2, &internal_format2, &format2, &type2);
+        if (internal_format2 == internal_format &&
+            format2 == format && type2 == type)
+            return fmt2;
+    }
+
+    return NULL;
+}
+
+static int mapper_init(struct ra_hwdec_mapper *mapper)
 {
-    struct priv *p = hw->priv;
-    assert(params->imgfmt == hw->driver->imgfmt);
+    struct priv *p = mapper->priv;
+    GL *gl = ra_gl_get(mapper->ra);
 
-    if (!params->hw_subfmt) {
-        MP_ERR(hw, "Unsupported CVPixelBuffer format.\n");
+    mapper->dst_params = mapper->src_params;
+    mapper->dst_params.imgfmt = mapper->src_params.hw_subfmt;
+    mapper->dst_params.hw_subfmt = 0;
+
+    if (!mapper->dst_params.imgfmt) {
+        MP_ERR(mapper, "Unsupported CVPixelBuffer format.\n");
         return -1;
     }
 
-    if (!gl_get_imgfmt_desc(hw->gl, params->hw_subfmt, &p->desc)) {
-        MP_ERR(hw, "Unsupported texture format.\n");
+    if (!ra_get_imgfmt_desc(mapper->ra, mapper->dst_params.imgfmt, &p->desc)) {
+        MP_ERR(mapper, "Unsupported texture format.\n");
+        return -1;
+    }
+
+    for (int n = 0; n < p->desc.num_planes; n++) {
+        p->desc.planes[n] = find_la_variant(mapper->ra, p->desc.planes[n]);
+        if (!p->desc.planes[n] || p->desc.planes[n]->ctype != RA_CTYPE_UNORM) {
+            MP_ERR(mapper, "Format unsupported.\n");
+            return -1;
+        }
+    }
+
+    CVReturn err = CVOpenGLESTextureCacheCreate(
+        kCFAllocatorDefault,
+        NULL,
+        [EAGLContext currentContext],
+        NULL,
+        &p->gl_texture_cache);
+
+    if (err != noErr) {
+        MP_ERR(mapper, "Failure in CVOpenGLESTextureCacheCreate: %d\n", err);
         return -1;
     }
 
-    params->imgfmt = params->hw_subfmt;
-    params->hw_subfmt = 0;
     return 0;
 }
 
-static void cleanup_textures(struct gl_hwdec *hw)
+static void mapper_unmap(struct ra_hwdec_mapper *mapper)
 {
-    struct priv *p = hw->priv;
-    int i;
+    struct priv *p = mapper->priv;
 
-    for (i = 0; i < MP_MAX_PLANES; i++) {
+    for (int i = 0; i < p->desc.num_planes; i++) {
+        ra_tex_free(mapper->ra, &mapper->tex[i]);
         if (p->gl_planes[i]) {
             CFRelease(p->gl_planes[i]);
             p->gl_planes[i] = NULL;
@@ -127,37 +186,26 @@ static void cleanup_textures(struct gl_hwdec *hw)
     CVOpenGLESTextureCacheFlush(p->gl_texture_cache, 0);
 }
 
-static int map_frame(struct gl_hwdec *hw, struct mp_image *hw_image,
-                     struct gl_hwdec_frame *out_frame)
+static int mapper_map(struct ra_hwdec_mapper *mapper)
 {
-    struct priv *p = hw->priv;
-    GL *gl = hw->gl;
+    struct priv *p = mapper->priv;
+    GL *gl = ra_gl_get(hw->ra);
 
     CVPixelBufferRelease(p->pbuf);
-    p->pbuf = (CVPixelBufferRef)hw_image->planes[3];
+    p->pbuf = (CVPixelBufferRef)mapper->src->planes[3];
     CVPixelBufferRetain(p->pbuf);
 
     const bool planar = CVPixelBufferIsPlanar(p->pbuf);
     const int planes  = CVPixelBufferGetPlaneCount(p->pbuf);
     assert((planar && planes == p->desc.num_planes) || p->desc.num_planes == 1);
 
-    cleanup_textures(hw);
-
     for (int i = 0; i < p->desc.num_planes; i++) {
-        const struct gl_format *fmt = p->desc.planes[i];
-        GLenum format = fmt->format;
-        GLenum internal_format = fmt->internal_format;
-
-        if (hw->gl->es >= 300) {
-            // In GLES3 mode, CVOpenGLESTextureCacheCreateTextureFromImage()
-            // will return error -6683 unless invoked with GL_LUMINANCE and
-            // GL_LUMINANCE_ALPHA (http://stackoverflow.com/q/36213994/332798)
-            if (format == GL_RED) {
-                format = internal_format = GL_LUMINANCE;
-            } else if (format == GL_RG) {
-                format = internal_format = GL_LUMINANCE_ALPHA;
-            }
-        }
+        const struct ra_format *fmt = p->desc.planes[i];
+
+        GLint internal_format;
+        GLenum format;
+        GLenum type;
+        ra_gl_get_format(fmt, &internal_format, &format, &type);
 
         CVReturn err = CVOpenGLESTextureCacheCreateTextureFromImage(
             kCFAllocatorDefault,
@@ -169,12 +217,12 @@ static int map_frame(struct gl_hwdec *hw, struct mp_image *hw_image,
             CVPixelBufferGetWidthOfPlane(p->pbuf, i),
             CVPixelBufferGetHeightOfPlane(p->pbuf, i),
             format,
-            fmt->type,
+            type,
             i,
             &p->gl_planes[i]);
 
         if (err != noErr) {
-            MP_ERR(hw, "error creating texture for plane %d: %d\n", i, err);
+            MP_ERR(mapper, "error creating texture for plane %d: %d\n", i, err);
             return -1;
         }
 
@@ -185,39 +233,46 @@ static int map_frame(struct gl_hwdec *hw, struct mp_image *hw_image,
         gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
         gl->BindTexture(GL_TEXTURE_2D, 0);
 
-        out_frame->planes[i] = (struct gl_hwdec_plane){
-            .gl_texture = CVOpenGLESTextureGetName(p->gl_planes[i]),
-            .gl_target = GL_TEXTURE_2D,
-            .gl_format = format,
-            .tex_w = CVPixelBufferGetWidthOfPlane(p->pbuf, i),
-            .tex_h = CVPixelBufferGetHeightOfPlane(p->pbuf, i),
+        struct ra_tex_params params = {
+            .dimensions = 2,
+            .w = CVPixelBufferGetWidthOfPlane(p->pbuf, i),
+            .h = CVPixelBufferGetHeightOfPlane(p->pbuf, i),
+            .d = 1,
+            .format = fmt,
+            .render_src = true,
+            .src_linear = true,
         };
+
+        mapper->tex[i] = ra_create_wrapped_tex(mapper->ra, &params,
+                                               p->gl_planes[i]);
+        if (!mapper->tex[i])
+            return -1;
     }
 
     return 0;
 }
 
-static void destroy(struct gl_hwdec *hw)
+static void mapper_uninit(struct ra_hwdec_mapper *mapper)
 {
-    struct priv *p = hw->priv;
-
-    cleanup_textures(hw);
+    struct priv *p = mapper->priv;
 
     CVPixelBufferRelease(p->pbuf);
     CFRelease(p->gl_texture_cache);
     p->gl_texture_cache = NULL;
-
-    av_buffer_unref(&p->hwctx.av_device_ref);
-
-    hwdec_devices_remove(hw->devs, &p->hwctx);
 }
 
-const struct gl_hwdec_driver gl_hwdec_videotoolbox = {
+const struct ra_hwdec_driver ra_hwdec_videotoolbox = {
     .name = "videotoolbox",
+    .priv_size = sizeof(struct priv_owner),
     .api = HWDEC_VIDEOTOOLBOX,
-    .imgfmt = IMGFMT_VIDEOTOOLBOX,
-    .create = create_hwdec,
-    .reinit = reinit,
-    .map_frame = map_frame,
-    .destroy = destroy,
+    .imgfmts = {IMGFMT_VIDEOTOOLBOX, 0},
+    .init = init,
+    .uninit = uninit,
+    .mapper = &(const struct ra_hwdec_mapper_driver){
+        .priv_size = sizeof(struct priv),
+        .init = mapper_init,
+        .uninit = mapper_uninit,
+        .map = mapper_map,
+        .unmap = mapper_unmap,
+    },
 };