vo_opengl: hwdec_osx: use new format setup function

We can drop the custom table.

For some reason, the interop does not accept GL_RGB_RAW_422_APPLE as
internal format for GL_RGB_422_APPLE, so switch the format table to use
GL_RGB (this way both interop and real textures work the same).

Another victim of the apparent requirement of exactly matching texture
formats is kCVPixelFormatType_32BGRA. vo_opengl wants to handle this as
normal RGBA texture, with a swizzle applied in the shader.
CGLTexImageIOSurface2D() rejects this, because it wants the exact
internal format. Just drop the format, because it's useless anyway.

(Maybe this is a bit too fragile...)

4 files changed, 23 insertions, 90 deletions

diff --git a/DOCS/man/options.rst b/DOCS/man/options.rst
index 9cbe6799cc..7db72b9fef 100644
--- a/DOCS/man/options.rst
+++ b/DOCS/man/options.rst
@@ -776,8 +776,8 @@ Video
     choice of the format can influence performance considerably. On the other
     hand, there doesn't appear to be a good way to detect the best format for
     the given hardware. ``nv12``, the default, works better on modern hardware,
-    while ``uyvy422`` appears to be better for old hardware. ``rgb0`` and
-    ``yuv420p`` also work.
+    while ``uyvy422`` appears to be better for old hardware. ``yuv420p`` also
+    works.
 
 ``--panscan=<0.0-1.0>``
     Enables pan-and-scan functionality (cropping the sides of e.g. a 16:9
diff --git a/video/out/opengl/formats.c b/video/out/opengl/formats.c
index 229b3b6ad1..45cb96f8cb 100644
--- a/video/out/opengl/formats.c
+++ b/video/out/opengl/formats.c
@@ -88,9 +88,9 @@ const struct gl_format gl_formats[] = {
     // Special formats.
     {GL_RGB8,                GL_RGB,
      GL_UNSIGNED_SHORT_5_6_5,                           F_TF | F_GL2 | F_GL3},
-    {GL_RGB_RAW_422_APPLE,   GL_RGB_422_APPLE,
+    {GL_RGB,                 GL_RGB_422_APPLE,
      GL_UNSIGNED_SHORT_8_8_APPLE,                       F_TF | F_APPL},
-    {GL_RGB_RAW_422_APPLE,   GL_RGB_422_APPLE,
+    {GL_RGB,                 GL_RGB_422_APPLE,
      GL_UNSIGNED_SHORT_8_8_REV_APPLE,                   F_TF | F_APPL},
 
     {0}
diff --git a/video/out/opengl/hwdec_osx.c b/video/out/opengl/hwdec_osx.c
index eb1634b930..463f9c3f14 100644
--- a/video/out/opengl/hwdec_osx.c
+++ b/video/out/opengl/hwdec_osx.c
@@ -26,86 +26,18 @@
 
 #include "video/mp_image_pool.h"
 #include "video/vt.h"
+#include "formats.h"
 #include "hwdec.h"
 
-struct vt_gl_plane_format {
-    GLenum gl_format;
-    GLenum gl_type;
-    GLenum gl_internal_format;
-};
-
-struct vt_format {
-    uint32_t cvpixfmt;
-    int imgfmt;
-    int planes;
-    struct vt_gl_plane_format gl[MP_MAX_PLANES];
-    char swizzle[5];
-};
-
 struct priv {
     struct mp_hwdec_ctx hwctx;
 
     CVPixelBufferRef pbuf;
     GLuint gl_planes[MP_MAX_PLANES];
-};
 
-static struct vt_format vt_formats[] = {
-    {
-        .cvpixfmt = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange,
-        .imgfmt = IMGFMT_NV12,
-        .planes = 2,
-        .gl = {
-            { GL_RED, GL_UNSIGNED_BYTE, GL_RED },
-            { GL_RG,  GL_UNSIGNED_BYTE, GL_RG } ,
-        }
-    },
-    {
-        .cvpixfmt = kCVPixelFormatType_422YpCbCr8,
-        .imgfmt = IMGFMT_UYVY,
-        .planes = 1,
-        .gl = {
-            { GL_RGB_422_APPLE, GL_UNSIGNED_SHORT_8_8_APPLE, GL_RGB }
-        },
-        .swizzle = "gbra",
-    },
-    {
-        .cvpixfmt = kCVPixelFormatType_420YpCbCr8Planar,
-        .imgfmt = IMGFMT_420P,
-        .planes = 3,
-        .gl = {
-            { GL_RED, GL_UNSIGNED_BYTE, GL_RED },
-            { GL_RED, GL_UNSIGNED_BYTE, GL_RED },
-            { GL_RED, GL_UNSIGNED_BYTE, GL_RED },
-        }
-    },
-    {
-        .cvpixfmt = kCVPixelFormatType_32BGRA,
-        .imgfmt = IMGFMT_BGR0,
-        .planes = 1,
-        .gl = {
-            { GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, GL_RGBA }
-        }
-    },
+    struct gl_imgfmt_desc desc;
 };
 
-static struct vt_format *vt_get_gl_format(uint32_t cvpixfmt)
-{
-    for (int i = 0; i < MP_ARRAY_SIZE(vt_formats); i++) {
-        if (vt_formats[i].cvpixfmt == cvpixfmt)
-            return &vt_formats[i];
-    }
-    return NULL;
-}
-
-static struct vt_format *vt_get_gl_format_from_imgfmt(uint32_t imgfmt)
-{
-    for (int i = 0; i < MP_ARRAY_SIZE(vt_formats); i++) {
-        if (vt_formats[i].imgfmt == imgfmt)
-            return &vt_formats[i];
-    }
-    return NULL;
-}
-
 static bool check_hwdec(struct gl_hwdec *hw)
 {
     if (hw->gl->version < 300) {
@@ -143,15 +75,21 @@ static int create(struct gl_hwdec *hw)
 
 static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
 {
+    struct priv *p = hw->priv;
+
     assert(params->imgfmt == hw->driver->imgfmt);
 
-    struct vt_format *f = vt_get_gl_format_from_imgfmt(params->hw_subfmt);
-    if (!f) {
+    if (!params->hw_subfmt) {
         MP_ERR(hw, "Unsupported CVPixelBuffer format.\n");
         return -1;
     }
 
-    params->imgfmt = f->imgfmt;
+    if (!gl_get_imgfmt_desc(hw->gl, params->hw_subfmt, &p->desc)) {
+        MP_ERR(hw, "Unsupported texture format.\n");
+        return -1;
+    }
+
+    params->imgfmt = params->hw_subfmt;
     params->hw_subfmt = 0;
     return 0;
 }
@@ -171,28 +109,23 @@ static int map_frame(struct gl_hwdec *hw, struct mp_image *hw_image,
         return -1;
     }
 
-    uint32_t cvpixfmt = CVPixelBufferGetPixelFormatType(p->pbuf);
-    struct vt_format *f = vt_get_gl_format(cvpixfmt);
-    if (!f) {
-        MP_ERR(hw, "CVPixelBuffer has unsupported format type\n");
-        return -1;
-    }
-
     const bool planar = CVPixelBufferIsPlanar(p->pbuf);
     const int planes  = CVPixelBufferGetPlaneCount(p->pbuf);
-    assert(planar && planes == f->planes || f->planes == 1);
+    assert((planar && planes == p->desc.num_planes) || p->desc.num_planes == 1);
 
     GLenum gl_target = GL_TEXTURE_RECTANGLE;
 
-    for (int i = 0; i < f->planes; i++) {
+    for (int i = 0; i < p->desc.num_planes; i++) {
+        const struct gl_format *fmt = p->desc.planes[i];
+
         gl->BindTexture(gl_target, p->gl_planes[i]);
 
         CGLError err = CGLTexImageIOSurface2D(
             CGLGetCurrentContext(), gl_target,
-            f->gl[i].gl_internal_format,
+            fmt->internal_format,
             IOSurfaceGetWidthOfPlane(surface, i),
             IOSurfaceGetHeightOfPlane(surface, i),
-            f->gl[i].gl_format, f->gl[i].gl_type, surface, i);
+            fmt->format, fmt->type, surface, i);
 
         if (err != kCGLNoError)
             MP_ERR(hw, "error creating IOSurface texture for plane %d: %s (%x)\n",
@@ -208,7 +141,8 @@ static int map_frame(struct gl_hwdec *hw, struct mp_image *hw_image,
         };
     }
 
-    snprintf(out_frame->swizzle, sizeof(out_frame->swizzle), "%s", f->swizzle);
+    snprintf(out_frame->swizzle, sizeof(out_frame->swizzle), "%s",
+             p->desc.swizzle);
 
     return 0;
 }
diff --git a/video/vt.c b/video/vt.c
index f0804d38a0..8488256399 100644
--- a/video/vt.c
+++ b/video/vt.c
@@ -9,7 +9,6 @@
 static const uint32_t map_imgfmt_cvpixfmt[][2] = {
     {IMGFMT_420P,   kCVPixelFormatType_420YpCbCr8Planar},
     {IMGFMT_UYVY,   kCVPixelFormatType_422YpCbCr8},
-    {IMGFMT_BGR0,   kCVPixelFormatType_32BGRA},
     {IMGFMT_NV12,   kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange},
     {0}
 };