vo_opengl: reorganize texture format handling

This merges all knowledge about texture format into a central table.

Most of the work done here is actually identifying which formats exactly
are supported by OpenGL(ES) under which circumstances, and keeping this
information in the format table in a somewhat declarative way. (Although
only to the extend needed by mpv.) In particular, ES and float formats
are a horrible mess.

Again this is a big refactor that might cause regression on "obscure"
configurations.

9 files changed, 443 insertions, 311 deletions

diff --git a/video/out/opengl/common.c b/video/out/opengl/common.c
index ee8b4c6468..c181a8ed13 100644
--- a/video/out/opengl/common.c
+++ b/video/out/opengl/common.c
@@ -72,6 +72,8 @@ struct gl_functions {
     int provides;               // bitfield of MPGL_CAP_* constants
     int ver_core;               // introduced as required function
     int ver_es_core;            // introduced as required GL ES function
+    int ver_exclude;            // not applicable to versions >= ver_exclude
+    int ver_es_exclude;         // same for GLES
     const struct gl_function *functions;
 };
 
@@ -228,9 +230,28 @@ static const struct gl_functions gl_functions[] = {
     },
     // GL_R16 etc.
     {
-        .ver_core = 300,
         .extension = "GL_EXT_texture_norm16",
         .provides = MPGL_CAP_EXT16,
+        .ver_exclude = 1, // never in desktop GL
+    },
+    // Float texture support for GL 2.x
+    {
+        .extension = "GL_ARB_texture_float",
+        .provides = MPGL_CAP_ARB_FLOAT,
+        .ver_exclude = 300,
+        .ver_es_exclude = 1,
+    },
+    // 16 bit float textures filterable with GL_LINEAR in GLES
+    {
+        .extension = "GL_OES_texture_half_float_linear",
+        .provides = MPGL_CAP_OES_HFLOAT_LIN,
+        .ver_exclude = 1,
+    },
+    // 16 bit float textures that can be rendered to in GLES
+    {
+        .extension = "GL_EXT_color_buffer_half_float",
+        .provides = MPGL_CAP_EXT_CR_HFLOAT,
+        .ver_exclude = 1,
     },
     {
         .ver_core = 320,
@@ -439,6 +460,13 @@ void mpgl_load_functions2(GL *gl, void *(*get_fn)(void *ctx, const char *n),
         // NOTE: Function entrypoints can exist, even if they do not work.
         //       We must always check extension strings and versions.
 
+        if (gl->version && section->ver_exclude &&
+            gl->version >= section->ver_exclude)
+            continue;
+        if (gl->es && section->ver_es_exclude &&
+            gl->es >= section->ver_es_exclude)
+            continue;
+
         bool exists = false, must_exist = false;
         if (ver_core)
             must_exist = version >= ver_core;
@@ -505,14 +533,6 @@ void mpgl_load_functions2(GL *gl, void *(*get_fn)(void *ctx, const char *n),
         mp_verbose(log, "Detected suspected software renderer.\n");
     }
 
-    // Detect 16F textures that work with GL_LINEAR filtering.
-    if ((!gl->es && (gl->version >= 300 || check_ext(gl, "GL_ARB_texture_float"))) ||
-        (gl->es && (gl->version >= 310 || check_ext(gl, "GL_OES_texture_half_float_linear"))))
-    {
-        mp_verbose(log, "Filterable half-float textures supported.\n");
-        gl->mpgl_caps |= MPGL_CAP_FLOAT_TEX;
-    }
-
     // Provided for simpler handling if no framebuffer support is available.
     if (!gl->BindFramebuffer)
         gl->BindFramebuffer = &dummy_glBindFramebuffer;
diff --git a/video/out/opengl/common.h b/video/out/opengl/common.h
index cc6f7b6459..4f21e25455 100644
--- a/video/out/opengl/common.h
+++ b/video/out/opengl/common.h
@@ -53,7 +53,6 @@ enum {
     MPGL_CAP_ROW_LENGTH         = (1 << 4),     // GL_[UN]PACK_ROW_LENGTH
     MPGL_CAP_FB                 = (1 << 5),
     MPGL_CAP_VAO                = (1 << 6),
-    MPGL_CAP_FLOAT_TEX          = (1 << 9),
     MPGL_CAP_TEX_RG             = (1 << 10),    // GL_ARB_texture_rg / GL 3.x
     MPGL_CAP_VDPAU              = (1 << 11),    // GL_NV_vdpau_interop
     MPGL_CAP_APPLE_RGB_422      = (1 << 12),    // GL_APPLE_rgb_422
@@ -62,6 +61,10 @@ enum {
     MPGL_CAP_DEBUG              = (1 << 16),
     MPGL_CAP_DXINTEROP          = (1 << 17),    // WGL_NV_DX_interop
     MPGL_CAP_EXT16              = (1 << 18),    // GL_EXT_texture_norm16
+    MPGL_CAP_ARB_FLOAT          = (1 << 19),    // GL_ARB_texture_float
+    MPGL_CAP_EXT_CR_HFLOAT      = (1 << 20),    // GL_EXT_color_buffer_half_float
+    MPGL_CAP_OES_HFLOAT_LIN     = (1 << 21),    // GL_OES_texture_half_float_linear
+
     MPGL_CAP_SW                 = (1 << 30),    // indirect or sw renderer
 };
 
diff --git a/video/out/opengl/formats.c b/video/out/opengl/formats.c
new file mode 100644
index 0000000000..36f76df459
--- /dev/null
+++ b/video/out/opengl/formats.c
@@ -0,0 +1,274 @@
+#include "common/common.h"
+#include "formats.h"
+
+enum {
+    // --- GL type aliases (for readability)
+    T_U8        = GL_UNSIGNED_BYTE,
+    T_U16       = GL_UNSIGNED_SHORT,
+    T_FL        = GL_FLOAT,
+};
+
+// List of allowed formats, and their usability for bilinear filtering and FBOs.
+// This is limited to combinations that are useful for our renderer.
+const struct gl_format gl_formats[] = {
+    // These are used for desktop GL 3+, and GLES 3+ with GL_EXT_texture_norm16.
+    {GL_R8,                  GL_RED,             T_U8,  F_CF | F_GL3 | F_GL2F | F_ES3},
+    {GL_RG8,                 GL_RG,              T_U8,  F_CF | F_GL3 | F_GL2F | F_ES3},
+    {GL_RGB8,                GL_RGB,             T_U8,  F_CF | F_GL3 | F_GL2F | F_ES3},
+    {GL_RGBA8,               GL_RGBA,            T_U8,  F_CF | F_GL3 | F_GL2F | F_ES3},
+    {GL_R16,                 GL_RED,             T_U16, F_CF | F_GL3 | F_GL2F | F_EXT16},
+    {GL_RG16,                GL_RG,              T_U16, F_CF | F_GL3 | F_GL2F | F_EXT16},
+    {GL_RGB16,               GL_RGB,             T_U16, F_CF | F_GL3 | F_GL2F},
+    {GL_RGBA16,              GL_RGBA,            T_U16, F_CF | F_GL3 | F_GL2F | F_EXT16},
+
+    // Specifically not color-renderable.
+    {GL_RGB16,               GL_RGB,             T_U16, F_TF | F_EXT16},
+
+    // GL2 legacy. Ignores possibly present FBO extensions (no CF flag set).
+    {GL_LUMINANCE8,          GL_LUMINANCE,       T_U8,  F_TF | F_GL2},
+    {GL_LUMINANCE8_ALPHA8,   GL_LUMINANCE_ALPHA, T_U8,  F_TF | F_GL2},
+    {GL_RGB8,                GL_RGB,             T_U8,  F_TF | F_GL2},
+    {GL_RGBA8,               GL_RGBA,            T_U8,  F_TF | F_GL2},
+    {GL_LUMINANCE16,         GL_LUMINANCE,       T_U16, F_TF | F_GL2},
+    {GL_LUMINANCE16_ALPHA16, GL_LUMINANCE_ALPHA, T_U16, F_TF | F_GL2},
+    {GL_RGB16,               GL_RGB,             T_U16, F_TF | F_GL2},
+    {GL_RGBA16,              GL_RGBA,            T_U16, F_TF | F_GL2},
+
+    // ES2 legacy
+    {GL_LUMINANCE,           GL_LUMINANCE,       T_U8,  F_CF | F_ES2},
+    {GL_LUMINANCE_ALPHA,     GL_LUMINANCE_ALPHA, T_U8,  F_CF | F_ES2},
+    {GL_RGB,                 GL_RGB,             T_U8,  F_CF | F_ES2},
+    {GL_RGBA,                GL_RGBA,            T_U8,  F_CF | F_ES2},
+
+    // Non-normalized integer formats.
+    // Follows ES 3.0 as to which are color-renderable.
+    {GL_R8UI,                GL_RED_INTEGER,     T_U8,  F_CR | F_GL3 | F_ES3},
+    {GL_RG8UI,               GL_RG_INTEGER,      T_U8,  F_CR | F_GL3 | F_ES3},
+    {GL_RGB8UI,              GL_RGB_INTEGER,     T_U8,         F_GL3 | F_ES3},
+    {GL_RGBA8UI,             GL_RGBA_INTEGER,    T_U8,  F_CR | F_GL3 | F_ES3},
+    {GL_R16UI,               GL_RED_INTEGER,     T_U16, F_CR | F_GL3 | F_ES3},
+    {GL_RG16UI,              GL_RG_INTEGER,      T_U16, F_CR | F_GL3 | F_ES3},
+    {GL_RGB16UI,             GL_RGB_INTEGER,     T_U16,        F_GL3 | F_ES3},
+    {GL_RGBA16UI,            GL_RGBA_INTEGER,    T_U16, F_CR | F_GL3 | F_ES3},
+
+    // On GL3+ or GL2.1 with GL_ARB_texture_float, floats work fully.
+    {GL_R16F,                GL_RED,             T_FL,  F_F16 | F_CF | F_GL3 | F_GL2F},
+    {GL_RG16F,               GL_RG,              T_FL,  F_F16 | F_CF | F_GL3 | F_GL2F},
+    {GL_RGB16F,              GL_RGB,             T_FL,  F_F16 | F_CF | F_GL3 | F_GL2F},
+    {GL_RGBA16F,             GL_RGBA,            T_FL,  F_F16 | F_CF | F_GL3 | F_GL2F},
+    {GL_R32F,                GL_RED,             T_FL,          F_CF | F_GL3 | F_GL2F},
+    {GL_RG32F,               GL_RG,              T_FL,          F_CF | F_GL3 | F_GL2F},
+    {GL_RGB32F,              GL_RGB,             T_FL,          F_CF | F_GL3 | F_GL2F},
+    {GL_RGBA32F,             GL_RGBA,            T_FL,          F_CF | F_GL3 | F_GL2F},
+
+    // Note: we simply don't support float anything on ES2, despite extensions.
+    // We also don't bother with non-filterable float formats, and we ignore
+    // 32 bit float formats that are not blendable when rendering to them.
+
+    // On ES3.2+, both 16 bit floats work fully (except 3-component formats).
+    // F_EXTF16 implies extensions that also enable 16 bit floats fully.
+    {GL_R16F,                GL_RED,             T_FL,  F_F16 | F_CF | F_ES32 | F_EXTF16},
+    {GL_RG16F,               GL_RG,              T_FL,  F_F16 | F_CF | F_ES32 | F_EXTF16},
+    {GL_RGB16F,              GL_RGB,             T_FL,  F_F16 | F_TF | F_ES32 | F_EXTF16},
+    {GL_RGBA16F,             GL_RGBA,            T_FL,  F_F16 | F_CF | F_ES32 | F_EXTF16},
+
+    // On ES3.0+, 16 bit floats are texture-filterable.
+    // Don't bother with 32 bit floats; they exist but are neither CR nor TF.
+    {GL_R16F,                GL_RED,             T_FL,  F_F16 | F_TF | F_ES3},
+    {GL_RG16F,               GL_RG,              T_FL,  F_F16 | F_TF | F_ES3},
+    {GL_RGB16F,              GL_RGB,             T_FL,  F_F16 | F_TF | F_ES3},
+    {GL_RGBA16F,             GL_RGBA,            T_FL,  F_F16 | F_TF | F_ES3},
+
+    // These might be useful as FBO formats.
+    {GL_RGB10_A2,            GL_RGBA,
+     GL_UNSIGNED_INT_2_10_10_10_REV,                    F_CF | F_GL3 | F_ES3},
+    {GL_RGBA12,              GL_RGBA,            T_U16, F_CF | F_GL2 | F_GL3},
+    {GL_RGB10,               GL_RGB,             T_U16, F_CF | F_GL2 | F_GL3},
+
+    // Special formats.
+    {GL_RGB8,                GL_RGB,
+     GL_UNSIGNED_SHORT_5_6_5,                           F_TF | F_GL2 | F_GL3},
+    {GL_RGB_RAW_422_APPLE,   GL_RGB_422_APPLE,
+     GL_UNSIGNED_SHORT_8_8_APPLE,                       F_TF | F_APPL},
+    {GL_RGB_RAW_422_APPLE,   GL_RGB_422_APPLE,
+     GL_UNSIGNED_SHORT_8_8_REV_APPLE,                   F_TF | F_APPL},
+
+    {0}
+};
+
+// Pairs of mpv formats and OpenGL types that match directly. Code using this
+// is supposed to look through the gl_formats table, and there is supposed to
+// be exactly 1 matching entry (which tells you format/internal format).
+static const int special_formats[][2] = {
+    {IMGFMT_RGB565,     GL_UNSIGNED_SHORT_5_6_5},
+    {IMGFMT_UYVY,       GL_UNSIGNED_SHORT_8_8_APPLE},
+    {IMGFMT_YUYV,       GL_UNSIGNED_SHORT_8_8_REV_APPLE},
+    {0}
+};
+
+// Return an or-ed combination of all F_ flags that apply.
+int gl_format_feature_flags(GL *gl)
+{
+    return (gl->version == 210 ? F_GL2 : 0)
+         | (gl->version >= 300 ? F_GL3 : 0)
+         | (gl->es == 200 ? F_ES2 : 0)
+         | (gl->es >= 300 ? F_ES3 : 0)
+         | (gl->es >= 320 ? F_ES32 : 0)
+         | (gl->mpgl_caps & MPGL_CAP_EXT16 ? F_EXT16 : 0)
+         | ((gl->es &&
+            (gl->mpgl_caps & MPGL_CAP_TEX_RG) &&
+            (gl->mpgl_caps & MPGL_CAP_EXT_CR_HFLOAT) &&
+            (gl->mpgl_caps & MPGL_CAP_OES_HFLOAT_LIN)) ? F_EXTF16 : 0)
+         | ((gl->version == 210 &&
+            (gl->mpgl_caps & MPGL_CAP_ARB_FLOAT) &&
+            (gl->mpgl_caps & MPGL_CAP_TEX_RG) &&
+            (gl->mpgl_caps & MPGL_CAP_FB)) ? F_GL2F : 0)
+         | (gl->mpgl_caps & MPGL_CAP_APPLE_RGB_422 ? F_APPL : 0);
+}
+
+// Return the entry for the given internal format. Return NULL if unsupported.
+const struct gl_format *gl_find_internal_format(GL *gl, GLint internal_format)
+{
+    int features = gl_format_feature_flags(gl);
+    for (int n = 0; gl_formats[n].type; n++) {
+        const struct gl_format *f = &gl_formats[n];
+        if (f->internal_format == internal_format && (f->flags & features))
+            return f;
+    }
+    return NULL;
+}
+
+const struct gl_format *gl_find_special_format(GL *gl, int mpfmt)
+{
+    int features = gl_format_feature_flags(gl);
+    for (int n = 0; special_formats[n][0]; n++) {
+        if (special_formats[n][0] == mpfmt) {
+            GLenum type = special_formats[n][1];
+            for (int i = 0; gl_formats[i].type; i++) {
+                const struct gl_format *f = &gl_formats[i];
+                if (f->type == type && (f->flags & features))
+                    return f;
+            }
+            break;
+        }
+    }
+    return NULL;
+}
+
+// type: one of MPGL_TYPE_*
+// flags: bitset of F_*, all flags must be present
+const struct gl_format *gl_find_format(GL *gl, int type, int flags,
+                                       int bytes_per_component, int n_components)
+{
+    if (!bytes_per_component || !n_components || !type)
+        return NULL;
+    int features = gl_format_feature_flags(gl);
+    for (int n = 0; gl_formats[n].type; n++) {
+        const struct gl_format *f = &gl_formats[n];
+        if ((f->flags & features) &&
+            ((f->flags & flags) == flags) &&
+            gl_format_type(f) == type &&
+            gl_component_size(f->type) == bytes_per_component &&
+            gl_format_components(f->format) == n_components)
+            return f;
+    }
+    return NULL;
+}
+
+// Return a texture-filterable unsigned normalized fixed point format.
+const struct gl_format *gl_find_unorm_format(GL *gl, int bytes_per_component,
+                                             int n_components)
+{
+    return gl_find_format(gl, MPGL_TYPE_UNORM, F_TF, bytes_per_component,
+                          n_components);
+}
+
+// Return an unsigned integer format.
+const struct gl_format *gl_find_uint_format(GL *gl, int bytes_per_component,
+                                            int n_components)
+{
+    return gl_find_format(gl, MPGL_TYPE_UINT, 0, bytes_per_component,
+                          n_components);
+}
+
+// Return a 16 bit float format. Note that this will return a GL_FLOAT format
+// with 32 bit per component; just the internal representation is smaller.
+// Some GL versions will allow upload with GL_HALF_FLOAT as well.
+const struct gl_format *gl_find_float16_format(GL *gl, int n_components)
+{
+    return gl_find_format(gl, MPGL_TYPE_FLOAT, F_F16, 4, n_components);
+}
+
+int gl_format_type(const struct gl_format *format)
+{
+    if (!format)
+        return 0;
+    if (format->type == GL_FLOAT)
+        return MPGL_TYPE_FLOAT;
+    if (gl_integer_format_to_base(format->format))
+        return MPGL_TYPE_UINT;
+    return MPGL_TYPE_UNORM;
+}
+
+// Return an integer pixel "format" to a base internal format.
+// Return 0 if it's not an integer format.
+GLenum gl_integer_format_to_base(GLenum format)
+{
+    switch (format) {
+    case GL_RED_INTEGER:        return GL_RED;
+    case GL_RG_INTEGER:         return GL_RG;
+    case GL_RGB_INTEGER:        return GL_RGB;
+    case GL_RGBA_INTEGER:       return GL_RGBA;
+    }
+    return 0;
+}
+
+// Return the number of bytes per component this format implies.
+// Returns 0 for formats with non-byte alignments and formats which
+// merge multiple components (like GL_UNSIGNED_SHORT_5_6_5).
+int gl_component_size(GLenum type)
+{
+    switch (type) {
+    case GL_UNSIGNED_BYTE:                      return 1;
+    case GL_UNSIGNED_SHORT:                     return 2;
+    case GL_FLOAT:                              return 4;
+    }
+    return 0;
+}
+
+// Return the number of a pixel "format".
+int gl_format_components(GLenum format)
+{
+    switch (format) {
+    case GL_RED:
+    case GL_RED_INTEGER:
+    case GL_LUMINANCE:
+        return 1;
+    case GL_RG:
+    case GL_RG_INTEGER:
+    case GL_LUMINANCE_ALPHA:
+        return 2;
+    case GL_RGB:
+    case GL_RGB_INTEGER:
+        return 3;
+    case GL_RGBA:
+    case GL_RGBA_INTEGER:
+        return 4;
+    }
+    return 0;
+}
+
+// return the number of bytes per pixel for the given format
+// does not handle all possible variants, just those used by mpv
+int gl_bytes_per_pixel(GLenum format, GLenum type)
+{
+    // Formats with merged components are special.
+    switch (type) {
+    case GL_UNSIGNED_INT_2_10_10_10_REV:        return 4;
+    case GL_UNSIGNED_SHORT_5_6_5:               return 2;
+    case GL_UNSIGNED_SHORT_8_8_APPLE:           return 2;
+    case GL_UNSIGNED_SHORT_8_8_REV_APPLE:       return 2;
+    }
+
+    return gl_format_components(format) * gl_component_size(type);
+}
diff --git a/video/out/opengl/formats.h b/video/out/opengl/formats.h
new file mode 100644
index 0000000000..f62f96cff0
--- /dev/null
+++ b/video/out/opengl/formats.h
@@ -0,0 +1,60 @@
+#ifndef MPGL_FORMATS_H_
+#define MPGL_FORMATS_H_
+
+#include "common.h"
+
+struct gl_format {
+    GLint internal_format;      // glTexImage argument
+    GLenum format;              // glTexImage argument
+    GLenum type;                // e.g. GL_UNSIGNED_SHORT
+    int flags;
+};
+
+extern const struct gl_format gl_formats[];
+
+enum {
+    // --- gl_format.flags
+
+    // Version flags. If at least 1 flag matches, the format entry is considered
+    // supported on the current GL context.
+    F_GL2       = 1 << 0, // GL2.1-only
+    F_GL3       = 1 << 1, // GL3.0 or later
+    F_ES2       = 1 << 2, // ES2-only
+    F_ES3       = 1 << 3, // ES3.0 or later
+    F_ES32      = 1 << 4, // ES3.2 or later
+    F_EXT16     = 1 << 5, // ES with GL_EXT_texture_norm16
+    F_EXTF16    = 1 << 6, // GL_OES_texture_half_float_linear +
+                          // GL_EXT_color_buffer_half_float
+    F_GL2F      = 1 << 7, // GL2.1-only with texture_rg + texture_float + FBOs
+    F_APPL      = 1 << 8, // GL_APPLE_rgb_422
+
+    // Feature flags. They are additional and signal presence of features.
+    F_CR        = 1 << 16, // color-renderable
+    F_TF        = 1 << 17, // texture-filterable with GL_LINEAR
+    F_CF        = F_CR | F_TF,
+    F_F16       = 1 << 18, // uses half-floats (16 bit) internally, even though
+                           // the format is still GL_FLOAT (32 bit)
+
+    // --- Other constants.
+    MPGL_TYPE_UNORM = 1,
+    MPGL_TYPE_UINT = 2,
+    MPGL_TYPE_FLOAT = 3,
+};
+
+int gl_format_feature_flags(GL *gl);
+const struct gl_format *gl_find_internal_format(GL *gl, GLint internal_format);
+const struct gl_format *gl_find_special_format(GL *gl, int mpfmt);
+const struct gl_format *gl_find_format(GL *gl, int type, int flags,
+                                       int bytes_per_component, int n_components);
+const struct gl_format *gl_find_unorm_format(GL *gl, int bytes_per_component,
+                                             int n_components);
+const struct gl_format *gl_find_uint_format(GL *gl, int bytes_per_component,
+                                            int n_components);
+const struct gl_format *gl_find_float16_format(GL *gl, int n_components);
+int gl_format_type(const struct gl_format *format);
+GLenum gl_integer_format_to_base(GLenum format);
+int gl_component_size(GLenum type);
+int gl_format_components(GLenum format);
+int gl_bytes_per_pixel(GLenum format, GLenum type);
+
+#endif
diff --git a/video/out/opengl/header_fixes.h b/video/out/opengl/header_fixes.h
index 92867a0d01..964b33a3fa 100644
--- a/video/out/opengl/header_fixes.h
+++ b/video/out/opengl/header_fixes.h
@@ -98,6 +98,10 @@
 #define GL_TRANSLATED_SHADER_SOURCE_LENGTH_ANGLE 0x93A0
 #endif
 
+#ifndef GL_RGB_RAW_422_APPLE
+#define GL_RGB_RAW_422_APPLE 0x8A51
+#endif
+
 #undef MP_GET_GL_WORKAROUNDS
 
 #endif // MP_GET_GL_WORKAROUNDS
diff --git a/video/out/opengl/osd.c b/video/out/opengl/osd.c
index c554425e0f..02a0fde329 100644
--- a/video/out/opengl/osd.c
+++ b/video/out/opengl/osd.c
@@ -21,15 +21,10 @@
 
 #include "video/out/bitmap_packer.h"
 
+#include "formats.h"
 #include "utils.h"
 #include "osd.h"
 
-struct osd_fmt_entry {
-    GLint internal_format;
-    GLint format;
-    GLenum type;
-};
-
 // glBlendFuncSeparate() arguments
 static const int blend_factors[SUBBITMAP_COUNT][4] = {
     [SUBBITMAP_LIBASS] = {GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA,
@@ -38,21 +33,6 @@ static const int blend_factors[SUBBITMAP_COUNT][4] = {
                           GL_ONE,       GL_ONE_MINUS_SRC_ALPHA},
 };
 
-static const struct osd_fmt_entry osd_to_gl3_formats[SUBBITMAP_COUNT] = {
-    [SUBBITMAP_LIBASS] = {GL_RED,   GL_RED,   GL_UNSIGNED_BYTE},
-    [SUBBITMAP_RGBA] =   {GL_RGBA,  GL_RGBA,  GL_UNSIGNED_BYTE},
-};
-
-static const struct osd_fmt_entry osd_to_gles3_formats[SUBBITMAP_COUNT] = {
-    [SUBBITMAP_LIBASS] = {GL_R8,    GL_RED,   GL_UNSIGNED_BYTE},
-    [SUBBITMAP_RGBA] =   {GL_RGBA8, GL_RGBA,  GL_UNSIGNED_BYTE},
-};
-
-static const struct osd_fmt_entry osd_to_gl2_formats[SUBBITMAP_COUNT] = {
-    [SUBBITMAP_LIBASS] = {GL_LUMINANCE, GL_LUMINANCE,   GL_UNSIGNED_BYTE},
-    [SUBBITMAP_RGBA] =   {GL_RGBA,      GL_RGBA,        GL_UNSIGNED_BYTE},
-};
-
 struct vertex {
     float position[2];
     float texcoord[2];
@@ -86,7 +66,7 @@ struct mpgl_osd {
     bool use_pbo;
     bool scaled;
     struct mpgl_osd_part *parts[MAX_OSD_PARTS];
-    const struct osd_fmt_entry *fmt_table;
+    const struct gl_format *fmt_table[SUBBITMAP_COUNT];
     bool formats[SUBBITMAP_COUNT];
     struct gl_vao vao;
     int64_t change_counter;
@@ -106,15 +86,11 @@ struct mpgl_osd *mpgl_osd_init(GL *gl, struct mp_log *log, struct osd_state *osd
         .log = log,
         .osd = osd,
         .gl = gl,
-        .fmt_table = osd_to_gl3_formats,
         .scratch = talloc_zero_size(ctx, 1),
     };
 
-    if (gl->es >= 300) {
-        ctx->fmt_table = osd_to_gles3_formats;
-    } else if (!(gl->mpgl_caps & MPGL_CAP_TEX_RG)) {
-        ctx->fmt_table = osd_to_gl2_formats;
-    }
+    ctx->fmt_table[SUBBITMAP_LIBASS] = gl_find_unorm_format(gl, 1, 1);
+    ctx->fmt_table[SUBBITMAP_RGBA]   = gl_find_unorm_format(gl, 1, 4);
 
     for (int n = 0; n < MAX_OSD_PARTS; n++) {
         struct mpgl_osd_part *p = talloc_ptrtype(ctx, p);
@@ -128,7 +104,7 @@ struct mpgl_osd *mpgl_osd_init(GL *gl, struct mp_log *log, struct osd_state *osd
     }
 
     for (int n = 0; n < SUBBITMAP_COUNT; n++)
-        ctx->formats[n] = ctx->fmt_table[n].type != 0;
+        ctx->formats[n] = !!ctx->fmt_table[n];
 
     gl_vao_init(&ctx->vao, gl, sizeof(struct vertex), vertex_vao);
 
@@ -163,8 +139,8 @@ static bool upload_pbo(struct mpgl_osd *ctx, struct mpgl_osd_part *osd,
 {
     GL *gl = ctx->gl;
     bool success = true;
-    struct osd_fmt_entry fmt = ctx->fmt_table[imgs->format];
-    int pix_stride = glFmt2bpp(fmt.format, fmt.type);
+    const struct gl_format *fmt = ctx->fmt_table[imgs->format];
+    int pix_stride = gl_bytes_per_pixel(fmt->format, fmt->type);
 
     if (!osd->buffer) {
         gl->GenBuffers(1, &osd->buffer);
@@ -185,7 +161,7 @@ static bool upload_pbo(struct mpgl_osd *ctx, struct mpgl_osd_part *osd,
         packer_copy_subbitmaps(osd->packer, imgs, data, pix_stride, stride);
         if (!gl->UnmapBuffer(GL_PIXEL_UNPACK_BUFFER))
             success = false;
-        glUploadTex(gl, GL_TEXTURE_2D, fmt.format, fmt.type, NULL, stride,
+        glUploadTex(gl, GL_TEXTURE_2D, fmt->format, fmt->type, NULL, stride,
                     bb[0].x, bb[0].y, bb[1].x - bb[0].x, bb[1].y - bb[0].y, 0);
     }
     gl->BindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
@@ -201,11 +177,11 @@ static bool upload_pbo(struct mpgl_osd *ctx, struct mpgl_osd_part *osd,
 static void upload_tex(struct mpgl_osd *ctx, struct mpgl_osd_part *osd,
                        struct sub_bitmaps *imgs)
 {
-    struct osd_fmt_entry fmt = ctx->fmt_table[imgs->format];
+    const struct gl_format *fmt = ctx->fmt_table[imgs->format];
     if (osd->packer->padding) {
         struct pos bb[2];
         packer_get_bb(osd->packer, bb);
-        glClearTex(ctx->gl, GL_TEXTURE_2D, fmt.format, fmt.type,
+        glClearTex(ctx->gl, GL_TEXTURE_2D, fmt->format, fmt->type,
                    bb[0].x, bb[0].y, bb[1].x - bb[0].y, bb[1].y - bb[0].y,
                    0, &ctx->scratch);
     }
@@ -213,7 +189,7 @@ static void upload_tex(struct mpgl_osd *ctx, struct mpgl_osd_part *osd,
         struct sub_bitmap *s = &imgs->parts[n];
         struct pos p = osd->packer->result[n];
 
-        glUploadTex(ctx->gl, GL_TEXTURE_2D, fmt.format, fmt.type,
+        glUploadTex(ctx->gl, GL_TEXTURE_2D, fmt->format, fmt->type,
                     s->bitmap, s->stride, p.x, p.y, s->w, s->h, 0);
     }
 }
@@ -232,8 +208,8 @@ static bool upload_osd(struct mpgl_osd *ctx, struct mpgl_osd_part *osd,
         return false;
     }
 
-    struct osd_fmt_entry fmt = ctx->fmt_table[imgs->format];
-    assert(fmt.type != 0);
+    const struct gl_format *fmt = ctx->fmt_table[imgs->format];
+    assert(fmt);
 
     if (!osd->texture)
         gl->GenTextures(1, &osd->texture);
@@ -247,8 +223,8 @@ static bool upload_osd(struct mpgl_osd *ctx, struct mpgl_osd_part *osd,
         osd->w = FFMAX(32, osd->packer->w);
         osd->h = FFMAX(32, osd->packer->h);
 
-        gl->TexImage2D(GL_TEXTURE_2D, 0, fmt.internal_format, osd->w, osd->h,
-                       0, fmt.format, fmt.type, NULL);
+        gl->TexImage2D(GL_TEXTURE_2D, 0, fmt->internal_format, osd->w, osd->h,
+                       0, fmt->format, fmt->type, NULL);
 
         gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
         gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
diff --git a/video/out/opengl/utils.c b/video/out/opengl/utils.c
index 60e1792a14..1085110802 100644
--- a/video/out/opengl/utils.c
+++ b/video/out/opengl/utils.c
@@ -24,6 +24,7 @@
 #include <assert.h>
 
 #include "common/common.h"
+#include "formats.h"
 #include "utils.h"
 
 // GLU has this as gluErrorString (we don't use GLU, as it is legacy-OpenGL)
@@ -50,52 +51,6 @@ void glCheckError(GL *gl, struct mp_log *log, const char *info)
     }
 }
 
-// return the number of bytes per pixel for the given format
-// does not handle all possible variants, just those used by mpv
-int glFmt2bpp(GLenum format, GLenum type)
-{
-    int component_size = 0;
-    switch (type) {
-    case GL_UNSIGNED_BYTE_3_3_2:
-    case GL_UNSIGNED_BYTE_2_3_3_REV:
-        return 1;
-    case GL_UNSIGNED_SHORT_5_5_5_1:
-    case GL_UNSIGNED_SHORT_1_5_5_5_REV:
-    case GL_UNSIGNED_SHORT_5_6_5:
-    case GL_UNSIGNED_SHORT_5_6_5_REV:
-        return 2;
-    case GL_UNSIGNED_BYTE:
-        component_size = 1;
-        break;
-    case GL_UNSIGNED_SHORT:
-        component_size = 2;
-        break;
-    }
-    switch (format) {
-    case GL_LUMINANCE:
-    case GL_ALPHA:
-        return component_size;
-    case GL_RGB_422_APPLE:
-        return 2;
-    case GL_RGB:
-    case GL_BGR:
-    case GL_RGB_INTEGER:
-        return 3 * component_size;
-    case GL_RGBA:
-    case GL_BGRA:
-    case GL_RGBA_INTEGER:
-        return 4 * component_size;
-    case GL_RED:
-    case GL_RED_INTEGER:
-        return component_size;
-    case GL_RG:
-    case GL_LUMINANCE_ALPHA:
-    case GL_RG_INTEGER:
-        return 2 * component_size;
-    }
-    abort(); // unknown
-}
-
 static int get_alignment(int stride)
 {
     if (stride % 8 == 0)
@@ -117,9 +72,10 @@ void glUploadTex(GL *gl, GLenum target, GLenum format, GLenum type,
                  const void *dataptr, int stride,
                  int x, int y, int w, int h, int slice)
 {
+    int bpp = gl_bytes_per_pixel(format, type);
     const uint8_t *data = dataptr;
     int y_max = y + h;
-    if (w <= 0 || h <= 0)
+    if (w <= 0 || h <= 0 || !bpp)
         return;
     if (slice <= 0)
         slice = h;
@@ -131,9 +87,9 @@ void glUploadTex(GL *gl, GLenum target, GLenum format, GLenum type,
     bool use_rowlength = slice > 1 && (gl->mpgl_caps & MPGL_CAP_ROW_LENGTH);
     if (use_rowlength) {
         // this is not always correct, but should work for MPlayer
-        gl->PixelStorei(GL_UNPACK_ROW_LENGTH, stride / glFmt2bpp(format, type));
+        gl->PixelStorei(GL_UNPACK_ROW_LENGTH, stride / bpp);
     } else {
-        if (stride != glFmt2bpp(format, type) * w)
+        if (stride != bpp * w)
             slice = 1; // very inefficient, but at least it works
     }
     for (; y + slice <= y_max; y += slice) {
@@ -153,10 +109,10 @@ void glUploadTex(GL *gl, GLenum target, GLenum format, GLenum type,
 void glClearTex(GL *gl, GLenum target, GLenum format, GLenum type,
                 int x, int y, int w, int h, uint8_t val, void **scratch)
 {
-    int bpp = glFmt2bpp(format, type);
+    int bpp = gl_bytes_per_pixel(format, type);
     int stride = w * bpp;
     int size = h * stride;
-    if (size < 1)
+    if (size < 1 || !bpp)
         return;
     void *data = scratch ? *scratch : NULL;
     if (talloc_get_size(data) < size)
@@ -307,32 +263,6 @@ void gl_vao_draw_data(struct gl_vao *vao, GLenum prim, void *ptr, size_t num)
     gl_vao_unbind(vao);
 }
 
-struct gl_format {
-    GLenum format;
-    GLenum type;
-    GLint internal_format;
-};
-
-static const struct gl_format gl_formats[] = {
-    // GLES 3.0
-    {GL_RGB,    GL_UNSIGNED_BYTE,               GL_RGB},
-    {GL_RGBA,   GL_UNSIGNED_BYTE,               GL_RGBA},
-    {GL_RGB,    GL_UNSIGNED_BYTE,               GL_RGB8},
-    {GL_RGBA,   GL_UNSIGNED_BYTE,               GL_RGBA8},
-    {GL_RGB,    GL_UNSIGNED_SHORT,              GL_RGB16},
-    {GL_RGBA,   GL_UNSIGNED_INT_2_10_10_10_REV, GL_RGB10_A2},
-    // not texture filterable in GLES 3.0
-    {GL_RGB,    GL_FLOAT,                       GL_RGB16F},
-    {GL_RGBA,   GL_FLOAT,                       GL_RGBA16F},
-    {GL_RGB,    GL_FLOAT,                       GL_RGB32F},
-    {GL_RGBA,   GL_FLOAT,                       GL_RGBA32F},
-    // Desktop GL
-    {GL_RGB,    GL_UNSIGNED_SHORT,              GL_RGB10},
-    {GL_RGBA,   GL_UNSIGNED_SHORT,              GL_RGBA12},
-    {GL_RGBA,   GL_UNSIGNED_SHORT,              GL_RGBA16},
-    {0}
-};
-
 // Create a texture and a FBO using the texture as color attachments.
 //  iformat: texture internal format
 // Returns success.
@@ -373,19 +303,16 @@ bool fbotex_change(struct fbotex *fbo, GL *gl, struct mp_log *log, int w, int h,
     if (flags & FBOTEX_FUZZY_H)
         h = MP_ALIGN_UP(h, 256);
 
-    GLenum filter = fbo->tex_filter;
+    mp_verbose(log, "Create FBO: %dx%d (%dx%d)\n", lw, lh, w, h);
 
-    struct gl_format format = {
-        .format = GL_RGBA,
-        .type = GL_UNSIGNED_BYTE,
-        .internal_format = iformat,
-    };
-    for (int n = 0; gl_formats[n].format; n++) {
-        if (gl_formats[n].internal_format == format.internal_format) {
-            format = gl_formats[n];
-            break;
-        }
+    const struct gl_format *format = gl_find_internal_format(gl, iformat);
+    if (!format || (format->flags & F_CF) != F_CF) {
+        mp_verbose(log, "Format 0x%x not supported.\n", (unsigned)iformat);
+        return false;
     }
+    assert(gl->mpgl_caps & MPGL_CAP_FB);
+
+    GLenum filter = fbo->tex_filter;
 
     *fbo = (struct fbotex) {
         .gl = gl,
@@ -396,17 +323,11 @@ bool fbotex_change(struct fbotex *fbo, GL *gl, struct mp_log *log, int w, int h,
         .iformat = iformat,
     };
 
-    mp_verbose(log, "Create FBO: %dx%d -> %dx%d\n", fbo->lw, fbo->lh,
-                                                    fbo->rw, fbo->rh);
-
-    if (!(gl->mpgl_caps & MPGL_CAP_FB))
-        return false;
-
     gl->GenFramebuffers(1, &fbo->fbo);
     gl->GenTextures(1, &fbo->texture);
     gl->BindTexture(GL_TEXTURE_2D, fbo->texture);
-    gl->TexImage2D(GL_TEXTURE_2D, 0, format.internal_format, fbo->rw, fbo->rh, 0,
-                   format.format, format.type, NULL);
+    gl->TexImage2D(GL_TEXTURE_2D, 0, format->internal_format, fbo->rw, fbo->rh, 0,
+                   format->format, format->type, NULL);
     gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
     gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
     gl->BindTexture(GL_TEXTURE_2D, 0);
diff --git a/video/out/opengl/utils.h b/video/out/opengl/utils.h
index 170e24d71a..19edfe4b24 100644
--- a/video/out/opengl/utils.h
+++ b/video/out/opengl/utils.h
@@ -25,7 +25,6 @@ struct mp_log;
 
 void glCheckError(GL *gl, struct mp_log *log, const char *info);
 
-int glFmt2bpp(GLenum format, GLenum type);
 void glUploadTex(GL *gl, GLenum target, GLenum format, GLenum type,
                  const void *dataptr, int stride,
                  int x, int y, int w, int h, int slice);
diff --git a/video/out/opengl/video.c b/video/out/opengl/video.c
index 4aef3280be..cf1a5c9432 100644
--- a/video/out/opengl/video.c
+++ b/video/out/opengl/video.c
@@ -31,6 +31,7 @@
 #include "common/global.h"
 #include "options/options.h"
 #include "common.h"
+#include "formats.h"
 #include "utils.h"
 #include "hwdec.h"
 #include "osd.h"
@@ -248,90 +249,6 @@ struct gl_video {
     bool custom_shader_fn_warned;
 };
 
-struct fmt_entry {
-    int mp_format;
-    GLint internal_format;
-    GLenum format;
-    GLenum type;
-};
-
-// Very special formats, for which OpenGL happens to have direct support
-static const struct fmt_entry mp_to_gl_formats[] = {
-    {IMGFMT_RGB565,  GL_RGB,   GL_RGB,  GL_UNSIGNED_SHORT_5_6_5},
-    {0},
-};
-
-// These are used for desktop GL 3+, and GLES 3+ with GL_EXT_texture_norm16.
-static const struct fmt_entry gl_byte