vo_opengl_old: fix OSD regression

Commit 0e8fbdbd removed the rg_texture requirement from vo_opengl;
commit 541f6731 changed to a more convenient method. Both commits broke
vo_opengl_old in some ways. vo_opengl_old always requires GL_ALPHA for
single-channel texture, because it draws the OSD without shaders and by
using certain blend modes.

So we need to explicitly distinguish between vo_opengl and vo_opengl_old
in the OSD renderer, and force fixed texture formats for vo_opengl_old.
The other logic is specific to the internals of vo_opengl. (Although it
might be possible to get the same result by playing with the old GL
fixed-function functions in vo_opengl_old. But seems like a waste of
time.)

Fixes #1370.

4 files changed, 16 insertions, 7 deletions

diff --git a/video/out/gl_osd.c b/video/out/gl_osd.c
index 2bac4ef357..5219e3fbf8 100644
--- a/video/out/gl_osd.c
+++ b/video/out/gl_osd.c
@@ -47,12 +47,18 @@ static const struct osd_fmt_entry osd_to_gles3_formats[SUBBITMAP_COUNT] = {
     [SUBBITMAP_RGBA] =   {GL_RGBA8, GL_BGRA,  GL_UNSIGNED_BYTE},
 };
 
-static const struct osd_fmt_entry osd_to_gl_legacy_formats[SUBBITMAP_COUNT] = {
+static const struct osd_fmt_entry osd_to_gl2_formats[SUBBITMAP_COUNT] = {
     [SUBBITMAP_LIBASS] = {GL_LUMINANCE, GL_LUMINANCE,   GL_UNSIGNED_BYTE},
     [SUBBITMAP_RGBA] =   {GL_RGBA,      GL_BGRA,        GL_UNSIGNED_BYTE},
 };
 
-struct mpgl_osd *mpgl_osd_init(GL *gl, struct mp_log *log, struct osd_state *osd)
+static const struct osd_fmt_entry osd_to_gl_legacy_formats[SUBBITMAP_COUNT] = {
+    [SUBBITMAP_LIBASS] = {GL_ALPHA, GL_ALPHA,   GL_UNSIGNED_BYTE},
+    [SUBBITMAP_RGBA] =   {GL_RGBA,  GL_BGRA,    GL_UNSIGNED_BYTE},
+};
+
+struct mpgl_osd *mpgl_osd_init(GL *gl, struct mp_log *log, struct osd_state *osd,
+                               bool legacy)
 {
     GLint max_texture_size;
     gl->GetIntegerv(GL_MAX_TEXTURE_SIZE, &max_texture_size);
@@ -66,10 +72,12 @@ struct mpgl_osd *mpgl_osd_init(GL *gl, struct mp_log *log, struct osd_state *osd
         .scratch = talloc_zero_size(ctx, 1),
     };
 
-    if (gl->es >= 300) {
+    if (legacy) {
+        ctx->fmt_table = osd_to_gl_legacy_formats;
+    } else if (gl->es >= 300) {
         ctx->fmt_table = osd_to_gles3_formats;
     } else if (!(gl->mpgl_caps & MPGL_CAP_TEX_RG)) {
-        ctx->fmt_table = osd_to_gl_legacy_formats;
+        ctx->fmt_table = osd_to_gl2_formats;
     }
 
     for (int n = 0; n < MAX_OSD_PARTS; n++) {
diff --git a/video/out/gl_osd.h b/video/out/gl_osd.h
index c91cc1e53f..f05385b547 100644
--- a/video/out/gl_osd.h
+++ b/video/out/gl_osd.h
@@ -30,7 +30,8 @@ struct mpgl_osd {
     void *scratch;
 };
 
-struct mpgl_osd *mpgl_osd_init(GL *gl, struct mp_log *log, struct osd_state *osd);
+struct mpgl_osd *mpgl_osd_init(GL *gl, struct mp_log *log, struct osd_state *osd,
+                               bool legacy);
 void mpgl_osd_destroy(struct mpgl_osd *ctx);
 
 struct mpgl_osd_part *mpgl_osd_generate(struct mpgl_osd *ctx,
diff --git a/video/out/gl_video.c b/video/out/gl_video.c
index bb1e37b288..64c1ac328e 100644
--- a/video/out/gl_video.c
+++ b/video/out/gl_video.c
@@ -1356,7 +1356,7 @@ static void recreate_osd(struct gl_video *p)
 {
     if (p->osd)
         mpgl_osd_destroy(p->osd);
-    p->osd = mpgl_osd_init(p->gl, p->log, p->osd_state);
+    p->osd = mpgl_osd_init(p->gl, p->log, p->osd_state, false);
     p->osd->use_pbo = p->opts.pbo;
 }
 
diff --git a/video/out/vo_opengl_old.c b/video/out/vo_opengl_old.c
index bf117fdbd9..7f81a727e5 100644
--- a/video/out/vo_opengl_old.c
+++ b/video/out/vo_opengl_old.c
@@ -1778,7 +1778,7 @@ static int initGl(struct vo *vo, uint32_t d_width, uint32_t d_height)
     }
 
     if (gl->BindTexture) {
-        p->osd = mpgl_osd_init(gl, vo->log, vo->osd);
+        p->osd = mpgl_osd_init(gl, vo->log, vo->osd, true);
         p->osd->scaled = p->scaled_osd;
     }