vo_opengl: dynamically manage texture units

A minor cleanup that makes the code simpler, and guarantees that we
cleanup the GL state properly at any point.

We do this by reusing the uniform caching, and assigning each sampler
uniform its own texture unit by incrementing a counter. This has various
subtle consequences for the GL driver, which hopefully don't matter. For
example, it will bind fewer textures at a time, but also rebind them
more often.

For some reason we keep TEXUNIT_VIDEO_NUM, because it limits the number
of hook passes that can be bound at the same time.

OSD rendering is an exception: we do many passes with the same shader,
and rebinding the texture each pass. For now, this is handled in an
unclean way, and we make the shader cache reserve texture unit 0 for the
OSD texture. At a later point, we should allocate that one dynamically
too, and just pass the texture unit to the OSD rendering code. Right now
I feel like vo_rpi.c (may it rot in hell) is in the way.

5 files changed, 54 insertions, 30 deletions

diff --git a/video/out/opengl/utils.c b/video/out/opengl/utils.c
index fa1131e815..c0bd07fb91 100644
--- a/video/out/opengl/utils.c
+++ b/video/out/opengl/utils.c
@@ -445,6 +445,9 @@ struct sc_uniform {
     int size;
     GLint loc;
     union uniform_val v;
+    // Set for sampler uniforms.
+    GLenum tex_target;
+    GLuint tex_handle;
 };
 
 struct sc_cached_uniform {
@@ -473,6 +476,7 @@ struct gl_shader_cache {
     bstr prelude_text;
     bstr header_text;
     bstr text;
+    int next_texture_unit;
     struct gl_vao *vao;
 
     struct sc_entry *entries;
@@ -507,15 +511,26 @@ void gl_sc_reset(struct gl_shader_cache *sc)
 {
     GL *gl = sc->gl;
 
-    if (sc->needs_reset)
+    if (sc->needs_reset) {
         gl->UseProgram(0);
 
+        for (int n = 0; n < sc->num_uniforms; n++) {
+            struct sc_uniform *u = &sc->uniforms[n];
+            if (u->type == UT_i && u->tex_target) {
+                gl->ActiveTexture(GL_TEXTURE0 + u->v.i[0]);
+                gl->BindTexture(u->tex_target, 0);
+            }
+        }
+        gl->ActiveTexture(GL_TEXTURE0);
+    }
+
     sc->prelude_text.len = 0;
     sc->header_text.len = 0;
     sc->text.len = 0;
     for (int n = 0; n < sc->num_uniforms; n++)
         talloc_free(sc->uniforms[n].name);
     sc->num_uniforms = 0;
+    sc->next_texture_unit = 1; // not 0, as 0 is "free for use"
     sc->needs_reset = false;
 }
 
@@ -622,6 +637,7 @@ const char* mp_sampler_type(GLenum texture_target)
     }
 }
 
+// gl_sc_uniform_tex() should be preferred.
 void gl_sc_uniform_sampler(struct gl_shader_cache *sc, char *name, GLenum target,
                            int unit)
 {
@@ -630,15 +646,31 @@ void gl_sc_uniform_sampler(struct gl_shader_cache *sc, char *name, GLenum target
     u->size = 1;
     u->glsl_type = mp_sampler_type(target);
     u->v.i[0] = unit;
+    u->tex_target = 0;
+    u->tex_handle = 0;
+}
+
+void gl_sc_uniform_tex(struct gl_shader_cache *sc, char *name, GLenum target,
+                       GLuint texture)
+{
+    struct sc_uniform *u = find_uniform(sc, name);
+    u->type = UT_i;
+    u->size = 1;
+    u->glsl_type = mp_sampler_type(target);
+    u->v.i[0] = sc->next_texture_unit++;
+    u->tex_target = target;
+    u->tex_handle = texture;
 }
 
-void gl_sc_uniform_sampler_ui(struct gl_shader_cache *sc, char *name, int unit)
+void gl_sc_uniform_tex_ui(struct gl_shader_cache *sc, char *name, GLuint texture)
 {
     struct sc_uniform *u = find_uniform(sc, name);
     u->type = UT_i;
     u->size = 1;
     u->glsl_type = sc->gl->es ? "highp usampler2D" : "usampler2D";
-    u->v.i[0] = unit;
+    u->v.i[0] = sc->next_texture_unit++;
+    u->tex_target = GL_TEXTURE_2D;
+    u->tex_handle = texture;
 }
 
 void gl_sc_uniform_f(struct gl_shader_cache *sc, char *name, GLfloat f)
@@ -755,6 +787,11 @@ static void update_uniform(GL *gl, struct sc_entry *e, struct sc_uniform *u, int
             memcpy(un->v.i, u->v.i, sizeof(u->v.i));
             gl->Uniform1i(loc, u->v.i[0]);
         }
+        // For samplers: set the actual texture.
+        if (u->tex_target) {
+            gl->ActiveTexture(GL_TEXTURE0 + u->v.i[0]);
+            gl->BindTexture(u->tex_target, u->tex_handle);
+        }
         break;
     case UT_f:
         if (memcmp(un->v.f, u->v.f, sizeof(u->v.f)) != 0) {
diff --git a/video/out/opengl/utils.h b/video/out/opengl/utils.h
index ab3c13c915..33c2daa5be 100644
--- a/video/out/opengl/utils.h
+++ b/video/out/opengl/utils.h
@@ -156,7 +156,9 @@ void gl_sc_haddf(struct gl_shader_cache *sc, const char *textf, ...);
 void gl_sc_hadd_bstr(struct gl_shader_cache *sc, struct bstr text);
 void gl_sc_uniform_sampler(struct gl_shader_cache *sc, char *name, GLenum target,
                            int unit);
-void gl_sc_uniform_sampler_ui(struct gl_shader_cache *sc, char *name, int unit);
+void gl_sc_uniform_tex(struct gl_shader_cache *sc, char *name, GLenum target,
+                       GLuint texture);
+void gl_sc_uniform_tex_ui(struct gl_shader_cache *sc, char *name, GLuint texture);
 void gl_sc_uniform_f(struct gl_shader_cache *sc, char *name, GLfloat f);
 void gl_sc_uniform_i(struct gl_shader_cache *sc, char *name, GLint f);
 void gl_sc_uniform_vec2(struct gl_shader_cache *sc, char *name, GLfloat f[2]);
diff --git a/video/out/opengl/video.c b/video/out/opengl/video.c
index b000b2e7ce..4121ff785c 100644
--- a/video/out/opengl/video.c
+++ b/video/out/opengl/video.c
@@ -673,7 +673,6 @@ static bool gl_video_get_lut3d(struct gl_video *p, enum mp_csp_prim prim,
     if (!p->lut_3d_texture)
         gl->GenTextures(1, &p->lut_3d_texture);
 
-    gl->ActiveTexture(GL_TEXTURE0 + TEXUNIT_3DLUT);
     gl->BindTexture(GL_TEXTURE_3D, p->lut_3d_texture);
     gl->PixelStorei(GL_UNPACK_ALIGNMENT, 1);
     gl->TexImage3D(GL_TEXTURE_3D, 0, GL_RGB16, lut3d->size[0], lut3d->size[1],
@@ -684,7 +683,7 @@ static bool gl_video_get_lut3d(struct gl_video *p, enum mp_csp_prim prim,
     gl->TexParameteri(GL_TEXTURE_3D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
     gl->TexParameteri(GL_TEXTURE_3D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
     gl->TexParameteri(GL_TEXTURE_3D, GL_TEXTURE_WRAP_R, GL_CLAMP_TO_EDGE);
-    gl->ActiveTexture(GL_TEXTURE0);
+    gl->BindTexture(GL_TEXTURE_3D, 0);
 
     debug_check_gl(p, "after 3d lut creation");
 
@@ -946,7 +945,6 @@ static void uninit_video(struct gl_video *p)
 
 static void pass_prepare_src_tex(struct gl_video *p)
 {
-    GL *gl = p->gl;
     struct gl_shader_cache *sc = p->sc;
 
     for (int n = 0; n < p->pass_tex_num; n++) {
@@ -964,9 +962,9 @@ static void pass_prepare_src_tex(struct gl_video *p)
         snprintf(pixel_size, sizeof(pixel_size), "pixel_size%d", n);
 
         if (s->use_integer) {
-            gl_sc_uniform_sampler_ui(sc, texture_name, n);
+            gl_sc_uniform_tex_ui(sc, texture_name, s->gl_tex);
         } else {
-            gl_sc_uniform_sampler(sc, texture_name, s->gl_target, n);
+            gl_sc_uniform_tex(sc, texture_name, s->gl_target, s->gl_tex);
         }
         float f[2] = {1, 1};
         if (s->gl_target != GL_TEXTURE_RECTANGLE) {
@@ -977,11 +975,7 @@ static void pass_prepare_src_tex(struct gl_video *p)
         gl_sc_uniform_mat2(sc, texture_rot, true, (float *)s->transform.m);
         gl_sc_uniform_vec2(sc, pixel_size, (GLfloat[]){1.0f / f[0],
                                                        1.0f / f[1]});
-
-        gl->ActiveTexture(GL_TEXTURE0 + n);
-        gl->BindTexture(s->gl_target, s->gl_tex);
     }
-    gl->ActiveTexture(GL_TEXTURE0);
 }
 
 static void render_pass_quad(struct gl_video *p, int vp_w, int vp_h,
@@ -1401,8 +1395,6 @@ static void reinit_scaler(struct gl_video *p, struct scaler *scaler,
     const struct gl_format *fmt = gl_find_float16_format(gl, elems_per_pixel);
     GLenum target = scaler->gl_target;
 
-    gl->ActiveTexture(GL_TEXTURE0 + TEXUNIT_SCALERS + scaler->index);
-
     if (!scaler->gl_lut)
         gl->GenTextures(1, &scaler->gl_lut);
 
@@ -1429,7 +1421,7 @@ static void reinit_scaler(struct gl_video *p, struct scaler *scaler,
     if (target != GL_TEXTURE_1D)
         gl->TexParameteri(target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
 
-    gl->ActiveTexture(GL_TEXTURE0);
+    gl->BindTexture(target, 0);
 
     debug_check_gl(p, "after initializing scaler");
 }
@@ -2184,7 +2176,7 @@ static void pass_colormanage(struct gl_video *p, struct mp_colorspace src, bool
                    p->opts.tone_mapping_param);
 
     if (p->use_lut_3d) {
-        gl_sc_uniform_sampler(p->sc, "lut_3d", GL_TEXTURE_3D, TEXUNIT_3DLUT);
+        gl_sc_uniform_tex(p->sc, "lut_3d", GL_TEXTURE_3D, p->lut_3d_texture);
         GLSL(vec3 cpos;)
         for (int i = 0; i < 3; i++)
             GLSLF("cpos[%d] = LUT_POS(color[%d], %d.0);\n", i, i, p->lut_3d_size[i]);
@@ -2250,7 +2242,6 @@ static void pass_dither(struct gl_video *p)
 
         p->dither_size = tex_size;
 
-        gl->ActiveTexture(GL_TEXTURE0 + TEXUNIT_DITHER);
         gl->GenTextures(1, &p->dither_texture);
         gl->BindTexture(GL_TEXTURE_2D, p->dither_texture);
         gl->PixelStorei(GL_UNPACK_ALIGNMENT, 1);
@@ -2261,7 +2252,7 @@ static void pass_dither(struct gl_video *p)
         gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT);
         gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT);
         gl->PixelStorei(GL_UNPACK_ALIGNMENT, 4);
-        gl->ActiveTexture(GL_TEXTURE0);
+        gl->BindTexture(GL_TEXTURE_2D, 0);
 
         debug_check_gl(p, "dither setup");
     }
@@ -2274,7 +2265,7 @@ static void pass_dither(struct gl_video *p)
     // dither patterns can be visible.
     int dither_quantization = (1 << dst_depth) - 1;
 
-    gl_sc_uniform_sampler(p->sc, "dither", GL_TEXTURE_2D, TEXUNIT_DITHER);
+    gl_sc_uniform_tex(p->sc, "dither", GL_TEXTURE_2D, p->dither_texture);
 
     GLSLF("vec2 dither_pos = gl_FragCoord.xy / %d.0;\n", p->dither_size);
 
diff --git a/video/out/opengl/video.h b/video/out/opengl/video.h
index a6c7ca2898..9c76c4a454 100644
--- a/video/out/opengl/video.h
+++ b/video/out/opengl/video.h
@@ -27,14 +27,10 @@
 #include "lcms.h"
 #include "video/out/filter_kernels.h"
 
-// Texture units 0-5 are used by the video, and for free use by the passes
+// Assume we have this many texture units for sourcing additional passes.
+// The actual texture unit assignment is dynamic.
 #define TEXUNIT_VIDEO_NUM 6
 
-// Other texture units are reserved for specific purposes
-#define TEXUNIT_SCALERS  TEXUNIT_VIDEO_NUM
-#define TEXUNIT_3DLUT    (TEXUNIT_SCALERS+SCALER_COUNT)
-#define TEXUNIT_DITHER   (TEXUNIT_3DLUT+1)
-
 struct scaler_fun {
     char *name;
     float params[2];
diff --git a/video/out/opengl/video_shaders.c b/video/out/opengl/video_shaders.c
index ff87b99b62..7d668dc4b8 100644
--- a/video/out/opengl/video_shaders.c
+++ b/video/out/opengl/video_shaders.c
@@ -38,8 +38,7 @@ void sampler_prelude(struct gl_shader_cache *sc, int tex_num)
 static void pass_sample_separated_get_weights(struct gl_shader_cache *sc,
                                               struct scaler *scaler)
 {
-    gl_sc_uniform_sampler(sc, "lut", scaler->gl_target,
-                          TEXUNIT_SCALERS + scaler->index);
+    gl_sc_uniform_tex(sc, "lut", scaler->gl_target, scaler->gl_lut);
     // Define a new variable to cache the corrected fcoord.
     GLSLF("float fcoord_lut = LUT_POS(fcoord, %d.0);\n", scaler->lut_size);
 
@@ -121,8 +120,7 @@ void pass_sample_polar(struct gl_shader_cache *sc, struct scaler *scaler)
         GLSL(vec4 lo = vec4(1.0);)
         GLSL(vec4 hi = vec4(0.0);)
     }
-    gl_sc_uniform_sampler(sc, "lut", scaler->gl_target,
-                          TEXUNIT_SCALERS + scaler->index);
+    gl_sc_uniform_tex(sc, "lut", scaler->gl_target, scaler->gl_lut);
     GLSLF("// scaler samples\n");
     for (int y = 1-bound; y <= bound; y++) {
         for (int x = 1-bound; x <= bound; x++) {