vo_opengl: support loading custom user textures

Parsing the texture data as raw strings makes the textures the most
portable and self-contained. In order to facilitate different types of
shaders, the parse_user_shader interaction has been changed to instead
have it loop through blocks and call the passed functions for each valid
block parsed. This is more modular and also cleaner, with better code
separation.

Closes #4586.

3 files changed, 328 insertions, 77 deletions

diff --git a/video/out/opengl/user_shaders.c b/video/out/opengl/user_shaders.c
index 5cfd89b5ef..1b6fb42ab1 100644
--- a/video/out/opengl/user_shaders.c
+++ b/video/out/opengl/user_shaders.c
@@ -19,6 +19,7 @@
 
 #include "misc/ctype.h"
 #include "user_shaders.h"
+#include "formats.h"
 
 static bool parse_rpn_szexpr(struct bstr line, struct szexp out[MAX_SZEXP_SIZE])
 {
@@ -158,13 +159,10 @@ done:
     return true;
 }
 
-bool parse_user_shader_pass(struct mp_log *log, struct bstr *body,
-                            struct gl_user_shader *out)
+static bool parse_hook(struct mp_log *log, struct bstr *body,
+                       struct gl_user_shader_hook *out)
 {
-    if (!body || !out || !body->start || body->len == 0)
-        return false;
-
-    *out = (struct gl_user_shader){
+    *out = (struct gl_user_shader_hook){
         .pass_desc = bstr0("(unknown)"),
         .offset = identity_trans,
         .width = {{ SZEXP_VAR_W, { .varname = bstr0("HOOKED") }}},
@@ -175,14 +173,6 @@ bool parse_user_shader_pass(struct mp_log *log, struct bstr *body,
     int hook_idx = 0;
     int bind_idx = 0;
 
-    // Skip all garbage (e.g. comments) before the first header
-    int pos = bstr_find(*body, bstr0("//!"));
-    if (pos < 0) {
-        mp_warn(log, "Shader appears to contain no headers!\n");
-        return false;
-    }
-    *body = bstr_cut(*body, pos);
-
     // Parse all headers
     while (true) {
         struct bstr rest;
@@ -295,3 +285,162 @@ bool parse_user_shader_pass(struct mp_log *log, struct bstr *body,
 
     return true;
 }
+
+static bool parse_tex(struct mp_log *log, struct bstr *body,
+                      struct gl_user_shader_tex *out)
+{
+    *out = (struct gl_user_shader_tex){
+        .name = bstr0("USER_TEX"),
+        .w = 1, .h = 1, .d = 1,
+        .components = 1,
+        .bytes = 1,
+        .mpgl_type = MPGL_TYPE_UINT,
+        .gl_filter = GL_LINEAR,
+        .gl_target = GL_TEXTURE_1D,
+        .gl_border = GL_CLAMP_TO_EDGE,
+    };
+
+    while (true) {
+        struct bstr rest;
+        struct bstr line = bstr_strip(bstr_getline(*body, &rest));
+
+        if (!bstr_eatstart0(&line, "//!"))
+            break;
+
+        *body = rest;
+
+        if (bstr_eatstart0(&line, "TEXTURE")) {
+            out->name = bstr_strip(line);
+            continue;
+        }
+
+        if (bstr_eatstart0(&line, "SIZE")) {
+            int num = bstr_sscanf(line, "%d %d %d", &out->w, &out->h, &out->d);
+            if (num < 1 || num > 3 || out->w < 1 || out->h < 1 || out->d < 1) {
+                mp_err(log, "Error while parsing SIZE!\n");
+                return false;
+            }
+            static GLenum tgt[] = {GL_TEXTURE_1D, GL_TEXTURE_2D, GL_TEXTURE_3D};
+            out->gl_target = tgt[num - 1];
+            continue;
+        }
+
+        if (bstr_eatstart0(&line, "COMPONENTS")) {
+            if (bstr_sscanf(line, "%d", &out->components) != 1) {
+                mp_err(log, "Error while parsing COMPONENTS!\n");
+                return false;
+            }
+            continue;
+        }
+
+        if (bstr_eatstart0(&line, "FORMAT")) {
+            int bits;
+            char fmt;
+            if (bstr_sscanf(line, "%d%c", &bits, &fmt) != 2) {
+                mp_err(log, "Error while parsing FORMAT!\n");
+                return false;
+            }
+
+            out->bytes = bits / 8;
+            switch (fmt) {
+            case 'f': out->mpgl_type = MPGL_TYPE_FLOAT; break;
+            case 'i': out->mpgl_type = MPGL_TYPE_UINT;  break;
+            case 'u': out->mpgl_type = MPGL_TYPE_UNORM; break;
+            default:
+                mp_err(log, "Unrecognized FORMAT description: '%c'!\n", fmt);
+                return false;
+            }
+            continue;
+        }
+
+        if (bstr_eatstart0(&line, "FILTER")) {
+            line = bstr_strip(line);
+            if (bstr_equals0(line, "LINEAR")) {
+                out->gl_filter = GL_LINEAR;
+            } else if (bstr_equals0(line, "NEAREST")) {
+                out->gl_filter = GL_NEAREST;
+            } else {
+                mp_err(log, "Unrecognized FILTER: '%.*s'!\n", BSTR_P(line));
+                return false;
+            }
+            continue;
+        }
+
+        if (bstr_eatstart0(&line, "BORDER")) {
+            line = bstr_strip(line);
+            if (bstr_equals0(line, "CLAMP")) {
+                out->gl_border = GL_CLAMP_TO_EDGE;
+            } else if (bstr_equals0(line, "REPEAT")) {
+                out->gl_border = GL_REPEAT;
+            } else if (bstr_equals0(line, "MIRROR")) {
+                out->gl_border = GL_MIRRORED_REPEAT;
+            } else {
+                mp_err(log, "Unrecognized BORDER: '%.*s'!\n", BSTR_P(line));
+                return false;
+            }
+            continue;
+        }
+
+        mp_err(log, "Unrecognized command '%.*s'!\n", BSTR_P(line));
+        return false;
+    }
+
+    // Decode the rest of the section (up to the next //! marker) as raw hex
+    // data for the texture
+    struct bstr hexdata;
+    if (bstr_split_tok(*body, "//!", &hexdata, body)) {
+        // Make sure the magic line is part of the rest
+        body->start -= 3;
+        body->len += 3;
+    }
+
+    struct bstr tex;
+    if (!bstr_decode_hex(NULL, bstr_strip(hexdata), &tex)) {
+        mp_err(log, "Error while parsing TEXTURE body: must be a valid "
+                    "hexadecimal sequence, on a single line!\n");
+        return false;
+    }
+
+    int expected_len = out->w * out->h * out->d * out->components * out->bytes;
+    if (tex.len != expected_len) {
+        mp_err(log, "Shader TEXTURE size mismatch: got %zd bytes, expected %d!\n",
+               tex.len, expected_len);
+        talloc_free(tex.start);
+        return false;
+    }
+
+    out->texdata = tex.start;
+    return true;
+}
+
+void parse_user_shader(struct mp_log *log, struct bstr shader, void *priv,
+                       bool (*dohook)(void *p, struct gl_user_shader_hook hook),
+                       bool (*dotex)(void *p, struct gl_user_shader_tex tex))
+{
+    if (!dohook || !dotex || !shader.len)
+        return;
+
+    // Skip all garbage (e.g. comments) before the first header
+    int pos = bstr_find(shader, bstr0("//!"));
+    if (pos < 0) {
+        mp_warn(log, "Shader appears to contain no headers!\n");
+        return;
+    }
+    shader = bstr_cut(shader, pos);
+
+    // Loop over the file
+    while (shader.len > 0)
+    {
+        // Peek at the first header to dispatch the right type
+        if (bstr_startswith0(shader, "//!TEXTURE")) {
+            struct gl_user_shader_tex t;
+            if (!parse_tex(log, &shader, &t) || !dotex(priv, t))
+                return;
+            continue;
+        }
+
+        struct gl_user_shader_hook h;
+        if (!parse_hook(log, &shader, &h) || !dohook(priv, h))
+            return;
+    }
+}
diff --git a/video/out/opengl/user_shaders.h b/video/out/opengl/user_shaders.h
index 7192309c54..bb550de2b8 100644
--- a/video/out/opengl/user_shaders.h
+++ b/video/out/opengl/user_shaders.h
@@ -55,7 +55,7 @@ struct szexp {
     } val;
 };
 
-struct gl_user_shader {
+struct gl_user_shader_hook {
     struct bstr pass_desc;
     struct bstr hook_tex[SHADER_MAX_HOOKS];
     struct bstr bind_tex[SHADER_MAX_BINDS];
@@ -70,10 +70,25 @@ struct gl_user_shader {
     int compute_h;
 };
 
-// Parse the next shader pass from `body`. The `struct bstr` is modified by the
-// function. Returns false if the end of the string was reached (or on error).
-bool parse_user_shader_pass(struct mp_log *log, struct bstr *body,
-                            struct gl_user_shader *out);
+struct gl_user_shader_tex {
+    struct bstr name;
+    int w, h, d;
+    int components;
+    int bytes;
+    int mpgl_type;
+    GLenum gl_target;
+    GLenum gl_filter;
+    GLenum gl_border;
+    void *texdata;
+    // for video.c
+    GLenum gl_tex;
+};
+
+// Parse the next shader block from `body`. The callbacks are invoked on every
+// valid shader block parsed.
+void parse_user_shader(struct mp_log *log, struct bstr shader, void *priv,
+                       bool (*dohook)(void *p, struct gl_user_shader_hook hook),
+                       bool (*dotex)(void *p, struct gl_user_shader_tex tex));
 
 // Evaluate a szexp, given a lookup function for named textures
 bool eval_szexpr(struct mp_log *log, void *priv,
diff --git a/video/out/opengl/video.c b/video/out/opengl/video.c
index 417c1b62b0..1a55d8393d 100644
--- a/video/out/opengl/video.c
+++ b/video/out/opengl/video.c
@@ -240,6 +240,12 @@ struct gl_video {
     struct fbotex vdpau_deinterleave_fbo[2];
     GLuint hdr_peak_ssbo;
 
+    // user pass descriptions and textures
+    struct tex_hook tex_hooks[SHADER_MAX_PASSES];
+    int tex_hook_num;
+    struct gl_user_shader_tex user_textures[SHADER_MAX_PASSES];
+    int user_tex_num;
+
     int surface_idx;
     int surface_now;
     int frames_drawn;
@@ -274,9 +280,7 @@ struct gl_video {
     struct gl_timer *upload_timer;
     struct gl_timer *blit_timer;
 
-    // hooks and saved textures
-    struct tex_hook tex_hooks[SHADER_MAX_PASSES];
-    int tex_hook_num;
+    // intermediate textures
     struct saved_tex saved_tex[SHADER_MAX_SAVED];
     int saved_tex_num;
     struct fbotex hook_fbos[SHADER_MAX_SAVED];
@@ -503,7 +507,11 @@ static void gl_video_reset_hooks(struct gl_video *p)
     for (int i = 0; i < p->tex_hook_num; i++)
         talloc_free(p->tex_hooks[i].priv);
 
+    for (int i = 0; i < p->user_tex_num; i++)
+        p->gl->DeleteTextures(1, &p->user_textures[i].gl_tex);
+
     p->tex_hook_num = 0;
+    p->user_tex_num = 0;
 }
 
 static inline int fbosurface_wrap(int id)
@@ -1378,6 +1386,51 @@ static void saved_tex_store(struct gl_video *p, const char *name,
     };
 }
 
+static bool pass_hook_setup_binds(struct gl_video *p, const char *name,
+                                  struct img_tex tex, struct tex_hook *hook)
+{
+    for (int t = 0; t < TEXUNIT_VIDEO_NUM; t++) {
+        char *bind_name = (char *)hook->bind_tex[t];
+
+        if (!bind_name)
+            continue;
+
+        // This is a special name that means "currently hooked texture"
+        if (strcmp(bind_name, "HOOKED") == 0) {
+            int id = pass_bind(p, tex);
+            hook_prelude(p, "HOOKED", id, tex);
+            hook_prelude(p, name, id, tex);
+            continue;
+        }
+
+        // BIND can also be used to load user-defined textures, in which
+        // case we will directly load them as a uniform instead of
+        // generating the hook_prelude boilerplate
+        for (int u = 0; u < p->user_tex_num; u++) {
+            struct gl_user_shader_tex *utex = &p->user_textures[u];
+            if (bstr_equals0(utex->name, bind_name)) {
+                gl_sc_uniform_tex(p->sc, bind_name, utex->gl_target, utex->gl_tex);
+                goto next_bind;
+            }
+        }
+
+        struct img_tex bind_tex;
+        if (!saved_tex_find(p, bind_name, &bind_tex)) {
+            // Clean up texture bindings and move on to the next hook
+            MP_DBG(p, "Skipping hook on %s due to no texture named %s.\n",
+                   name, bind_name);
+            p->pass_tex_num -= t;
+            return false;
+        }
+
+        hook_prelude(p, bind_name, pass_bind(p, bind_tex), bind_tex);
+
+next_bind: ;
+    }
+
+    return true;
+}
+
 // Process hooks for a plane, saving the result and returning a new img_tex
 // If 'trans' is NULL, the shader is forbidden from transforming tex
 static struct img_tex pass_hook(struct gl_video *p, const char *name,
@@ -1407,32 +1460,8 @@ found:
             continue;
         }
 
-        // Bind all necessary textures and add them to the prelude
-        for (int t = 0; t < TEXUNIT_VIDEO_NUM; t++) {
-            const char *bind_name = hook->bind_tex[t];
-            struct img_tex bind_tex;
-
-            if (!bind_name)
-                continue;
-
-            // This is a special name that means "currently hooked texture"
-            if (strcmp(bind_name, "HOOKED") == 0) {
-                int id = pass_bind(p, tex);
-                hook_prelude(p, "HOOKED", id, tex);
-                hook_prelude(p, name, id, tex);
-                continue;
-            }
-
-            if (!saved_tex_find(p, bind_name, &bind_tex)) {
-                // Clean up texture bindings and move on to the next hook
-                MP_DBG(p, "Skipping hook on %s due to no texture named %s.\n",
-                       name, bind_name);
-                p->pass_tex_num -= t;
-                goto next_hook;
-            }
-
-            hook_prelude(p, bind_name, pass_bind(p, bind_tex), bind_tex);
-        }
+        if (!pass_hook_setup_binds(p, name, tex, hook))
+            continue;
 
         // Run the actual hook. This generates a series of GLSL shader
         // instructions sufficient for drawing the hook's output
@@ -1471,8 +1500,6 @@ found:
         }
 
         saved_tex_store(p, store_name, saved_tex);
-
-next_hook: ;
     }
 
     return tex;
@@ -1825,13 +1852,15 @@ static bool img_tex_equiv(struct img_tex a, struct img_tex b)
            gl_transform_eq(a.transform, b.transform);
 }
 
-static void pass_add_hook(struct gl_video *p, struct tex_hook hook)
+static bool add_hook(struct gl_video *p, struct tex_hook hook)
 {
     if (p->tex_hook_num < SHADER_MAX_PASSES) {
         p->tex_hooks[p->tex_hook_num++] = hook;
+        return true;
     } else {
         MP_ERR(p, "Too many passes! Limit is %d.\n", SHADER_MAX_PASSES);
         talloc_free(hook.priv);
+        return false;
     }
 }
 
@@ -1896,7 +1925,7 @@ static bool szexp_lookup(void *priv, struct bstr var, float size[2])
 
 static bool user_hook_cond(struct gl_video *p, struct img_tex tex, void *priv)
 {
-    struct gl_user_shader *shader = priv;
+    struct gl_user_shader_hook *shader = priv;
     assert(shader);
 
     float res = false;
@@ -1907,7 +1936,7 @@ static bool user_hook_cond(struct gl_video *p, struct img_tex tex, void *priv)
 static void user_hook(struct gl_video *p, struct img_tex tex,
                       struct gl_transform *trans, void *priv)
 {
-    struct gl_user_shader *shader = priv;
+    struct gl_user_shader_hook *shader = priv;
     assert(shader);
 
     pass_describe(p, "user shader: %.*s (%s)", BSTR_P(shader->pass_desc),
@@ -1928,33 +1957,91 @@ static void user_hook(struct gl_video *p, struct img_tex tex,
     gl_transform_trans(shader->offset, trans);
 }
 
-static void pass_hook_user_shaders(struct gl_video *p, char **shaders)
+static bool add_user_hook(void *priv, struct gl_user_shader_hook hook)
+{
+    struct gl_video *p = priv;
+    struct gl_user_shader_hook *copy = talloc_ptrtype(p, copy);
+    *copy = hook;
+
+    struct tex_hook texhook = {
+        .save_tex = bstrdup0(copy, hook.save_tex),
+        .components = hook.components,
+        .hook = user_hook,
+        .cond = user_hook_cond,
+        .priv = copy,
+    };
+
+    for (int h = 0; h < SHADER_MAX_HOOKS; h++)
+        texhook.hook_tex[h] = bstrdup0(copy, hook.hook_tex[h]);
+    for (int h = 0; h < SHADER_MAX_BINDS; h++)
+        texhook.bind_tex[h] = bstrdup0(copy, hook.bind_tex[h]);
+
+    return add_hook(p, texhook);
+}
+
+static bool add_user_tex(void *priv, struct gl_user_shader_tex tex)
+{
+    struct gl_video *p = priv;
+    GL *gl = p->gl;
+
+    if (p->user_tex_num == SHADER_MAX_PASSES) {
+        MP_ERR(p, "Too many textures! Limit is %d.\n", SHADER_MAX_PASSES);
+        goto err;
+    }
+
+    const struct gl_format *format = gl_find_format(gl, tex.mpgl_type,
+            tex.gl_filter == GL_LINEAR ? F_TF : 0, tex.bytes, tex.components);
+
+    if (!format) {
+        MP_ERR(p, "Could not satisfy format requirements for user "
+                  "shader texture '%.*s'!\n", BSTR_P(tex.name));
+        goto err;
+    }
+
+    GLenum type = format->type,
+           ifmt = format->internal_format,
+            fmt = format->format;
+
+    GLenum tgt = tex.gl_target;
+    gl->GenTextures(1, &tex.gl_tex);
+    gl->BindTexture(tgt, tex.gl_tex);
+    gl->PixelStorei(GL_UNPACK_ALIGNMENT, 1);
+
+    if (tgt == GL_TEXTURE_3D) {
+        gl->TexImage3D(tgt, 0, ifmt, tex.w, tex.h, tex.d, 0, fmt, type, tex.texdata);
+        gl->TexParameteri(tgt, GL_TEXTURE_WRAP_S, tex.gl_border);
+        gl->TexParameteri(tgt, GL_TEXTURE_WRAP_T, tex.gl_border);
+        gl->TexParameteri(tgt, GL_TEXTURE_WRAP_R, tex.gl_border);
+    } else if (tgt == GL_TEXTURE_2D) {
+        gl->TexImage2D(tgt, 0, ifmt, tex.w, tex.h, 0, fmt, type, tex.texdata);
+        gl->TexParameteri(tgt, GL_TEXTURE_WRAP_S, tex.gl_border);
+        gl->TexParameteri(tgt, GL_TEXTURE_WRAP_T, tex.gl_border);
+    } else {
+        gl->TexImage1D(tgt, 0, ifmt, tex.w, 0, fmt, type, tex.texdata);
+        gl->TexParameteri(tgt, GL_TEXTURE_WRAP_S, tex.gl_border);
+    }
+    talloc_free(tex.texdata);
+
+    gl->TexParameteri(tgt, GL_TEXTURE_MIN_FILTER, tex.gl_filter);
+    gl->TexParameteri(tgt, GL_TEXTURE_MAG_FILTER, tex.gl_filter);
+    gl->BindTexture(tgt, 0);
+
+    p->user_textures[p->user_tex_num++] = tex;
+    return true;
+
+err:
+    talloc_free(tex.texdata);
+    return false;
+}
+
+static void load_user_shaders(struct gl_video *p, char **shaders)
 {
     if (!shaders)
         return;
 
     for (int n = 0; shaders[n] != NULL; n++) {
         struct bstr file = load_cached_file(p, shaders[n]);
-        struct gl_user_shader out;
-        while (parse_user_shader_pass(p->log, &file, &out)) {
-            struct gl_user_shader *hook = talloc_ptrtype(p, hook);
-            *hook = out;
-
-            struct tex_hook texhook = {
-                .save_tex = bstrdup0(hook, hook->save_tex),
-                .components = hook->components,
-                .hook = user_hook,
-                .cond = user_hook_cond,
-                .priv = hook,
-            };
-
-            for (int h = 0; h < SHADER_MAX_HOOKS; h++)
-                texhook.hook_tex[h] = bstrdup0(hook, hook->hook_tex[h]);
-            for (int h = 0; h < SHADER_MAX_BINDS; h++)
-                texhook.bind_tex[h] = bstrdup0(hook, hook->bind_tex[h]);
-
-            pass_add_hook(p, texhook);
-        }
+        parse_user_shader(p->log, file, p, add_user_hook, add_user_tex);
     }
 }
 
@@ -1963,7 +2050,7 @@ static void gl_video_setup_hooks(struct gl_video *p)
     gl_video_reset_hooks(p);
 
     if (p->opts.deband) {
-        pass_add_hook(p, (struct tex_hook) {
+        add_hook(p, (struct tex_hook) {
             .hook_tex = {"LUMA", "CHROMA", "RGB", "XYZ"},
             .bind_tex = {"HOOKED"},
             .hook = deband_hook,
@@ -1971,14 +2058,14 @@ static void gl_video_setup_hooks(struct gl_video *p)
     }
 
     if (p->opts.unsharp != 0.0) {
-        pass_add_hook(p, (struct tex_hook) {
+        add_hook(p, (struct tex_hook) {
             .hook_tex = {"MAIN"},
             .bind_tex = {"HOOKED"},
             .hook = unsharp_hook,
         });
     }
 
-    pass_hook_user_shaders(p, p->opts.user_shaders);
+    load_user_shaders(p, p->opts.user_shaders);
 }
 
 // sample from video textures, set "color" variable to yuv value