vo_opengl: use RPN expressions for user hook sizes

This replaces the previous TRANSFORM by WIDTH, HEIGHT and OFFSET where
WIDTH and HEIGHT are RPN expressions. This allows for more fine-grained
control over the output size, and also makes sure that overwriting
existing textures works more cleanly.

(Also add some more useful bstr functions)

6 files changed, 238 insertions, 24 deletions

diff --git a/DOCS/man/vo.rst b/DOCS/man/vo.rst
index 3f19c4c2ab..4d6972e5a3 100644
--- a/DOCS/man/vo.rst
+++ b/DOCS/man/vo.rst
@@ -739,11 +739,20 @@ Available video output drivers are:
             into. By default, this is set to the special name HOOKED which has
             the effect of overwriting the hooked texture.
 
-        TRANSFORM sx sy ox oy
-            Specifies how this pass intends to transform the hooked texture.
-            ``sx``/``sy`` refer to a linear scale factor, and ``ox``/``oy``
-            refer to a constant pixel shift that the shader will introduce. The
-            default values are 1 1 0 0 which leave the texture size unchanged.
+        WIDTH <szexpr>, HEIGHT <szexpr>
+            Specifies the size of the resulting texture for this pass.
+            ``szexpr`` refers to an expression in RPN (reverse polish
+            notation), using the operators + - * /, floating point literals,
+            and references to existing texture sizes such as MAIN.width or
+            CHROMA.height. By default, these are set to HOOKED.w and HOOKED.h,
+            respectively.
+
+        OFFSET ox oy
+            Indicates a pixel shift (offset) introduced by this pass. These
+            pixel offsets will be accumulated and corrected during the
+            next scaling pass (``cscale`` or ``scale``). The default values
+            are 0 0 which correspond to no shift. Note that offsets are ignored
+            when not overwriting the hooked texture.
 
         COMPONENTS n
             Specifies how many components of this pass's output are relevant
@@ -810,8 +819,9 @@ Available video output drivers are:
             The final output image, after color management but before
             dithering and drawing to screen.
 
-        Only the textures labelled with (resizable) may be transformed by
-        the pass. For all others, the TRANSFORM must be 1 1 0 0 (default).
+        Only the textures labelled with ``resizable`` may be transformed by the
+        pass. When overwriting a texture marked ``fixed``, the WIDTH, HEIGHT
+        and OFFSET must be left at their default values.
 
     ``deband``
         Enable the debanding algorithm. This greatly reduces the amount of
diff --git a/misc/bstr.c b/misc/bstr.c
index 1a2676c537..0ef0c357e8 100644
--- a/misc/bstr.c
+++ b/misc/bstr.c
@@ -215,9 +215,9 @@ struct bstr *bstr_splitlines(void *talloc_ctx, struct bstr str)
     return r;
 }
 
-struct bstr bstr_getline(struct bstr str, struct bstr *rest)
+struct bstr bstr_splitchar(struct bstr str, struct bstr *rest, const char c)
 {
-    int pos = bstrchr(str, '\n');
+    int pos = bstrchr(str, c);
     if (pos < 0)
         pos = str.len;
     if (rest)
@@ -243,6 +243,14 @@ bool bstr_eatstart(struct bstr *s, struct bstr prefix)
     return true;
 }
 
+bool bstr_eatend(struct bstr *s, struct bstr prefix)
+{
+    if (!bstr_endswith(*s, prefix))
+        return false;
+    s->len -= prefix.len;
+    return true;
+}
+
 void bstr_lower(struct bstr str)
 {
     for (int i = 0; i < str.len; i++)
diff --git a/misc/bstr.h b/misc/bstr.h
index f56516663c..2785520b87 100644
--- a/misc/bstr.h
+++ b/misc/bstr.h
@@ -116,10 +116,15 @@ int bstr_validate_utf8(struct bstr s);
 // talloc, with talloc_ctx as parent.
 struct bstr bstr_sanitize_utf8_latin1(void *talloc_ctx, struct bstr s);
 
-// Return the text before the next line break, and return it. Change *rest to
-// point to the text following this line break. (rest can be NULL.)
-// Line break characters are not stripped.
-struct bstr bstr_getline(struct bstr str, struct bstr *rest);
+// Return the text before the occurance of a character, and return it. Change
+// *rest to point to the text following this character. (rest can be NULL.)
+struct bstr bstr_splitchar(struct bstr str, struct bstr *rest, const char c);
+
+// Like bstr_splitchar. Trailing newlines are not stripped.
+static inline struct bstr bstr_getline(struct bstr str, struct bstr *rest)
+{
+    return bstr_splitchar(str, rest, '\n');
+}
 
 // Strip one trailing line break. This is intended for use with bstr_getline,
 // and will remove the trailing \n or \r\n sequence.
@@ -131,8 +136,10 @@ void bstr_xappend_asprintf(void *talloc_ctx, bstr *s, const char *fmt, ...)
 void bstr_xappend_vasprintf(void *talloc_ctx, bstr *s, const char *fmt, va_list va)
     PRINTF_ATTRIBUTE(3, 0);
 
-// If s starts with prefix, return true and return the rest of the string in s.
+// If s starts/ends with prefix, return true and return the rest of the string
+// in s.
 bool bstr_eatstart(struct bstr *s, struct bstr prefix);
+bool bstr_eatend(struct bstr *s, struct bstr prefix);
 
 bool bstr_case_startswith(struct bstr s, struct bstr prefix);
 bool bstr_case_endswith(struct bstr s, struct bstr suffix);
@@ -200,11 +207,16 @@ static inline int bstr_find0(struct bstr haystack, const char *needle)
     return bstr_find(haystack, bstr0(needle));
 }
 
-static inline int bstr_eatstart0(struct bstr *s, const char *prefix)
+static inline bool bstr_eatstart0(struct bstr *s, const char *prefix)
 {
     return bstr_eatstart(s, bstr0(prefix));
 }
 
+static inline bool bstr_eatend0(struct bstr *s, const char *prefix)
+{
+    return bstr_eatend(s, bstr0(prefix));
+}
+
 // create a pair (not single value!) for "%.*s" printf syntax
 #define BSTR_P(bstr) (int)((bstr).len), ((bstr).start ? (char*)(bstr).start : "")
 
diff --git a/video/out/opengl/user_shaders.c b/video/out/opengl/user_shaders.c
index 0c1b765400..0cf80af115 100644
--- a/video/out/opengl/user_shaders.c
+++ b/video/out/opengl/user_shaders.c
@@ -16,6 +16,54 @@
  */
 
 #include "user_shaders.h"
+#include "ctype.h"
+
+static bool parse_rpn_szexpr(struct bstr line, struct szexp out[MAX_SZEXP_SIZE])
+{
+    int pos = 0;
+
+    while (line.len > 0) {
+        struct bstr word = bstr_strip(bstr_splitchar(line, &line, ' '));
+        if (word.len == 0)
+            continue;
+
+        if (pos >= MAX_SZEXP_SIZE)
+            return false;
+
+        struct szexp *exp = &out[pos++];
+
+        if (bstr_eatend0(&word, ".w") || bstr_eatend0(&word, ".width")) {
+            exp->tag = SZEXP_VAR_W;
+            exp->val.varname = word;
+            continue;
+        }
+
+        if (bstr_eatend0(&word, ".h") || bstr_eatend0(&word, ".height")) {
+            exp->tag = SZEXP_VAR_H;
+            exp->val.varname = word;
+            continue;
+        }
+
+        switch (word.start[0]) {
+        case '+': exp->tag = SZEXP_OP2; exp->val.op = SZEXP_OP_ADD; continue;
+        case '-': exp->tag = SZEXP_OP2; exp->val.op = SZEXP_OP_SUB; continue;
+        case '*': exp->tag = SZEXP_OP2; exp->val.op = SZEXP_OP_MUL; continue;
+        case '/': exp->tag = SZEXP_OP2; exp->val.op = SZEXP_OP_DIV; continue;
+        }
+
+        if (isdigit(word.start[0])) {
+            exp->tag = SZEXP_CONST;
+            if (bstr_sscanf(word, "%f", &exp->val.cval) != 1)
+                return false;
+            continue;
+        }
+
+        // Some sort of illegal expression
+        return false;
+    }
+
+    return true;
+}
 
 // Returns false if no more shaders could be parsed
 bool parse_user_shader_pass(struct mp_log *log, struct bstr *body,
@@ -24,14 +72,19 @@ bool parse_user_shader_pass(struct mp_log *log, struct bstr *body,
     if (!body || !out || !body->start || body->len == 0)
         return false;
 
-    *out = (struct gl_user_shader){ .transform = identity_trans };
+    *out = (struct gl_user_shader){
+        .offset = identity_trans,
+        .width = {{ SZEXP_VAR_W, { .varname = bstr0("HOOKED") }}},
+        .height = {{ SZEXP_VAR_H, { .varname = bstr0("HOOKED") }}},
+    };
+
     int hook_idx = 0;
     int bind_idx = 0;
 
     // First parse all the headers
     while (true) {
         struct bstr rest;
-        struct bstr line = bstr_getline(*body, &rest);
+        struct bstr line = bstr_strip(bstr_getline(*body, &rest));
 
         // Check for the presence of the magic line beginning
         if (!bstr_eatstart0(&line, "//!"))
@@ -65,13 +118,30 @@ bool parse_user_shader_pass(struct mp_log *log, struct bstr *body,
             continue;
         }
 
-        if (bstr_eatstart0(&line, "TRANSFORM")) {
-            float sx, sy, ox, oy;
-            if (bstr_sscanf(line, "%f %f %f %f", &sx, &sy, &ox, &oy) != 4) {
-                mp_err(log, "Error while parsing TRANSFORM!\n");
+        if (bstr_eatstart0(&line, "OFFSET")) {
+            float ox, oy;
+            if (bstr_sscanf(line, "%f %f", &ox, &oy) != 2) {
+                mp_err(log, "Error while parsing OFFSET!\n");
+                return false;
+            }
+            out->offset.t[0] = ox;
+            out->offset.t[1] = oy;
+            continue;
+        }
+
+        if (bstr_eatstart0(&line, "WIDTH")) {
+            if (!parse_rpn_szexpr(line, out->width)) {
+                mp_err(log, "Error while parsing WIDTH!\n");
+                return false;
+            }
+            continue;
+        }
+
+        if (bstr_eatstart0(&line, "HEIGHT")) {
+            if (!parse_rpn_szexpr(line, out->height)) {
+                mp_err(log, "Error while parsing HEIGHT!\n");
                 return false;
             }
-            out->transform = (struct gl_transform){{{sx, 0}, {0, sy}}, {ox, oy}};
             continue;
         }
 
diff --git a/video/out/opengl/user_shaders.h b/video/out/opengl/user_shaders.h
index 051dcaaa58..0e32f53c57 100644
--- a/video/out/opengl/user_shaders.h
+++ b/video/out/opengl/user_shaders.h
@@ -24,13 +24,40 @@
 #define SHADER_API 1
 #define SHADER_MAX_HOOKS 16
 #define SHADER_MAX_BINDS 6
+#define MAX_SZEXP_SIZE 32
+
+enum szexp_op {
+    SZEXP_OP_ADD,
+    SZEXP_OP_SUB,
+    SZEXP_OP_MUL,
+    SZEXP_OP_DIV,
+};
+
+enum szexp_tag {
+    SZEXP_END = 0, // End of an RPN expression
+    SZEXP_CONST, // Push a constant value onto the stack
+    SZEXP_VAR_W, // Get the width/height of a named texture (variable)
+    SZEXP_VAR_H,
+    SZEXP_OP2, // Pop two elements and push the result of a dyadic operation
+} tag;
+
+struct szexp {
+    enum szexp_tag tag;
+    union {
+        float cval;
+        struct bstr varname;
+        enum szexp_op op;
+    } val;
+};
 
 struct gl_user_shader {
     struct bstr hook_tex[SHADER_MAX_HOOKS];
     struct bstr bind_tex[SHADER_MAX_BINDS];
     struct bstr save_tex;
     struct bstr pass_body;
-    struct gl_transform transform;
+    struct gl_transform offset;
+    struct szexp width[MAX_SZEXP_SIZE];
+    struct szexp height[MAX_SZEXP_SIZE];
     int components;
 };
 
diff --git a/video/out/opengl/video.c b/video/out/opengl/video.c
index f154fdf074..2a1d623f0b 100644
--- a/video/out/opengl/video.c
+++ b/video/out/opengl/video.c
@@ -1618,6 +1618,89 @@ static void user_hook_old(struct gl_video *p, struct img_tex tex,
     GLSLF("color = %s(HOOKED, HOOKED_pos, HOOKED_size);\n", fn_name);
 }
 
+// Returns 1.0 on failure to at least create a legal FBO
+static float eval_szexpr(struct gl_video *p, struct img_tex tex,
+                         struct szexp expr[MAX_SZEXP_SIZE])
+{
+    float stack[MAX_SZEXP_SIZE] = {0};
+    int idx = 0; // points to next element to push
+
+    for (int i = 0; i < MAX_SZEXP_SIZE; i++) {
+        switch (expr[i].tag) {
+        case SZEXP_END:
+            goto done;
+
+        case SZEXP_CONST:
+            // Since our SZEXPs are bound by MAX_SZEXP_SIZE, it should be
+            // impossible to overflow the stack
+            assert(idx < MAX_SZEXP_SIZE);
+            stack[idx++] = expr[i].val.cval;
+            continue;
+
+        case SZEXP_OP2:
+            if (idx < 2) {
+                MP_WARN(p, "Stack underflow in RPN expression!\n");
+                return 1.0;
+            }
+
+            // Pop the operands in reverse order
+            float op2 = stack[--idx], op1 = stack[--idx], res = 0.0;
+            switch (expr[i].val.op) {
+            case SZEXP_OP_ADD: res = op1 + op2; break;
+            case SZEXP_OP_SUB: res = op1 - op2; break;
+            case SZEXP_OP_MUL: res = op1 * op2; break;
+            case SZEXP_OP_DIV: res = op1 / op2; break;
+            default: abort();
+            }
+
+            if (isnan(res)) {
+                MP_WARN(p, "Illegal operation in RPN expression!\n");
+                return 1.0;
+            }
+
+            stack[idx++] = res;
+            continue;
+
+        case SZEXP_VAR_W:
+        case SZEXP_VAR_H: {
+            struct bstr name = expr[i].val.varname;
+            struct img_tex var_tex;
+
+            // HOOKED is a special case
+            if (bstr_equals0(name, "HOOKED")) {
+                var_tex = tex;
+                goto found_tex;
+            }
+
+            for (int o = 0; o < p->saved_tex_num; o++) {
+                if (bstr_equals0(name, p->saved_tex[o].name)) {
+                    var_tex = p->saved_tex[o].tex;
+                    goto found_tex;
+                }
+            }
+
+            char *errname = bstrto0(NULL, name);
+            MP_WARN(p, "Texture %s not found in RPN expression!\n", errname);
+            talloc_free(errname);
+            return 1.0;
+
+found_tex:
+            stack[idx++] = (expr[i].tag == SZEXP_VAR_W) ? var_tex.w : var_tex.h;
+            continue;
+            }
+        }
+    }
+
+done:
+    // Return the single stack element
+    if (idx != 1) {
+        MP_WARN(p, "Malformed stack after RPN expression!\n");
+        return 1.0;
+    }
+
+    return stack[0];
+}
+
 static void user_hook(struct gl_video *p, struct img_tex tex,
                       struct gl_transform *trans, void *priv)
 {
@@ -1628,7 +1711,11 @@ static void user_hook(struct gl_video *p, struct img_tex tex,
     GLSLF("// custom hook\n");
     GLSLF("color = hook();\n");
 
-    *trans = shader->transform;
+    float w = eval_szexpr(p, tex, shader->width);
+    float h = eval_szexpr(p, tex, shader->height);
+
+    *trans = (struct gl_transform){{{w / tex.w, 0}, {0, h / tex.h}}};
+    gl_transform_trans(shader->offset, trans);
 }
 
 static void user_hook_free(struct tex_hook *hook)