vo_opengl: use RPN expressions for user hook sizes

This replaces the previous TRANSFORM by WIDTH, HEIGHT and OFFSET where
WIDTH and HEIGHT are RPN expressions. This allows for more fine-grained
control over the output size, and also makes sure that overwriting
existing textures works more cleanly.

(Also add some more useful bstr functions)

1 files changed, 88 insertions, 1 deletions

diff --git a/video/out/opengl/video.c b/video/out/opengl/video.c
index f154fdf074..2a1d623f0b 100644
--- a/video/out/opengl/video.c
+++ b/video/out/opengl/video.c
@@ -1618,6 +1618,89 @@ static void user_hook_old(struct gl_video *p, struct img_tex tex,
     GLSLF("color = %s(HOOKED, HOOKED_pos, HOOKED_size);\n", fn_name);
 }
 
+// Returns 1.0 on failure to at least create a legal FBO
+static float eval_szexpr(struct gl_video *p, struct img_tex tex,
+                         struct szexp expr[MAX_SZEXP_SIZE])
+{
+    float stack[MAX_SZEXP_SIZE] = {0};
+    int idx = 0; // points to next element to push
+
+    for (int i = 0; i < MAX_SZEXP_SIZE; i++) {
+        switch (expr[i].tag) {
+        case SZEXP_END:
+            goto done;
+
+        case SZEXP_CONST:
+            // Since our SZEXPs are bound by MAX_SZEXP_SIZE, it should be
+            // impossible to overflow the stack
+            assert(idx < MAX_SZEXP_SIZE);
+            stack[idx++] = expr[i].val.cval;
+            continue;
+
+        case SZEXP_OP2:
+            if (idx < 2) {
+                MP_WARN(p, "Stack underflow in RPN expression!\n");
+                return 1.0;
+            }
+
+            // Pop the operands in reverse order
+            float op2 = stack[--idx], op1 = stack[--idx], res = 0.0;
+            switch (expr[i].val.op) {
+            case SZEXP_OP_ADD: res = op1 + op2; break;
+            case SZEXP_OP_SUB: res = op1 - op2; break;
+            case SZEXP_OP_MUL: res = op1 * op2; break;
+            case SZEXP_OP_DIV: res = op1 / op2; break;
+            default: abort();
+            }
+
+            if (isnan(res)) {
+                MP_WARN(p, "Illegal operation in RPN expression!\n");
+                return 1.0;
+            }
+
+            stack[idx++] = res;
+            continue;
+
+        case SZEXP_VAR_W:
+        case SZEXP_VAR_H: {
+            struct bstr name = expr[i].val.varname;
+            struct img_tex var_tex;
+
+            // HOOKED is a special case
+            if (bstr_equals0(name, "HOOKED")) {
+                var_tex = tex;
+                goto found_tex;
+            }
+
+            for (int o = 0; o < p->saved_tex_num; o++) {
+                if (bstr_equals0(name, p->saved_tex[o].name)) {
+                    var_tex = p->saved_tex[o].tex;
+                    goto found_tex;
+                }
+            }
+
+            char *errname = bstrto0(NULL, name);
+            MP_WARN(p, "Texture %s not found in RPN expression!\n", errname);
+            talloc_free(errname);
+            return 1.0;
+
+found_tex:
+            stack[idx++] = (expr[i].tag == SZEXP_VAR_W) ? var_tex.w : var_tex.h;
+            continue;
+            }
+        }
+    }
+
+done:
+    // Return the single stack element
+    if (idx != 1) {
+        MP_WARN(p, "Malformed stack after RPN expression!\n");
+        return 1.0;
+    }
+
+    return stack[0];
+}
+
 static void user_hook(struct gl_video *p, struct img_tex tex,
                       struct gl_transform *trans, void *priv)
 {
@@ -1628,7 +1711,11 @@ static void user_hook(struct gl_video *p, struct img_tex tex,
     GLSLF("// custom hook\n");
     GLSLF("color = hook();\n");
 
-    *trans = shader->transform;
+    float w = eval_szexpr(p, tex, shader->width);
+    float h = eval_szexpr(p, tex, shader->height);
+
+    *trans = (struct gl_transform){{{w / tex.w, 0}, {0, h / tex.h}}};
+    gl_transform_trans(shader->offset, trans);
 }
 
 static void user_hook_free(struct tex_hook *hook)