vo_opengl: Fix minor LUT sampling error

Define a macro to correct the coordinate for lookup texture. Cache
the corrected coordinate for 1D filter and use mix() to minimize the
performance impact.

3 files changed, 22 insertions, 11 deletions

diff --git a/video/out/opengl/video.c b/video/out/opengl/video.c
index 6f3aba249c..de20984cdc 100644
--- a/video/out/opengl/video.c
+++ b/video/out/opengl/video.c
@@ -1081,14 +1081,16 @@ static void reinit_scaler(struct gl_video *p, struct scaler *scaler,
 
     gl->BindTexture(target, scaler->gl_lut);
 
-    float *weights = talloc_array(NULL, float, LOOKUP_TEXTURE_SIZE * size);
-    mp_compute_lut(scaler->kernel, LOOKUP_TEXTURE_SIZE, weights);
+    scaler->lut_size = LOOKUP_TEXTURE_SIZE;
+
+    float *weights = talloc_array(NULL, float, scaler->lut_size * size);
+    mp_compute_lut(scaler->kernel, scaler->lut_size, weights);
 
     if (target == GL_TEXTURE_1D) {
-        gl->TexImage1D(target, 0, fmt->internal_format, LOOKUP_TEXTURE_SIZE,
+        gl->TexImage1D(target, 0, fmt->internal_format, scaler->lut_size,
                        0, fmt->format, GL_FLOAT, weights);
     } else {
-        gl->TexImage2D(target, 0, fmt->internal_format, width, LOOKUP_TEXTURE_SIZE,
+        gl->TexImage2D(target, 0, fmt->internal_format, width, scaler->lut_size,
                        0, fmt->format, GL_FLOAT, weights);
     }
 
diff --git a/video/out/opengl/video.h b/video/out/opengl/video.h
index c7c567d50c..a56c0c1c09 100644
--- a/video/out/opengl/video.h
+++ b/video/out/opengl/video.h
@@ -62,6 +62,7 @@ struct scaler {
     GLenum gl_target;
     struct fbotex sep_fbo;
     bool insufficient;
+    int lut_size;
 
     // kernel points here
     struct filter_kernel kernel_storage;
diff --git a/video/out/opengl/video_shaders.c b/video/out/opengl/video_shaders.c
index 34948f4fe8..e5b6f917d8 100644
--- a/video/out/opengl/video_shaders.c
+++ b/video/out/opengl/video_shaders.c
@@ -30,7 +30,7 @@
 #define GLSLH(x) gl_sc_hadd(sc, #x "\n");
 #define GLSLHF(...) gl_sc_haddf(sc, __VA_ARGS__)
 
-// Set up shared/commonly used variables
+// Set up shared/commonly used variables and macros
 void sampler_prelude(struct gl_shader_cache *sc, int tex_num)
 {
     GLSLF("#undef tex\n");
@@ -38,6 +38,10 @@ void sampler_prelude(struct gl_shader_cache *sc, int tex_num)
     GLSLF("vec2 pos = texcoord%d;\n", tex_num);
     GLSLF("vec2 size = texture_size%d;\n", tex_num);
     GLSLF("vec2 pt = vec2(1.0) / size;\n");
+    GLSLF("#undef LUT_POS\n");
+    // The variant of mix() with three floats as parameters is supported in
+    // all GLSL versions.
+    GLSLF("#define LUT_POS(x, lut_size) mix(0.5 / (lut_size), 1.0 - 0.5 / (lut_size), (x))\n");
 }
 
 static void pass_sample_separated_get_weights(struct gl_shader_cache *sc,
@@ -45,19 +49,21 @@ static void pass_sample_separated_get_weights(struct gl_shader_cache *sc,
 {
     gl_sc_uniform_sampler(sc, "lut", scaler->gl_target,
                           TEXUNIT_SCALERS + scaler->index);
+    // Define a new variable to cache the corrected fcoord.
+    GLSLF("float fcoord_lut = LUT_POS(fcoord, %d.0);\n", scaler->lut_size);
 
     int N = scaler->kernel->size;
     if (N == 2) {
-        GLSL(vec2 c1 = texture(lut, vec2(0.5, fcoord)).RG;)
+        GLSL(vec2 c1 = texture(lut, vec2(0.5, fcoord_lut)).RG;)
         GLSL(float weights[2] = float[](c1.r, c1.g);)
     } else if (N == 6) {
-        GLSL(vec4 c1 = texture(lut, vec2(0.25, fcoord));)
-        GLSL(vec4 c2 = texture(lut, vec2(0.75, fcoord));)
+        GLSL(vec4 c1 = texture(lut, vec2(0.25, fcoord_lut));)
+        GLSL(vec4 c2 = texture(lut, vec2(0.75, fcoord_lut));)
         GLSL(float weights[6] = float[](c1.r, c1.g, c1.b, c2.r, c2.g, c2.b);)
     } else {
         GLSLF("float weights[%d];\n", N);
         for (int n = 0; n < N / 4; n++) {
-            GLSLF("c = texture(lut, vec2(1.0 / %d.0 + %d.0 / %d.0, fcoord));\n",
+            GLSLF("c = texture(lut, vec2(1.0 / %d.0 + %d.0 / %d.0, fcoord_lut));\n",
                     N / 2, n, N / 4);
             GLSLF("weights[%d] = c.r;\n", n * 4 + 0);
             GLSLF("weights[%d] = c.g;\n", n * 4 + 1);
@@ -142,9 +148,11 @@ void pass_sample_polar(struct gl_shader_cache *sc, struct scaler *scaler)
             if (dmax >= radius - M_SQRT2)
                 GLSLF("if (d < 1.0) {\n");
             if (scaler->gl_target == GL_TEXTURE_1D) {
-                GLSL(w = texture1D(lut, d).r;)
+                GLSLF("w = texture1D(lut, LUT_POS(d, %d.0)).r;\n",
+                      scaler->lut_size);
             } else {
-                GLSL(w = texture(lut, vec2(0.5, d)).r;)
+                GLSLF("w = texture(lut, vec2(0.5, LUT_POS(d, %d.0))).r;\n",
+                      scaler->lut_size);
             }
             GLSL(wsum += w;)
             GLSLF("c = texture(tex, base + pt * vec2(%d.0, %d.0));\n", x, y);