vo_opengl: change the way unaligned chroma size is handled

This deals with subsampled YUV video that has odd sizes, for example a
5x5 image with 4:2:0 subsampling.

It would be easy to handle if we actually passed separate texture
coordinates for each plane to the shader, but as of now the luma
coordinates are implicitly rescaled to chroma one. If luma and chroma
sizes don't match up, and this is not handled, you'd get a chroma shift
by 1 pixel.

The existing hack worked, but broke separable scaling. This was exposed
by a recent commit which switched to GL_NEAREST sampling for FBOs. The
rendering was accidentally scaled by 1 pixel, because the FBO size used
the original video size, while textures_sizes[0] was set to the padded
texture size (i.e. one pixel larger).

It could be fixed by setting the padded texture size only on the first
shader. But somehow that is annoying, so do something else. Don't pad
textures anymore, and rescale the chroma coordinates in the shader
instead.

Seems like this somehow doesn't work with rectangle textures (and
introduces a chroma shift), but since it's only used when doing VDA
hardware decoding, and the bug occurs only with unaligned video sizes, I
don't care much.

Fixes #1523.

2 files changed, 24 insertions, 10 deletions

diff --git a/video/out/gl_video.c b/video/out/gl_video.c
index 9e25af376b..66830777b5 100644
--- a/video/out/gl_video.c
+++ b/video/out/gl_video.c
@@ -794,6 +794,21 @@ static void update_uniforms(struct gl_video *p, GLuint program)
         gl->Uniform2f(loc, 1.0 / (1 << xs), 1.0 / (1 << ys));
     }
 
+    loc = gl->GetUniformLocation(program, "chroma_fix");
+    if (loc >= 0) {
+        // If the dimensions of the Y plane are not aligned on the luma.
+        // Assume 4:2:0 with size (3,3). The last luma pixel is (2,2).
+        // The last chroma pixel is (1,1), not (0,0). So for luma, the
+        // coordinate range is [0,3), for chroma it is [0,2). This means the
+        // texture coordinates for chroma are stretched by adding 1 luma pixel
+        // to the range. Undo this.
+        double fx = p->image.planes[0].tex_w / (double)p->image.planes[1].tex_w
+                    / (1 << p->image_desc.chroma_xs);
+        double fy = p->image.planes[0].tex_h / (double)p->image.planes[1].tex_h
+                    / (1 << p->image_desc.chroma_ys);
+        gl->Uniform2f(loc, fx, fy);
+    }
+
     loc = gl->GetUniformLocation(program, "chroma_center_offset");
     if (loc >= 0) {
         int chr = p->opts.chroma_location;
@@ -1228,6 +1243,9 @@ static void compile_shaders(struct gl_video *p)
         shader_def(&header_conv, "USE_ALPHA_PLANE", "3");
     if (p->opts.alpha_mode == 2 && p->has_alpha)
         shader_def(&header_conv, "USE_ALPHA_BLEND", "1");
+    shader_def_opt(&header_conv, "USE_CHROMA_FIX",
+                   (p->image.planes[0].tex_w & ~(1u << p->image_desc.chroma_xs) ||
+                    p->image.planes[0].tex_h & ~(1u << p->image_desc.chroma_ys)));
 
     shader_def_opt(&header_final, "USE_SIGMOID_INV", use_sigmoid);
     shader_def_opt(&header_final, "USE_GAMMA_POW", p->opts.gamma > 0);
@@ -1716,21 +1734,13 @@ static void init_video(struct gl_video *p, const struct mp_image_params *params)
 
     debug_check_gl(p, "before video texture creation");
 
-    // For video with odd sizes: enlarge the luma texture so that it covers all
-    // chroma pixels - then we can render these correctly by cropping the final
-    // image (conceptually).
-    // Image allocations are always such that the "additional" luma border
-    // exists and can be accessed.
-    int full_w = MP_ALIGN_UP(p->image_w, 1 << p->image_desc.chroma_xs);
-    int full_h = MP_ALIGN_UP(p->image_h, 1 << p->image_desc.chroma_ys);
-
     struct video_image *vimg = &p->image;
 
     for (int n = 0; n < p->plane_count; n++) {
         struct texplane *plane = &vimg->planes[n];
 
-        plane->w = full_w >> p->image_desc.xs[n];
-        plane->h = full_h >> p->image_desc.ys[n];
+        plane->w = mp_chroma_div_up(p->image_w, p->image_desc.xs[n]);
+        plane->h = mp_chroma_div_up(p->image_h, p->image_desc.ys[n]);
 
         if (p->hwdec_active) {
             // We expect hwdec backends to allocate exact size
diff --git a/video/out/gl_video_shaders.glsl b/video/out/gl_video_shaders.glsl
index c9398c8cd5..9a24c5bb4e 100644
--- a/video/out/gl_video_shaders.glsl
+++ b/video/out/gl_video_shaders.glsl
@@ -163,6 +163,7 @@ uniform VIDEO_SAMPLER texture3;
 uniform vec2 textures_size[4];
 uniform vec2 chroma_center_offset;
 uniform vec2 chroma_div;
+uniform vec2 chroma_fix;
 uniform sampler2D lut_2d_c;
 uniform sampler2D lut_2d_l;
 #if HAVE_1DTEX
@@ -365,6 +366,9 @@ vec4 sample_sharpen5(VIDEO_SAMPLER tex, vec2 texsize, vec2 texcoord, float param
 
 void main() {
     vec2 chr_texcoord = texcoord;
+#ifdef USE_CHROMA_FIX
+    chr_texcoord = chr_texcoord * chroma_fix;
+#endif
 #ifdef USE_RECTANGLE
     chr_texcoord = chr_texcoord * chroma_div;
 #else