vo_opengl: adjust the rules for linearization

Two changes, compounded into one since they affect the same logic:

1. Never use linearization for HDR downscaling
2. Always use linearization for interpolation

Instead of fixing p->use_linear at the beginning of pass_render_frame,
we flip it on "dynamically" as needed. I plan on killing this
p->use_linear frame (along with other per-pass metadata) and moving them
into their own struct for tracking the "current" state of the video, but
that's a separate/upcoming refactor.

As a small bonus, reduce some code duplication in the interpolation
logic.

Fixes #4631

2 files changed, 45 insertions, 22 deletions

diff --git a/DOCS/man/options.rst b/DOCS/man/options.rst
index d174dc4c5a..bf72d90fc0 100644
--- a/DOCS/man/options.rst
+++ b/DOCS/man/options.rst
@@ -4110,7 +4110,8 @@ The following video options are currently all specific to ``--vo=opengl`` and
 
 ``--linear-scaling``
     Scale in linear light. It should only be used with a
-    ``--opengl-fbo-format`` that has at least 16 bit precision.
+    ``--opengl-fbo-format`` that has at least 16 bit precision. This option
+    has no effect on HDR content.
 
 ``--correct-downscaling``
     When using convolution based filters, extend the filter size when
@@ -4133,7 +4134,8 @@ The following video options are currently all specific to ``--vo=opengl`` and
     the ``--tscale`` setting.
 
     Note that this relies on vsync to work, see ``--opengl-swapinterval`` for
-    more information.
+    more information. It should also only be used with an ``--opengl-fbo-format``
+    that has at least 16 bit precision.
 
 ``--interpolation-threshold=<0..1,-1>``
     Threshold below which frame ratio interpolation gets disabled (default:
diff --git a/video/out/opengl/video.c b/video/out/opengl/video.c
index e1fd60646a..d0efcd067d 100644
--- a/video/out/opengl/video.c
+++ b/video/out/opengl/video.c
@@ -2326,12 +2326,21 @@ static void pass_scale_main(struct gl_video *p)
 
     // Pre-conversion, like linear light/sigmoidization
     GLSLF("// scaler pre-conversion\n");
-    if (p->use_linear) {
+    bool use_linear = p->opts.linear_scaling || p->opts.sigmoid_upscaling;
+
+    // Linear light downscaling results in nasty artifacts for HDR curves due
+    // to the potentially extreme brightness differences severely compounding
+    // any ringing. So just scale in gamma light instead.
+    if (mp_trc_is_hdr(p->image_params.color.gamma) && downscaling)
+        use_linear = false;
+
+    if (use_linear) {
+        p->use_linear = true;
         pass_linearize(p->sc, p->image_params.color.gamma);
         pass_opt_hook_point(p, "LINEAR", NULL);
     }
 
-    bool use_sigmoid = p->use_linear && p->opts.sigmoid_upscaling && upscaling;
+    bool use_sigmoid = use_linear && p->opts.sigmoid_upscaling && upscaling;
     float sig_center, sig_slope, sig_offset, sig_scale;
     if (use_sigmoid) {
         // Coefficients for the sigmoidal transform are taken from the
@@ -2692,7 +2701,6 @@ static bool pass_render_frame(struct gl_video *p, struct mp_image *mpi, uint64_t
     if (p->dumb_mode)
         return true;
 
-    p->use_linear = p->opts.linear_scaling || p->opts.sigmoid_upscaling;
     pass_read_video(p);
     pass_opt_hook_point(p, "NATIVE", &p->texture_offset);
     pass_convert_yuv(p);
@@ -2800,13 +2808,34 @@ static void pass_draw_to_screen(struct gl_video *p, int fbo)
     finish_pass_direct(p, fbo, p->vp_w, p->vp_h, &p->dst_rect);
 }
 
-// Draws an interpolate frame to fbo, based on the frame timing in t
-static void gl_video_interpolate_frame(struct gl_video *p, struct vo_frame *t,
-                                       int fbo)
+static bool update_fbosurface(struct gl_video *p, struct mp_image *mpi,
+                              uint64_t id, struct fbosurface *surf)
 {
     int vp_w = p->dst_rect.x1 - p->dst_rect.x0,
         vp_h = p->dst_rect.y1 - p->dst_rect.y0;
 
+    pass_info_reset(p, false);
+    if (!pass_render_frame(p, mpi, id))
+        return false;
+
+    // Frame blending should always be done in linear light to preserve the
+    // overall brightness, otherwise this will result in flashing dark frames
+    // because mixing in compressed light artificially darkens the results
+    if (!p->use_linear) {
+        p->use_linear = true;
+        pass_linearize(p->sc, p->image_params.color.gamma);
+    }
+
+    finish_pass_fbo(p, &surf->fbotex, vp_w, vp_h, FBOTEX_FUZZY);
+    surf->id  = id;
+    surf->pts = mpi->pts;
+    return true;
+}
+
+// Draws an interpolate frame to fbo, based on the frame timing in t
+static void gl_video_interpolate_frame(struct gl_video *p, struct vo_frame *t,
+                                       int fbo)
+{
     bool is_new = false;
 
     // Reset the queue completely if this is a still image, to avoid any
@@ -2818,15 +2847,11 @@ static void gl_video_interpolate_frame(struct gl_video *p, struct vo_frame *t,
     // First of all, figure out if we have a frame available at all, and draw
     // it manually + reset the queue if not
     if (p->surfaces[p->surface_now].id == 0) {
-        is_new = true;
-        pass_info_reset(p, false);
-        if (!pass_render_frame(p, t->current, t->frame_id))
+        struct fbosurface *now = &p->surfaces[p->surface_now];
+        if (!update_fbosurface(p, t->current, t->frame_id, now))
             return;
-        finish_pass_fbo(p, &p->surfaces[p->surface_now].fbotex,
-                        vp_w, vp_h, FBOTEX_FUZZY);
-        p->surfaces[p->surface_now].id = p->image.id;
-        p->surfaces[p->surface_now].pts = p->image.mpi->pts;
         p->surface_idx = p->surface_now;
+        is_new = true;
     }
 
     // Find the right frame for this instant
@@ -2881,16 +2906,12 @@ static void gl_video_interpolate_frame(struct gl_video *p, struct vo_frame *t,
             continue;
 
         if (f_id > p->surfaces[p->surface_idx].id) {
-            is_new = true;
-            pass_info_reset(p, false);
-            if (!pass_render_frame(p, f, f_id))
+            struct fbosurface *dst = &p->surfaces[surface_dst];
+            if (!update_fbosurface(p, f, f_id, dst))
                 return;
-            finish_pass_fbo(p, &p->surfaces[surface_dst].fbotex,
-                            vp_w, vp_h, FBOTEX_FUZZY);
-            p->surfaces[surface_dst].id = f_id;
-            p->surfaces[surface_dst].pts = f->pts;
             p->surface_idx = surface_dst;
             surface_dst = fbosurface_wrap(surface_dst + 1);
+            is_new = true;
         }
     }