vo_gpu/vo_gpu_next: rework --alpha into --background option

The --alpha option currently covers two related but different concepts:
whether or not to ignore the alpha component and possibly blending it
with a background. Because of the way the option currently works, it is
impossible to have a transparent window (which requires setting
--alpha=yes) while blending it with the background at the same time. To
solve this, let's rework it so it it superseded by the background
option.

--background controls what kind of background to set for the image if
possible. It can be blended with the set background color, with tiles,
or not blended at all (the last one is still broken on X11/mesa except
for GLX, *sigh*). In this new paradigm, --alpha=no has no real purpose
because you can simply set the background to color and whatever color
you want for exactly the same effect. So the option is removed. Instead,
the hint set by windowing backends (i.e. setting
ra_ctx->opts.want_alpha) can by done with the --background option.

As an aside, the colors in vo_gpu are currently bugged due to not
pre-multiplying the alpha and it seems no one ever noticed. The next
commit fixes that. vo_gpu_next support happens latter since it requires
new things from libplacebo.

Fixes #9615.

6 files changed, 45 insertions, 47 deletions

diff --git a/DOCS/interface-changes.rst b/DOCS/interface-changes.rst
index e9984f7ce5..b8a904d995 100644
--- a/DOCS/interface-changes.rst
+++ b/DOCS/interface-changes.rst
@@ -58,6 +58,8 @@ Interface changes
       parameter to the fourth (after `index`)
     - add `--drag-and-drop=insert-next` option
     - rename `--background` to `--background-color`
+    - remove `--alpha` and reintroduce `--background` option for better control
+      over blending alpha components into specific background types
  --- mpv 0.37.0 ---
     - `--save-position-on-quit` and its associated commands now store state files
       in %LOCALAPPDATA% instead of %APPDATA% directory by default on Windows.
diff --git a/DOCS/man/options.rst b/DOCS/man/options.rst
index e484b183ab..4e9392861b 100644
--- a/DOCS/man/options.rst
+++ b/DOCS/man/options.rst
@@ -6921,35 +6921,31 @@ them.
                  softsubbed ASS signs to match the video colors, but may cause
                  SRT subtitles or similar to look slightly off.
 
-``--alpha=<blend-tiles|blend|yes|no>``
-    Decides what to do if the input has an alpha component.
-
-    blend-tiles
+``--background=<none|color|tiles>``
+    If the frame has an alpha component, decide what kind of background, if any,
+    to blend it with. This does nothing if there is no alpha component.
+
+    color
+        Blend the frame against the background color (``--background-color``,
+        normally black).
+    tiles
         Blend the frame against a 16x16 gray/white tiles background (default).
-    blend
-        Blend the frame against the background color (``--background``, normally
-        black).
-    yes
-        Try to create a framebuffer with alpha component. This only makes sense
-        if the video contains alpha information (which is extremely rare) or if
-        you make the background color transparent. May not be supported on all
-        platforms. If alpha framebuffers are unavailable, it silently falls
-        back on a normal framebuffer. Note that if you set the ``--fbo-format``
-        option to a non-default value, a format with alpha must be specified,
-        or this won't work. Whether this really works depends on the windowing
-        system and desktop environment.
-    no
-        Ignore alpha component.
+    none
+        Do not blend the frame and leave the alpha as is.
 
-``--opengl-rectangle-textures``
-    Force use of rectangle textures (default: no). Normally this shouldn't have
-    any advantages over normal textures. Note that hardware decoding overrides
-    this flag. Could be removed any time.
+    Before mpv 0.38.0, this option used to accept a color value specifying the
+    background color. This is now done by the ``--background-color`` option.
+    Use that instead.
 
 ``--background-color=<color>``
     Color used to draw parts of the mpv window not covered by video. See the
     ``--sub-color`` option for how colors are defined.
 
+``--opengl-rectangle-textures``
+    Force use of rectangle textures (default: no). Normally this shouldn't have
+    any advantages over normal textures. Note that hardware decoding overrides
+    this flag. Could be removed any time.
+
 ``--gpu-tex-pad-x``, ``--gpu-tex-pad-y``
     Enlarge the video source textures by this many pixels. For debugging only
     (normally textures are sized exactly, but due to hardware decoding interop
diff --git a/video/out/gpu/video.c b/video/out/gpu/video.c
index 75eca4a314..1da7124fe3 100644
--- a/video/out/gpu/video.c
+++ b/video/out/gpu/video.c
@@ -312,7 +312,7 @@ static const struct gl_video_opts gl_video_opts_def = {
     .linear_downscaling = true,
     .sigmoid_upscaling = true,
     .interpolation_threshold = 0.01,
-    .alpha_mode = ALPHA_BLEND_TILES,
+    .background = BACKGROUND_TILES,
     .background_color = {0, 0, 0, 255},
     .gamma = 1.0f,
     .tone_map = {
@@ -447,11 +447,10 @@ const struct m_sub_options gl_video_conf = {
             M_RANGE(1, 128)},
         {"error-diffusion",
             OPT_STRING_VALIDATE(error_diffusion, validate_error_diffusion_opt)},
-        {"alpha", OPT_CHOICE(alpha_mode,
-            {"no", ALPHA_NO},
-            {"yes", ALPHA_YES},
-            {"blend", ALPHA_BLEND},
-            {"blend-tiles", ALPHA_BLEND_TILES})},
+        {"background", OPT_CHOICE(background,
+            {"none", BACKGROUND_NONE},
+            {"color", BACKGROUND_COLOR},
+            {"tiles", BACKGROUND_TILES})},
         {"opengl-rectangle-textures", OPT_BOOL(use_rectangle)},
         {"background-color", OPT_COLOR(background_color)},
         {"interpolation", OPT_BOOL(interpolation)},
@@ -2396,7 +2395,7 @@ static void pass_convert_yuv(struct gl_video *p)
     }
 
     p->components = 3;
-    if (!p->has_alpha || p->opts.alpha_mode == ALPHA_NO) {
+    if (!p->has_alpha) {
         GLSL(color.a = 1.0;)
     } else if (p->image_params.repr.alpha == PL_ALPHA_PREMULTIPLIED) {
         p->components = 4;
@@ -3076,8 +3075,8 @@ static void pass_draw_to_screen(struct gl_video *p, const struct ra_fbo *fbo, in
         copy_image(p, &(int){0}, tmp);
     }
 
-    if (p->has_alpha){
-        if (p->opts.alpha_mode == ALPHA_BLEND_TILES) {
+    if (p->has_alpha) {
+        if (p->opts.background == BACKGROUND_TILES) {
             // Draw checkerboard pattern to indicate transparency
             GLSLF("// transparency checkerboard\n");
             GLSLF("vec2 tile_coord = vec2(gl_FragCoord.x, %d.0 + %f * gl_FragCoord.y);",
@@ -3086,13 +3085,12 @@ static void pass_draw_to_screen(struct gl_video *p, const struct ra_fbo *fbo, in
             GLSL(vec3 background = vec3(tile.x == tile.y ? 0.93 : 0.87);)
             GLSL(color.rgb += background.rgb * (1.0 - color.a);)
             GLSL(color.a = 1.0;)
-        } else if (p->opts.alpha_mode == ALPHA_BLEND) {
+        } else if (p->opts.background == BACKGROUND_COLOR) {
             // Blend into background color (usually black)
             struct m_color c = p->opts.background_color;
             GLSLF("vec4 background = vec4(%f, %f, %f, %f);\n",
                   c.r / 255.0, c.g / 255.0, c.b / 255.0, c.a / 255.0);
-            GLSL(color.rgb += background.rgb * (1.0 - color.a);)
-            GLSL(color.a = background.a;)
+            GLSL(color += background * (1.0 - color.a);)
         }
     }
 
@@ -3804,9 +3802,8 @@ static void check_gl_features(struct gl_video *p)
         p->opts.dither_algo = DITHER_NONE;
         MP_WARN(p, "Disabling dithering (no gl_FragCoord).\n");
     }
-    if (!have_fragcoord && p->opts.alpha_mode == ALPHA_BLEND_TILES) {
-        p->opts.alpha_mode = ALPHA_BLEND;
-        // Verbose, since this is the default setting
+    if (!have_fragcoord && p->opts.background == BACKGROUND_TILES) {
+        p->opts.background = BACKGROUND_COLOR;
         MP_VERBOSE(p, "Disabling alpha checkerboard (no gl_FragCoord).\n");
     }
     if (!have_fbo && have_compute) {
@@ -3862,7 +3859,7 @@ static void check_gl_features(struct gl_video *p)
             .gamma_auto = p->opts.gamma_auto,
             .pbo = p->opts.pbo,
             .fbo_format = p->opts.fbo_format,
-            .alpha_mode = p->opts.alpha_mode,
+            .background = p->opts.background,
             .use_rectangle = p->opts.use_rectangle,
             .background_color = p->opts.background_color,
             .dither_algo = p->opts.dither_algo,
diff --git a/video/out/gpu/video.h b/video/out/gpu/video.h
index 672f80cf85..a929970686 100644
--- a/video/out/gpu/video.h
+++ b/video/out/gpu/video.h
@@ -72,11 +72,10 @@ enum dither_algo {
     DITHER_ERROR_DIFFUSION,
 };
 
-enum alpha_mode {
-    ALPHA_NO = 0,
-    ALPHA_YES,
-    ALPHA_BLEND,
-    ALPHA_BLEND_TILES,
+enum background_type {
+    BACKGROUND_NONE = 0,
+    BACKGROUND_COLOR,
+    BACKGROUND_TILES,
 };
 
 enum blend_subs_mode {
@@ -155,7 +154,7 @@ struct gl_video_opts {
     int temporal_dither_period;
     char *error_diffusion;
     char *fbo_format;
-    int alpha_mode;
+    int background;
     bool use_rectangle;
     struct m_color background_color;
     bool interpolation;
diff --git a/video/out/vo_gpu.c b/video/out/vo_gpu.c
index ec70f45da9..2f62876db6 100644
--- a/video/out/vo_gpu.c
+++ b/video/out/vo_gpu.c
@@ -173,7 +173,9 @@ static void update_ra_ctx_options(struct vo *vo)
 {
     struct gpu_priv *p = vo->priv;
     struct gl_video_opts *gl_opts = mp_get_config_group(p->ctx, vo->global, &gl_video_conf);
-    p->ctx->opts.want_alpha = gl_opts->alpha_mode == 1;
+    p->ctx->opts.want_alpha = (gl_opts->background == BACKGROUND_COLOR &&
+                               gl_opts->background_color.a != 255) ||
+                               gl_opts->background == BACKGROUND_NONE;
     talloc_free(gl_opts);
 }
 
diff --git a/video/out/vo_gpu_next.c b/video/out/vo_gpu_next.c
index a4b92eb38d..9a63596f64 100644
--- a/video/out/vo_gpu_next.c
+++ b/video/out/vo_gpu_next.c
@@ -1458,7 +1458,9 @@ static void update_ra_ctx_options(struct vo *vo)
 {
     struct priv *p = vo->priv;
     struct gl_video_opts *gl_opts = p->opts_cache->opts;
-    p->ra_ctx->opts.want_alpha = gl_opts->alpha_mode == ALPHA_YES;
+    p->ra_ctx->opts.want_alpha = (gl_opts->background == BACKGROUND_COLOR &&
+                                  gl_opts->background_color.a != 255) ||
+                                  gl_opts->background == BACKGROUND_NONE;
 }
 
 static int control(struct vo *vo, uint32_t request, void *data)
@@ -2090,7 +2092,7 @@ static void update_render_options(struct vo *vo)
     pars->params.skip_anti_aliasing = !opts->correct_downscaling;
     pars->params.disable_linear_scaling = !opts->linear_downscaling && !opts->linear_upscaling;
     pars->params.disable_fbos = opts->dumb_mode == 1;
-    pars->params.blend_against_tiles = opts->alpha_mode == ALPHA_BLEND_TILES;
+    pars->params.blend_against_tiles = opts->background == BACKGROUND_TILES;
     pars->params.corner_rounding = p->next_opts->corner_rounding;
     pars->params.correct_subpixel_offsets = !opts->scaler_resizes_only;