vo_vdpau: allow setting colorkey

Formally, this sets the "background color" of the presentation queue.
But in practice, this color is also used as colorkey.

This commit doesn't change the VDPAU default yet.

3 files changed, 23 insertions, 0 deletions

diff --git a/DOCS/man/en/vo.rst b/DOCS/man/en/vo.rst
index 2e24b670d8..d37b679303 100644
--- a/DOCS/man/en/vo.rst
+++ b/DOCS/man/en/vo.rst
@@ -133,6 +133,12 @@ Available video output drivers are:
     ``output_surfaces=<2-15>``
         Allocate this many output surfaces to display video frames (default:
         3). See below for additional information.
+    ``colorkey=<#RRGGBB|#AARRGGBB>``
+        Set the VDPAU presentation queue background color, which in practice
+        is the colorkey used if VDPAU operates in overlay mode (default:
+        ``#00000000``, meaning do not change the VDPAU default). If the alpha
+        component of this value is 0, the default VDPAU colorkey will be used
+        instead (which is usually green).
 
     Using the VDPAU frame queueing functionality controlled by the queuetime
     options makes mpv's frame flip timing less sensitive to system CPU load and
diff --git a/video/out/vo_vdpau.c b/video/out/vo_vdpau.c
index 588a1ceac5..addb155c56 100644
--- a/video/out/vo_vdpau.c
+++ b/video/out/vo_vdpau.c
@@ -72,6 +72,8 @@ struct vdpctx {
     struct vdp_functions              *vdp;
     VdpDevice                          vdp_device;
 
+    struct m_color                     colorkey;
+
     bool                               is_preempted;
     bool                               preemption_acked;
     bool                               preemption_user_notified;
@@ -489,6 +491,18 @@ static int win_x11_init_vdpau_flip_queue(struct vo *vo)
             CHECK_ST_ERROR("Error when calling vdp_presentation_queue_create");
     }
 
+    if (vc->colorkey.a > 0) {
+        VdpColor color = {
+            .red = vc->colorkey.r / 255.0,
+            .green = vc->colorkey.g / 255.0,
+            .blue = vc->colorkey.b / 255.0,
+            .alpha = 0,
+        };
+        vdp_st = vdp->presentation_queue_set_background_color(vc->flip_queue,
+                                                              &color);
+        CHECK_ST_WARNING("Error setting colorkey");
+    }
+
     VdpTime vdp_time;
     vdp_st = vdp->presentation_queue_get_time(vc->flip_queue, &vdp_time);
     CHECK_ST_ERROR("Error when calling vdp_presentation_queue_get_time");
@@ -1542,6 +1556,7 @@ const struct vo_driver video_out_vdpau = {
         OPT_INT("queuetime_fs", flip_offset_fs, 0, OPTDEF_INT(50)),
         OPT_INTRANGE("output_surfaces", num_output_surfaces, 0,
                      2, MAX_OUTPUT_SURFACES, OPTDEF_INT(3)),
+        OPT_COLOR("colorkey", colorkey, 0),
         {NULL},
     }
 };
diff --git a/video/vdpau_functions.inc b/video/vdpau_functions.inc
index 528fe999e2..bbac30dedf 100644
--- a/video/vdpau_functions.inc
+++ b/video/vdpau_functions.inc
@@ -28,6 +28,8 @@ VDP_FUNCTION(VdpPresentationQueueDestroy, VDP_FUNC_ID_PRESENTATION_QUEUE_DESTROY
 VDP_FUNCTION(VdpPresentationQueueDisplay, VDP_FUNC_ID_PRESENTATION_QUEUE_DISPLAY, presentation_queue_display)
 VDP_FUNCTION(VdpPresentationQueueGetTime, VDP_FUNC_ID_PRESENTATION_QUEUE_GET_TIME, presentation_queue_get_time)
 VDP_FUNCTION(VdpPresentationQueueQuerySurfaceStatus, VDP_FUNC_ID_PRESENTATION_QUEUE_QUERY_SURFACE_STATUS, presentation_queue_query_surface_status)
+VDP_FUNCTION(VdpPresentationQueueSetBackgroundColor, VDP_FUNC_ID_PRESENTATION_QUEUE_SET_BACKGROUND_COLOR, presentation_queue_set_background_color)
+VDP_FUNCTION(VdpPresentationQueueGetBackgroundColor, VDP_FUNC_ID_PRESENTATION_QUEUE_GET_BACKGROUND_COLOR, presentation_queue_get_background_color)
 VDP_FUNCTION(VdpPresentationQueueTargetCreateX11, VDP_FUNC_ID_PRESENTATION_QUEUE_TARGET_CREATE_X11, presentation_queue_target_create_x11)
 VDP_FUNCTION(VdpPresentationQueueTargetDestroy, VDP_FUNC_ID_PRESENTATION_QUEUE_TARGET_DESTROY, presentation_queue_target_destroy)
 VDP_FUNCTION(VdpVideoMixerCreate, VDP_FUNC_ID_VIDEO_MIXER_CREATE, video_mixer_create)