video: handle hwdec screenshots differently

Instead of converting the hw surface to an image in the VO, provide a
generic way to convet hw surfaces, and use this in the screenshot code.

It's all relatively straightforward, except vdpau is being terrible. It
needs a huge chunk of new code, because copying back is not simple.

5 files changed, 48 insertions, 90 deletions

diff --git a/video/out/gl_hwdec.h b/video/out/gl_hwdec.h
index 8d2f563d5e..ffe685fa97 100644
--- a/video/out/gl_hwdec.h
+++ b/video/out/gl_hwdec.h
@@ -43,9 +43,7 @@ struct gl_hwdec_driver {
     // Undo map_image(). The user of map_image() calls this when the textures
     // are not needed anymore.
     void (*unmap_image)(struct gl_hwdec *hw);
-    // Return a mp_image downloaded from the GPU (optional)
-    struct mp_image *(*download_image)(struct gl_hwdec *hw,
-                                       struct mp_image *hw_image);
+
     void (*destroy)(struct gl_hwdec *hw);
 };
 
diff --git a/video/out/gl_hwdec_vda.c b/video/out/gl_hwdec_vda.c
index 4264dff975..d90b3419ae 100644
--- a/video/out/gl_hwdec_vda.c
+++ b/video/out/gl_hwdec_vda.c
@@ -22,14 +22,42 @@
 #include <OpenGL/OpenGL.h>
 #include <OpenGL/CGLIOSurface.h>
 
-#include "video/decode/dec_video.h"
+#include "video/mp_image_pool.h"
 #include "gl_hwdec.h"
 
 struct priv {
     CVPixelBufferRef pbuf;
     GLuint gl_texture;
+    struct mp_hwdec_ctx hwctx;
 };
 
+static struct mp_image *download_image(struct mp_hwdec_ctx *ctx,
+                                       struct mp_image *hw_image,
+                                       struct mp_image_pool *swpool)
+{
+    if (hw_image->imgfmt != IMGFMT_VDA)
+        return NULL;
+
+    CVPixelBufferRef pbuf = (CVPixelBufferRef)hw_image->planes[3];
+    CVPixelBufferLockBaseAddress(pbuf, 0);
+    void *base = CVPixelBufferGetBaseAddress(pbuf);
+    size_t width  = CVPixelBufferGetWidth(pbuf);
+    size_t height = CVPixelBufferGetHeight(pbuf);
+    size_t stride = CVPixelBufferGetBytesPerRow(pbuf);
+
+    struct mp_image img = {0};
+    mp_image_setfmt(&img, IMGFMT_UYVY);
+    mp_image_set_size(&img, width, height);
+    img.planes[0] = base;
+    img.stride[0] = stride;
+    mp_image_copy_attributes(&img, hw_image);
+
+    struct mp_image *image = mp_image_pool_new_copy(swpool, &img);
+    CVPixelBufferUnlockBaseAddress(pbuf, 0);
+
+    return image;
+}
+
 static bool check_hwdec(struct gl_hwdec *hw)
 {
     if (hw->gl_texture_target != GL_TEXTURE_RECTANGLE) {
@@ -60,6 +88,9 @@ static int create(struct gl_hwdec *hw)
     if (!check_hwdec(hw))
         return -1;
 
+    hw->hwctx = &p->hwctx;
+    hw->hwctx->download_image = download_image;
+
     GL *gl = hw->gl;
     gl->GenTextures(1, &p->gl_texture);
 
@@ -104,28 +135,6 @@ static int map_image(struct gl_hwdec *hw, struct mp_image *hw_image,
 
 static void unmap_image(struct gl_hwdec *hw) { }
 
-static struct mp_image *download_image(struct gl_hwdec *hw,
-                                       struct mp_image *hw_image)
-{
-    CVPixelBufferRef pbuf = (CVPixelBufferRef)hw_image->planes[3];
-    CVPixelBufferLockBaseAddress(pbuf, 0);
-    void *base = CVPixelBufferGetBaseAddress(pbuf);
-    size_t width  = CVPixelBufferGetWidth(pbuf);
-    size_t height = CVPixelBufferGetHeight(pbuf);
-    size_t stride = CVPixelBufferGetBytesPerRow(pbuf);
-
-    struct mp_image img = {0};
-    mp_image_setfmt(&img, IMGFMT_UYVY);
-    mp_image_set_size(&img, width, height);
-    img.planes[0] = base;
-    img.stride[0] = stride;
-
-    struct mp_image *image = mp_image_new_copy(&img);
-    CVPixelBufferUnlockBaseAddress(pbuf, 0);
-
-    return image;
-}
-
 static void destroy(struct gl_hwdec *hw)
 {
     struct priv *p = hw->priv;
@@ -136,7 +145,6 @@ static void destroy(struct gl_hwdec *hw)
     p->gl_texture = 0;
 }
 
-
 const struct gl_hwdec_driver gl_hwdec_vda = {
     .api_name = "vda",
     .imgfmt = IMGFMT_VDA,
@@ -144,6 +152,5 @@ const struct gl_hwdec_driver gl_hwdec_vda = {
     .reinit = reinit,
     .map_image = map_image,
     .unmap_image = unmap_image,
-    .download_image = download_image,
     .destroy = destroy,
 };
diff --git a/video/out/gl_video.c b/video/out/gl_video.c
index e3eb4d1abf..1f32844c07 100644
--- a/video/out/gl_video.c
+++ b/video/out/gl_video.c
@@ -1671,7 +1671,7 @@ static void uninit_video(struct gl_video *p)
 
     fbotex_uninit(p, &p->indirect_fbo);
     fbotex_uninit(p, &p->scale_sep_fbo);
-    
+
     // Invalidate image_params to ensure that gl_video_config() will call
     // init_video() on uninitialized gl_video.
     p->image_params = (struct mp_image_params){0};
@@ -2017,16 +2017,14 @@ struct mp_image *gl_video_download_image(struct gl_video *p)
 
     struct video_image *vimg = &p->image;
 
-    if (!p->have_image || !gl->GetTexImage)
+    if (!p->have_image)
         return NULL;
 
-    if (p->hwdec_active && p->hwdec->driver->download_image) {
-        struct mp_image *dlimage =
-            p->hwdec->driver->download_image(p->hwdec, vimg->hwimage);
-        if (dlimage)
-            mp_image_set_attributes(dlimage, &p->image_params);
-        return dlimage;
-    }
+    if (p->hwdec_active)
+        return mp_image_new_ref(vimg->hwimage);
+
+    if (!gl->GetTexImage)
+        return NULL;
 
     set_image_textures(p, vimg, NULL);
 
diff --git a/video/out/vo_vaapi.c b/video/out/vo_vaapi.c
index cf07787a48..06c7b3c9f4 100644
--- a/video/out/vo_vaapi.c
+++ b/video/out/vo_vaapi.c
@@ -303,14 +303,7 @@ static struct mp_image *get_screenshot(struct priv *p)
     struct mp_image *hwimg = p->output_surfaces[p->visible_surface];
     if (!hwimg)
         return NULL;
-    struct mp_image *img = va_surface_download(hwimg, NULL);
-    if (!img)
-        return NULL;
-    struct mp_image_params params = p->image_params;
-    params.imgfmt = img->imgfmt;
-    mp_image_params_guess_csp(&params); // ensure colorspace consistency
-    mp_image_set_params(img, &params);
-    return img;
+    return mp_image_new_ref(hwimg);
 }
 
 static void free_subpicture(struct priv *p, struct vaapi_osd_image *img)
diff --git a/video/out/vo_vdpau.c b/video/out/vo_vdpau.c
index a022946c1b..e89cda6490 100644
--- a/video/out/vo_vdpau.c
+++ b/video/out/vo_vdpau.c
@@ -80,7 +80,6 @@ struct vdpctx {
     VdpPresentationQueue               flip_queue;
 
     VdpOutputSurface                   output_surfaces[MAX_OUTPUT_SURFACES];
-    VdpOutputSurface                   screenshot_surface;
     int                                num_output_surfaces;
     VdpOutputSurface                   black_pixel;
 
@@ -176,24 +175,16 @@ static int render_video_to_output_surface(struct vo *vo,
                       "vdp_presentation_queue_block_until_surface_idle");
 
     if (vc->rgb_mode) {
-        VdpOutputSurface surface = (uintptr_t)mpi->planes[3];
+        // Clear the borders between video and window (if there are any).
+        // For some reason, video_mixer_render doesn't need it for YUV.
         int flags = VDP_OUTPUT_SURFACE_RENDER_ROTATE_0;
         vdp_st = vdp->output_surface_render_output_surface(output_surface,
                                                            NULL, vc->black_pixel,
                                                            NULL, NULL, NULL,
                                                            flags);
         CHECK_VDP_WARNING(vo, "Error clearing screen");
-        vdp_st = vdp->output_surface_render_output_surface(output_surface,
-                                                           output_rect,
-                                                           surface,
-                                                           video_rect,
-                                                           NULL, NULL, flags);
-        CHECK_VDP_WARNING(vo, "Error when calling "
-                          "vdp_output_surface_render_output_surface");
-        return 0;
     }
 
-
     struct mp_vdpau_mixer_frame *frame = mp_vdpau_mixed_frame_get(mpi);
     struct mp_vdpau_mixer_opts opts = {0};
     if (frame)
@@ -344,12 +335,6 @@ static void free_video_specific(struct vo *vo)
 
     forget_frames(vo, false);
 
-    if (vc->screenshot_surface != VDP_INVALID_HANDLE) {
-        vdp_st = vdp->output_surface_destroy(vc->screenshot_surface);
-        CHECK_VDP_WARNING(vo, "Error when calling vdp_output_surface_destroy");
-    }
-    vc->screenshot_surface = VDP_INVALID_HANDLE;
-
     if (vc->black_pixel != VDP_INVALID_HANDLE) {
         vdp_st = vdp->output_surface_destroy(vc->black_pixel);
         CHECK_VDP_WARNING(vo, "Error when calling vdp_output_surface_destroy");
@@ -399,7 +384,6 @@ static void mark_vdpau_objects_uninitialized(struct vo *vo)
     vc->flip_target = VDP_INVALID_HANDLE;
     for (int i = 0; i < MAX_OUTPUT_SURFACES; i++)
         vc->output_surfaces[i] = VDP_INVALID_HANDLE;
-    vc->screenshot_surface = VDP_INVALID_HANDLE;
     vc->vdp_device = VDP_INVALID_HANDLE;
     for (int i = 0; i < MAX_OSD_PARTS; i++) {
         struct osd_bitmap_surface *sfc = &vc->osd_surfaces[i];
@@ -878,30 +862,6 @@ static struct mp_image *read_output_surface(struct vo *vo,
     return image;
 }
 
-static struct mp_image *get_screenshot(struct vo *vo)
-{
-    struct vdpctx *vc = vo->priv;
-    VdpStatus vdp_st;
-    struct vdp_functions *vdp = vc->vdp;
-
-    if (!vo->params)
-        return NULL;
-
-    if (vc->screenshot_surface == VDP_INVALID_HANDLE) {
-        vdp_st = vdp->output_surface_create(vc->vdp_device,
-                                            OUTPUT_RGBA_FORMAT,
-                                            vo->params->d_w, vo->params->d_h,
-                                            &vc->screenshot_surface);
-        CHECK_VDP_WARNING(vo, "Error when calling vdp_output_surface_create");
-    }
-
-    VdpRect in = { .x1 = vo->params->w, .y1 = vo->params->h };
-    VdpRect out = { .x1 = vo->params->d_w, .y1 = vo->params->d_h };
-    render_video_to_output_surface(vo, vc->screenshot_surface, &out, &in);
-
-    return read_output_surface(vo, vc->screenshot_surface, out.x1, out.y1);
-}
-
 static struct mp_image *get_window_screenshot(struct vo *vo)
 {
     struct vdpctx *vc = vo->priv;
@@ -1092,10 +1052,12 @@ static int control(struct vo *vo, uint32_t request, void *data)
         if (!status_ok(vo))
             return false;
         struct voctrl_screenshot_args *args = data;
-        if (args->full_window)
+        if (args->full_window) {
             args->out_image = get_window_screenshot(vo);
-        else
-            args->out_image = get_screenshot(vo);
+        } else {
+            args->out_image =
+                vc->current_image ? mp_image_new_ref(vc->current_image) : NULL;
+        }
         return true;
     }
     case VOCTRL_GET_PREF_DEINT: