vo_gpu: make screenshots use the GL renderer

Using the GL renderer for color conversion will make sure screenshots
will use the same conversion as normal video rendering. It can do this
for all types of screenshots.

The logic when to write 16 bit PNGs changes. To approximate the old
behavior, we decide by looking whether the source video format has more
than 8 bits per component. We apply this logic even for window
screenshots. Also, 16 bit PNGs now always include an unused alpha
channel. The reason is that FFmpeg has RGB48 and RGBA64 formats, but no
RGB064. RGB48 is 3 bytes and usually not supported by GPUs for
rendering, so we have to use RGBA64, which forces an alpha channel.

Will break for users who use --target-trc and similar options.

I considered creating a new gl_video context, but it could double GPU
memory use, so I didn't.

This uses FBOs instead of glGetTexImage(), because that increases the
chance it could work on GLES (e.g. ANGLE). Untested. No support for the
Vulkan and D3D11 backends yet.

Fixes #5498. Also fixes #5240, because the code for reading back is not
used with the new code path.

4 files changed, 51 insertions, 24 deletions

diff --git a/video/out/opengl/context.c b/video/out/opengl/context.c
index cdaf6320cd..8f44119210 100644
--- a/video/out/opengl/context.c
+++ b/video/out/opengl/context.c
@@ -251,16 +251,21 @@ struct mp_image *ra_gl_ctx_screenshot(struct ra_swapchain *sw)
 {
     struct priv *p = sw->priv;
 
+    struct mp_image *screen = mp_image_alloc(IMGFMT_RGB24, p->wrapped_fb->params.w,
+                                             p->wrapped_fb->params.h);
+    if (!screen)
+        return NULL;
+
+    int dir = p->params.flipped ? 1 : -1;
+
     assert(p->wrapped_fb);
-    struct mp_image *screen = gl_read_fbo_contents(p->gl, p->main_fb,
-                                                   p->wrapped_fb->params.w,
-                                                   p->wrapped_fb->params.h);
-
-    // OpenGL FB is also read in flipped order, so we need to flip when the
-    // rendering is *not* flipped, which in our case is whenever
-    // p->params.flipped is true. I hope that made sense
-    if (screen && p->params.flipped)
-        mp_image_vflip(screen);
+    if (!gl_read_fbo_contents(p->gl, p->main_fb, dir, GL_RGB, GL_UNSIGNED_BYTE,
+                              p->wrapped_fb->params.w, p->wrapped_fb->params.h,
+                              screen->planes[0], screen->stride[0]))
+    {
+        talloc_free(screen);
+        return NULL;
+    }
 
     return screen;
 }
diff --git a/video/out/opengl/ra_gl.c b/video/out/opengl/ra_gl.c
index 356ed81d03..7112464e87 100644
--- a/video/out/opengl/ra_gl.c
+++ b/video/out/opengl/ra_gl.c
@@ -277,6 +277,13 @@ static struct ra_tex *gl_tex_create_blank(struct ra *ra,
         tex_gl->target = GL_TEXTURE_EXTERNAL_OES;
     }
 
+    if (params->downloadable && !(params->dimensions == 2 &&
+                                  params->format->renderable))
+    {
+        gl_tex_destroy(ra, tex);
+        return NULL;
+    }
+
     return tex;
 }
 
@@ -329,8 +336,11 @@ static struct ra_tex *gl_tex_create(struct ra *ra,
 
     gl_check_error(gl, ra->log, "after creating texture");
 
-    // Even blitting needs an FBO in OpenGL for strange reasons
-    if (tex->params.render_dst || tex->params.blit_src || tex->params.blit_dst) {
+    // Even blitting needs an FBO in OpenGL for strange reasons.
+    // Download is handled by reading from an FBO.
+    if (tex->params.render_dst || tex->params.blit_src ||
+        tex->params.blit_dst || tex->params.downloadable)
+    {
         if (!tex->params.format->renderable) {
             MP_ERR(ra, "Trying to create renderable texture with unsupported "
                    "format.\n");
@@ -512,6 +522,18 @@ static bool gl_tex_upload(struct ra *ra,
     return true;
 }
 
+static bool gl_tex_download(struct ra *ra, struct ra_tex_download_params *params)
+{
+    GL *gl = ra_gl_get(ra);
+    struct ra_tex *tex = params->tex;
+    struct ra_tex_gl *tex_gl = tex->priv;
+    if (!tex_gl->fbo)
+        return false;
+    return gl_read_fbo_contents(gl, tex_gl->fbo, 1, tex_gl->format, tex_gl->type,
+                                tex->params.w, tex->params.h, params->dst,
+                                params->stride);
+}
+
 static void gl_buf_destroy(struct ra *ra, struct ra_buf *buf)
 {
     if (!buf)
@@ -1134,6 +1156,7 @@ static struct ra_fns ra_fns_gl = {
     .tex_create             = gl_tex_create,
     .tex_destroy            = gl_tex_destroy,
     .tex_upload             = gl_tex_upload,
+    .tex_download           = gl_tex_download,
     .buf_create             = gl_buf_create,
     .buf_destroy            = gl_buf_destroy,
     .buf_update             = gl_buf_update,
diff --git a/video/out/opengl/utils.c b/video/out/opengl/utils.c
index 34f4736705..a551ce4299 100644
--- a/video/out/opengl/utils.c
+++ b/video/out/opengl/utils.c
@@ -105,25 +105,23 @@ void gl_upload_tex(GL *gl, GLenum target, GLenum format, GLenum type,
     gl->PixelStorei(GL_UNPACK_ALIGNMENT, 4);
 }
 
-mp_image_t *gl_read_fbo_contents(GL *gl, int fbo, int w, int h)
+bool gl_read_fbo_contents(GL *gl, int fbo, int dir, GLenum format, GLenum type,
+                          int w, int h, uint8_t *dst, int dst_stride)
 {
-    if (gl->es)
-        return NULL; // ES can't read from front buffer
-    mp_image_t *image = mp_image_alloc(IMGFMT_RGB24, w, h);
-    if (!image)
-        return NULL;
+    assert(dir == 1 || dir == -1);
+    if (fbo == 0 && gl->es)
+        return false; // ES can't read from front buffer
     gl->BindFramebuffer(GL_FRAMEBUFFER, fbo);
     GLenum obj = fbo ? GL_COLOR_ATTACHMENT0 : GL_FRONT;
     gl->PixelStorei(GL_PACK_ALIGNMENT, 1);
     gl->ReadBuffer(obj);
-    //flip image while reading (and also avoid stride-related trouble)
-    for (int y = 0; y < h; y++) {
-        gl->ReadPixels(0, h - y - 1, w, 1, GL_RGB, GL_UNSIGNED_BYTE,
-                       image->planes[0] + y * image->stride[0]);
-    }
+    // reading by line allows flipping, and avoids stride-related trouble
+    int y1 = dir > 0 ? 0 : h;
+    for (int y = 0; y < h; y++)
+        gl->ReadPixels(0, y, w, 1, format, type, dst + (y1 + dir * y) * dst_stride);
     gl->PixelStorei(GL_PACK_ALIGNMENT, 4);
     gl->BindFramebuffer(GL_FRAMEBUFFER, 0);
-    return image;
+    return true;
 }
 
 static void gl_vao_enable_attribs(struct gl_vao *vao)
diff --git a/video/out/opengl/utils.h b/video/out/opengl/utils.h
index 53127e479e..9bcadae91c 100644
--- a/video/out/opengl/utils.h
+++ b/video/out/opengl/utils.h
@@ -32,7 +32,8 @@ void gl_upload_tex(GL *gl, GLenum target, GLenum format, GLenum type,
                    const void *dataptr, int stride,
                    int x, int y, int w, int h);
 
-mp_image_t *gl_read_fbo_contents(GL *gl, int fbo, int w, int h);
+bool gl_read_fbo_contents(GL *gl, int fbo, int dir, GLenum format, GLenum type,
+                          int w, int h, uint8_t *dst, int dst_stride);
 
 struct gl_vao {
     GL *gl;