vo_opengl: vdpau interop without RGB conversion

Until now, we've always converted vdpau video surfaces to RGB, and then
mapped the resulting RGB texture. Change this so that the surface is
mapped as NV12 plane textures.

The reason this wasn't done until now is because vdpau surfaces are
mapped in an "interlaced" way as separate fields, even for progressive
video. This requires messy reinterleraving. It turns out that even
though it's an extra processing step, the result can be faster than
going through the video mixer for RGB conversion.

Other than some potential speed-gain, doing this has multiple other
advantages. We can apply our own color conversion, which is important in
more complex cases. We can correctly apply debanding and potentially
other processing that requires chroma-specific or in-YUV handling.

If deinterlacing is enabled, this switches back to the old RGB
conversion method. Until we have at least a primitive deinterlacer in
vo_opengl, this will stay this way. The d3d11 and vaapi code paths are
similar. (Of course these don't require any crazy field reinterleaving.)

1 files changed, 89 insertions, 36 deletions

diff --git a/video/out/opengl/hwdec_vdpau.c b/video/out/opengl/hwdec_vdpau.c
index e1bccd7771..83f664a756 100644
--- a/video/out/opengl/hwdec_vdpau.c
+++ b/video/out/opengl/hwdec_vdpau.c
@@ -36,11 +36,12 @@ struct priv {
     struct mp_vdpau_ctx *ctx;
     uint64_t preemption_counter;
     struct mp_image_params image_params;
-    GLuint gl_texture;
+    GLuint gl_textures[4];
     bool vdpgl_initialized;
     GLvdpauSurfaceNV vdpgl_surface;
     VdpOutputSurface vdp_surface;
     struct mp_vdpau_mixer *mixer;
+    bool direct_mode;
     bool mapped;
 };
 
@@ -49,8 +50,13 @@ static void unmap(struct gl_hwdec *hw)
     struct priv *p = hw->priv;
     GL *gl = hw->gl;
 
-    if (p->mapped)
+    if (p->mapped) {
         gl->VDPAUUnmapSurfacesNV(1, &p->vdpgl_surface);
+        if (p->direct_mode) {
+            gl->VDPAUUnregisterSurfaceNV(p->vdpgl_surface);
+            p->vdpgl_surface = 0;
+        }
+    }
     p->mapped = false;
 }
 
@@ -75,8 +81,9 @@ static void destroy_objects(struct gl_hwdec *hw)
         gl->VDPAUUnregisterSurfaceNV(p->vdpgl_surface);
     p->vdpgl_surface = 0;
 
-    glDeleteTextures(1, &p->gl_texture);
-    p->gl_texture = 0;
+    glDeleteTextures(4, p->gl_textures);
+    for (int n = 0; n < 4; n++)
+        p->gl_textures[n] = 0;
 
     if (p->vdp_surface != VDP_INVALID_HANDLE) {
         vdp_st = vdp->output_surface_destroy(p->vdp_surface);
@@ -151,30 +158,39 @@ static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
 
     p->vdpgl_initialized = true;
 
-    vdp_st = vdp->output_surface_create(p->ctx->vdp_device,
-                                        VDP_RGBA_FORMAT_B8G8R8A8,
-                                        params->w, params->h, &p->vdp_surface);
-    CHECK_VDP_ERROR(p, "Error when calling vdp_output_surface_create");
-
-    gl->GenTextures(1, &p->gl_texture);
-    gl->BindTexture(GL_TEXTURE_2D, p->gl_texture);
-    gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
-    gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
-    gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
-    gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+    p->direct_mode = params->hw_subfmt == IMGFMT_NV12;
+
+    gl->GenTextures(4, p->gl_textures);
+    for (int n = 0; n < 4; n++) {
+        gl->BindTexture(GL_TEXTURE_2D, p->gl_textures[n]);
+        GLenum filter = p->direct_mode ? GL_NEAREST : GL_LINEAR;
+        gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, filter);
+        gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, filter);
+        gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+        gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+    }
     gl->BindTexture(GL_TEXTURE_2D, 0);
 
-    p->vdpgl_surface = gl->VDPAURegisterOutputSurfaceNV(BRAINDEATH(p->vdp_surface),
-                                                        GL_TEXTURE_2D,
-                                                        1, &p->gl_texture);
-    if (!p->vdpgl_surface)
-        return -1;
+    if (p->direct_mode) {
+        params->imgfmt = IMGFMT_NV12;
+    } else {
+        vdp_st = vdp->output_surface_create(p->ctx->vdp_device,
+                                            VDP_RGBA_FORMAT_B8G8R8A8,
+                                            params->w, params->h, &p->vdp_surface);
+        CHECK_VDP_ERROR(p, "Error when calling vdp_output_surface_create");
+
+        p->vdpgl_surface = gl->VDPAURegisterOutputSurfaceNV(BRAINDEATH(p->vdp_surface),
+                                                            GL_TEXTURE_2D,
+                                                            1, p->gl_textures);
+        if (!p->vdpgl_surface)
+            return -1;
 
-    gl->VDPAUSurfaceAccessNV(p->vdpgl_surface, GL_READ_ONLY);
+        gl->VDPAUSurfaceAccessNV(p->vdpgl_surface, GL_READ_ONLY);
 
-    gl_check_error(gl, hw->log, "After initializing vdpau OpenGL interop");
+        params->imgfmt = IMGFMT_RGB0;
+    }
 
-    params->imgfmt = IMGFMT_RGB0;
+    gl_check_error(gl, hw->log, "After initializing vdpau OpenGL interop");
 
     return 0;
 }
@@ -184,6 +200,8 @@ static int map_frame(struct gl_hwdec *hw, struct mp_image *hw_image,
 {
     struct priv *p = hw->priv;
     GL *gl = hw->gl;
+    struct vdp_functions *vdp = &p->ctx->vdp;
+    VdpStatus vdp_st;
 
     int pe = mp_vdpau_handle_preemption(p->ctx, &p->preemption_counter);
     if (pe < 1) {
@@ -194,23 +212,58 @@ static int map_frame(struct gl_hwdec *hw, struct mp_image *hw_image,
             return -1;
     }
 
-    if (!p->vdpgl_surface)
-        return -1;
+    if (p->direct_mode) {
+        VdpVideoSurface surface = (intptr_t)hw_image->planes[3];
+
+        // We need the uncropped size.
+        VdpChromaType s_chroma_type;
+        uint32_t s_w, s_h;
+        vdp_st = vdp->video_surface_get_parameters(surface, &s_chroma_type, &s_w, &s_h);
+        CHECK_VDP_ERROR(hw, "Error when calling vdp_video_surface_get_parameters");
 
-    mp_vdpau_mixer_render(p->mixer, NULL, p->vdp_surface, NULL, hw_image, NULL);
+        p->vdpgl_surface = gl->VDPAURegisterVideoSurfaceNV(BRAINDEATH(surface),
+                                                           GL_TEXTURE_2D,
+                                                           4, p->gl_textures);
+        if (!p->vdpgl_surface)
+            return -1;
 
-    gl->VDPAUMapSurfacesNV(1, &p->vdpgl_surface);
-    p->mapped = true;
-    *out_frame = (struct gl_hwdec_frame){
-        .planes = {
-            {
-                .gl_texture = p->gl_texture,
+        gl->VDPAUSurfaceAccessNV(p->vdpgl_surface, GL_READ_ONLY);
+        gl->VDPAUMapSurfacesNV(1, &p->vdpgl_surface);
+
+        p->mapped = true;
+        *out_frame = (struct gl_hwdec_frame){
+            .vdpau_fields = true,
+        };
+        for (int n = 0; n < 4; n++) {
+            bool chroma = n >= 2;
+            out_frame->planes[n] = (struct gl_hwdec_plane){
+                .gl_texture = p->gl_textures[n],
                 .gl_target = GL_TEXTURE_2D,
-                .tex_w = p->image_params.w,
-                .tex_h = p->image_params.h,
+                .tex_w = s_w / (chroma ? 2 : 1),
+                .tex_h = s_h / (chroma ? 4 : 2),
+            };
+        };
+    } else {
+        if (!p->vdpgl_surface)
+            return -1;
+
+        mp_vdpau_mixer_render(p->mixer, NULL, p->vdp_surface, NULL, hw_image, NULL);
+
+        gl->VDPAUMapSurfacesNV(1, &p->vdpgl_surface);
+
+        p->mapped = true;
+        *out_frame = (struct gl_hwdec_frame){
+            .planes = {
+                {
+                    .gl_texture = p->gl_textures[0],
+                    .gl_target = GL_TEXTURE_2D,
+                    .tex_w = p->image_params.w,
+                    .tex_h = p->image_params.h,
+                },
             },
-        },
-    };
+        };
+    }
+
     return 0;
 }