vf_vavpp: add advanced deint bug compatibility for Intel vaapi drivers

I'm not sure what's going on here, but it appears kodi switches forward
and backwards references for advanced VPP deinterlacing modes. This in
turn makes deinterlacing with these modes apparently work. If you don't
switch the directions, you get a stuttering mess.

As far as the libva trace dump is concerned, this makes mpv's libva
deinterlacing API use behave like kodi's, and appears to reproduce
smooth video with advanced libva deinterlacing enabled.

I'm hearing that Mesa actually does it correctly, and I'm not sure what
will happen there. For now, passing "reversal-bug=no" as sub-option to
the vavpp filter will undo this behavior.

1 files changed, 20 insertions, 4 deletions

diff --git a/video/filter/vf_vavpp.c b/video/filter/vf_vavpp.c
index 35172f0344..8f43b7f4ce 100644
--- a/video/filter/vf_vavpp.c
+++ b/video/filter/vf_vavpp.c
@@ -54,6 +54,7 @@ struct pipeline {
 struct vf_priv_s {
     int deint_type; // 0: none, 1: discard, 2: double fps
     int interlaced_only;
+    int reversal_bug;
     bool do_deint;
     VABufferID buffers[VAProcFilterCount];
     int num_buffers;
@@ -74,6 +75,7 @@ static const struct vf_priv_s vf_priv_default = {
     .context = VA_INVALID_ID,
     .deint_type = 2,
     .interlaced_only = 1,
+    .reversal_bug = 1,
 };
 
 static void add_surfaces(struct vf_priv_s *p, struct surface_refs *refs, int dir)
@@ -136,8 +138,13 @@ static void update_pipeline(struct vf_instance *vf)
     p->pipe.num_output_colors = caps.num_output_color_standards;
     p->pipe.forward.max_surfaces = caps.num_forward_references;
     p->pipe.backward.max_surfaces = caps.num_backward_references;
-    mp_refqueue_set_refs(p->queue, caps.num_backward_references,
-                                   caps.num_forward_references);
+    if (p->reversal_bug) {
+        int max = MPMAX(caps.num_forward_references, caps.num_backward_references);
+        mp_refqueue_set_refs(p->queue, max, max);
+    } else {
+        mp_refqueue_set_refs(p->queue, p->pipe.backward.max_surfaces,
+                                       p->pipe.forward.max_surfaces);
+    }
     mp_refqueue_set_mode(p->queue,
         (p->do_deint ? MP_MODE_DEINT : 0) |
         (p->deint_type >= 2 ? MP_MODE_OUTPUT_FIELDS : 0) |
@@ -221,14 +228,22 @@ static struct mp_image *render(struct vf_instance *vf)
     param->filters = p->pipe.filters;
     param->num_filters = p->pipe.num_filters;
 
-    add_surfaces(p, &p->pipe.forward, 1);
+    int dir = p->reversal_bug ? -1 : 1;
+
+    add_surfaces(p, &p->pipe.forward, 1 * dir);
     param->forward_references = p->pipe.forward.surfaces;
     param->num_forward_references = p->pipe.forward.num_surfaces;
 
-    add_surfaces(p, &p->pipe.backward, -1);
+    add_surfaces(p, &p->pipe.backward, -1 * dir);
     param->backward_references = p->pipe.backward.surfaces;
     param->num_backward_references = p->pipe.backward.num_surfaces;
 
+    MP_TRACE(vf, "in=0x%x\n", (unsigned)in_id);
+    for (int n = 0; n < param->num_backward_references; n++)
+        MP_TRACE(vf, " b%d=0x%x\n", n, (unsigned)param->backward_references[n]);
+    for (int n = 0; n < param->num_forward_references; n++)
+        MP_TRACE(vf, " f%d=0x%x\n", n, (unsigned)param->forward_references[n]);
+
     vaUnmapBuffer(p->display, buffer);
 
     status = vaRenderPicture(p->display, p->context, &buffer, 1);
@@ -481,6 +496,7 @@ static const m_option_t vf_opts_fields[] = {
                 {"motion-adaptive", 4},
                 {"motion-compensated", 5})),
     OPT_FLAG("interlaced-only", interlaced_only, 0),
+    OPT_FLAG("reversal-bug", reversal_bug, 0),
     {0}
 };