video: make vdpau hardware decoding not use DR code path

vdpau hardware decoding used the DR (direct rendering) path to let the
decoder query a surface from the VO. Special-case the HW decoding path
instead, to make it separate from DR.

5 files changed, 117 insertions, 66 deletions

diff --git a/video/decode/vd_lavc.c b/video/decode/vd_lavc.c
index 24f797beea..408170dbab 100644
--- a/video/decode/vd_lavc.c
+++ b/video/decode/vd_lavc.c
@@ -62,9 +62,12 @@ static const vd_info_t info = {
 #error palette too large, adapt libmpcodecs/vf.c:vf_get_image
 #endif
 
+#define MAX_NUM_MPI 50
+
 typedef struct {
     AVCodecContext *avctx;
     AVFrame *pic;
+    struct mp_image hwdec_mpi[MAX_NUM_MPI];
     enum PixelFormat pix_fmt;
     int do_dr1;
     int vo_initialized;
@@ -82,11 +85,15 @@ typedef struct {
 
 static int get_buffer(AVCodecContext *avctx, AVFrame *pic);
 static void release_buffer(AVCodecContext *avctx, AVFrame *pic);
+
+static int get_buffer_hwdec(AVCodecContext *avctx, AVFrame *pic);
+static void release_buffer_hwdec(AVCodecContext *avctx, AVFrame *pic);
 static void draw_slice_hwdec(struct AVCodecContext *s, const AVFrame *src,
                              int offset[4], int y, int type, int height);
 
-static enum PixelFormat get_format(struct AVCodecContext *avctx,
-                                   const enum PixelFormat *pix_fmt);
+static enum PixelFormat get_format_hwdec(struct AVCodecContext *avctx,
+                                         const enum PixelFormat *pix_fmt);
+
 static void uninit(struct sh_video *sh);
 
 const m_option_t lavc_decode_opts_conf[] = {
@@ -267,10 +274,9 @@ static int init(sh_video_t *sh)
     if (lavc_codec->capabilities & CODEC_CAP_HWACCEL_VDPAU) {
         ctx->do_dr1            = true;
         avctx->thread_count    = 1;
-        avctx->get_format      = get_format;
-        avctx->get_buffer      = get_buffer;
-        avctx->release_buffer  = release_buffer;
-        avctx->reget_buffer    = get_buffer;
+        avctx->get_format      = get_format_hwdec;
+        avctx->get_buffer      = get_buffer_hwdec;
+        avctx->release_buffer  = release_buffer_hwdec;
         avctx->draw_horiz_band = draw_slice_hwdec;
         if (lavc_codec->capabilities & CODEC_CAP_HWACCEL_VDPAU)
             mp_msg(MSGT_DECVIDEO, MSGL_V, "[VD_FFMPEG] VDPAU hardware "
@@ -296,7 +302,7 @@ static int init(sh_video_t *sh)
                 "%d threads if supported.\n", avctx->thread_count);
     }
 
-    if (ctx->do_dr1) {
+    if (ctx->do_dr1 && avctx->get_buffer != get_buffer_hwdec) {
         avctx->flags |= CODEC_FLAG_EMU_EDGE;
         avctx->get_buffer = get_buffer;
         avctx->release_buffer = release_buffer;
@@ -444,16 +450,6 @@ static void uninit(sh_video_t *sh)
     talloc_free(ctx);
 }
 
-static void draw_slice_hwdec(struct AVCodecContext *s,
-                             const AVFrame *src, int offset[4],
-                             int y, int type, int height)
-{
-    sh_video_t *sh = s->opaque;
-    struct vf_instance *vf = sh->vfilter;
-    void *state_ptr = src->data[0];
-    vf->control(vf, VFCTRL_HWDEC_DECODER_RENDER, state_ptr);
-}
-
 static int init_vo(sh_video_t *sh, enum PixelFormat pix_fmt)
 {
     vd_ffmpeg_ctx *ctx = sh->context;
@@ -499,6 +495,99 @@ static int init_vo(sh_video_t *sh, enum PixelFormat pix_fmt)
     return 0;
 }
 
+static enum PixelFormat get_format_hwdec(struct AVCodecContext *avctx,
+                                         const enum PixelFormat *fmt)
+{
+    sh_video_t *sh = avctx->opaque;
+    int i;
+
+    for (i = 0; fmt[i] != PIX_FMT_NONE; i++) {
+        int imgfmt = pixfmt2imgfmt(fmt[i]);
+        if (!IMGFMT_IS_HWACCEL(imgfmt))
+            continue;
+        mp_msg(MSGT_DECVIDEO, MSGL_V, "[VD_FFMPEG] Trying pixfmt=%d.\n", i);
+        if (init_vo(sh, fmt[i]) >= 0)
+            break;
+    }
+    return fmt[i];
+}
+
+static void draw_slice_hwdec(struct AVCodecContext *s,
+                             const AVFrame *src, int offset[4],
+                             int y, int type, int height)
+{
+    sh_video_t *sh = s->opaque;
+    struct vf_instance *vf = sh->vfilter;
+    void *state_ptr = src->data[0];
+    vf->control(vf, VFCTRL_HWDEC_DECODER_RENDER, state_ptr);
+}
+
+static struct mp_image *get_image_hwdec(vd_ffmpeg_ctx *ctx)
+{
+    for (int n = 0; n < MAX_NUM_MPI; n++) {
+        struct mp_image *cur = &ctx->hwdec_mpi[n];
+        if (cur->usage_count == 0) {
+            *cur = (struct mp_image) {
+                .number = n,
+                .imgfmt = ctx->best_csp,
+                .usage_count = 1,
+            };
+            return cur;
+        }
+    }
+    return NULL;
+}
+
+static int get_buffer_hwdec(AVCodecContext *avctx, AVFrame *pic)
+{
+    sh_video_t *sh = avctx->opaque;
+    vd_ffmpeg_ctx *ctx = sh->context;
+
+    assert(IMGFMT_IS_HWACCEL(ctx->best_csp));
+
+    // Uncertain whether this is needed; at least deals with VO/filter failures
+    if (init_vo(sh, avctx->pix_fmt) < 0) {
+        avctx->release_buffer = avcodec_default_release_buffer;
+        avctx->get_buffer = avcodec_default_get_buffer;
+        avctx->reget_buffer = avcodec_default_reget_buffer;
+        return avctx->get_buffer(avctx, pic);
+    }
+
+    struct mp_image *mpi = get_image_hwdec(ctx);
+    if (!mpi)
+        return -1;
+
+    struct vf_instance *vf = sh->vfilter;
+    vf->control(vf, VFCTRL_HWDEC_GET_SURFACE, mpi);
+
+    for (int i = 0; i < 4; i++)
+        pic->data[i] = mpi->planes[i];
+    pic->opaque = mpi;
+    pic->type = FF_BUFFER_TYPE_USER;
+
+    /* The libavcodec reordered_opaque functionality is implemented by
+     * a similar copy in avcodec_default_get_buffer() and without a
+     * workaround like this it'd stop working when a custom buffer
+     * callback is used.
+     */
+    pic->reordered_opaque = avctx->reordered_opaque;
+    return 0;
+}
+
+static void release_buffer_hwdec(AVCodecContext *avctx, AVFrame *pic)
+{
+    mp_image_t *mpi = pic->opaque;
+
+    assert(pic->type == FF_BUFFER_TYPE_USER);
+    assert(mpi);
+    assert(mpi->usage_count > 0);
+
+    mpi->usage_count--;
+
+    for (int i = 0; i < 4; i++)
+        pic->data[i] = NULL;
+}
+
 static int get_buffer(AVCodecContext *avctx, AVFrame *pic)
 {
     sh_video_t *sh = avctx->opaque;
@@ -652,12 +741,8 @@ static struct mp_image *decode(struct sh_video *sh, struct demux_packet *packet,
     AVFrame *pic = ctx->pic;
     AVCodecContext *avctx = ctx->avctx;
     mp_image_t *mpi = NULL;
-    int dr1 = ctx->do_dr1;
     AVPacket pkt;
 
-    if (!dr1)
-        avctx->draw_horiz_band = NULL;
-
     if (flags & 2)
         avctx->skip_frame = AVDISCARD_ALL;
     else if (flags & 1)
@@ -682,7 +767,7 @@ static struct mp_image *decode(struct sh_video *sh, struct demux_packet *packet,
     ret = avcodec_decode_video2(avctx, pic, &got_picture, &pkt);
     *reordered_pts = (union pts){.i = pic->reordered_opaque}.d;
 
-    dr1 = ctx->do_dr1;
+    int dr1 = ctx->do_dr1;
     if (ret < 0)
         mp_msg(MSGT_DECVIDEO, MSGL_WARN, "Error while decoding frame!\n");
 
@@ -751,23 +836,6 @@ static struct mp_image *decode(struct sh_video *sh, struct demux_packet *packet,
     return mpi;
 }
 
-static enum PixelFormat get_format(struct AVCodecContext *avctx,
-                                   const enum PixelFormat *fmt)
-{
-    sh_video_t *sh = avctx->opaque;
-    int i;
-
-    for (i = 0; fmt[i] != PIX_FMT_NONE; i++) {
-        int imgfmt = pixfmt2imgfmt(fmt[i]);
-        if (!IMGFMT_IS_HWACCEL(imgfmt))
-            continue;
-        mp_msg(MSGT_DECVIDEO, MSGL_V, "[VD_FFMPEG] Trying pixfmt=%d.\n", i);
-        if (init_vo(sh, fmt[i]) >= 0)
-            break;
-    }
-    return fmt[i];
-}
-
 static int control(sh_video_t *sh, int cmd, void *arg)
 {
     vd_ffmpeg_ctx *ctx = sh->context;
diff --git a/video/filter/vf.h b/video/filter/vf.h
index 29727259ba..1a850995f3 100644
--- a/video/filter/vf.h
+++ b/video/filter/vf.h
@@ -98,6 +98,7 @@ struct vf_ctrl_screenshot {
 #define VFCTRL_SET_EQUALIZER 6 // set color options (brightness,contrast etc)
 #define VFCTRL_GET_EQUALIZER 8 // get color options (brightness,contrast etc)
 #define VFCTRL_HWDEC_DECODER_RENDER 9 // vdpau hw decoding
+#define VFCTRL_HWDEC_GET_SURFACE 10 // vdpau hw decoding
 #define VFCTRL_SCREENSHOT      14  // Take screenshot, arg is vf_ctrl_screenshot
 #define VFCTRL_INIT_OSD        15  // Filter OSD renderer present?
 #define VFCTRL_SET_DEINTERLACE 18  // Set deinterlacing status
diff --git a/video/filter/vf_vo.c b/video/filter/vf_vo.c
index b8eb7bb13d..731d1bbcff 100644
--- a/video/filter/vf_vo.c
+++ b/video/filter/vf_vo.c
@@ -105,6 +105,8 @@ static int control(struct vf_instance *vf, int request, void *data)
     }
     case VFCTRL_HWDEC_DECODER_RENDER:
         return vo_control(video_out, VOCTRL_HWDEC_DECODER_RENDER, data);
+    case VFCTRL_HWDEC_GET_SURFACE:
+        return vo_control(video_out, VOCTRL_HWDEC_GET_SURFACE, data);
     }
     return CONTROL_UNKNOWN;
 }
@@ -119,16 +121,6 @@ static int query_format(struct vf_instance *vf, unsigned int fmt)
     return flags;
 }
 
-static void get_image(struct vf_instance *vf,
-                      mp_image_t *mpi)
-{
-    if (!video_out->config_ok)
-        return;
-    // GET_IMAGE is required for hardware-accelerated formats
-    if (IMGFMT_IS_HWACCEL(mpi->imgfmt))
-        vo_control(video_out, VOCTRL_GET_IMAGE, mpi);
-}
-
 static int put_image(struct vf_instance *vf, mp_image_t *mpi, double pts)
 {
     if (!video_out->config_ok)
@@ -151,7 +143,6 @@ static int vf_open(vf_instance_t *vf, char *args)
     vf->config = config;
     vf->control = control;
     vf->query_format = query_format;
-    vf->get_image = get_image;
     vf->put_image = put_image;
     vf->uninit = uninit;
     vf->priv = calloc(1, sizeof(struct vf_priv_s));
diff --git a/video/out/vo.h b/video/out/vo.h
index bcc751e5f2..7b929f985d 100644
--- a/video/out/vo.h
+++ b/video/out/vo.h
@@ -44,8 +44,7 @@ enum mp_voctrl {
     VOCTRL_PAUSE,
     /* start/resume playback */
     VOCTRL_RESUME,
-    /* libmpcodecs direct rendering */
-    VOCTRL_GET_IMAGE,
+
     VOCTRL_GET_PANSCAN,
     VOCTRL_SET_PANSCAN,
     VOCTRL_SET_EQUALIZER,               // struct voctrl_set_equalizer_args
@@ -54,6 +53,7 @@ enum mp_voctrl {
 
     /* for vdpau hardware decoding */
     VOCTRL_HWDEC_DECODER_RENDER,        // pointer to hw state
+    VOCTRL_HWDEC_GET_SURFACE,           // struct mp_image
 
     VOCTRL_NEWFRAME,
     VOCTRL_SKIPFRAME,
diff --git a/video/out/vo_vdpau.c b/video/out/vo_vdpau.c
index 13a352f790..67ac0593f8 100644
--- a/video/out/vo_vdpau.c
+++ b/video/out/vo_vdpau.c
@@ -1307,7 +1307,7 @@ static void draw_image(struct vo *vo, mp_image_t *mpi, double pts)
     struct vdpau_render_state *rndr;
 
     if (IMGFMT_IS_VDPAU(vc->image_format)) {
-        rndr = mpi->priv;
+        rndr = (struct vdpau_render_state *)mpi->planes[0];
         reserved_mpi = mpi;
     } else {
         rndr = get_surface(vo, vc->deint_counter);
@@ -1394,31 +1394,22 @@ static struct mp_image *get_window_screenshot(struct vo *vo)
     return image;
 }
 
-static uint32_t get_image(struct vo *vo, mp_image_t *mpi)
+static uint32_t get_decoder_surface(struct vo *vo, mp_image_t *mpi)
 {
     struct vdpctx *vc = vo->priv;
     struct vdpau_render_state *rndr;
 
-    // no dr for non-decoding for now
     if (!IMGFMT_IS_VDPAU(vc->image_format))
         return VO_FALSE;
-    if (mpi->type != MP_IMGTYPE_NUMBERED)
-        return VO_FALSE;
 
     rndr = get_surface(vo, mpi->number);
     if (!rndr) {
         mp_msg(MSGT_VO, MSGL_ERR, "[vdpau] no surfaces available in "
-               "get_image\n");
+               "get_decoder_surface\n");
         // TODO: this probably breaks things forever, provide a dummy buffer?
         return VO_FALSE;
     }
-    mpi->flags |= MP_IMGFLAG_DIRECT;
-    mpi->stride[0] = mpi->stride[1] = mpi->stride[2] = 0;
-    mpi->planes[0] = mpi->planes[1] = mpi->planes[2] = NULL;
-    // hack to get around a check and to avoid a special-case in vd_ffmpeg.c
     mpi->planes[0] = (void *)rndr;
-    mpi->num_planes = 1;
-    mpi->priv = rndr;
     return VO_TRUE;
 }
 
@@ -1598,8 +1589,8 @@ static int control(struct vo *vo, uint32_t request, void *data)
         if (vc->dropped_frame)
             vo->want_redraw = true;
         return true;
-    case VOCTRL_GET_IMAGE:
-        return get_image(vo, data);
+    case VOCTRL_HWDEC_GET_SURFACE:
+        return get_decoder_surface(vo, data);
     case VOCTRL_HWDEC_DECODER_RENDER:
         return decoder_render(vo, data);
     case VOCTRL_BORDER: