video: initialize hw decoder in get_format

Apparently the "right" place to initialize the hardware decoder is in
the libavcodec get_format callback.

This doesn't change vda.c and vdpau_old.c, because I don't have OSX, and
vdpau_old.c is probably going to be removed soon (if Libav ever manages
to release Libav 10). So for now the init_decoder callback added with
this commit is optional.

This also means vdpau.c and vaapi.c don't have to manage and check the
image parameters anymore.

This change is probably needed for when libavcodec VDA supports gets a
new iteration of its API.

1 files changed, 20 insertions, 2 deletions

diff --git a/video/decode/vd_lavc.c b/video/decode/vd_lavc.c
index df0c098155..eed79e81b1 100644
--- a/video/decode/vd_lavc.c
+++ b/video/decode/vd_lavc.c
@@ -339,6 +339,7 @@ static void init_avctx(struct dec_video *vd, const char *decoder,
     ctx->do_dr1 = ctx->do_hw_dr1 = 0;
     ctx->pix_fmt = AV_PIX_FMT_NONE;
     ctx->hwdec = hwdec;
+    ctx->hwdec_fmt = 0;
     ctx->avctx = avcodec_alloc_context3(lavc_codec);
     AVCodecContext *avctx = ctx->avctx;
     avctx->opaque = vd;
@@ -511,8 +512,20 @@ static enum AVPixelFormat get_format_hwdec(struct AVCodecContext *avctx,
     for (int i = 0; fmt[i] != AV_PIX_FMT_NONE; i++) {
         const int *okfmt = ctx->hwdec->image_formats;
         for (int n = 0; okfmt && okfmt[n]; n++) {
-            if (imgfmt2pixfmt(okfmt[n]) == fmt[i])
+            if (imgfmt2pixfmt(okfmt[n]) == fmt[i]) {
+                ctx->hwdec_w = avctx->width;
+                ctx->hwdec_h = avctx->height;
+                ctx->hwdec_fmt = okfmt[n];
+                if (ctx->hwdec->init_decoder) {
+                    if (ctx->hwdec->init_decoder(ctx, ctx->hwdec_fmt,
+                                                 ctx->hwdec_w, ctx->hwdec_h) < 0)
+                    {
+                        ctx->hwdec_fmt = 0;
+                        break;
+                    }
+                }
                 return fmt[i];
+            }
         }
     }
 
@@ -535,7 +548,7 @@ static struct mp_image *get_surface_hwdec(struct dec_video *vd, AVFrame *pic)
      * get_buffer callback.
      */
     int imgfmt = pixfmt2imgfmt(pic->format);
-    if (!IMGFMT_IS_HWACCEL(imgfmt))
+    if (!IMGFMT_IS_HWACCEL(imgfmt) || !ctx->hwdec)
         return NULL;
 
     // Using frame->width/height is bad. For non-mod 16 video (which would
@@ -545,6 +558,11 @@ static struct mp_image *get_surface_hwdec(struct dec_video *vd, AVFrame *pic)
     int w = ctx->avctx->width;
     int h = ctx->avctx->height;
 
+    if (ctx->hwdec->init_decoder) {
+        if (imgfmt != ctx->hwdec_fmt && w != ctx->hwdec_w && h != ctx->hwdec_h)
+            return NULL;
+    }
+
     struct mp_image *mpi = ctx->hwdec->allocate_image(ctx, imgfmt, w, h);
 
     if (mpi) {