build: handle insane libavcodec API bullshit

The oldest supported FFmpeg release doesn't provide
av_vdpau_alloc_context(). With these versions, the application has no
other choice than to hard code the size of AVVDPAUContext. (On the other
hand, there's av_alloc_vdpaucontext(), which does the same thing, but is
FFmpeg specific - not sure if it was available early enough, so I'm not
touching it.)

Newer FFmpeg and Libav releases require you to call this function, for
ABI compatibility reasons. It's the typcal lakc of foresight that make
FFmpeg APIs terrible. mpv successfully pretended that this crap didn't
exist (ABI compat. is near impossible to reach anyway) - but it appears
newer developments in Libav change the function from initializing the
struct with all-zeros to something else, and mpv vdpau decoding would
stop working as soon as this new work is relewased.

So, add a configure test (sigh).

CC: @mpv-player/stable

3 files changed, 37 insertions, 11 deletions

diff --git a/old-configure b/old-configure
index 910ebf7cc8..7fe8229b9c 100755
--- a/old-configure
+++ b/old-configure
@@ -833,6 +833,12 @@ api_statement_check \
     libavutil/frame.h \
     'av_frame_get_qp_table(NULL, NULL, NULL)'
 
+api_statement_check \
+    "libavcodec av_vdpau_alloc_context()" \
+    HAVE_AVCODEC_VDPAU_ALLOC_CONTEXT \
+    libavcodec/vdpau.h \
+    'AVVDPAUContext *x = av_vdpau_alloc_context()'
+
 check_pkg_config "libavfilter" $_libavfilter LIBAVFILTER 'libavfilter >= 3.90.100'
 
 check_pkg_config "libavdevice" $_libavdevice LIBAVDEVICE 'libavdevice >= 54.0.0'
diff --git a/video/decode/vdpau.c b/video/decode/vdpau.c
index 44850dfaec..f144176520 100644
--- a/video/decode/vdpau.c
+++ b/video/decode/vdpau.c
@@ -22,6 +22,7 @@
 #include <libavcodec/vdpau.h>
 #include <libavutil/common.h>
 
+#include "config.h"
 #include "lavc.h"
 #include "common/common.h"
 #include "common/av_common.h"
@@ -36,7 +37,7 @@ struct priv {
     uint64_t                    preemption_counter;
     int                         fmt, w, h;
 
-    AVVDPAUContext              context;
+    AVVDPAUContext             *context;
 };
 
 #define PE(av_codec_id, ff_profile, vdp_profile)                \
@@ -76,8 +77,8 @@ static int init_decoder(struct lavc_ctx *ctx, int fmt, int w, int h)
     if (mp_vdpau_handle_preemption(p->mpvdp, &p->preemption_counter) < 0)
         return 0;
 
-    if (p->context.decoder != VDP_INVALID_HANDLE)
-        vdp->decoder_destroy(p->context.decoder);
+    if (p->context->decoder != VDP_INVALID_HANDLE)
+        vdp->decoder_destroy(p->context->decoder);
 
     const struct hwdec_profile_entry *pe = hwdec_find_profile(ctx, profiles);
     if (!pe) {
@@ -104,14 +105,14 @@ static int init_decoder(struct lavc_ctx *ctx, int fmt, int w, int h)
     int maxrefs = hwdec_get_max_refs(ctx);
 
     vdp_st = vdp->decoder_create(vdp_device, pe->hw_profile, w, h, maxrefs,
-                                 &p->context.decoder);
+                                 &p->context->decoder);
     CHECK_VDP_WARNING(p, "Failed creating VDPAU decoder");
     if (vdp_st != VDP_STATUS_OK)
         goto fail;
     return 0;
 
 fail:
-    p->context.decoder = VDP_INVALID_HANDLE;
+    p->context->decoder = VDP_INVALID_HANDLE;
     return -1;
 }
 
@@ -138,9 +139,10 @@ static void uninit(struct lavc_ctx *ctx)
     if (!p)
         return;
 
-    if (p->context.decoder != VDP_INVALID_HANDLE)
-        p->vdp->decoder_destroy(p->context.decoder);
+    if (p->context && p->context->decoder != VDP_INVALID_HANDLE)
+        p->vdp->decoder_destroy(p->context->decoder);
 
+    av_free(p->context);
     talloc_free(p);
 
     ctx->hwdec_priv = NULL;
@@ -155,16 +157,28 @@ static int init(struct lavc_ctx *ctx)
     };
     ctx->hwdec_priv = p;
 
+#if HAVE_AVCODEC_VDPAU_ALLOC_CONTEXT
+    p->context = av_vdpau_alloc_context();
+#else
+    p->context = av_mallocz(sizeof(AVVDPAUContext));
+#endif
+    if (!p->context)
+        goto error;
+
     p->vdp = &p->mpvdp->vdp;
-    p->context.render = p->vdp->decoder_render;
-    p->context.decoder = VDP_INVALID_HANDLE;
+    p->context->render = p->vdp->decoder_render;
+    p->context->decoder = VDP_INVALID_HANDLE;
 
     if (mp_vdpau_handle_preemption(p->mpvdp, &p->preemption_counter) < 1)
-        return -1;
+        goto error;
 
-    ctx->avctx->hwaccel_context = &p->context;
+    ctx->avctx->hwaccel_context = p->context;
 
     return 0;
+
+error:
+    uninit(ctx);
+    return -1;
 }
 
 static int probe(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
diff --git a/wscript b/wscript
index 7786ebf7d2..54b6cd4322 100644
--- a/wscript
+++ b/wscript
@@ -369,6 +369,12 @@ Libav libraries ({0}). Aborting.".format(" ".join(libav_pkg_config_checks))
                                 'av_frame_get_qp_table(NULL, NULL, NULL)',
                                 use='libav')
     }, {
+        'name': 'avcodec-vdpau-alloc-context',
+        'desc': 'libavcodec vdpau non-sense',
+        'func': check_statement('libavcodec/vdpau.h',
+                                'AVVDPAUContext *x = av_vdpau_alloc_context()',
+                                use='libav')
+    }, {
         'name': '--libavfilter',
         'desc': 'libavfilter',
         'func': check_pkg_config('libavfilter', '>= 3.90.100'),