From 26d25d567f49d0f98f7c21a18be7bd921e351f8d Mon Sep 17 00:00:00 2001
From: wm4 <wm4@nowhere>
Date: Thu, 12 Jan 2017 09:40:16 +0100
Subject: vaapi: properly set hw_subfmt field with new decode API

This fixes direct rendering with hwdec_vaegl.c.

The code duplication between update_image_params() and
mp_image_copy_fields_from_av_frame() is quite annoying,
bit will have to be resolved in another commit.
---
 video/decode/vaapi.c   | 7 -------
 video/decode/vd_lavc.c | 6 ++++++
 video/mp_image.c       | 6 ++++++
 3 files changed, 12 insertions(+), 7 deletions(-)

(limited to 'video')

diff --git a/video/decode/vaapi.c b/video/decode/vaapi.c
index 8d7af19bfa..bd84fcc253 100644
--- a/video/decode/vaapi.c
+++ b/video/decode/vaapi.c
@@ -101,12 +101,6 @@ static int init_decoder(struct lavc_ctx *ctx, int w, int h)
     return 0;
 }
 
-static struct mp_image *update_format(struct lavc_ctx *ctx, struct mp_image *img)
-{
-    va_surface_init_subformat(img);
-    return img;
-}
-
 static void uninit(struct lavc_ctx *ctx)
 {
     struct priv *p = ctx->hwdec_priv;
@@ -222,7 +216,6 @@ const struct vd_lavc_hwdec mp_vd_lavc_vaapi = {
     .init_decoder = init_decoder,
     .lock = intel_shit_lock,
     .unlock = intel_crap_unlock,
-    .process_image = update_format,
 };
 
 const struct vd_lavc_hwdec mp_vd_lavc_vaapi_copy = {
diff --git a/video/decode/vd_lavc.c b/video/decode/vd_lavc.c
index 056a170d36..227401a677 100644
--- a/video/decode/vd_lavc.c
+++ b/video/decode/vd_lavc.c
@@ -24,6 +24,7 @@
 
 #include <libavutil/common.h>
 #include <libavutil/opt.h>
+#include <libavutil/hwcontext.h>
 #include <libavutil/intreadwrite.h>
 #include <libavutil/pixdesc.h>
 
@@ -659,6 +660,11 @@ static void update_image_params(struct dec_video *vd, AVFrame *frame,
         .rotate = vd->codec->rotate,
         .stereo_in = vd->codec->stereo_mode,
     };
+
+    if (frame->hw_frames_ctx) {
+        AVHWFramesContext *fctx = (void *)frame->hw_frames_ctx->data;
+        out_params->hw_subfmt = pixfmt2imgfmt(fctx->sw_format);
+    }
 }
 
 static enum AVPixelFormat get_format_hwdec(struct AVCodecContext *avctx,
diff --git a/video/mp_image.c b/video/mp_image.c
index 2c4627c33e..9781386129 100644
--- a/video/mp_image.c
+++ b/video/mp_image.c
@@ -27,6 +27,7 @@
 #include <libavutil/mem.h>
 #include <libavutil/common.h>
 #include <libavutil/bswap.h>
+#include <libavutil/hwcontext.h>
 #include <libavutil/rational.h>
 #include <libavcodec/avcodec.h>
 
@@ -706,6 +707,11 @@ static void mp_image_copy_fields_from_av_frame(struct mp_image *dst,
         dst->fields |= MP_IMGFIELD_TOP_FIRST;
     if (src->repeat_pict == 1)
         dst->fields |= MP_IMGFIELD_REPEAT_FIRST;
+
+    if (src->hw_frames_ctx) {
+        AVHWFramesContext *fctx = (void *)src->hw_frames_ctx->data;
+        dst->params.hw_subfmt = pixfmt2imgfmt(fctx->sw_format);
+    }
 }
 
 // Copy properties and data of the mp_image into the AVFrame, without taking
-- 
cgit v1.2.3