vaapi: move legacy code to vo_vaapi.c

Done for license reasons. vo_vaapi.c is turned into some kind of
dumpster fire, and we'll remove it as soon as I'm mentally ready for
unkind users to complain about removal of this old POS.

1 files changed, 310 insertions, 4 deletions

diff --git a/video/out/vo_vaapi.c b/video/out/vo_vaapi.c
index 3468ac6643..a3f701519d 100644
--- a/video/out/vo_vaapi.c
+++ b/video/out/vo_vaapi.c
@@ -96,6 +96,8 @@ struct priv {
     VADisplayAttribute      *va_display_attrs;
     int                     *mp_display_attr;
     int                      va_num_display_attrs;
+
+    struct va_image_formats *image_formats;
 };
 
 #define OSD_VA_FORMAT VA_FOURCC_BGRA
@@ -108,6 +110,306 @@ static const bool osd_formats[SUBBITMAP_COUNT] = {
 
 static void draw_osd(struct vo *vo);
 
+
+struct fmtentry {
+    uint32_t va;
+    enum mp_imgfmt mp;
+};
+
+static const struct fmtentry va_to_imgfmt[] = {
+    {VA_FOURCC_NV12, IMGFMT_NV12},
+    {VA_FOURCC_YV12, IMGFMT_420P},
+    {VA_FOURCC_IYUV, IMGFMT_420P},
+    {VA_FOURCC_UYVY, IMGFMT_UYVY},
+    // Note: not sure about endian issues (the mp formats are byte-addressed)
+    {VA_FOURCC_RGBA, IMGFMT_RGBA},
+    {VA_FOURCC_RGBX, IMGFMT_RGBA},
+    {VA_FOURCC_BGRA, IMGFMT_BGRA},
+    {VA_FOURCC_BGRX, IMGFMT_BGRA},
+    {0             , IMGFMT_NONE}
+};
+
+static enum mp_imgfmt va_fourcc_to_imgfmt(uint32_t fourcc)
+{
+    for (const struct fmtentry *entry = va_to_imgfmt; entry->va; ++entry) {
+        if (entry->va == fourcc)
+            return entry->mp;
+    }
+    return IMGFMT_NONE;
+}
+
+static uint32_t va_fourcc_from_imgfmt(int imgfmt)
+{
+    for (const struct fmtentry *entry = va_to_imgfmt; entry->va; ++entry) {
+        if (entry->mp == imgfmt)
+            return entry->va;
+    }
+    return 0;
+}
+
+struct va_image_formats {
+    VAImageFormat *entries;
+    int num;
+};
+
+static void va_get_formats(struct priv *ctx)
+{
+    struct va_image_formats *formats = talloc_ptrtype(ctx, formats);
+    formats->num = vaMaxNumImageFormats(ctx->display);
+    formats->entries = talloc_array(formats, VAImageFormat, formats->num);
+    VAStatus status = vaQueryImageFormats(ctx->display, formats->entries,
+                                          &formats->num);
+    if (!CHECK_VA_STATUS(ctx, "vaQueryImageFormats()"))
+        return;
+    MP_VERBOSE(ctx, "%d image formats available:\n", formats->num);
+    for (int i = 0; i < formats->num; i++)
+        MP_VERBOSE(ctx, "  %s\n", mp_tag_str(formats->entries[i].fourcc));
+    ctx->image_formats = formats;
+}
+
+static VAImageFormat *va_image_format_from_imgfmt(struct priv *ctx,
+                                                  int imgfmt)
+{
+    struct va_image_formats *formats = ctx->image_formats;
+    const int fourcc = va_fourcc_from_imgfmt(imgfmt);
+    if (!formats || !formats->num || !fourcc)
+        return NULL;
+    for (int i = 0; i < formats->num; i++) {
+        if (formats->entries[i].fourcc == fourcc)
+            return &formats->entries[i];
+    }
+    return NULL;
+}
+
+struct va_surface {
+    struct mp_vaapi_ctx *ctx;
+    VADisplay display;
+
+    VASurfaceID id;
+    int rt_format;
+
+    // The actually allocated surface size (needed for cropping).
+    // mp_images can have a smaller size than this, which means they are
+    // cropped down to a smaller size by removing right/bottom pixels.
+    int w, h;
+
+    VAImage image;       // used for software decoding case
+    bool is_derived;     // is image derived by vaDeriveImage()?
+};
+
+static struct va_surface *va_surface_in_mp_image(struct mp_image *mpi)
+{
+    return mpi && mpi->imgfmt == IMGFMT_VAAPI ?
+        (struct va_surface*)mpi->planes[0] : NULL;
+}
+
+static void release_va_surface(void *arg)
+{
+    struct va_surface *surface = arg;
+
+    if (surface->id != VA_INVALID_ID) {
+        if (surface->image.image_id != VA_INVALID_ID)
+            vaDestroyImage(surface->display, surface->image.image_id);
+        vaDestroySurfaces(surface->display, &surface->id, 1);
+    }
+
+    talloc_free(surface);
+}
+
+static struct mp_image *alloc_surface(struct mp_vaapi_ctx *ctx, int rt_format,
+                                      int w, int h)
+{
+    VASurfaceID id = VA_INVALID_ID;
+    VAStatus status;
+    status = vaCreateSurfaces(ctx->display, rt_format, w, h, &id, 1, NULL, 0);
+    if (!CHECK_VA_STATUS(ctx, "vaCreateSurfaces()"))
+        return NULL;
+
+    struct va_surface *surface = talloc_ptrtype(NULL, surface);
+    if (!surface)
+        return NULL;
+
+    *surface = (struct va_surface){
+        .ctx = ctx,
+        .id = id,
+        .rt_format = rt_format,
+        .w = w,
+        .h = h,
+        .display = ctx->display,
+        .image = { .image_id = VA_INVALID_ID, .buf = VA_INVALID_ID },
+    };
+
+    struct mp_image img = {0};
+    mp_image_setfmt(&img, IMGFMT_VAAPI);
+    mp_image_set_size(&img, w, h);
+    img.planes[0] = (uint8_t*)surface;
+    img.planes[3] = (uint8_t*)(uintptr_t)surface->id;
+    return mp_image_new_custom_ref(&img, surface, release_va_surface);
+}
+
+static void va_surface_image_destroy(struct va_surface *surface)
+{
+    if (!surface || surface->image.image_id == VA_INVALID_ID)
+        return;
+    vaDestroyImage(surface->display, surface->image.image_id);
+    surface->image.image_id = VA_INVALID_ID;
+    surface->is_derived = false;
+}
+
+static int va_surface_image_alloc(struct va_surface *p, VAImageFormat *format)
+{
+    VADisplay *display = p->display;
+
+    if (p->image.image_id != VA_INVALID_ID &&
+        p->image.format.fourcc == format->fourcc)
+        return 0;
+
+    int r = 0;
+
+    va_surface_image_destroy(p);
+
+    VAStatus status = vaDeriveImage(display, p->id, &p->image);
+    if (status == VA_STATUS_SUCCESS) {
+        /* vaDeriveImage() is supported, check format */
+        if (p->image.format.fourcc == format->fourcc &&
+            p->image.width == p->w && p->image.height == p->h)
+        {
+            p->is_derived = true;
+            MP_TRACE(p->ctx, "Using vaDeriveImage()\n");
+        } else {
+            vaDestroyImage(p->display, p->image.image_id);
+            status = VA_STATUS_ERROR_OPERATION_FAILED;
+        }
+    }
+    if (status != VA_STATUS_SUCCESS) {
+        p->image.image_id = VA_INVALID_ID;
+        status = vaCreateImage(p->display, format, p->w, p->h, &p->image);
+        if (!CHECK_VA_STATUS(p->ctx, "vaCreateImage()")) {
+            p->image.image_id = VA_INVALID_ID;
+            r = -1;
+        }
+    }
+
+    return r;
+}
+
+// img must be a VAAPI surface; make sure its internal VAImage is allocated
+// to a format corresponding to imgfmt (or return an error).
+static int va_surface_alloc_imgfmt(struct priv *priv, struct mp_image *img,
+                                   int imgfmt)
+{
+    struct va_surface *p = va_surface_in_mp_image(img);
+    if (!p)
+        return -1;
+    // Multiple FourCCs can refer to the same imgfmt, so check by doing the
+    // surjective conversion first.
+    if (p->image.image_id != VA_INVALID_ID &&
+        va_fourcc_to_imgfmt(p->image.format.fourcc) == imgfmt)
+        return 0;
+    VAImageFormat *format = va_image_format_from_imgfmt(priv, imgfmt);
+    if (!format)
+        return -1;
+    if (va_surface_image_alloc(p, format) < 0)
+        return -1;
+    return 0;
+}
+
+static bool va_image_map(struct mp_vaapi_ctx *ctx, VAImage *image,
+                         struct mp_image *mpi)
+{
+    int imgfmt = va_fourcc_to_imgfmt(image->format.fourcc);
+    if (imgfmt == IMGFMT_NONE)
+        return false;
+    void *data = NULL;
+    const VAStatus status = vaMapBuffer(ctx->display, image->buf, &data);
+    if (!CHECK_VA_STATUS(ctx, "vaMapBuffer()"))
+        return false;
+
+    *mpi = (struct mp_image) {0};
+    mp_image_setfmt(mpi, imgfmt);
+    mp_image_set_size(mpi, image->width, image->height);
+
+    for (int p = 0; p < image->num_planes; p++) {
+        mpi->stride[p] = image->pitches[p];
+        mpi->planes[p] = (uint8_t *)data + image->offsets[p];
+    }
+
+    if (image->format.fourcc == VA_FOURCC_YV12) {
+        MPSWAP(int, mpi->stride[1], mpi->stride[2]);
+        MPSWAP(uint8_t *, mpi->planes[1], mpi->planes[2]);
+    }
+
+    return true;
+}
+
+static bool va_image_unmap(struct mp_vaapi_ctx *ctx, VAImage *image)
+{
+    const VAStatus status = vaUnmapBuffer(ctx->display, image->buf);
+    return CHECK_VA_STATUS(ctx, "vaUnmapBuffer()");
+}
+
+// va_dst: copy destination, must be IMGFMT_VAAPI
+// sw_src: copy source, must be a software pixel format
+static int va_surface_upload(struct priv *priv, struct mp_image *va_dst,
+                             struct mp_image *sw_src)
+{
+    struct va_surface *p = va_surface_in_mp_image(va_dst);
+    if (!p)
+        return -1;
+
+    if (va_surface_alloc_imgfmt(priv, va_dst, sw_src->imgfmt) < 0)
+        return -1;
+
+    struct mp_image img;
+    if (!va_image_map(p->ctx, &p->image, &img))
+        return -1;
+    assert(sw_src->w <= img.w && sw_src->h <= img.h);
+    mp_image_set_size(&img, sw_src->w, sw_src->h); // copy only visible part
+    mp_image_copy(&img, sw_src);
+    va_image_unmap(p->ctx, &p->image);
+
+    if (!p->is_derived) {
+        VAStatus status = vaPutImage(p->display, p->id,
+                                     p->image.image_id,
+                                     0, 0, sw_src->w, sw_src->h,
+                                     0, 0, sw_src->w, sw_src->h);
+        if (!CHECK_VA_STATUS(p->ctx, "vaPutImage()"))
+            return -1;
+    }
+
+    if (p->is_derived)
+        va_surface_image_destroy(p);
+    return 0;
+}
+
+struct pool_alloc_ctx {
+    struct mp_vaapi_ctx *vaapi;
+    int rt_format;
+};
+
+static struct mp_image *alloc_pool(void *pctx, int fmt, int w, int h)
+{
+    struct pool_alloc_ctx *alloc_ctx = pctx;
+    if (fmt != IMGFMT_VAAPI)
+        return NULL;
+
+    return alloc_surface(alloc_ctx->vaapi, alloc_ctx->rt_format, w, h);
+}
+
+// The allocator of the given image pool to allocate VAAPI surfaces, using
+// the given rt_format.
+static void va_pool_set_allocator(struct mp_image_pool *pool,
+                                  struct mp_vaapi_ctx *ctx, int rt_format)
+{
+    struct pool_alloc_ctx *alloc_ctx = talloc_ptrtype(pool, alloc_ctx);
+    *alloc_ctx = (struct pool_alloc_ctx){
+        .vaapi = ctx,
+        .rt_format = rt_format,
+    };
+    mp_image_pool_set_allocator(pool, alloc_pool, alloc_ctx);
+    mp_image_pool_set_lru(pool);
+}
+
 static void flush_output_surfaces(struct priv *p)
 {
     for (int n = 0; n < MAX_OUTPUT_SURFACES; n++)
@@ -135,7 +437,7 @@ static bool alloc_swdec_surfaces(struct priv *p, int w, int h, int imgfmt)
     free_video_specific(p);
     for (int i = 0; i < MAX_OUTPUT_SURFACES; i++) {
         p->swdec_surfaces[i] = mp_image_pool_get(p->pool, IMGFMT_VAAPI, w, h);
-        if (va_surface_alloc_imgfmt(p->swdec_surfaces[i], imgfmt) < 0)
+        if (va_surface_alloc_imgfmt(p, p->swdec_surfaces[i], imgfmt) < 0)
             return false;
     }
     return true;
@@ -172,7 +474,7 @@ static int reconfig(struct vo *vo, struct mp_image_params *params)
 static int query_format(struct vo *vo, int imgfmt)
 {
     struct priv *p = vo->priv;
-    if (imgfmt == IMGFMT_VAAPI || va_image_format_from_imgfmt(p->mpvaapi, imgfmt))
+    if (imgfmt == IMGFMT_VAAPI || va_image_format_from_imgfmt(p, imgfmt))
         return 1;
 
     return 0;
@@ -193,7 +495,7 @@ static bool render_to_screen(struct priv *p, struct mp_image *mpi)
                 struct mp_image *img = mp_image_alloc(fmt, w, h);
                 if (img) {
                     mp_image_clear(img, 0, 0, w, h);
-                    if (va_surface_upload(p->black_surface, img) < 0)
+                    if (va_surface_upload(p, p->black_surface, img) < 0)
                         mp_image_unrefp(&p->black_surface);
                     talloc_free(img);
                 }
@@ -268,7 +570,7 @@ static void draw_image(struct vo *vo, struct mp_image *mpi)
 
     if (mpi->imgfmt != IMGFMT_VAAPI) {
         struct mp_image *dst = p->swdec_surfaces[p->output_surface];
-        if (!dst || va_surface_upload(dst, mpi) < 0) {
+        if (!dst || va_surface_upload(p, dst, mpi) < 0) {
             MP_WARN(vo, "Could not upload surface.\n");
             talloc_free(mpi);
             return;
@@ -510,6 +812,10 @@ static int preinit(struct vo *vo)
                     "It's better to use VDPAU directly with: --vo=vdpau\n");
     }
 
+    va_get_formats(p);
+    if (!p->image_formats)
+        goto fail;
+
     p->pool = mp_image_pool_new(MAX_OUTPUT_SURFACES + 3);
     va_pool_set_allocator(p->pool, p->mpvaapi, VA_RT_FORMAT_YUV420);