vo_vaapi: use new overlay API

This will probably make it slower. But since I don't care about
vo_vaapi, that's perfectly OK. It serves mostly as a test for the
previous commit. In addition, this code was pretty bad (custom broken
scaling and not-blending that probably broke in some situation). If that
wasn't enough, some vaapi drivers also provide only a single overlay at
a time, while this code required a bunch.

There also seems to be a Mesa bug: the overlay gets stretched when
src_x/y was not 0. Or maybe I misunderstood how this is supposed to
work. A bug is probably more likely? Nobody cares about this API.

1 files changed, 85 insertions, 123 deletions

diff --git a/video/out/vo_vaapi.c b/video/out/vo_vaapi.c
index 91ce7bc534..95c4dd52a3 100644
--- a/video/out/vo_vaapi.c
+++ b/video/out/vo_vaapi.c
@@ -33,6 +33,7 @@
 #include "video/out/vo.h"
 #include "video/mp_image_pool.h"
 #include "video/sws_utils.h"
+#include "sub/draw_bmp.h"
 #include "sub/img_convert.h"
 #include "sub/osd.h"
 #include "x11_common.h"
@@ -83,8 +84,9 @@ struct priv {
     int                      force_scaled_osd;
 
     VAImageFormat            osd_format; // corresponds to OSD_VA_FORMAT
-    struct vaapi_osd_part    osd_parts[MAX_OSD_PARTS];
+    struct vaapi_osd_part    osd_part;
     bool                     osd_screen;
+    struct mp_draw_sub_cache *osd_cache;
 
     struct mp_image_pool    *pool;
 
@@ -102,12 +104,6 @@ struct priv {
 
 #define OSD_VA_FORMAT VA_FOURCC_BGRA
 
-static const bool osd_formats[SUBBITMAP_COUNT] = {
-    // Actually BGRA, but only on little endian.
-    // This will break on big endian, I think.
-    [SUBBITMAP_RGBA] = true,
-};
-
 static void draw_osd(struct vo *vo);
 
 
@@ -507,22 +503,20 @@ static bool render_to_screen(struct priv *p, struct mp_image *mpi)
     if (surface == VA_INVALID_ID)
         return false;
 
-    for (int n = 0; n < MAX_OSD_PARTS; n++) {
-        struct vaapi_osd_part *part = &p->osd_parts[n];
-        if (part->active) {
-            struct vaapi_subpic *sp = &part->subpic;
-            int flags = 0;
-            if (p->osd_screen)
-                flags |= VA_SUBPICTURE_DESTINATION_IS_SCREEN_COORD;
-            status = vaAssociateSubpicture(p->display,
-                                           sp->id, &surface, 1,
-                                           sp->src_x, sp->src_y,
-                                           sp->src_w, sp->src_h,
-                                           sp->dst_x, sp->dst_y,
-                                           sp->dst_w, sp->dst_h,
-                                           flags);
-            CHECK_VA_STATUS(p, "vaAssociateSubpicture()");
-        }
+    struct vaapi_osd_part *part = &p->osd_part;
+    if (part->active) {
+        struct vaapi_subpic *sp = &part->subpic;
+        int flags = 0;
+        if (p->osd_screen)
+            flags |= VA_SUBPICTURE_DESTINATION_IS_SCREEN_COORD;
+        status = vaAssociateSubpicture(p->display,
+                                       sp->id, &surface, 1,
+                                       sp->src_x, sp->src_y,
+                                       sp->src_w, sp->src_h,
+                                       sp->dst_x, sp->dst_y,
+                                       sp->dst_w, sp->dst_h,
+                                       flags);
+        CHECK_VA_STATUS(p, "vaAssociateSubpicture()");
     }
 
     int flags = va_get_colorspace_flag(p->image_params.color.space) |
@@ -542,14 +536,11 @@ static bool render_to_screen(struct priv *p, struct mp_image *mpi)
                           flags);
     CHECK_VA_STATUS(p, "vaPutSurface()");
 
-    for (int n = 0; n < MAX_OSD_PARTS; n++) {
-        struct vaapi_osd_part *part = &p->osd_parts[n];
-        if (part->active) {
-            struct vaapi_subpic *sp = &part->subpic;
-            status = vaDeassociateSubpicture(p->display, sp->id,
-                                             &surface, 1);
-            CHECK_VA_STATUS(p, "vaDeassociateSubpicture()");
-        }
+    if (part->active) {
+        struct vaapi_subpic *sp = &part->subpic;
+        status = vaDeassociateSubpicture(p->display, sp->id,
+                                         &surface, 1);
+        CHECK_VA_STATUS(p, "vaDeassociateSubpicture()");
     }
 
     return true;
@@ -626,114 +617,89 @@ error:
     return -1;
 }
 
-static void draw_osd_cb(void *pctx, struct sub_bitmaps *imgs)
+static void draw_osd(struct vo *vo)
 {
-    struct priv *p = pctx;
-
-    struct vaapi_osd_part *part = &p->osd_parts[imgs->render_index];
-    if (imgs->change_id != part->change_id) {
-        part->change_id = imgs->change_id;
-
-        struct mp_rect bb;
-        if (!mp_sub_bitmaps_bb(imgs, &bb))
-            goto error;
-
-        // Prevent filtering artifacts on borders
-        int pad = 2;
-
-        int w = bb.x1 - bb.x0;
-        int h = bb.y1 - bb.y0;
-        if (part->image.w < w + pad || part->image.h < h + pad) {
-            int sw = MP_ALIGN_UP(w + pad, 64);
-            int sh = MP_ALIGN_UP(h + pad, 64);
-            if (new_subpicture(p, sw, sh, &part->image) < 0)
-                goto error;
-        }
+    struct priv *p = vo->priv;
 
-        struct vaapi_osd_image *img = &part->image;
-        struct mp_image vaimg;
-        if (!va_image_map(p->mpvaapi, &img->image, &vaimg))
-            goto error;
+    struct mp_image *cur = p->output_surfaces[p->output_surface];
+    double pts = cur ? cur->pts : 0;
 
-        // Clear borders and regions uncovered by sub-bitmaps
-        mp_image_clear(&vaimg, 0, 0, w + pad, h + pad);
+    if (!p->osd_format.fourcc)
+        return;
 
-        for (int n = 0; n < imgs->num_parts; n++) {
-            struct sub_bitmap *sub = &imgs->parts[n];
+    struct mp_osd_res vid_res = osd_res_from_image_params(vo->params);
 
-            struct mp_image src = {0};
-            mp_image_setfmt(&src, IMGFMT_BGRA);
-            mp_image_set_size(&src, sub->w, sub->h);
-            src.planes[0] = sub->bitmap;
-            src.stride[0] = sub->stride;
+    struct mp_osd_res *res;
+    if (p->osd_screen) {
+        res = &p->screen_osd_res;
+    } else {
+        res = &vid_res;
+    }
 
-            struct mp_image *bmp = &src;
+    p->osd_part.active = false;
 
-            struct mp_image *tmp = NULL;
-            if (sub->dw != sub->w || sub->dh != sub->h) {
-                tmp = mp_image_alloc(IMGFMT_BGRA, sub->dw, sub->dh);
-                if (!tmp)
-                    goto error;
+    if (!p->osd_cache)
+        p->osd_cache = mp_draw_sub_alloc(p);
 
-                mp_image_swscale(tmp, &src, mp_sws_fast_flags);
+    struct sub_bitmap_list *sbs = osd_render(vo->osd, *res, pts, 0,
+                                             mp_draw_sub_formats);
 
-                bmp = tmp;
-            }
+    struct mp_rect act_rc[1], mod_rc[64];
+    int num_act_rc = 0, num_mod_rc = 0;
 
-            // Note: nothing guarantees that the sub-bitmaps don't overlap.
-            //       But in all currently existing cases, they don't.
-            //       We simply hope that this won't change, and nobody will
-            //       ever notice our little shortcut here.
+    struct mp_image *osd = mp_draw_sub_overlay(p->osd_cache, sbs,
+                    act_rc, MP_ARRAY_SIZE(act_rc), &num_act_rc,
+                    mod_rc, MP_ARRAY_SIZE(mod_rc), &num_mod_rc);
 
-            size_t dst = (sub->y - bb.y0) * vaimg.stride[0] +
-                         (sub->x - bb.x0) * 4;
+    if (!osd)
+        goto error;
 
-            memcpy_pic(vaimg.planes[0] + dst, bmp->planes[0], sub->dw * 4,
-                       sub->dh, vaimg.stride[0], bmp->stride[0]);
+    struct vaapi_osd_part *part = &p->osd_part;
 
-            talloc_free(tmp);
-        }
+    part->active = false;
 
-        if (!va_image_unmap(p->mpvaapi, &img->image))
+    int w = res->w;
+    int h = res->h;
+    if (part->image.w != w || part->image.h != h) {
+        if (new_subpicture(p, w, h, &part->image) < 0)
             goto error;
-
-        part->subpic = (struct vaapi_subpic) {
-            .id = img->subpic_id,
-            .src_x = 0,     .src_y = 0,
-            .src_w = w,     .src_h = h,
-            .dst_x = bb.x0, .dst_y = bb.y0,
-            .dst_w = w,     .dst_h = h,
-        };
     }
 
-    part->active = true;
+    struct vaapi_osd_image *img = &part->image;
+    struct mp_image vaimg;
+    if (!va_image_map(p->mpvaapi, &img->image, &vaimg))
+        goto error;
 
-error:
-    ;
-}
+    for (int n = 0; n < num_mod_rc; n++) {
+        struct mp_rect *rc = &mod_rc[n];
 
-static void draw_osd(struct vo *vo)
-{
-    struct priv *p = vo->priv;
+        int rw = mp_rect_w(*rc);
+        int rh = mp_rect_h(*rc);
 
-    struct mp_image *cur = p->output_surfaces[p->output_surface];
-    double pts = cur ? cur->pts : 0;
+        void *src = mp_image_pixel_ptr(osd, 0, rc->x0, rc->y0);
+        void *dst = vaimg.planes[0] + rc->y0 * vaimg.stride[0] + rc->x0 * 4;
 
-    if (!p->osd_format.fourcc)
-        return;
+        memcpy_pic(dst, src, rw * 4, rh, vaimg.stride[0], osd->stride[0]);
+    }
 
-    struct mp_osd_res vid_res = osd_res_from_image_params(vo->params);
+    if (!va_image_unmap(p->mpvaapi, &img->image))
+        goto error;
 
-    struct mp_osd_res *res;
-    if (p->osd_screen) {
-        res = &p->screen_osd_res;
-    } else {
-        res = &vid_res;
+    if (num_act_rc) {
+        struct mp_rect rc = act_rc[0];
+        rc.x0 = rc.y0 = 0; // must be a Mesa bug
+        part->subpic = (struct vaapi_subpic) {
+            .id = img->subpic_id,
+            .src_x = rc.x0,         .src_y = rc.y0,
+            .src_w = mp_rect_w(rc), .src_h = mp_rect_h(rc),
+            .dst_x = rc.x0,         .dst_y = rc.y0,
+            .dst_w = mp_rect_w(rc), .dst_h = mp_rect_h(rc),
+        };
+        part->active = true;
     }
 
-    for (int n = 0; n < MAX_OSD_PARTS; n++)
-        p->osd_parts[n].active = false;
-    osd_draw(vo->osd, *res, pts, 0, osd_formats, draw_osd_cb, p);
+error:
+    talloc_free(sbs);
 }
 
 static int control(struct vo *vo, uint32_t request, void *data)
@@ -767,10 +733,8 @@ static void uninit(struct vo *vo)
     free_video_specific(p);
     talloc_free(p->pool);
 
-    for (int n = 0; n < MAX_OSD_PARTS; n++) {
-        struct vaapi_osd_part *part = &p->osd_parts[n];
-        free_subpicture(p, &part->image);
-    }
+    struct vaapi_osd_part *part = &p->osd_part;
+    free_subpicture(p, &part->image);
 
     if (vo->hwdec_devs) {
         hwdec_devices_remove(vo->hwdec_devs, &p->mpvaapi->hwctx);
@@ -847,11 +811,9 @@ static int preinit(struct vo *vo)
     if (!p->osd_format.fourcc)
         MP_ERR(vo, "OSD format not supported. Disabling OSD.\n");
 
-    for (int n = 0; n < MAX_OSD_PARTS; n++) {
-        struct vaapi_osd_part *part = &p->osd_parts[n];
-        part->image.image.image_id = VA_INVALID_ID;
-        part->image.subpic_id = VA_INVALID_ID;
-    }
+    struct vaapi_osd_part *part = &p->osd_part;
+    part->image.image.image_id = VA_INVALID_ID;
+    part->image.subpic_id = VA_INVALID_ID;
 
     int max_display_attrs = vaMaxNumDisplayAttributes(p->display);
     p->va_display_attrs = talloc_array(vo, VADisplayAttribute, max_display_attrs);