client API: add software rendering API

This can be used to make vo_libmpv render video to a memory buffer. It
only adds a new backend API that takes memory surfaces. All the render
API (such as frame rendering control and so on) is reused.

I'm not quite convinced of the usefulness of this, and until now I
always resisted providing something like this. It only seems to
facilitate inefficient implementation. But whatever.

Unfortunately, this duplicates the software rendering glue code yet
again (like it exists in vo_x11, vo_wlshm, vo_drm, and probably more).
But in theory, these could reuse this backend in the future, just like
vo_gpu could reuse the render_gl API.

Fixes: #7852

8 files changed, 315 insertions, 2 deletions

diff --git a/DOCS/client-api-changes.rst b/DOCS/client-api-changes.rst
index ade0234e79..35ffc58086 100644
--- a/DOCS/client-api-changes.rst
+++ b/DOCS/client-api-changes.rst
@@ -33,6 +33,7 @@ API changes
 ::
 
  --- mpv 0.33.0 ---
+ 1.109  - add MPV_RENDER_API_TYPE_SW and related (software rendering API)
  1.108  - Deprecate MPV_EVENT_IDLE
         - add mpv_event_start_file
         - add the following fields to mpv_event_end_file: playlist_entry_id,
diff --git a/libmpv/client.h b/libmpv/client.h
index b23a69598d..339cee138b 100644
--- a/libmpv/client.h
+++ b/libmpv/client.h
@@ -232,7 +232,7 @@ extern "C" {
  * relational operators (<, >, <=, >=).
  */
 #define MPV_MAKE_VERSION(major, minor) (((major) << 16) | (minor) | 0UL)
-#define MPV_CLIENT_API_VERSION MPV_MAKE_VERSION(1, 108)
+#define MPV_CLIENT_API_VERSION MPV_MAKE_VERSION(1, 109)
 
 /**
  * The API user is allowed to "#define MPV_ENABLE_DEPRECATED 0" before
diff --git a/libmpv/render.h b/libmpv/render.h
index 293de3c9f0..c25b2d2524 100644
--- a/libmpv/render.h
+++ b/libmpv/render.h
@@ -50,6 +50,7 @@ extern "C" {
  * ------------------
  *
  * OpenGL: via MPV_RENDER_API_TYPE_OPENGL, see render_gl.h header.
+ * Software: via MPV_RENDER_API_TYPE_SW, see section "Software renderer"
  *
  * Threading
  * ---------
@@ -120,6 +121,40 @@ extern "C" {
  *
  * You must free the context with mpv_render_context_free() before the mpv core
  * is destroyed. If this doesn't happen, undefined behavior will result.
+ *
+ * Software renderer
+ * -----------------
+ *
+ * MPV_RENDER_API_TYPE_SW provides an extremely simple (but slow) renderer to
+ * memory surfaces. You probably don't want to use this. Use other render API
+ * types, or other methods of video embedding.
+ *
+ * Use mpv_render_context_create() with MPV_RENDER_PARAM_API_TYPE set to
+ * MPV_RENDER_API_TYPE_SW.
+ *
+ * Call mpv_render_context_render() with various MPV_RENDER_PARAM_SW_* fields
+ * to render the video frame to an in-memory surface. The following fields are
+ * required: MPV_RENDER_PARAM_SW_SIZE, MPV_RENDER_PARAM_SW_FORMAT,
+ * MPV_RENDER_PARAM_SW_STRIDE, MPV_RENDER_PARAM_SW_POINTER.
+ *
+ * This method of rendering is very slow, because everything, including color
+ * conversion, scaling, and OSD rendering, is done on the CPU, single-threaded.
+ * In particular, large video or display sizes, as well as presence of OSD or
+ * subtitles can make it too slow for realtime. As with other software rendering
+ * VOs, setting "sw-fast" may help. Enabling or disabling zimg may help,
+ * depending on the platform.
+ *
+ * In addition, certain multimedia job creation measures like HDR may not work
+ * properly, and will have to be manually handled by for example inserting
+ * filters.
+ *
+ * This API is not really suitable to extract individual frames from video etc.
+ * (basically non-playback uses) - there are better libraries for this. It can
+ * be used this way, but it may be clunky and tricky.
+ *
+ * Further notes:
+ * - MPV_RENDER_PARAM_FLIP_Y is currently ignored (unsupported)
+ * - MPV_RENDER_PARAM_DEPTH is ignored (meaningless)
  */
 
 /**
@@ -312,6 +347,68 @@ typedef enum mpv_render_param_type {
      * Type : struct mpv_opengl_drm_params_v2*
     */
     MPV_RENDER_PARAM_DRM_DISPLAY_V2 = 16,
+    /**
+     * MPV_RENDER_API_TYPE_SW only: rendering target surface size, mandatory.
+     * Valid for MPV_RENDER_API_TYPE_SW & mpv_render_context_create().
+     * Type: int[2] (e.g.: int s[2] = {w, h}; param.data = &s[0];)
+     *
+     * The video frame is transformed as with other VOs. Typically, this means
+     * the video gets scaled and black bars are added if the video size or
+     * aspect ratio mismatches with the target size.
+     */
+    MPV_RENDER_PARAM_SW_SIZE = 17,
+    /**
+     * MPV_RENDER_API_TYPE_SW only: rendering target surface pixel format,
+     * mandatory.
+     * Valid for MPV_RENDER_API_TYPE_SW & mpv_render_context_create().
+     * Type: char* (e.g.: char *f = "rgb0"; param.data = f;)
+     *
+     * Valid values are:
+     *  "rgb0", "bgr0", "0bgr", "0rgb"
+     *      4 bytes per pixel RGB, 1 byte (8 bit) per component, component bytes
+     *      with increasing address from left to right (e.g. "rgb0" has r at
+     *      address 0), the "0" component contains uninitialized garbage (often
+     *      the value 0, but not necessarily; the bad naming is inherited from
+     *      FFmpeg)
+     *  "rgb24"
+     *      3 bytes per pixel RGB. This is strongly discouraged because it is
+     *      very slow.
+     *  other
+     *      The API may accept other pixel formats, using mpv internal format
+     *      names, as long as it's internally marked as RGB, has exactly 1
+     *      plane, and is supported as conversion output. It is not a good idea
+     *      to rely on any of these. Their semantics and handling could change.
+     */
+    MPV_RENDER_PARAM_SW_FORMAT = 18,
+    /**
+     * MPV_RENDER_API_TYPE_SW only: rendering target surface bytes per line,
+     * mandatory.
+     * Valid for MPV_RENDER_API_TYPE_SW & mpv_render_context_create().
+     * Type: size_t*
+     *
+     * This is the number of bytes between a pixel (x, y) and (x, y + 1) on the
+     * target surface. It must be a multiple of the pixel size, and have space
+     * for the surface width as specified by MPV_RENDER_PARAM_SW_SIZE.
+     *
+     * It should be a multiple of 64 to facilitate fast SIMD operation.
+     */
+    MPV_RENDER_PARAM_SW_STRIDE = 19,
+    /*
+     * MPV_RENDER_API_TYPE_SW only: rendering target surface pixel data pointer,
+     * mandatory.
+     * Valid for MPV_RENDER_API_TYPE_SW & mpv_render_context_create().
+     * Type: void*
+     *
+     * This points to the first pixel at the left/top corner (0, 0). In
+     * particular, each line y starts at (pointer + stride * y). Upon rendering,
+     * all data between pointer and (pointer + stride * h) is overwritten.
+     * Whether the padding between (w, y) and (0, y + 1) is overwritten is left
+     * unspecified (it should not be, but unfortunately some scaler backends
+     * will do it anyway). It is assumed that even the padding after the last
+     * line (starting at bytepos(w, h) until (pointer + stride * h)) is
+     * writable.
+     */
+    MPV_RENDER_PARAM_SW_POINTER = 20,
 } mpv_render_param_type;
 
 /**
@@ -354,7 +451,10 @@ typedef struct mpv_render_param {
 /**
  * Predefined values for MPV_RENDER_PARAM_API_TYPE.
  */
+// See render_gl.h
 #define MPV_RENDER_API_TYPE_OPENGL "opengl"
+// See section "Software renderer"
+#define MPV_RENDER_API_TYPE_SW "sw"
 
 /**
  * Flags used in mpv_render_frame_info.flags. Each value represents a bit in it.
diff --git a/video/out/gpu/libmpv_gpu.c b/video/out/gpu/libmpv_gpu.c
index b8edc694db..8b93dcadfd 100644
--- a/video/out/gpu/libmpv_gpu.c
+++ b/video/out/gpu/libmpv_gpu.c
@@ -65,7 +65,7 @@ static int init(struct render_backend *ctx, mpv_render_param *params)
     }
 
     if (!p->context)
-        return MPV_ERROR_INVALID_PARAMETER;
+        return MPV_ERROR_NOT_IMPLEMENTED;
 
     int err = p->context->fns->init(p->context, params);
     if (err < 0)
diff --git a/video/out/libmpv.h b/video/out/libmpv.h
index 329b2dffec..485ba8a76c 100644
--- a/video/out/libmpv.h
+++ b/video/out/libmpv.h
@@ -80,3 +80,4 @@ struct render_backend_fns {
 };
 
 extern const struct render_backend_fns render_backend_gpu;
+extern const struct render_backend_fns render_backend_sw;
diff --git a/video/out/libmpv_sw.c b/video/out/libmpv_sw.c
new file mode 100644
index 0000000000..901bd4e663
--- /dev/null
+++ b/video/out/libmpv_sw.c
@@ -0,0 +1,209 @@
+#include "config.h"
+#include "libmpv/render_gl.h"
+#include "libmpv.h"
+#include "sub/osd.h"
+#include "video/sws_utils.h"
+
+struct priv {
+    struct libmpv_gpu_context *context;
+
+    struct mp_sws_context *sws;
+    struct osd_state *osd;
+
+    struct mp_image_params src_params, dst_params;
+    struct mp_rect src_rc, dst_rc;
+    struct mp_osd_res osd_rc;
+    bool anything_changed;
+};
+
+static int init(struct render_backend *ctx, mpv_render_param *params)
+{
+    ctx->priv = talloc_zero(NULL, struct priv);
+    struct priv *p = ctx->priv;
+
+    char *api = get_mpv_render_param(params, MPV_RENDER_PARAM_API_TYPE, NULL);
+    if (!api)
+        return MPV_ERROR_INVALID_PARAMETER;
+
+    if (strcmp(api, MPV_RENDER_API_TYPE_SW) != 0)
+        return MPV_ERROR_NOT_IMPLEMENTED;
+
+    p->sws = mp_sws_alloc(p);
+    mp_sws_enable_cmdline_opts(p->sws, ctx->global);
+
+    p->anything_changed = true;
+
+    return 0;
+}
+
+static bool check_format(struct render_backend *ctx, int imgfmt)
+{
+    struct priv *p = ctx->priv;
+
+    // Note: we don't know the output format yet. Using an arbitrary supported
+    //       format is fine, because we know that any supported input format can
+    //       be converted to any supported output format.
+    return mp_sws_supports_formats(p->sws, IMGFMT_RGB0, imgfmt);
+}
+
+static int set_parameter(struct render_backend *ctx, mpv_render_param param)
+{
+    return MPV_ERROR_NOT_IMPLEMENTED;
+}
+
+static void reconfig(struct render_backend *ctx, struct mp_image_params *params)
+{
+    struct priv *p = ctx->priv;
+
+    p->src_params = *params;
+    p->anything_changed = true;
+}
+
+static void reset(struct render_backend *ctx)
+{
+    // stateless
+}
+
+static void update_external(struct render_backend *ctx, struct vo *vo)
+{
+    struct priv *p = ctx->priv;
+
+    p->osd = vo ? vo->osd : NULL;
+}
+
+static void resize(struct render_backend *ctx, struct mp_rect *src,
+                   struct mp_rect *dst, struct mp_osd_res *osd)
+{
+    struct priv *p = ctx->priv;
+
+    p->src_rc = *src;
+    p->dst_rc = *dst;
+    p->osd_rc = *osd;
+    p->anything_changed = true;
+}
+
+static int get_target_size(struct render_backend *ctx, mpv_render_param *params,
+                           int *out_w, int *out_h)
+{
+    int *sz = get_mpv_render_param(params, MPV_RENDER_PARAM_SW_SIZE, NULL);
+    if (!sz)
+        return MPV_ERROR_INVALID_PARAMETER;
+
+    *out_w = sz[0];
+    *out_h = sz[1];
+    return 0;
+}
+
+static int render(struct render_backend *ctx, mpv_render_param *params,
+                  struct vo_frame *frame)
+{
+    struct priv *p = ctx->priv;
+
+    int *sz = get_mpv_render_param(params, MPV_RENDER_PARAM_SW_SIZE, NULL);
+    char *fmt = get_mpv_render_param(params, MPV_RENDER_PARAM_SW_FORMAT, NULL);
+    size_t *stride = get_mpv_render_param(params, MPV_RENDER_PARAM_SW_STRIDE, NULL);
+    void *ptr = get_mpv_render_param(params, MPV_RENDER_PARAM_SW_POINTER, NULL);
+
+    if (!sz || !fmt || !stride || !ptr)
+        return MPV_ERROR_INVALID_PARAMETER;
+
+    char *prev_fmt = mp_imgfmt_to_name(p->dst_params.imgfmt);
+    if (strcmp(prev_fmt, fmt) != 0)
+        p->anything_changed = true;
+
+    if (sz[0] != p->dst_params.w || sz[1] != p->dst_params.h)
+        p->anything_changed = true;
+
+    if (p->anything_changed) {
+        p->dst_params = (struct mp_image_params){
+            .imgfmt = mp_imgfmt_from_name(bstr0(fmt)),
+            .w = sz[0],
+            .h = sz[1],
+        };
+
+        // Exclude "problematic" formats. In particular, reject multi-plane and
+        // hw formats. Exclude non-byte-aligned formats for easier stride
+        // checking.
+        struct mp_imgfmt_desc desc = mp_imgfmt_get_desc(p->dst_params.imgfmt);
+        if (!(desc.flags & MP_IMGFLAG_COLOR_RGB) ||
+            !(desc.flags & (MP_IMGFLAG_TYPE_UINT | MP_IMGFLAG_TYPE_FLOAT)) ||
+            (desc.flags & MP_IMGFLAG_TYPE_PAL8) ||
+            !(desc.flags & MP_IMGFLAG_BYTE_ALIGNED) ||
+            desc.num_planes != 1)
+            return MPV_ERROR_UNSUPPORTED;
+
+        mp_image_params_guess_csp(&p->dst_params);
+
+        // Can be unset if rendering before any video was loaded.
+        if (p->src_params.imgfmt) {
+            p->sws->src = p->src_params;
+            p->sws->src.w = mp_rect_w(p->src_rc);
+            p->sws->src.h = mp_rect_h(p->src_rc);
+
+            p->sws->dst = p->dst_params;
+            p->sws->dst.w = mp_rect_w(p->dst_rc);
+            p->sws->dst.h = mp_rect_h(p->dst_rc);
+
+            if (mp_sws_reinit(p->sws) < 0)
+                return MPV_ERROR_UNSUPPORTED; // probably
+        }
+
+        p->anything_changed = false;
+    }
+
+    struct mp_image wrap_img = {0};
+    mp_image_set_params(&wrap_img, &p->dst_params);
+
+    size_t bpp = wrap_img.fmt.bpp[0] / 8;
+    if (!bpp || bpp * wrap_img.w > *stride || *stride % bpp)
+        return MPV_ERROR_INVALID_PARAMETER;
+
+    wrap_img.planes[0] = ptr;
+    wrap_img.stride[0] = *stride;
+
+    struct mp_image *img = frame->current;
+    if (img) {
+        assert(p->src_params.imgfmt);
+
+        mp_image_clear_rc_inv(&wrap_img, p->dst_rc);
+
+        struct mp_image src = *img;
+        struct mp_rect src_rc = p->src_rc;
+        src_rc.x0 = MP_ALIGN_DOWN(src_rc.x0, src.fmt.align_x);
+        src_rc.y0 = MP_ALIGN_DOWN(src_rc.y0, src.fmt.align_y);
+        mp_image_crop_rc(&src, src_rc);
+
+        struct mp_image dst = wrap_img;
+        mp_image_crop_rc(&dst, p->dst_rc);
+
+        if (mp_sws_scale(p->sws, &dst, &src) < 0) {
+            mp_image_clear(&wrap_img, 0, 0, wrap_img.w, wrap_img.h);
+            return MPV_ERROR_GENERIC;
+        }
+    } else {
+        mp_image_clear(&wrap_img, 0, 0, wrap_img.w, wrap_img.h);
+    }
+
+    if (p->osd)
+        osd_draw_on_image(p->osd, p->osd_rc, img ? img->pts : 0, 0, &wrap_img);
+
+    return 0;
+}
+
+static void destroy(struct render_backend *ctx)
+{
+    // nop
+}
+
+const struct render_backend_fns render_backend_sw = {
+    .init = init,
+    .check_format = check_format,
+    .set_parameter = set_parameter,
+    .reconfig = reconfig,
+    .reset = reset,
+    .update_external = update_external,
+    .resize = resize,
+    .get_target_size = get_target_size,
+    .render = render,
+    .destroy = destroy,
+};
diff --git a/video/out/vo_libmpv.c b/video/out/vo_libmpv.c
index fffb65b29a..d07ab4c4eb 100644
--- a/video/out/vo_libmpv.c
+++ b/video/out/vo_libmpv.c
@@ -111,6 +111,7 @@ struct mpv_render_context {
 
 const struct render_backend_fns *render_backends[] = {
     &render_backend_gpu,
+    &render_backend_sw,
     NULL
 };
 
diff --git a/wscript_build.py b/wscript_build.py
index 635bd117bc..525d744bfb 100644
--- a/wscript_build.py
+++ b/wscript_build.py
@@ -452,6 +452,7 @@ def build(ctx):
         ( "video/out/hwdec/hwdec_vaapi.c",       "vaapi-egl || vaapi-vulkan" ),
         ( "video/out/hwdec/hwdec_vaapi_gl.c",    "vaapi-egl" ),
         ( "video/out/hwdec/hwdec_vaapi_vk.c",    "vaapi-vulkan" ),
+        ( "video/out/libmpv_sw.c" ),
         ( "video/out/placebo/ra_pl.c",           "libplacebo" ),
         ( "video/out/placebo/utils.c",           "libplacebo" ),
         ( "video/out/opengl/angle_dynamic.c",    "egl-angle" ),