vo_opengl_cb: introduce frame queue

The previous implementation of opengl-cb kept only latest flipped frame.
This can cause massive frame drops because rendering is done asynchronously
and only the latest frame can be rendered.

This commit introduces frame queue and releated options to opengl-cb.

frame-queue-size: the maximum size of frame queue (1-100, default: 1)
frame-drop-mode: behavior when frame queue is full (pop, clear, default: pop)

The frame queue holds delayed frames and drops frames if the frame queue is
overflowed with next method:

'pop' mode: drops all the oldest frames overflown.
'clear' mode: drops all frames in queue and clear it.

With default options(frame-queue-size=1:frame-drop-mode=pop),
opengl-cb behaves in the same way as previous implementation effectively.

For frame-queue-size > 1, opengl-cb tries to calls update() without waiting
next flip_page() in order to consume queued frames.

Signed-off-by: wm4 <wm4@nowhere>

2 files changed, 110 insertions, 19 deletions

diff --git a/DOCS/man/vo.rst b/DOCS/man/vo.rst
index e845eef239..c296e705be 100644
--- a/DOCS/man/vo.rst
+++ b/DOCS/man/vo.rst
@@ -915,8 +915,27 @@ Available video output drivers are:
 ``opengl-cb``
     For use with libmpv direct OpenGL embedding; useless in any other contexts.
     (See ``<mpv/opengl_cb.h>``.)
+    Usually, ``opengl-cb`` renders frames asynchronously by client and this
+    can cause some frame drops. In order to provide a way to handle this
+    situation, ``opengl-cb`` has its own frame queue and calls update callback
+    more frequently if the queue is not empty regardless of existence of new frame.
+    Once the queue is filled, ``opengl-cb`` drops frames automatically.
 
-    This supports many of the suboptions the ``opengl`` VO has. Run
+    With default options, ``opengl-cb`` renders only the latest frame and drops
+    all frames handed over while waiting render function after update callback.
+
+    ``frame-queue-size=<1..100>``
+        The maximum count of frames which the frame queue can hold (default: 1)
+
+    ``frame-drop-mode=<pop|clear>``
+        Select the behavior when the frame queue is full.
+
+        pop
+            Drop the oldest frame in the frame queue. (default)
+        clear
+            Drop all frames in the frame queue.
+
+    This also supports many of the suboptions the ``opengl`` VO has. Runs
     ``mpv --vo=opengl-cb:help`` for a list.
 
     This also supports the ``vo_cmdline`` command.
diff --git a/video/out/vo_opengl_cb.c b/video/out/vo_opengl_cb.c
index a689a1dcbe..b49f3fdb9d 100644
--- a/video/out/vo_opengl_cb.c
+++ b/video/out/vo_opengl_cb.c
@@ -39,6 +39,9 @@
  * here to transfer the video frames somehow.
  */
 
+#define FRAME_DROP_POP      0 // drop the oldest frame in queue
+#define FRAME_DROP_CLEAR    1 // drop all frames in queue
+
 struct vo_priv {
     struct vo *vo;
 
@@ -47,6 +50,8 @@ struct vo_priv {
     // Immutable after VO init
     int use_gl_debug;
     struct gl_video_opts *renderer_opts;
+    int frame_queue_size;
+    int frame_drop_mode;
 };
 
 struct mpv_opengl_cb_context {
@@ -60,7 +65,9 @@ struct mpv_opengl_cb_context {
     mpv_opengl_cb_update_fn update_cb;
     void *update_cb_ctx;
     struct mp_image *waiting_frame;
-    struct mp_image *next_frame;
+    struct mp_image **frame_queue;
+    int queued_frames;
+    uint64_t dropped_frames;
     struct mp_image_params img_params;
     bool reconfigured;
     struct mp_rect wnd;
@@ -90,6 +97,53 @@ struct mpv_opengl_cb_context {
     struct vo *active;
 };
 
+static void update(struct vo_priv *p);
+
+// all queue manipulation functions shold be called under locked state
+
+static struct mp_image *frame_queue_pop(struct mpv_opengl_cb_context *ctx)
+{
+    if (ctx->queued_frames == 0)
+        return NULL;
+    struct mp_image *ret = ctx->frame_queue[0];
+    MP_TARRAY_REMOVE_AT(ctx->frame_queue, ctx->queued_frames, 0);
+    return ret;
+}
+
+static void frame_queue_drop(struct mpv_opengl_cb_context *ctx)
+{
+    talloc_free(frame_queue_pop(ctx));
+    ctx->dropped_frames++;
+}
+
+static void frame_queue_clear(struct mpv_opengl_cb_context *ctx)
+{
+    for (int i = 0; i < ctx->queued_frames; i++)
+        talloc_free(ctx->frame_queue[i]);
+    talloc_free(ctx->frame_queue);
+    ctx->frame_queue = NULL;
+    ctx->dropped_frames += ctx->queued_frames;
+    ctx->queued_frames = 0;
+}
+
+static void frame_queue_push(struct mpv_opengl_cb_context *ctx, struct mp_image *mpi)
+{
+    MP_TARRAY_APPEND(ctx, ctx->frame_queue, ctx->queued_frames, mpi);
+}
+
+static void frame_queue_shrink(struct mpv_opengl_cb_context *ctx, int size)
+{
+    while (ctx->queued_frames > size)
+        frame_queue_drop(ctx);
+}
+
+static void forget_frames(struct mpv_opengl_cb_context *ctx)
+{
+    frame_queue_clear(ctx);
+    mp_image_unrefp(&ctx->waiting_frame);
+    ctx->dropped_frames = 0;
+}
+
 static void free_ctx(void *ptr)
 {
     mpv_opengl_cb_context *ctx = ptr;
@@ -185,7 +239,9 @@ int mpv_opengl_cb_uninit_gl(struct mpv_opengl_cb_context *ctx)
 {
     // Bring down the decoder etc., which still might be using the hwdec
     // context. Setting initialized=false guarantees it can't come back.
+
     pthread_mutex_lock(&ctx->lock);
+    forget_frames(ctx);
     ctx->initialized = false;
     pthread_mutex_unlock(&ctx->lock);
 
@@ -252,6 +308,7 @@ int mpv_opengl_cb_render(struct mpv_opengl_cb_context *ctx, int fbo, int vp[4])
             gl_video_set_options(ctx->renderer, opts->renderer_opts);
             ctx->gl->debug_context = opts->use_gl_debug;
             gl_video_set_debug(ctx->renderer, opts->use_gl_debug);
+            frame_queue_shrink(ctx, opts->frame_queue_size);
         }
         ctx->reconfigured = false;
         ctx->update_new_opts = false;
@@ -265,8 +322,7 @@ int mpv_opengl_cb_render(struct mpv_opengl_cb_context *ctx, int fbo, int vp[4])
     ctx->eq_changed = false;
     ctx->eq = *eq;
 
-    struct mp_image *mpi = ctx->next_frame;
-    ctx->next_frame = NULL;
+    struct mp_image *mpi = frame_queue_pop(ctx);
 
     pthread_mutex_unlock(&ctx->lock);
 
@@ -277,6 +333,11 @@ int mpv_opengl_cb_render(struct mpv_opengl_cb_context *ctx, int fbo, int vp[4])
 
     gl_video_unset_gl_state(ctx->renderer);
 
+    pthread_mutex_lock(&ctx->lock);
+    if (vo && ctx->queued_frames > 0)
+        update(vo->priv);
+    pthread_mutex_unlock(&ctx->lock);
+
     return 0;
 }
 
@@ -302,8 +363,13 @@ static void flip_page(struct vo *vo)
     struct vo_priv *p = vo->priv;
 
     pthread_mutex_lock(&p->ctx->lock);
-    mp_image_unrefp(&p->ctx->next_frame);
-    p->ctx->next_frame = p->ctx->waiting_frame;
+    if (p->ctx->queued_frames >= p->frame_queue_size) {
+        if (p->frame_drop_mode == FRAME_DROP_CLEAR)
+            frame_queue_clear(p->ctx);
+        else // FRAME_DROP_POP mode
+            frame_queue_shrink(p->ctx, p->frame_queue_size - 1);
+    }
+    frame_queue_push(p->ctx, p->ctx->waiting_frame);
     p->ctx->waiting_frame = NULL;
     update(p);
     pthread_mutex_unlock(&p->ctx->lock);
@@ -326,7 +392,7 @@ static int reconfig(struct vo *vo, struct mp_image_params *params, int flags)
     struct vo_priv *p = vo->priv;
 
     pthread_mutex_lock(&p->ctx->lock);
-    mp_image_unrefp(&p->ctx->next_frame);
+    forget_frames(p->ctx);
     p->ctx->img_params = *params;
     p->ctx->reconfigured = true;
     pthread_mutex_unlock(&p->ctx->lock);
@@ -334,6 +400,19 @@ static int reconfig(struct vo *vo, struct mp_image_params *params, int flags)
     return 0;
 }
 
+// list of options which can be changed at runtime
+#define OPT_BASE_STRUCT struct vo_priv
+static const struct m_option change_opts[] = {
+    OPT_FLAG("debug", use_gl_debug, 0),
+    OPT_INTRANGE("frame-queue-size", frame_queue_size, 0, 1, 100, OPTDEF_INT(1)),
+    OPT_CHOICE("frame-drop-mode", frame_drop_mode, 0,
+               ({"pop", FRAME_DROP_POP},
+                {"clear", FRAME_DROP_CLEAR})),
+    OPT_SUBSTRUCT("", renderer_opts, gl_video_conf, 0),
+    {0}
+};
+#undef OPT_BASE_STRUCT
+
 static bool reparse_cmdline(struct vo_priv *p, char *args)
 {
     struct m_config *cfg = NULL;
@@ -341,16 +420,6 @@ static bool reparse_cmdline(struct vo_priv *p, char *args)
     int r = 0;
 
     pthread_mutex_lock(&p->ctx->lock);
-
-    // list of options which can be changed at runtime
-#define OPT_BASE_STRUCT struct vo_priv
-    static const struct m_option change_opts[] = {
-        OPT_FLAG("debug", use_gl_debug, 0),
-        OPT_SUBSTRUCT("", renderer_opts, gl_video_conf, 0),
-        {0}
-    };
-#undef OPT_BASE_STRUCT
-
     const struct vo_priv *vodef = p->vo->driver->priv_defaults;
     cfg = m_config_new(NULL, p->vo->log, sizeof(*opts), vodef, change_opts);
     opts = cfg->optstruct;
@@ -426,8 +495,7 @@ static void uninit(struct vo *vo)
     struct vo_priv *p = vo->priv;
 
     pthread_mutex_lock(&p->ctx->lock);
-    mp_image_unrefp(&p->ctx->next_frame);
-    mp_image_unrefp(&p->ctx->waiting_frame);
+    forget_frames(p->ctx);
     p->ctx->img_params = (struct mp_image_params){0};
     p->ctx->reconfigured = true;
     p->ctx->active = NULL;
@@ -462,6 +530,10 @@ static int preinit(struct vo *vo)
 #define OPT_BASE_STRUCT struct vo_priv
 static const struct m_option options[] = {
     OPT_FLAG("debug", use_gl_debug, 0),
+    OPT_INTRANGE("frame-queue-size", frame_queue_size, 0, 1, 100, OPTDEF_INT(1)),
+    OPT_CHOICE("frame-drop-mode", frame_drop_mode, 0,
+               ({"pop", FRAME_DROP_POP},
+                {"clear", FRAME_DROP_CLEAR})),
     OPT_SUBSTRUCT("", renderer_opts, gl_video_conf, 0),
     {0},
 };