path: root/player/video.c

/*
 * This file is part of mpv.
 *
 * mpv is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * mpv is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with mpv.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <stddef.h>
#include <stdbool.h>
#include <inttypes.h>
#include <math.h>
#include <assert.h>

#include "config.h"
#include "mpv_talloc.h"

#include "common/msg.h"
#include "options/options.h"
#include "options/m_config.h"
#include "options/m_option.h"
#include "common/common.h"
#include "common/encode.h"
#include "options/m_property.h"
#include "osdep/timer.h"

#include "audio/out/ao.h"
#include "audio/format.h"
#include "demux/demux.h"
#include "stream/stream.h"
#include "sub/osd.h"
#include "video/hwdec.h"
#include "filters/f_decoder_wrapper.h"
#include "video/out/vo.h"

#include "core.h"
#include "command.h"
#include "screenshot.h"

enum {
    // update_video() - code also uses: <0 error, 0 eof, >0 progress
    VD_ERROR = -1,
    VD_EOF = 0,         // end of file - no new output
    VD_PROGRESS = 1,    // progress, but no output; repeat call with no waiting
    VD_NEW_FRAME = 2,   // the call produced a new frame
    VD_WAIT = 3,        // no EOF, but no output; wait until wakeup
};

static const char av_desync_help_text[] =
"\n"
"Audio/Video desynchronisation detected! Possible reasons include too slow\n"
"hardware, temporary CPU spikes, broken drivers, and broken files. Audio\n"
"position will not match to the video (see A-V status field).\n"
"\n";

static bool recreate_video_filters(struct MPContext *mpctx)
{
    struct MPOpts *opts = mpctx->opts;
    struct vo_chain *vo_c = mpctx->vo_chain;
    assert(vo_c);

    return mp_output_chain_update_filters(vo_c->filter, opts->vf_settings);
}

int reinit_video_filters(struct MPContext *mpctx)
{
    struct vo_chain *vo_c = mpctx->vo_chain;

    if (!vo_c)
        return 0;

    if (!recreate_video_filters(mpctx))
        return -1;

    mp_force_video_refresh(mpctx);

    mp_notify(mpctx, MPV_EVENT_VIDEO_RECONFIG, NULL);

    return 0;
}

static void vo_chain_reset_state(struct vo_chain *vo_c)
{
    vo_seek_reset(vo_c->vo);
}

void reset_video_state(struct MPContext *mpctx)
{
    if (mpctx->vo_chain)
        vo_chain_reset_state(mpctx->vo_chain);

    for (int n = 0; n < mpctx->num_next_frames; n++)
        mp_image_unrefp(&mpctx->next_frames[n]);
    mpctx->num_next_frames = 0;
    mp_image_unrefp(&mpctx->saved_frame);

    mpctx->delay = 0;
    mpctx->time_frame = 0;
    mpctx->video_pts = MP_NOPTS_VALUE;
    mpctx->last_frame_duration = 0;
    mpctx->num_past_frames = 0;
    mpctx->total_avsync_change = 0;
    mpctx->last_av_difference = 0;
    mpctx->mistimed_frames_total = 0;
    mpctx->drop_message_shown = 0;
    mpctx->display_sync_drift_dir = 0;
    mpctx->display_sync_broken = false;

    mpctx->video_status = mpctx->vo_chain ? STATUS_SYNCING : STATUS_EOF;
}

void uninit_video_out(struct MPContext *mpctx)
{
    uninit_video_chain(mpctx);
    if (mpctx->video_out) {
        vo_destroy(mpctx->video_out);
        mp_notify(mpctx, MPV_EVENT_VIDEO_RECONFIG, NULL);
    }
    mpctx->video_out = NULL;
}

static void vo_chain_uninit(struct vo_chain *vo_c)
{
    struct track *track = vo_c->track;
    if (track) {
        assert(track->vo_c == vo_c);
        track->vo_c = NULL;
        if (vo_c->dec_src)
            assert(track->dec->f->pins[0] == vo_c->dec_src);
        talloc_free(track->dec->f);
        track->dec = NULL;
    }

    if (vo_c->filter_src)
        mp_pin_disconnect(vo_c->filter_src);

    talloc_free(vo_c->filter->f);
    talloc_free(vo_c);
    // this does not free the VO
}

void uninit_video_chain(struct MPContext *mpctx)
{
    if (mpctx->vo_chain) {
        reset_video_state(mpctx);
        vo_chain_uninit(mpctx->vo_chain);
        mpctx->vo_chain = NULL;

        mpctx->video_status = STATUS_EOF;

        mp_notify(mpctx, MPV_EVENT_VIDEO_RECONFIG, NULL);
    }
}

int init_video_decoder(struct MPContext *mpctx, struct track *track)
{
    assert(!track->dec);
    if (!track->stream)
        goto err_out;

    struct mp_filter *parent = mpctx->filter_root;

    // If possible, set this as parent so the decoder gets the hwdec and DR
    // interfaces.
    // Note: at least mpv_opengl_cb_uninit_gl() relies on being able to get
    //       rid of all references to the VO by destroying the VO chain. Thus,
    //       decoders not linked to vo_chain must not use the hwdec context.
    if (track->vo_c)
        parent = track->vo_c->filter->f;

    track->dec = mp_decoder_wrapper_create(parent, track->stream);
    if (!track->dec)
        goto err_out;

    if (!mp_decoder_wrapper_reinit(track->dec))
        goto err_out;

    return 1;

err_out:
    if (track->sink)
        mp_pin_disconnect(track->sink);
    track->sink = NULL;
    error_on_track(mpctx, track);
    return 0;
}

void reinit_video_chain(struct MPContext *mpctx)
{
    struct track *track = mpctx->current_track[0][STREAM_VIDEO];
    if (!track || !track->stream) {
        error_on_track(mpctx, track);
        return;
    }
    reinit_video_chain_src(mpctx, track);
}

static void filter_update_subtitles(void *ctx, double pts)
{
    struct MPContext *mpctx = ctx;

    if (osd_get_render_subs_in_filter(mpctx->osd))
        update_subtitles(mpctx, pts);
}

// (track=NULL creates a blank chain, used for lavfi-complex)
void reinit_video_chain_src(struct MPContext *mpctx, struct track *track)
{
    assert(!mpctx->vo_chain);

    if (!mpctx->video_out) {
        struct vo_extra ex = {
            .input_ctx = mpctx->input,
            .osd = mpctx->osd,
            .encode_lavc_ctx = mpctx->encode_lavc_ctx,
            .opengl_cb_context = mpctx->gl_cb_ctx,
            .wakeup_cb = mp_wakeup_core_cb,
            .wakeup_ctx = mpctx,
        };
        mpctx->video_out = init_best_video_out(mpctx->global, &ex);
        if (!mpctx->video_out) {
            MP_FATAL(mpctx, "Error opening/initializing "
                    "the selected video_out (--vo) device.\n");
            mpctx->error_playing = MPV_ERROR_VO_INIT_FAILED;
            goto err_out;
        }
        mpctx->mouse_cursor_visible = true;
    }

    update_window_title(mpctx, true);

    struct vo_chain *vo_c = talloc_zero(NULL, struct vo_chain);
    mpctx->vo_chain = vo_c;
    vo_c->log = mpctx->log;
    vo_c->vo = mpctx->video_out;
    vo_c->filter =
        mp_output_chain_create(mpctx->filter_root, MP_OUTPUT_CHAIN_VIDEO);
    mp_output_chain_set_vo(vo_c->filter, vo_c->vo);
    vo_c->filter->update_subtitles = filter_update_subtitles;
    vo_c->filter->update_subtitles_ctx = mpctx;

    if (track) {
        vo_c->track = track;
        track->vo_c = vo_c;
        if (!init_video_decoder(mpctx, track))
            goto err_out;

        vo_c->dec_src = track->dec->f->pins[0];
        vo_c->filter->container_fps = track->dec->fps;
        vo_c->is_coverart = !!track->stream->attached_picture;

        track->vo_c = vo_c;
        vo_c->track = track;

        mp_pin_connect(vo_c->filter->f->pins[0], vo_c->dec_src);
    }

#if HAVE_ENCODING
    if (mpctx->encode_lavc_ctx) {
        encode_lavc_set_video_fps(mpctx->encode_lavc_ctx,
                                  vo_c->filter->container_fps);
    }
#endif

    if (!recreate_video_filters(mpctx))
        goto err_out;

    update_screensaver_state(mpctx);

    vo_set_paused(vo_c->vo, mpctx->paused);

    // If we switch on video again, ensure audio position matches up.
    if (mpctx->ao_chain)
        mpctx->audio_status = STATUS_SYNCING;

    reset_video_state(mpctx);
    reset_subtitle_state(mpctx);

    return;

err_out:
    uninit_video_chain(mpctx);
    error_on_track(mpctx, track);
    handle_force_window(mpctx, true);
}

// Try to refresh the video by doing a precise seek to the currently displayed
// frame. This can go wrong in all sorts of ways, so use sparingly.
void mp_force_video_refresh(struct MPContext *mpctx)
{
    struct MPOpts *opts = mpctx->opts;
    struct vo_chain *vo_c = mpctx->vo_chain;

    if (!vo_c)
        return;

    // If not paused, the next frame should come soon enough.
    if (opts->pause || mpctx->time_frame >= 0.5 ||
        mpctx->video_status == STATUS_EOF)
    {
        issue_refresh_seek(mpctx, MPSEEK_VERY_EXACT);
    }
}

static void check_framedrop(struct MPContext *mpctx, struct vo_chain *vo_c)
{
    struct MPOpts *opts = mpctx->opts;
    // check for frame-drop:
    if (mpctx->video_status == STATUS_PLAYING && !mpctx->paused &&
        mpctx->audio_status == STATUS_PLAYING && !ao_untimed(mpctx->ao) &&
        vo_c->track && vo_c->track->dec && (opts->frame_dropping & 2))
    {
        float fps = vo_c->filter->container_fps;
        // it's a crappy heuristic; avoid getting upset by incorrect fps
        if (fps <= 20 || fps >= 500)
            return;
        double frame_time =  1.0 / fps;
        // try to drop as many frames as we appear to be behind
        vo_c->track->dec->attempt_framedrops =
            MPCLAMP((mpctx->last_av_difference - 0.010) / frame_time, 0, 100);
    }
}

/* Modify video timing to match the audio timeline. There are two main
 * reasons this is needed. First, video and audio can start from different
 * positions at beginning of file or after a seek (MPlayer starts both
 * immediately even if they have different pts). Second, the file can have
 * audio timestamps that are inconsistent with the duration of the audio
 * packets, for example two consecutive timestamp values differing by
 * one second but only a packet with enough samples for half a second
 * of playback between them.
 */
static void adjust_sync(struct MPContext *mpctx, double v_pts, double frame_time)
{
    struct MPOpts *opts = mpctx->opts;

    if (mpctx->audio_status != STATUS_PLAYING)
        return;

    double a_pts = written_audio_pts(mpctx) + opts->audio_delay - mpctx->delay;
    double av_delay = a_pts - v_pts;

    double change = av_delay * 0.1;
    double factor = fabs(av_delay) < 0.3 ? 0.1 : 0.4;
    double max_change = opts->default_max_pts_correction >= 0 ?
                        opts->default_max_pts_correction : frame_time * factor;
    if (change < -max_change)
        change = -max_change;
    else if (change > max_change)
        change = max_change;
    mpctx->delay += change;
    mpctx->total_avsync_change += change;

    if (mpctx->display_sync_active)
        mpctx->total_avsync_change = 0;
}

// Make the frame at position 0 "known" to the playback logic. This must happen
// only once for each frame, so this function has to be called carefully.
// Generally, if position 0 gets a new frame, this must be called.
static void handle_new_frame(struct MPContext *mpctx)
{
    assert(mpctx->num_next_frames >= 1);

    double frame_time = 0;
    double pts = mpctx->next_frames[0]->pts;
    if (mpctx->video_pts != MP_NOPTS_VALUE) {
        frame_time = pts - mpctx->video_pts;
        double tolerance = mpctx->demuxer->ts_resets_possible ? 5 : 1e4;
        if (frame_time <= 0 || frame_time >= tolerance) {
            // Assume a discontinuity.
            MP_WARN(mpctx, "Invalid video timestamp: %f -> %f\n",
                    mpctx->video_pts, pts);
            frame_time = 0;
        }
    }
    mpctx->delay -= frame_time;
    if (mpctx->video_status >= STATUS_PLAYING) {
        mpctx->time_frame += frame_time / mpctx->video_speed;
        adjust_sync(mpctx, pts, frame_time);
    }
    MP_TRACE(mpctx, "frametime=%5.3f\n", frame_time);
}

// Remove the first frame in mpctx->next_frames
static void shift_frames(struct MPContext *mpctx)
{
    if (mpctx->num_next_frames < 1)
        return;
    talloc_free(mpctx->next_frames[0]);
    for (int n = 0; n < mpctx->num_next_frames - 1; n++)
        mpctx->next_frames[n] = mpctx->next_frames[n + 1];
    mpctx->num_next_frames -= 1;
}

static int get_req_frames(struct MPContext *mpctx, bool eof)
{
    // On EOF, drain all frames.
    if (eof)
        return 1;

    // On the first frame, output a new frame as quickly as possible.
    // But display-sync likes to have a correct frame duration always.
    if (mpctx->video_pts == MP_NOPTS_VALUE)
        return mpctx->opts->video_sync == VS_DEFAULT ? 1 : 2;

    int req = vo_get_num_req_frames(mpctx->video_out);
    return MPCLAMP(req, 2, MP_ARRAY_SIZE(mpctx->next_frames) - 1);
}

// Whether it's fine to call add_new_frame() now.
static bool needs_new_frame(struct MPContext *mpctx)
{
    return mpctx->num_next_frames < get_req_frames(mpctx, false);
}

// Queue a frame to mpctx->next_frames[]. Call only if needs_new_frame() signals ok.
static void add_new_frame(struct MPContext *mpctx, struct mp_image *frame)
{
    assert(mpctx->num_next_frames < MP_ARRAY_SIZE(mpctx->next_frames));
    assert(frame);
    mpctx->next_frames[mpctx->num_next_frames++] = frame;
    if (mpctx->num_next_frames == 1)
        handle_new_frame(mpctx);
}

// Enough video filtered already to push one frame to the VO?
// Set eof to true if no new frames are to be expected.
static bool have_new_frame(struct MPContext *mpctx, bool eof)
{
    return mpctx->num_next_frames >= get_req_frames(mpctx, eof);
}

// Fill mpctx->next_frames[] with a newly filtered or decoded image.
// returns VD_* code
static int video_output_image(struct MPContext *mpctx)
{
    struct vo_chain *vo_c = mpctx->vo_chain;
    bool hrseek = mpctx->hrseek_active && mpctx->video_status == STATUS_SYNCING;

    if (vo_c->is_coverart) {
        if (vo_has_frame(mpctx->video_out))
            return VD_EOF;
        hrseek = false;
    }

    if (have_new_frame(mpctx, false))
        return VD_NEW_FRAME;

    // Get a new frame if we need one.
    int r = VD_PROGRESS;
    if (needs_new_frame(mpctx)) {
        // Filter a new frame.
        struct mp_image *img = NULL;
        struct mp_frame frame = mp_pin_out_read(vo_c->filter->f->pins[1]);
        if (frame.type == MP_FRAME_NONE) {
            r = vo_c->filter->got_output_eof ? VD_EOF : VD_WAIT;
        } else if (frame.type == MP_FRAME_EOF) {
            r = VD_EOF;
        } else if (frame.type == MP_FRAME_VIDEO) {
            img = frame.data;
        } else {
            MP_ERR(mpctx, "unexpected frame type %s\n",
                   mp_frame_type_str(frame.type));
            mp_frame_unref(&frame);
            return VD_ERROR;
        }
        if (img) {
            double endpts = get_play_end_pts(mpctx);
            if ((endpts != MP_NOPTS_VALUE && img->pts >= endpts) ||
                mpctx->max_frames == 0)
            {
                mp_pin_out_unread(vo_c->filter->f->pins[1], frame);
                img = NULL;
                r = VD_EOF;
            } else if (hrseek && mpctx->hrseek_lastframe) {
                mp_image_setrefp(&mpctx->saved_frame, img);
            } else if (hrseek && img->pts < mpctx->hrseek_pts - .005) {
                /* just skip - but save if backstep active */
                if (mpctx->hrseek_backstep)
                    mp_image_setrefp(&mpctx->saved_frame, img);
            } else if (mpctx->video_status == STATUS_SYNCING &&
                       mpctx->playback_pts != MP_NOPTS_VALUE &&
                       img->pts < mpctx->playback_pts && !vo_c->is_coverart)
            {
                /* skip after stream-switching */
            } else {
                if (hrseek && mpctx->hrseek_backstep) {
                    if (mpctx->saved_frame) {
                        add_new_frame(mpctx, mpctx->saved_frame);
                        mpctx->saved_frame = NULL;
                    } else {
                        MP_WARN(mpctx, "Backstep failed.\n");
                    }
                    mpctx->hrseek_backstep = false;
                }
                add_new_frame(mpctx, img);
                img = NULL;
            }
            talloc_free(img);
        }
    }

    // Last-frame seek
    if (r <= 0 && hrseek && mpctx->hrseek_lastframe && mpctx->saved_frame) {
        add_new_frame(mpctx, mpctx->saved_frame);
        mpctx->saved_frame = NULL;
        r = VD_PROGRESS;
    }

    return have_new_frame(mpctx, r <= 0) ? VD_NEW_FRAME : r;
}

static bool check_for_hwdec_fallback(struct MPContext *mpctx)
{
    struct vo_chain *vo_c = mpctx->vo_chain;

    if (!vo_c->filter->failed_output_conversion || !vo_c->track)
        return false;

    if (mp_decoder_wrapper_control(vo_c->track->dec,
                            VDCTRL_FORCE_HWDEC_FALLBACK, NULL) != CONTROL_OK)
        return false;

    mp_output_chain_reset_harder(vo_c->filter);
    return true;
}

/* Update avsync before a new video frame is displayed. Actually, this can be
 * called arbitrarily often before the actual display.
 * This adjusts the time of the next video frame */
static void update_avsync_before_frame(struct MPContext *mpctx)
{
    struct MPOpts *opts = mpctx->opts;
    struct vo *vo = mpctx->video_out;

    if (mpctx->vo_chain->is_coverart || mpctx->video_status < STATUS_READY) {
        mpctx->time_frame = 0;
    } else if (mpctx->display_sync_active || opts->video_sync == VS_NONE) {
        // don't touch the timing
    } else if (mpctx->audio_status == STATUS_PLAYING &&
               mpctx->video_status == STATUS_PLAYING &&
               !ao_untimed(mpctx->ao))
    {
        double buffered_audio = ao_get_delay(mpctx->ao);

        double predicted = mpctx->delay / mpctx->video_speed +
                           mpctx->time_frame;
        double difference = buffered_audio - predicted;
        MP_STATS(mpctx, "value %f audio-diff", difference);

        if (opts->autosync) {
            /* Smooth reported playback position from AO by averaging
             * it with the value expected based on previus value and
             * time elapsed since then. May help smooth video timing
             * with audio output that have inaccurate position reporting.
             * This is badly implemented; the behavior of the smoothing
             * now undesirably depends on how often this code runs
             * (mainly depends on video frame rate). */
            buffered_audio = predicted + difference / opts->autosync;
        }

        mpctx->time_frame = buffered_audio - mpctx->delay / mpctx->video_speed;
    } else {
        /* If we're more than 200 ms behind the right playback
         * position, don't try to speed up display of following
         * frames to catch up; continue with default speed from
         * the current frame instead.
         * If untimed is set always output frames immediately
         * without sleeping.
         */
        if (mpctx->time_frame < -0.2 |