path: root/sub/sd_ass.c

/*
 * This file is part of mpv.
 *
 * mpv is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * mpv is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with mpv.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <stdlib.h>
#include <assert.h>
#include <string.h>
#include <math.h>
#include <limits.h>

#include <libavutil/common.h>
#include <ass/ass.h>

#include "mpv_talloc.h"

#include "config.h"
#include "options/m_config.h"
#include "options/options.h"
#include "common/common.h"
#include "common/msg.h"
#include "demux/demux.h"
#include "video/csputils.h"
#include "video/mp_image.h"
#include "dec_sub.h"
#include "ass_mp.h"
#include "sd.h"

struct sd_ass_priv {
    struct ass_library *ass_library;
    struct ass_renderer *ass_renderer;
    struct ass_track *ass_track;
    struct ass_track *shadow_track; // for --sub-ass=no rendering
    bool ass_configured;
    bool is_converted;
    struct lavc_conv *converter;
    struct sd_filter **filters;
    int num_filters;
    bool clear_once;
    struct mp_ass_packer *packer;
    struct sub_bitmap_copy_cache *copy_cache;
    char last_text[500];
    struct mp_image_params video_params;
    struct mp_image_params last_params;
    struct mp_osd_res osd;
    struct seen_packet *seen_packets;
    int num_seen_packets;
    bool *packets_animated;
    int num_packets_animated;
    bool duration_unknown;
};

struct seen_packet {
    int64_t pos;
    double pts;
};

static void mangle_colors(struct sd *sd, struct sub_bitmaps *parts);
static void fill_plaintext(struct sd *sd, double pts);

static const struct sd_filter_functions *const filters[] = {
    // Note: list order defines filter order.
    &sd_filter_sdh,
#if HAVE_POSIX
    &sd_filter_regex,
#endif
#if HAVE_JAVASCRIPT
    &sd_filter_jsre,
#endif
    NULL,
};

// Add default styles, if the track does not have any styles yet.
// Apply style overrides if the user provides any.
static void mp_ass_add_default_styles(ASS_Track *track, struct mp_subtitle_opts *opts,
                                      struct mp_subtitle_shared_opts *shared_opts, int order)
{
    if (opts->ass_styles_file && shared_opts->ass_style_override[order])
        ass_read_styles(track, opts->ass_styles_file, NULL);

    if (track->n_styles == 0) {
        if (!track->PlayResY) {
            track->PlayResX = MP_ASS_FONT_PLAYRESX;
            track->PlayResY = MP_ASS_FONT_PLAYRESY;
        }
        track->Kerning = true;
        int sid = ass_alloc_style(track);
        track->default_style = sid;
        ASS_Style *style = track->styles + sid;
        style->Name = strdup("Default");
        mp_ass_set_style(style, track->PlayResY, opts->sub_style);
    }

    if (shared_opts->ass_style_override[order])
        ass_process_force_style(track);
}

static const char *const font_mimetypes[] = {
    "application/x-truetype-font",
    "application/vnd.ms-opentype",
    "application/x-font-ttf",
    "application/x-font", // probably incorrect
    "application/font-sfnt",
    "font/collection",
    "font/otf",
    "font/sfnt",
    "font/ttf",
    NULL
};

static const char *const font_exts[] = {".ttf", ".ttc", ".otf", ".otc", NULL};

static bool attachment_is_font(struct mp_log *log, struct demux_attachment *f)
{
    if (!f->name || !f->type || !f->data || !f->data_size)
        return false;
    for (int n = 0; font_mimetypes[n]; n++) {
        if (strcmp(font_mimetypes[n], f->type) == 0)
            return true;
    }
    // fallback: match against file extension
    char *ext = strlen(f->name) > 4 ? f->name + strlen(f->name) - 4 : "";
    for (int n = 0; font_exts[n]; n++) {
        if (strcasecmp(ext, font_exts[n]) == 0) {
            mp_warn(log, "Loading font attachment '%s' with MIME type %s. "
                    "Assuming this is a broken Matroska file, which was "
                    "muxed without setting a correct font MIME type.\n",
                    f->name, f->type);
            return true;
        }
    }
    return false;
}

static void add_subtitle_fonts(struct sd *sd)
{
    struct sd_ass_priv *ctx = sd->priv;
    struct mp_subtitle_opts *opts = sd->opts;
    if (!opts->ass_enabled || !opts->use_embedded_fonts || !sd->attachments)
        return;
    for (int i = 0; i < sd->attachments->num_entries; i++) {
        struct demux_attachment *f = &sd->attachments->entries[i];
        if (attachment_is_font(sd->log, f))
            ass_add_font(ctx->ass_library, f->name, f->data, f->data_size);
    }
}

static void filters_destroy(struct sd *sd)
{
    struct sd_ass_priv *ctx = sd->priv;

    for (int n = 0; n < ctx->num_filters; n++) {
        struct sd_filter *ft = ctx->filters[n];
        if (ft->driver->uninit)
            ft->driver->uninit(ft);
        talloc_free(ft);
    }
    ctx->num_filters = 0;
}

static void filters_init(struct sd *sd)
{
    struct sd_ass_priv *ctx = sd->priv;

    filters_destroy(sd);

    for (int n = 0; filters[n]; n++) {
        struct sd_filter *ft = talloc_ptrtype(ctx, ft);
        *ft = (struct sd_filter){
            .global = sd->global,
            .log = sd->log,
            .opts = mp_get_config_group(ft, sd->global, &mp_sub_filter_opts),
            .driver = filters[n],
            .codec = "ass",
            .event_format = talloc_strdup(ft, ctx->ass_track->event_format),
        };
        if (ft->driver->init(ft)) {
            MP_TARRAY_APPEND(ctx, ctx->filters, ctx->num_filters, ft);
        } else {
            talloc_free(ft);
        }
    }
}

static void enable_output(struct sd *sd, bool enable)
{
    struct sd_ass_priv *ctx = sd->priv;
    if (enable == !!ctx->ass_renderer)
        return;
    if (ctx->ass_renderer) {
        ass_renderer_done(ctx->ass_renderer);
        ctx->ass_renderer = NULL;
    } else {
        ctx->ass_renderer = ass_renderer_init(ctx->ass_library);

        mp_ass_configure_fonts(ctx->ass_renderer, sd->opts->sub_style,
                               sd->global, sd->log);
    }
}

static void assobjects_init(struct sd *sd)
{
    struct sd_ass_priv *ctx = sd->priv;
    struct mp_subtitle_opts *opts = sd->opts;
    struct mp_subtitle_shared_opts *shared_opts = sd->shared_opts;

    ctx->ass_library = mp_ass_init(sd->global, sd->opts->sub_style, sd->log);
    ass_set_extract_fonts(ctx->ass_library, opts->use_embedded_fonts);

    add_subtitle_fonts(sd);

    if (shared_opts->ass_style_override[sd->order])
        ass_set_style_overrides(ctx->ass_library, opts->ass_style_override_list);

    ctx->ass_track = ass_new_track(ctx->ass_library);
    ctx->ass_track->track_type = TRACK_TYPE_ASS;

    ctx->shadow_track = ass_new_track(ctx->ass_library);
    ctx->shadow_track->PlayResX = MP_ASS_FONT_PLAYRESX;
    ctx->shadow_track->PlayResY = MP_ASS_FONT_PLAYRESY;
    mp_ass_add_default_styles(ctx->shadow_track, opts, shared_opts, sd->order);

    char *extradata = sd->codec->extradata;
    int extradata_size = sd->codec->extradata_size;
    if (ctx->converter) {
        extradata = lavc_conv_get_extradata(ctx->converter);
        extradata_size = extradata ? strlen(extradata) : 0;
    }
    if (extradata)
        ass_process_codec_private(ctx->ass_track, extradata, extradata_size);

    mp_ass_add_default_styles(ctx->ass_track, opts, shared_opts, sd->order);

#if LIBASS_VERSION >= 0x01302000
    ass_set_check_readorder(ctx->ass_track, sd->opts->sub_clear_on_seek ? 0 : 1);
#endif

    enable_output(sd, true);
}

static void assobjects_destroy(struct sd *sd)
{
    struct sd_ass_priv *ctx = sd->priv;

    ass_free_track(ctx->ass_track);
    ass_free_track(ctx->shadow_track);
    enable_output(sd, false);
    ass_library_done(ctx->ass_library);
}

static int init(struct sd *sd)
{
    struct sd_ass_priv *ctx = talloc_zero(sd, struct sd_ass_priv);
    sd->priv = ctx;

    // Note: accept "null" as alias for "ass", so EDL delay_open subtitle
    //       streams work.
    if (strcmp(sd->codec->codec, "ass") != 0 &&
        strcmp(sd->codec->codec, "null") != 0)
    {
        ctx->is_converted = true;
        ctx->converter = lavc_conv_create(sd);
        if (!ctx->converter)
            return -1;

        if (strcmp(sd->codec->codec, "eia_608") == 0)
            ctx->duration_unknown = 1;
    }

    assobjects_init(sd);
    filters_init(sd);

    ctx->packer = mp_ass_packer_alloc(ctx);

    return 0;
}

// Check if subtitle has events that would cause it to be animated inside {}
static bool is_animated(char *s)
{
    bool in_tag = false;
    bool valid_event = false;
    bool valid_tag = false;
    while (*s) {
        if (!in_tag && s[0] == '{')
            in_tag = true;
        if (s[0] == '\\') {
            s++;
            if (!s[0])
                break;
            if (s[0] == 'k' || s[0] == 'K' || s[0] == 't') {
                valid_event = true;
                continue;
            // just bruteforce the multi-letter ones
            } else if (s[0] == 'f') {
                if (!strncmp(s, "fad", 3)) {
                    valid_event = true;
                    continue;
                }
            } else if (s[0] == 'm') {
                if (!strncmp(s, "move", 4)) {
                    valid_event = true;
                    continue;
                }
            }
        }
        if (in_tag && valid_event && s[0] == '}') {
            valid_tag = true;
            break;
        } else if (s[0] == '}') {
            in_tag = false;
            valid_event = false;
            valid_tag = false;
        }
        s++;
    }
    return valid_tag;
}

// Note: pkt is not necessarily a fully valid refcounted packet.
static void filter_and_add(struct sd *sd, struct demux_packet *pkt)
{
    struct sd_ass_priv *ctx = sd->priv;
    struct demux_packet *orig_pkt = pkt;
    ASS_Track *track = ctx->ass_track;
    int old_n_events = track->n_events;

    for (int n = 0; n < ctx->num_filters; n++) {
        struct sd_filter *ft = ctx->filters[n];
        struct demux_packet *npkt = ft->driver->filter(ft, pkt);
        if (pkt != npkt && pkt != orig_pkt)
            talloc_free(pkt);
        pkt = npkt;
        if (!pkt)
            return;
    }

    ass_process_chunk(ctx->ass_track, pkt->buffer, pkt->len,
                      llrint(pkt->pts * 1000),
                      llrint(pkt->duration * 1000));

    // This bookkeeping is only ever needed for ASS subs
    if (!ctx->is_converted) {
        if (!pkt->seen) {
            for (int n = track->n_events - 1; n >= 0; n--) {
                if (n + 1 == old_n_events || pkt->animated)
                    break;
                ASS_Event *event = &track->events[n];
                pkt->animated = (event->Effect && event->Effect[0]) ||
                                 is_animated(event->Text);
            }
            MP_TARRAY_APPEND(ctx, ctx->packets_animated, ctx->num_packets_animated, pkt->animated);
        } else {
            pkt->animated = ctx->packets_animated[pkt->seen_pos];
        }
    }

    if (pkt != orig_pkt)
        talloc_free(pkt);
}

// Test if the packet with the given file position and pts was already consumed.
// Return false if the packet is new (and add it to the internal list), and
// return true if it was already seen.
static bool check_packet_seen(struct sd *sd, struct demux_packet *packet)
{
    struct sd_ass_priv *priv = sd->priv;
    int a = 0;
    int b = priv->num_seen_packets;
    while (a < b) {
        int mid = a + (b - a) / 2;
        struct seen_packet *seen_packet = &priv->seen_packets[mid];
        if (packet->pos == seen_packet->pos && packet->pts == seen_packet->pts) {
            packet->seen_pos = mid;
            return true;
        }
        if (packet->pos > seen_packet->pos ||
            (packet->pos == seen_packet->pos && packet->pts > seen_packet->pts)) {
            a = mid + 1;
        } else {
            b = mid;
        }
    }
    packet->seen_pos = a;
    MP_TARRAY_INSERT_AT(priv, priv->seen_packets, priv->num_seen_packets, a,
                        (struct seen_packet){packet->pos, packet->pts});
    return false;
}

#define UNKNOWN_DURATION (INT_MAX / 1000)

static void decode(struct sd *sd, struct demux_packet *packet)
{
    struct sd_ass_priv *ctx = sd->priv;
    ASS_Track *track = ctx->ass_track;

    packet->sub_duration = packet->duration;

    if (ctx->converter) {
        if (!sd->opts->sub_clear_on_seek && packet->pos >= 0 &&
            check_packet_seen(sd, packet))
            return;

        double sub_pts = 0;
        double sub_duration = 0;
        char **r = lavc_conv_decode(ctx->converter, packet, &sub_pts,
                                    &sub_duration);
        if (sd->opts->sub_stretch_durations ||
            packet->duration < 0 || sub_duration == UINT32_MAX) {
            if (!ctx->duration_unknown) {
                MP_WARN(sd, "Subtitle with unknown duration.\n");
                ctx->duration_unknown = true;
            }
            sub_duration = UNKNOWN_DURATION;
        }

        for (int n = 0; r && r[n]; n++) {
            struct demux_packet pkt2 = {
                .pts = sub_pts,
                .duration = sub_duration,
                .buffer = r[n],
                .len = strlen(r[n]),
            };
            filter_and_add(sd, &pkt2);
        }
        if (ctx->duration_unknown) {
            for (int n = track->n_events - 2; n >= 0; n--) {
                if (track->events[n].Duration == UNKNOWN_DURATION * 1000) {
                    if (track->events[n].Start != track->events[n + 1].Start) {
                        track->events[n].Duration = track->events[n + 1].Start -
                                                    track->events[n].Start;
                    } else {
                        track->events[n].Duration = track->events[n + 1].Duration;
                    }
                }
            }
        }
    } else {
        // Note that for this packet format, libass has an internal mechanism
        // for discarding duplicate (already seen) packets but we check this
        // anyways for our purposes for ASS subtitles.
        packet->seen = check_packet_seen(sd, packet);
        filter_and_add(sd, packet);
    }
}

static void configure_ass(struct sd *sd, struct mp_osd_res *dim,
                          bool converted, ASS_Track *track)
{
    struct mp_subtitle_opts *opts = sd->opts;
    struct mp_subtitle_shared_opts *shared_opts = sd->shared_opts;
    struct sd_ass_priv *ctx = sd->priv;
    ASS_Renderer *priv = ctx->ass_renderer;

    ass_set_frame_size(priv, dim->w, dim->h);
    ass_set_margins(priv, dim->mt, dim->mb, dim->ml, dim->mr);

    bool set_use_margins = false;
    float set_sub_pos = 0.0f;
    float set_line_spacing = 0;
    float set_font_scale = 1;
    int set_hinting = 0;
    bool set_scale_with_window = false;
    bool set_scale_by_window = true;
    bool total_override = false;
    // With forced overrides, apply the --sub-* specific options
    if (converted || shared_opts->ass_style_override[sd->order] == 3) { // 'force'
        set_scale_with_window = opts->sub_scale_with_window;
        set_use_margins = opts->sub_use_margins;
        set_scale_by_window = opts->sub_scale_by_window;
        total_override = true;
    } else {
        set_scale_with_window = opts->ass_scale_with_window;
        set_use_margins = opts->ass_use_margins;
    }
    if (converted || shared_opts->ass_style_override[sd->order]) {
        set_sub_pos = 100.0f - shared_opts->sub_pos[sd->order];
        set_line_spacing = opts->ass_line_spacing;
        set_hinting = opts->ass_hinting;
        set_font_scale = opts->sub_scale;
    }
    if (set_scale_with_window) {
        int vidh = dim->h - (dim->mt + dim->mb);
        set_font_scale *= dim->h / (float)MPMAX(vidh, 1);
    }
    if (!set_scale_by_window) {
        double factor = dim->h / 720.0;
        if (factor != 0.0)
            set_font_scale /= factor;
    }
    ass_set_use_margins(priv, set_use_margins);
    ass_set_line_position(priv, set_sub_pos);
    ass_set_shaper(priv, opts->ass_shaper);
    int set_force_flags = 0;
    if (total_override)
        set_force_flags |= ASS_OVERRIDE_BIT_STYLE | ASS_OVERRIDE_BIT_SELECTIVE_FONT_SCALE;
    if (shared_opts->ass_style_override[sd->order] == 4) // 'scale'
        set_force_flags |= ASS_OVERRIDE_BIT_SELECTIVE_FONT_SCALE;
    if (converted)
        set_force_flags |= ASS_OVERRIDE_BIT_ALIGNMENT;
#if LIBASS_VERSION >= 0x01306000
    if ((converted || shared_opts->ass_style_override[sd->order]) && opts->ass_justify)
        set_force_flags |= ASS_OVERRIDE_BIT_JUSTIFY;
#endif
    ass_set_selective_style_override_enabled(priv, set_force_flags);
    ASS_Style style = {0};
    mp_ass_set_style(&style, MP_ASS_FONT_PLAYRESY, opts->sub_style);
    ass_set_selective_style_override(priv, &style);
    free(style.FontName);
    if (converted && track->default_style < track->n_styles) {
        mp_ass_set_style(track->styles + track->default_style,
                         track->PlayResY, opts->sub_style);
    }
    ass_set_font_scale(priv, set_font_scale);
    ass_set_hinting(priv, set_hinting);
    ass_set_line_spacing(priv, set_line_spacing);
#if LIBASS_VERSION >= 0x01600010
    if (converted) {
        ass_track_set_feature(track, ASS_FEATURE_WRAP_UNICODE, 1);
        if (!opts->sub_vsfilter_bidi_compat) {
            for (int n = 0; n < track->n_styles; n++) {
                track->styles[n].Encoding = -1;
            }
            ass_track_set_feature(track, ASS_FEATURE_BIDI_BRACKETS, 1);
            ass_track_set_feature(track, ASS_FEATURE_WHOLE_TEXT_LAYOUT, 1);
        }
    }
#endif
    if (converted) {
        bool override_playres = true;
        char **ass_style_override_list = opts->ass_style_override_list;
        for (int i = 0; ass_style_override_list && ass_style_override_list[i]; i++) {
            if (bstr_find0(bstr0(ass_style_override_list[i]), "PlayResX") >= 0)
                override_playres = false;
        }

        // srt to ass conversion from ffmpeg has fixed PlayResX of 384 with an
        // aspect of 4:3. Starting with libass f08f8ea5 (pre 0.17) PlayResX
        // affects shadow and border widths, among others, so to render borders
        // and shadows correctly, we adjust PlayResX according to the DAR.
        // But PlayResX also affects margins, so we adjust those too.
        // This should ensure basic srt-to-ass ffmpeg conversion has correct
        // borders, but there could be other issues with some srt extensions
        // and/or different source formats which would be exposed over time.
        // Make these adjustments only if the user didn't set PlayResX.
        if (override_playres) {
            int vidw = dim->w - (dim->ml + dim->mr);
            int vidh = dim->h - (dim->mt + dim->mb);
            int old_playresx = track->PlayResX;
            track->PlayResX = track->PlayResY * (double)vidw / MPMAX(vidh, 1);
            double fix_margins = track->PlayResX / (double)old_playresx;
            for (int n = 0; n < track->n_styles; n++) {
                track->styles[n].MarginL = round(track->styles[n].MarginL * fix_margins);
                track->styles[n].MarginR = round(track->styles[n].MarginR * fix_margins);
            }
        }
    }
}

static bool has_overrides(char *s)
{
    if (!s)
        return false;
    return strstr(s, "\\pos") || strstr(s, "\\move") || strstr(s, "\\clip") ||
           strstr(s, "\\iclip") || strstr(s, "\\org") || strstr(s, "\\p");
}

#define END(ev) ((ev)->Start + (ev)->Duration)

static long long find_timestamp(struct sd *sd, double pts)
{
    struct sd_ass_priv *priv = sd->priv;
    if (pts == MP_NOPTS_VALUE)
        return 0;

    long long ts = llrint(pts * 1000);

    if (!sd->opts->sub_fix_timing || sd->shared_opts->ass_style_override[sd->order] == 0)
        return ts;

    // Try to fix small gaps and overlaps.
    ASS_Track *track = priv->ass_track;
    int threshold = SUB_GAP_THRESHOLD * 1000;
    int keep = SUB_GAP_KEEP * 1000;

    // Find the "current" event.
    ASS_Event *ev[2] = {0};
    int n_ev = 0;
    for (int n = 0; n < track->n_events; n++) {
        ASS_Event *event = &track->events[n];
        if (ts >= event->Start - threshold && ts <= END(event) + threshold) {
            if (n_ev >= MP_ARRAY_SIZE(ev))
                return ts; // multiple overlaps - give up (probably complex subs)
            ev[n_ev++] = event;
        }
    }

    if (n_ev != 2)
        return ts;

    // Simple/minor heuristic against destroying typesetting.
    if (ev[0]->Style != ev[1]->Style || has_overrides(ev[0]->Text) ||
        has_overrides(ev[1]->Text))
        return ts;

    // Sort by start timestamps.
    if (ev[0]->Start > ev[1]->Start)
        MPSWAP(ASS_Event*, ev[0], ev[1]);

    // We want to fix partial overlaps only.
    if (END(ev[0]) >= END(ev[1]))
        return ts;

    if (ev[0]->Duration < keep || ev[1]->Duration < keep)
        return ts;

    // Gap between the events -> move ts to show the end of the first event.
    if (ts >= END(ev[0]) && ts < ev[1]->Start && END(ev[0]) < ev[1]->Start &&
        END(ev[0]) + threshold >= ev[1]->Start)
        return END(ev[0]) - 1;

    // Overlap -> move ts to the (exclusive) end of the first event.
    // Relies on the fact that the ASS_Renderer has no overlap registered, even
    // if there is one. This happens to work because we never render the
    // overlapped state, and libass never resolves a collision.
    if (ts >= ev[1]->Start && ts <= END(ev[0]) && END(ev[0]) > ev[1]->Start &&
        END(ev[0]) <= ev[1]->Start + threshold)
        return END(ev[0]);

    return ts;
}

#undef END

static struct sub_b