sub: switch to premultiplied alpha

Fixes problems with ugly borders.

Note that at least in the DVD sub case, we could have just set all
transparent pixels to black to solve this.

vo_direct3d.c change untested, because mingw is a miserable pile of
crap.

5 files changed, 35 insertions, 7 deletions

diff --git a/libvo/gl_osd.c b/libvo/gl_osd.c
index 74053afd4a..5266dca4ac 100644
--- a/libvo/gl_osd.c
+++ b/libvo/gl_osd.c
@@ -32,7 +32,7 @@ struct osd_fmt_entry {
 // glBlendFunc() arguments
 static const int blend_factors[SUBBITMAP_COUNT][2] = {
     [SUBBITMAP_LIBASS] = {GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA},
-    [SUBBITMAP_RGBA] =   {GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA},
+    [SUBBITMAP_RGBA] =   {GL_ONE,       GL_ONE_MINUS_SRC_ALPHA},
 };
 
 static const struct osd_fmt_entry osd_to_gl3_formats[SUBBITMAP_COUNT] = {
diff --git a/libvo/vo_direct3d.c b/libvo/vo_direct3d.c
index 4f16c6a6b1..2b8f3429e4 100644
--- a/libvo/vo_direct3d.c
+++ b/libvo/vo_direct3d.c
@@ -2049,6 +2049,9 @@ static void draw_osd(d3d_priv *priv, struct sub_bitmaps *imgs)
 
         IDirect3DDevice9_SetTextureStageState(priv->d3d_device, 0,
                                             D3DTSS_ALPHAOP, D3DTOP_MODULATE);
+    } else {
+        IDirect3DDevice9_SetRenderState(priv->d3d_device, D3DRS_SRCBLEND,
+                                        D3DBLEND_ONE);
     }
 
     IDirect3DDevice9_SetFVF(priv->d3d_device, D3DFVF_EOSD_VERTEX);
@@ -2060,6 +2063,8 @@ static void draw_osd(d3d_priv *priv, struct sub_bitmaps *imgs)
                                           D3DTSS_COLORARG1, D3DTA_TEXTURE);
     IDirect3DDevice9_SetTextureStageState(priv->d3d_device, 0,
                                           D3DTSS_ALPHAOP, D3DTOP_SELECTARG1);
+    IDirect3DDevice9_SetRenderState(priv->d3d_device,
+                                    D3DRS_SRCBLEND, D3DBLEND_SRCALPHA);
 
     IDirect3DDevice9_SetTexture(priv->d3d_device, 0, NULL);
 
diff --git a/libvo/vo_vdpau.c b/libvo/vo_vdpau.c
index 34aa9824dc..94d37e3018 100644
--- a/libvo/vo_vdpau.c
+++ b/libvo/vo_vdpau.c
@@ -970,14 +970,21 @@ static void draw_eosd(struct vo *vo, int index)
         .blend_equation_alpha = VDP_OUTPUT_SURFACE_RENDER_BLEND_EQUATION_ADD,
     };
 
+    VdpOutputSurfaceRenderBlendState blend_state_premultiplied = blend_state;
+    blend_state_premultiplied.blend_factor_source_color =
+            VDP_OUTPUT_SURFACE_RENDER_BLEND_FACTOR_ONE;
+
     for (i = 0; i < sfc->render_count; i++) {
+        VdpOutputSurfaceRenderBlendState *blend = &blend_state;
+        if (sfc->format == VDP_RGBA_FORMAT_B8G8R8A8)
+            blend = &blend_state_premultiplied;
         vdp_st = vdp->
             output_surface_render_bitmap_surface(output_surface,
                                                  &sfc->targets[i].dest,
                                                  sfc->surface,
                                                  &sfc->targets[i].source,
                                                  &sfc->targets[i].color,
-                                                 &blend_state,
+                                                 blend,
                                                  VDP_OUTPUT_SURFACE_RENDER_ROTATE_0);
         CHECK_ST_WARNING("EOSD: Error when rendering");
     }
diff --git a/sub/img_convert.c b/sub/img_convert.c
index b8a769f052..888380cf70 100644
--- a/sub/img_convert.c
+++ b/sub/img_convert.c
@@ -166,6 +166,21 @@ bool osd_conv_ass_to_old_p(struct osd_conv_cache *c, struct sub_bitmaps *imgs)
     return true;
 }
 
+static void rgba_to_premultiplied_rgba(uint32_t *colors, size_t count)
+{
+    for (int n = 0; n < count; n++) {
+        uint32_t c = colors[n];
+        int b = c & 0xFF;
+        int g = (c >> 8) & 0xFF;
+        int r = (c >> 16) & 0xFF;
+        int a = (c >> 24) & 0xFF;
+        b = b * a / 255;
+        g = g * a / 255;
+        r = r * a / 255;
+        colors[n] = b | (g << 8) | (r << 16) | (a << 24);
+    }
+}
+
 bool osd_conv_idx_to_rgba(struct osd_conv_cache *c, struct sub_bitmaps *imgs)
 {
     struct sub_bitmaps src = *imgs;
@@ -179,18 +194,19 @@ bool osd_conv_idx_to_rgba(struct osd_conv_cache *c, struct sub_bitmaps *imgs)
     for (int n = 0; n < src.num_parts; n++) {
         struct sub_bitmap *d = &imgs->parts[n];
         struct sub_bitmap *s = &src.parts[n];
-        struct osd_bmp_indexed *sb = s->bitmap;
+        struct osd_bmp_indexed sb = *(struct osd_bmp_indexed *)s->bitmap;
+
+        rgba_to_premultiplied_rgba(sb.palette, 256);
 
         *d = *s;
         d->stride = s->w * 4;
         d->bitmap = talloc_size(c->parts, s->h * d->stride);
 
-        uint32_t *palette = sb->palette;
         uint32_t *outbmp = d->bitmap;
         for (int y = 0; y < s->h; y++) {
-            uint8_t *inbmp = sb->bitmap + y * s->stride;
+            uint8_t *inbmp = sb.bitmap + y * s->stride;
             for (int x = 0; x < s->w; x++)
-                *outbmp++ = palette[*inbmp++];
+                *outbmp++ = sb.palette[*inbmp++];
         }
     }
     return true;
diff --git a/sub/sub.h b/sub/sub.h
index 8649018029..649f196414 100644
--- a/sub/sub.h
+++ b/sub/sub.h
@@ -30,7 +30,7 @@ struct sub_render_params;
 enum sub_bitmap_format {
     SUBBITMAP_EMPTY = 0,// no bitmaps; always has num_parts==0
     SUBBITMAP_LIBASS,   // A8, with a per-surface blend color (libass.color)
-    SUBBITMAP_RGBA,     // B8G8R8A8 (MSB=A, LSB=B), can be scaled
+    SUBBITMAP_RGBA,     // B8G8R8A8 (MSB=A, LSB=B), scaled, premultiplied alpha
     SUBBITMAP_INDEXED,  // scaled, bitmap points to osd_bmp_indexed
     SUBBITMAP_OLD_PLANAR, // like previous, but bitmap points to old_osd_planar