When converting from a non alpha format to an alpha format, defaults to all ones rather than all zeroes

git-svn-id: svn://svn.mplayerhq.hu/mplayer/trunk@28756 b3059339-0415-0410-9bf9-f77b7e298cf2

6 files changed, 40 insertions, 37 deletions

diff --git a/libswscale/rgb2rgb.c b/libswscale/rgb2rgb.c
index 99be2a4f27..ad69265c37 100644
--- a/libswscale/rgb2rgb.c
+++ b/libswscale/rgb2rgb.c
@@ -94,6 +94,7 @@ DECLARE_ASM_CONST(8, uint64_t, mmx_one)      = 0xFFFFFFFFFFFFFFFFULL;
 DECLARE_ASM_CONST(8, uint64_t, mask32b)      = 0x000000FF000000FFULL;
 DECLARE_ASM_CONST(8, uint64_t, mask32g)      = 0x0000FF000000FF00ULL;
 DECLARE_ASM_CONST(8, uint64_t, mask32r)      = 0x00FF000000FF0000ULL;
+DECLARE_ASM_CONST(8, uint64_t, mask32a)      = 0xFF000000FF000000ULL;
 DECLARE_ASM_CONST(8, uint64_t, mask32)       = 0x00FFFFFF00FFFFFFULL;
 DECLARE_ASM_CONST(8, uint64_t, mask3216br)   = 0x00F800F800F800F8ULL;
 DECLARE_ASM_CONST(8, uint64_t, mask3216g)    = 0x0000FC000000FC00ULL;
@@ -281,7 +282,7 @@ void rgb24to32(const uint8_t *src, uint8_t *dst, long src_size)
     {
         #ifdef WORDS_BIGENDIAN
             /* RGB24 (= R,G,B) -> BGR32 (= A,R,G,B) */
-            dst[4*i + 0] = 0;
+            dst[4*i + 0] = 255;
             dst[4*i + 1] = src[3*i + 0];
             dst[4*i + 2] = src[3*i + 1];
             dst[4*i + 3] = src[3*i + 2];
@@ -289,7 +290,7 @@ void rgb24to32(const uint8_t *src, uint8_t *dst, long src_size)
             dst[4*i + 0] = src[3*i + 2];
             dst[4*i + 1] = src[3*i + 1];
             dst[4*i + 2] = src[3*i + 0];
-            dst[4*i + 3] = 0;
+            dst[4*i + 3] = 255;
         #endif
     }
 }
@@ -305,7 +306,7 @@ void rgb16tobgr32(const uint8_t *src, uint8_t *dst, long src_size)
         register uint16_t bgr;
         bgr = *s++;
         #ifdef WORDS_BIGENDIAN
-            *d++ = 0;
+            *d++ = 255;
             *d++ = (bgr&0x1F)<<3;
             *d++ = (bgr&0x7E0)>>3;
             *d++ = (bgr&0xF800)>>8;
@@ -313,7 +314,7 @@ void rgb16tobgr32(const uint8_t *src, uint8_t *dst, long src_size)
             *d++ = (bgr&0xF800)>>8;
             *d++ = (bgr&0x7E0)>>3;
             *d++ = (bgr&0x1F)<<3;
-            *d++ = 0;
+            *d++ = 255;
         #endif
     }
 }
@@ -369,7 +370,7 @@ void rgb15tobgr32(const uint8_t *src, uint8_t *dst, long src_size)
         register uint16_t bgr;
         bgr = *s++;
         #ifdef WORDS_BIGENDIAN
-            *d++ = 0;
+            *d++ = 255;
             *d++ = (bgr&0x1F)<<3;
             *d++ = (bgr&0x3E0)>>2;
             *d++ = (bgr&0x7C00)>>7;
@@ -377,7 +378,7 @@ void rgb15tobgr32(const uint8_t *src, uint8_t *dst, long src_size)
             *d++ = (bgr&0x7C00)>>7;
             *d++ = (bgr&0x3E0)>>2;
             *d++ = (bgr&0x1F)<<3;
-            *d++ = 0;
+            *d++ = 255;
         #endif
     }
 }
diff --git a/libswscale/rgb2rgb_template.c b/libswscale/rgb2rgb_template.c
index f8915e8df2..e95b628049 100644
--- a/libswscale/rgb2rgb_template.c
+++ b/libswscale/rgb2rgb_template.c
@@ -83,7 +83,7 @@ static inline void RENAME(rgb24tobgr32)(const uint8_t *src, uint8_t *dst, long s
     #if HAVE_MMX
         __asm__ volatile(PREFETCH"    %0"::"m"(*s):"memory");
         mm_end = end - 23;
-        __asm__ volatile("movq        %0, %%mm7"::"m"(mask32):"memory");
+        __asm__ volatile("movq        %0, %%mm7"::"m"(mask32a):"memory");
         while (s < mm_end)
         {
             __asm__ volatile(
@@ -96,10 +96,10 @@ static inline void RENAME(rgb24tobgr32)(const uint8_t *src, uint8_t *dst, long s
             "punpckldq   15%1, %%mm2    \n\t"
             "movd        18%1, %%mm3    \n\t"
             "punpckldq   21%1, %%mm3    \n\t"
-            "pand       %%mm7, %%mm0    \n\t"
-            "pand       %%mm7, %%mm1    \n\t"
-            "pand       %%mm7, %%mm2    \n\t"
-            "pand       %%mm7, %%mm3    \n\t"
+            "por        %%mm7, %%mm0    \n\t"
+            "por        %%mm7, %%mm1    \n\t"
+            "por        %%mm7, %%mm2    \n\t"
+            "por        %%mm7, %%mm3    \n\t"
             MOVNTQ"     %%mm0,   %0     \n\t"
             MOVNTQ"     %%mm1,  8%0     \n\t"
             MOVNTQ"     %%mm2, 16%0     \n\t"
@@ -117,7 +117,7 @@ static inline void RENAME(rgb24tobgr32)(const uint8_t *src, uint8_t *dst, long s
     {
     #ifdef WORDS_BIGENDIAN
         /* RGB24 (= R,G,B) -> RGB32 (= A,B,G,R) */
-        *dest++ = 0;
+        *dest++ = 255;
         *dest++ = s[2];
         *dest++ = s[1];
         *dest++ = s[0];
@@ -126,7 +126,7 @@ static inline void RENAME(rgb24tobgr32)(const uint8_t *src, uint8_t *dst, long s
         *dest++ = *s++;
         *dest++ = *s++;
         *dest++ = *s++;
-        *dest++ = 0;
+        *dest++ = 255;
     #endif
     }
 }
@@ -1265,7 +1265,7 @@ static inline void RENAME(rgb15to32)(const uint8_t *src, uint8_t *dst, long src_
         register uint16_t bgr;
         bgr = *s++;
 #ifdef WORDS_BIGENDIAN
-        *d++ = 0;
+        *d++ = 255;
         *d++ = (bgr&0x7C00)>>7;
         *d++ = (bgr&0x3E0)>>2;
         *d++ = (bgr&0x1F)<<3;
@@ -1273,7 +1273,7 @@ static inline void RENAME(rgb15to32)(const uint8_t *src, uint8_t *dst, long src_
         *d++ = (bgr&0x1F)<<3;
         *d++ = (bgr&0x3E0)>>2;
         *d++ = (bgr&0x7C00)>>7;
-        *d++ = 0;
+        *d++ = 255;
 #endif
 
 #endif
@@ -1339,7 +1339,7 @@ static inline void RENAME(rgb16to32)(const uint8_t *src, uint8_t *dst, long src_
         register uint16_t bgr;
         bgr = *s++;
 #ifdef WORDS_BIGENDIAN
-        *d++ = 0;
+        *d++ = 255;
         *d++ = (bgr&0xF800)>>8;
         *d++ = (bgr&0x7E0)>>3;
         *d++ = (bgr&0x1F)<<3;
@@ -1347,7 +1347,7 @@ static inline void RENAME(rgb16to32)(const uint8_t *src, uint8_t *dst, long src_
         *d++ = (bgr&0x1F)<<3;
         *d++ = (bgr&0x7E0)>>3;
         *d++ = (bgr&0xF800)>>8;
-        *d++ = 0;
+        *d++ = 255;
 #endif
     }
 }
diff --git a/libswscale/swscale.c b/libswscale/swscale.c
index 1fe421e0d6..7c335f1680 100644
--- a/libswscale/swscale.c
+++ b/libswscale/swscale.c
@@ -922,12 +922,12 @@ static inline void yuv2rgbXinC_full(SwsContext *c, int16_t *lumFilter, int16_t *
     switch(c->dstFormat){
     case PIX_FMT_ARGB:
         dest++;
-        aidx= 0;
+        aidx= -1;
     case PIX_FMT_RGB24:
         aidx--;
     case PIX_FMT_RGBA:
         YSCALE_YUV_2_RGBX_FULL_C(1<<21)
-            dest[aidx]= 0;
+            dest[aidx]= 255;
             dest[0]= R>>22;
             dest[1]= G>>22;
             dest[2]= B>>22;
@@ -936,12 +936,12 @@ static inline void yuv2rgbXinC_full(SwsContext *c, int16_t *lumFilter, int16_t *
         break;
     case PIX_FMT_ABGR:
         dest++;
-        aidx= 0;
+        aidx= -1;
     case PIX_FMT_BGR24:
         aidx--;
     case PIX_FMT_BGRA:
         YSCALE_YUV_2_RGBX_FULL_C(1<<21)
-            dest[aidx]= 0;
+            dest[aidx]= 255;
             dest[0]= B>>22;
             dest[1]= G>>22;
             dest[2]= R>>22;
diff --git a/libswscale/swscale_template.c b/libswscale/swscale_template.c
index ba2b6f99c6..3262b6ee85 100644
--- a/libswscale/swscale_template.c
+++ b/libswscale/swscale_template.c
@@ -1031,7 +1031,7 @@ static inline void RENAME(yuv2packedX)(SwsContext *c, int16_t *lumFilter, int16_
             case PIX_FMT_RGB32:
                 YSCALEYUV2PACKEDX_ACCURATE
                 YSCALEYUV2RGBX
-                "pxor %%mm7, %%mm7 \n\t"
+                "pcmpeqd %%mm7, %%mm7 \n\t"
                 WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
 
                 YSCALEYUV2PACKEDX_END
@@ -1097,7 +1097,7 @@ static inline void RENAME(yuv2packedX)(SwsContext *c, int16_t *lumFilter, int16_
             case PIX_FMT_RGB32:
                 YSCALEYUV2PACKEDX
                 YSCALEYUV2RGBX
-                "pxor %%mm7, %%mm7 \n\t"
+                "pcmpeqd %%mm7, %%mm7 \n\t"
                 WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
                 YSCALEYUV2PACKEDX_END
                 return;
@@ -1196,7 +1196,7 @@ static inline void RENAME(yuv2packed2)(SwsContext *c, uint16_t *buf0, uint16_t *
                 "mov        %4, %%"REG_b"               \n\t"
                 "push %%"REG_BP"                        \n\t"
                 YSCALEYUV2RGB(%%REGBP, %5)
-                "pxor    %%mm7, %%mm7                   \n\t"
+                "pcmpeqd %%mm7, %%mm7                   \n\t"
                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
                 "pop %%"REG_BP"                         \n\t"
                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
@@ -1312,7 +1312,7 @@ static inline void RENAME(yuv2packed1)(SwsContext *c, uint16_t *buf0, uint16_t *
                 "mov        %4, %%"REG_b"               \n\t"
                 "push %%"REG_BP"                        \n\t"
                 YSCALEYUV2RGB1(%%REGBP, %5)
-                "pxor    %%mm7, %%mm7                   \n\t"
+                "pcmpeqd %%mm7, %%mm7                   \n\t"
                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
                 "pop %%"REG_BP"                         \n\t"
                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
@@ -1405,7 +1405,7 @@ static inline void RENAME(yuv2packed1)(SwsContext *c, uint16_t *buf0, uint16_t *
                 "mov        %4, %%"REG_b"               \n\t"
                 "push %%"REG_BP"                        \n\t"
                 YSCALEYUV2RGB1b(%%REGBP, %5)
-                "pxor    %%mm7, %%mm7                   \n\t"
+                "pcmpeqd %%mm7, %%mm7                   \n\t"
                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
                 "pop %%"REG_BP"                         \n\t"
                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
@@ -1642,7 +1642,7 @@ BGR2Y(uint16_t, bgr15ToY, 0, 0, 0, 0x001F, 0x03E0, 0x7C00, RY<<10, GY<<5, BY
 BGR2Y(uint16_t, rgb16ToY, 0, 0, 0, 0xF800, 0x07E0, 0x001F, RY    , GY<<5, BY<<11, RGB2YUV_SHIFT+8)
 BGR2Y(uint16_t, rgb15ToY, 0, 0, 0, 0x7C00, 0x03E0, 0x001F, RY    , GY<<5, BY<<10, RGB2YUV_SHIFT+7)
 
-#define BGR2UV(type, name, shr, shg, shb, maskr, maskg, maskb, RU, GU, BU, RV, GV, BV, S)\
+#define BGR2UV(type, name, shr, shg, shb, maska, maskr, maskg, maskb, RU, GU, BU, RV, GV, BV, S)\
 static inline void RENAME(name)(uint8_t *dstU, uint8_t *dstV, uint8_t *src, uint8_t *dummy, long width, uint32_t *unused)\
 {\
     int i;\
@@ -1663,9 +1663,10 @@ static inline void RENAME(name ## _half)(uint8_t *dstU, uint8_t *dstV, uint8_t *
     {\
         int pix0= ((type*)src)[2*i+0];\
         int pix1= ((type*)src)[2*i+1];\
-        int g= (pix0&maskg)+(pix1&maskg);\
+        int g= (pix0&(maskg|maska))+(pix1&(maskg|maska));\
         int b= ((pix0+pix1-g)&(maskb|(2*maskb)))>>shb;\
         int r= ((pix0+pix1-g)&(maskr|(2*maskr)))>>shr;\
+        g&= maskg|(2*maskg);\
 \
         g>>=shg;\
 \
@@ -1674,12 +1675,12 @@ static inline void RENAME(name ## _half)(uint8_t *dstU, uint8_t *dstV, uint8_t *
     }\
 }
 
-BGR2UV(uint32_t, bgr32ToUV,16, 0, 0, 0xFF0000, 0xFF00, 0x00FF, RU<< 8, GU   , BU<< 8, RV<< 8, GV   , BV<< 8, RGB2YUV_SHIFT+8)
-BGR2UV(uint32_t, rgb32ToUV, 0, 0,16, 0x00FF, 0xFF00, 0xFF0000, RU<< 8, GU   , BU<< 8, RV<< 8, GV   , BV<< 8, RGB2YUV_SHIFT+8)
-BGR2UV(uint16_t, bgr16ToUV, 0, 0, 0, 0x001F, 0x07E0, 0xF800, RU<<11, GU<<5, BU    , RV<<11, GV<<5, BV    , RGB2YUV_SHIFT+8)
-BGR2UV(uint16_t, bgr15ToUV, 0, 0, 0, 0x001F, 0x03E0, 0x7C00, RU<<10, GU<<5, BU    , RV<<10, GV<<5, BV    , RGB2YUV_SHIFT+7)
-BGR2UV(uint16_t, rgb16ToUV, 0, 0, 0, 0xF800, 0x07E0, 0x001F, RU    , GU<<5, BU<<11, RV    , GV<<5, BV<<11, RGB2YUV_SHIFT+8)
-BGR2UV(uint16_t, rgb15ToUV, 0, 0, 0, 0x7C00, 0x03E0, 0x001F, RU    , GU<<5, BU<<10, RV    , GV<<5, BV<<10, RGB2YUV_SHIFT+7)
+BGR2UV(uint32_t, bgr32ToUV,16, 0, 0, 0xFF000000, 0xFF0000, 0xFF00,   0x00FF, RU<< 8, GU   , BU<< 8, RV<< 8, GV   , BV<< 8, RGB2YUV_SHIFT+8)
+BGR2UV(uint32_t, rgb32ToUV, 0, 0,16, 0xFF000000,   0x00FF, 0xFF00, 0xFF0000, RU<< 8, GU   , BU<< 8, RV<< 8, GV   , BV<< 8, RGB2YUV_SHIFT+8)
+BGR2UV(uint16_t, bgr16ToUV, 0, 0, 0,          0,   0x001F, 0x07E0,   0xF800, RU<<11, GU<<5, BU    , RV<<11, GV<<5, BV    , RGB2YUV_SHIFT+8)
+BGR2UV(uint16_t, bgr15ToUV, 0, 0, 0,          0,   0x001F, 0x03E0,   0x7C00, RU<<10, GU<<5, BU    , RV<<10, GV<<5, BV    , RGB2YUV_SHIFT+7)
+BGR2UV(uint16_t, rgb16ToUV, 0, 0, 0,          0,   0xF800, 0x07E0,   0x001F, RU    , GU<<5, BU<<11, RV    , GV<<5, BV<<11, RGB2YUV_SHIFT+8)
+BGR2UV(uint16_t, rgb15ToUV, 0, 0, 0,          0,   0x7C00, 0x03E0,   0x001F, RU    , GU<<5, BU<<10, RV    , GV<<5, BV<<10, RGB2YUV_SHIFT+7)
 
 #if HAVE_MMX
 static inline void RENAME(bgr24ToY_mmx)(uint8_t *dst, uint8_t *src, long width, int srcFormat)
diff --git a/libswscale/yuv2rgb.c b/libswscale/yuv2rgb.c
index 53a0d4e7b8..0a8cd99a97 100644
--- a/libswscale/yuv2rgb.c
+++ b/libswscale/yuv2rgb.c
@@ -533,7 +533,7 @@ av_cold int sws_yuv2rgb_c_init_tables(SwsContext *c, const int inv_table[4], int
     uint8_t *y_table;
     uint16_t *y_table16;
     uint32_t *y_table32;
-    int i, base, rbase, gbase, bbase;
+    int i, base, rbase, gbase, bbase, abase;
     const int yoffs = fullRange ? 384 : 326;
 
     int64_t crv =  inv_table[0];
@@ -659,12 +659,13 @@ av_cold int sws_yuv2rgb_c_init_tables(SwsContext *c, const int inv_table[4], int
         rbase = base + (isRgb ? 16 : 0);
         gbase = base + 8;
         bbase = base + (isRgb ? 0 : 16);
+        abase = (c->dstFormat == PIX_FMT_RGBA || c->dstFormat == PIX_FMT_BGRA) ? 24 : 0;
         c->yuvTable = av_malloc(1024*3*4);
         y_table32 = c->yuvTable;
         yb = -(384<<16) - oy;
         for (i = 0; i < 1024; i++) {
             uint8_t yval = av_clip_uint8((yb + 0x8000) >> 16);
-            y_table32[i     ] = yval << rbase;
+            y_table32[i     ] = (yval << rbase) + (255 << abase);
             y_table32[i+1024] = yval << gbase;
             y_table32[i+2048] = yval << bbase;
             yb += cy;
diff --git a/libswscale/yuv2rgb_template.c b/libswscale/yuv2rgb_template.c
index 0b5e6f9ca1..f55568b0ab 100644
--- a/libswscale/yuv2rgb_template.c
+++ b/libswscale/yuv2rgb_template.c
@@ -446,7 +446,7 @@ static inline int RENAME(yuv420_rgb32)(SwsContext *c, uint8_t* src[], int srcStr
 
         YUV2RGB_INIT
         YUV2RGB
-        "pxor      %%mm3, %%mm3;"   /* zero mm3 */
+        "pcmpeqd   %%mm3, %%mm3;"   /* fill mm3 */
         RGB_PLANAR2PACKED32
 
     YUV2RGB_ENDLOOP(4)