mmx implementation of yadif. 3.5x faster.

git-svn-id: svn://svn.mplayerhq.hu/mplayer/trunk@19836 b3059339-0415-0410-9bf9-f77b7e298cf2

1 files changed, 228 insertions, 1 deletions

diff --git a/libmpcodecs/vf_yadif.c b/libmpcodecs/vf_yadif.c
index c79225d8f8..f58c7718d6 100644
--- a/libmpcodecs/vf_yadif.c
+++ b/libmpcodecs/vf_yadif.c
@@ -23,6 +23,7 @@
 #include <math.h>
 
 #include "config.h"
+#include "cpudetect.h"
 
 #include "mp_msg.h"
 
@@ -55,6 +56,8 @@ struct vf_priv_s {
     uint8_t *ref[4][3];
 };
 
+static void (*filter_line)(struct vf_priv_s *p, uint8_t *dst, uint8_t *prev, uint8_t *cur, uint8_t *next, int w, int refs, int parity);
+
 static void store_ref(struct vf_priv_s *p, uint8_t *src[3], int src_stride[3], int width, int height){
     int i;
 
@@ -68,7 +71,226 @@ static void store_ref(struct vf_priv_s *p, uint8_t *src[3], int src_stride[3], i
     }
 }
 
-static void filter_line(struct vf_priv_s *p, uint8_t *dst, uint8_t *prev, uint8_t *cur, uint8_t *next, int w, int refs, int parity){
+#if __GNUC__ < 3 || (__GNUC__ == 3 && __GNUC_MINOR__ == 0)
+#undef HAVE_MMX
+#endif
+
+#ifdef HAVE_MMX
+
+#define LOAD4(mem,dst) \
+            "movd      "mem", "#dst" \n\t"\
+            "punpcklbw %%mm7, "#dst" \n\t"
+
+#define PABS(tmp,dst) \
+            "pxor     "#tmp", "#tmp" \n\t"\
+            "psubw    "#dst", "#tmp" \n\t"\
+            "pmaxsw   "#tmp", "#dst" \n\t"
+
+#define CHECK(pj,mj) \
+            "movq "#pj"-1(%[cur],%[mrefs]), %%mm2 \n\t" /* cur[x-refs-1+j] */\
+            "movq "#mj"-1(%[cur],%[prefs]), %%mm3 \n\t" /* cur[x+refs-1-j] */\
+            "movq      %%mm2, %%mm4 \n\t"\
+            "movq      %%mm2, %%mm5 \n\t"\
+            "pxor      %%mm3, %%mm4 \n\t"\
+            "pavgb     %%mm3, %%mm5 \n\t"\
+            "pand     %[pb1], %%mm4 \n\t"\
+            "psubusb   %%mm4, %%mm5 \n\t"\
+            "psrlq     $8,    %%mm5 \n\t"\
+            "punpcklbw %%mm7, %%mm5 \n\t" /* (cur[x-refs+j] + cur[x+refs-j])>>1 */\
+            "movq      %%mm2, %%mm4 \n\t"\
+            "psubusb   %%mm3, %%mm2 \n\t"\
+            "psubusb   %%mm4, %%mm3 \n\t"\
+            "pmaxub    %%mm3, %%mm2 \n\t"\
+            "movq      %%mm2, %%mm3 \n\t"\
+            "movq      %%mm2, %%mm4 \n\t" /* ABS(cur[x-refs-1+j] - cur[x+refs-1-j]) */\
+            "psrlq      $8,   %%mm3 \n\t" /* ABS(cur[x-refs  +j] - cur[x+refs  -j]) */\
+            "psrlq     $16,   %%mm4 \n\t" /* ABS(cur[x-refs+1+j] - cur[x+refs+1-j]) */\
+            "punpcklbw %%mm7, %%mm2 \n\t"\
+            "punpcklbw %%mm7, %%mm3 \n\t"\
+            "punpcklbw %%mm7, %%mm4 \n\t"\
+            "paddw     %%mm3, %%mm2 \n\t"\
+            "paddw     %%mm4, %%mm2 \n\t" /* score */
+
+#define CHECK1 \
+            "movq      %%mm0, %%mm3 \n\t"\
+            "pcmpgtw   %%mm2, %%mm3 \n\t" /* if(score < spatial_score) */\
+            "pminsw    %%mm2, %%mm0 \n\t" /* spatial_score= score; */\
+            "movq      %%mm3, %%mm6 \n\t"\
+            "pand      %%mm3, %%mm5 \n\t"\
+            "pandn     %%mm1, %%mm3 \n\t"\
+            "por       %%mm5, %%mm3 \n\t"\
+            "movq      %%mm3, %%mm1 \n\t" /* spatial_pred= (cur[x-refs+j] + cur[x+refs-j])>>1; */
+
+#define CHECK2 /* pretend not to have checked dir=2 if dir=1 was bad.\
+                  hurts both quality and speed, but matches the C version. */\
+            "paddw    %[pw1], %%mm6 \n\t"\
+            "psllw     $14,   %%mm6 \n\t"\
+            "paddsw    %%mm6, %%mm2 \n\t"\
+            "movq      %%mm0, %%mm3 \n\t"\
+            "pcmpgtw   %%mm2, %%mm3 \n\t"\
+            "pminsw    %%mm2, %%mm0 \n\t"\
+            "pand      %%mm3, %%mm5 \n\t"\
+            "pandn     %%mm1, %%mm3 \n\t"\
+            "por       %%mm5, %%mm3 \n\t"\
+            "movq      %%mm3, %%mm1 \n\t"
+
+static void filter_line_mmx2(struct vf_priv_s *p, uint8_t *dst, uint8_t *prev, uint8_t *cur, uint8_t *next, int w, int refs, int parity){
+    static const uint64_t pw_1 = 0x0001000100010001ULL;
+    static const uint64_t pb_1 = 0x0101010101010101ULL;
+    const int mode = p->mode;
+    uint64_t tmp0, tmp1, tmp2, tmp3;
+    int x;
+
+#define FILTER\
+    for(x=0; x<w; x+=4){\
+        asm volatile(\
+            "pxor      %%mm7, %%mm7 \n\t"\
+            LOAD4("(%[cur],%[mrefs])", %%mm0) /* c = cur[x-refs] */\
+            LOAD4("(%[cur],%[prefs])", %%mm1) /* e = cur[x+refs] */\
+            LOAD4("(%["prev2"])", %%mm2) /* prev2[x] */\
+            LOAD4("(%["next2"])", %%mm3) /* next2[x] */\
+            "movq      %%mm3, %%mm4 \n\t"\
+            "paddw     %%mm2, %%mm3 \n\t"\
+            "psraw     $1,    %%mm3 \n\t" /* d = (prev2[x] + next2[x])>>1 */\
+            "movq      %%mm0, %[tmp0] \n\t" /* c */\
+            "movq      %%mm3, %[tmp1] \n\t" /* d */\
+            "movq      %%mm1, %[tmp2] \n\t" /* e */\
+            "psubw     %%mm4, %%mm2 \n\t"\
+            PABS(      %%mm4, %%mm2) /* temporal_diff0 */\
+            LOAD4("(%[prev],%[mrefs])", %%mm3) /* prev[x-refs] */\
+            LOAD4("(%[prev],%[prefs])", %%mm4) /* prev[x+refs] */\
+            "psubw     %%mm0, %%mm3 \n\t"\
+            "psubw     %%mm1, %%mm4 \n\t"\
+            PABS(      %%mm5, %%mm3)\
+            PABS(      %%mm5, %%mm4)\
+            "paddw     %%mm4, %%mm3 \n\t" /* temporal_diff1 */\
+            "psrlw     $1,    %%mm2 \n\t"\
+            "psrlw     $1,    %%mm3 \n\t"\
+            "pmaxsw    %%mm3, %%mm2 \n\t"\
+            LOAD4("(%[next],%[mrefs])", %%mm3) /* next[x-refs] */\
+            LOAD4("(%[next],%[prefs])", %%mm4) /* next[x+refs] */\
+            "psubw     %%mm0, %%mm3 \n\t"\
+            "psubw     %%mm1, %%mm4 \n\t"\
+            PABS(      %%mm5, %%mm3)\
+            PABS(      %%mm5, %%mm4)\
+            "paddw     %%mm4, %%mm3 \n\t" /* temporal_diff2 */\
+            "psrlw     $1,    %%mm3 \n\t"\
+            "pmaxsw    %%mm3, %%mm2 \n\t"\
+            "movq      %%mm2, %[tmp3] \n\t" /* diff */\
+\
+            "paddw     %%mm0, %%mm1 \n\t"\
+            "paddw     %%mm0, %%mm0 \n\t"\
+            "psubw     %%mm1, %%mm0 \n\t"\
+            "psrlw     $1,    %%mm1 \n\t" /* spatial_pred */\
+            PABS(      %%mm2, %%mm0)      /* ABS(c-e) */\
+\
+            "movq -1(%[cur],%[mrefs]), %%mm2 \n\t" /* cur[x-refs-1] */\
+            "movq -1(%[cur],%[prefs]), %%mm3 \n\t" /* cur[x+refs-1] */\
+            "movq      %%mm2, %%mm4 \n\t"\
+            "psubusb   %%mm3, %%mm2 \n\t"\
+            "psubusb   %%mm4, %%mm3 \n\t"\
+            "pmaxub    %%mm3, %%mm2 \n\t"\
+            "pshufw $9,%%mm2, %%mm3 \n\t"\
+            "punpcklbw %%mm7, %%mm2 \n\t" /* ABS(cur[x-refs-1] - cur[x+refs-1]) */\
+            "punpcklbw %%mm7, %%mm3 \n\t" /* ABS(cur[x-refs+1] - cur[x+refs+1]) */\
+            "paddw     %%mm2, %%mm0 \n\t"\
+            "paddw     %%mm3, %%mm0 \n\t"\
+            "psubw    %[pw1], %%mm0 \n\t" /* spatial_score */\
+\
+            CHECK(-1,1)\
+            CHECK1\
+            CHECK(-2,2)\
+            CHECK2\
+            CHECK(1,-1)\
+            CHECK1\
+            CHECK(2,-2)\
+            CHECK2\
+\
+            /* if(p->mode<2) ... */\
+            "movq    %[tmp3], %%mm6 \n\t" /* diff */\
+            "cmp       $2, %[mode] \n\t"\
+            "jge       1f \n\t"\
+            LOAD4("(%["prev2"],%[mrefs],2)", %%mm2) /* prev2[x-2*refs] */\
+            LOAD4("(%["next2"],%[mrefs],2)", %%mm4) /* next2[x-2*refs] */\
+            LOAD4("(%["prev2"],%[prefs],2)", %%mm3) /* prev2[x+2*refs] */\
+            LOAD4("(%["next2"],%[prefs],2)", %%mm5) /* next2[x+2*refs] */\
+            "paddw     %%mm4, %%mm2 \n\t"\
+            "paddw     %%mm5, %%mm3 \n\t"\
+            "psrlw     $1,    %%mm2 \n\t" /* b */\
+            "psrlw     $1,    %%mm3 \n\t" /* f */\
+            "movq    %[tmp0], %%mm4 \n\t" /* c */\
+            "movq    %[tmp1], %%mm5 \n\t" /* d */\
+            "movq    %[tmp2], %%mm7 \n\t" /* e */\
+            "psubw     %%mm4, %%mm2 \n\t" /* b-c */\
+            "psubw     %%mm7, %%mm3 \n\t" /* f-e */\
+            "movq      %%mm5, %%mm0 \n\t"\
+            "psubw     %%mm4, %%mm5 \n\t" /* d-c */\
+            "psubw     %%mm7, %%mm0 \n\t" /* d-e */\
+            "movq      %%mm2, %%mm4 \n\t"\
+            "pminsw    %%mm3, %%mm2 \n\t"\
+            "pmaxsw    %%mm4, %%mm3 \n\t"\
+            "pmaxsw    %%mm5, %%mm2 \n\t"\
+            "pminsw    %%mm5, %%mm3 \n\t"\
+            "pmaxsw    %%mm0, %%mm2 \n\t" /* max */\
+            "pminsw    %%mm0, %%mm3 \n\t" /* min */\
+            "pxor      %%mm4, %%mm4 \n\t"\
+            "pmaxsw    %%mm3, %%mm6 \n\t"\
+            "psubw     %%mm2, %%mm4 \n\t" /* -max */\
+            "pmaxsw    %%mm4, %%mm6 \n\t" /* diff= MAX3(diff, min, -max); */\
+            "1: \n\t"\
+\
+            "movq    %[tmp1], %%mm2 \n\t" /* d */\
+            "movq      %%mm2, %%mm3 \n\t"\
+            "psubw     %%mm6, %%mm2 \n\t" /* d-diff */\
+            "paddw     %%mm6, %%mm3 \n\t" /* d+diff */\
+            "pmaxsw    %%mm2, %%mm1 \n\t"\
+            "pminsw    %%mm3, %%mm1 \n\t" /* d = clip(spatial_pred, d-diff, d+diff); */\
+            "packuswb  %%mm1, %%mm1 \n\t"\
+\
+            :[tmp0]"=m"(tmp0),\
+             [tmp1]"=m"(tmp1),\
+             [tmp2]"=m"(tmp2),\
+             [tmp3]"=m"(tmp3)\
+            :[prev] "r"(prev),\
+             [cur]  "r"(cur),\
+             [next] "r"(next),\
+             [prefs]"r"((long)refs),\
+             [mrefs]"r"((long)-refs),\
+             [pw1]  "m"(pw_1),\
+             [pb1]  "m"(pb_1),\
+             [mode] "g"(mode)\
+        );\
+        asm volatile("movd %%mm1, %0" :"=m"(*dst));\
+        dst += 4;\
+        prev+= 4;\
+        cur += 4;\
+        next+= 4;\
+    }
+
+    if(parity){
+#define prev2 "prev"
+#define next2 "cur"
+        FILTER
+#undef prev2
+#undef next2
+    }else{
+#define prev2 "cur"
+#define next2 "next"
+        FILTER
+#undef prev2
+#undef next2
+    }
+}
+#undef LOAD4
+#undef PABS
+#undef CHECK
+#undef CHECK1
+#undef CHECK2
+#undef FILTER
+
+#endif // HAVE_MMX
+
+static void filter_line_c(struct vf_priv_s *p, uint8_t *dst, uint8_t *prev, uint8_t *cur, uint8_t *next, int w, int refs, int parity){
     int x;
     uint8_t *prev2= parity ? prev : cur ;
     uint8_t *next2= parity ? cur  : next;
@@ -262,6 +484,11 @@ static int open(vf_instance_t *vf, char* args){
 
     if (args) sscanf(args, "%d:%d", &vf->priv->mode, &vf->priv->parity);
 
+    filter_line = filter_line_c;
+#ifdef HAVE_MMX
+    if(gCpuCaps.hasMMX2) filter_line = filter_line_mmx2;
+#endif
+
     return 1;
 }