From 31bc8ba5cc9850cd77cba84e0932d11cd5607e5f Mon Sep 17 00:00:00 2001 From: 11rcombs Date: Sat, 25 Jan 2014 16:55:21 -0600 Subject: Added x86 ASM functions --- libass/Makefile.am | 46 ++ libass/ass_render.c | 33 +- libass/ass_utils.c | 36 ++ libass/ass_utils.h | 6 + libass/x86/be_blur.asm | 239 +++++++ libass/x86/be_blur.h | 30 + libass/x86/blend_bitmaps.asm | 290 +++++++++ libass/x86/blend_bitmaps.h | 56 ++ libass/x86/cpuid.asm | 32 + libass/x86/cpuid.h | 24 + libass/x86/x86inc.asm | 1450 ++++++++++++++++++++++++++++++++++++++++++ 11 files changed, 2238 insertions(+), 4 deletions(-) create mode 100644 libass/x86/be_blur.asm create mode 100644 libass/x86/be_blur.h create mode 100644 libass/x86/blend_bitmaps.asm create mode 100644 libass/x86/blend_bitmaps.h create mode 100644 libass/x86/cpuid.asm create mode 100644 libass/x86/cpuid.h create mode 100644 libass/x86/x86inc.asm (limited to 'libass') diff --git a/libass/Makefile.am b/libass/Makefile.am index 524d9a8..ddd5acf 100644 --- a/libass/Makefile.am +++ b/libass/Makefile.am @@ -6,6 +6,16 @@ LIBASS_LT_CURRENT = 5 LIBASS_LT_REVISION = 0 LIBASS_LT_AGE = 0 +.asm.lo: + $(LIBTOOL) --quiet --mode=compile $(AS) $(ASFLAGS) -o $@ $< -prefer-non-pic + +.S.lo: + $(LIBTOOL) --quiet --mode=compile $(AS) $(ASFLAGS)-o $@ $< -prefer-non-pic + +SRC_INTEL = x86/blend_bitmaps.asm x86/cpuid.asm +SRC_INTEL64 = x86/be_blur.asm +SRC_ARM = arm/blend_bitmaps.S + lib_LTLIBRARIES = libass.la libass_la_SOURCES = ass.c ass_cache.c ass_font.c ass_fontconfig.c ass_render.c \ ass_utils.c ass_bitmap.c ass_library.c ass_bitmap.h \ @@ -14,9 +24,45 @@ libass_la_SOURCES = ass.c ass_cache.c ass_font.c ass_fontconfig.c ass_render.c \ ass_drawing.h ass_cache_template.h ass_render.h \ ass_parse.c ass_parse.h ass_render_api.c ass_shaper.c \ ass_shaper.h ass_strtod.c + libass_la_LDFLAGS = -no-undefined -version-info $(LIBASS_LT_CURRENT):$(LIBASS_LT_REVISION):$(LIBASS_LT_AGE) libass_la_LDFLAGS += -export-symbols $(srcdir)/libass.sym +if ASM +if INTEL +AS = "yasm" +libass_la_SOURCES += $(SRC_INTEL) +if X86 +ASFLAGS += -DARCH_X86_64=0 -m x86 +if MACHO +ASFLAGS += -f macho32 -DPREFIX +endif +if ELF +ASFLAGS += -f elf +endif +if WIN32 +ASFLAGS += -f win32 -DPREFIX +endif +endif +if X64 +libass_la_SOURCES += $(SRC_INTEL64) +ASFLAGS += -DARCH_X86_64=1 -m amd64 +if MACHO +ASFLAGS += -f macho64 -DPREFIX +endif +if ELF +ASFLAGS += -f elf +endif +if WIN32 +ASFLAGS += -f win64 +endif +endif +endif +if ARM +libass_la_SOURCES += $(SRC_ARM) +endif +endif + assheadersdir = $(includedir)/ass dist_assheaders_HEADERS = ass.h ass_types.h diff --git a/libass/ass_render.c b/libass/ass_render.c index e6e6052..0f17404 100644 --- a/libass/ass_render.c +++ b/libass/ass_render.c @@ -33,6 +33,12 @@ #define SUBPIXEL_MASK 63 #define SUBPIXEL_ACCURACY 7 +#if (defined(__i386__) || defined(__x86_64__)) && CONFIG_ASM + +#include "x86/blend_bitmaps.h" +#include "x86/be_blur.h" + +#endif // ASM ASS_Renderer *ass_renderer_init(ASS_Library *library) { @@ -63,10 +69,29 @@ ASS_Renderer *ass_renderer_init(ASS_Library *library) priv->ftlibrary = ft; // images_root and related stuff is zero-filled in calloc - priv->add_bitmaps_func = add_bitmaps_c; - priv->sub_bitmaps_func = sub_bitmaps_c; - priv->mul_bitmaps_func = mul_bitmaps_c; - priv->be_blur_func = be_blur_c; + #if (defined(__i386__) || defined(__x86_64__)) && CONFIG_ASM + int sse2 = has_sse2(); + int avx2 = has_avx2(); + priv->add_bitmaps_func = avx2 ? ass_add_bitmaps_avx2 : + (sse2 ? ass_add_bitmaps_sse2 : ass_add_bitmaps_x86); + #ifdef __x86_64__ + priv->be_blur_func = avx2 ? ass_be_blur_avx2 : + (sse2 ? ass_be_blur_sse2 : be_blur_c); + priv->mul_bitmaps_func = avx2 ? ass_mul_bitmaps_avx2 : + (sse2 ? ass_mul_bitmaps_sse2 : mul_bitmaps_c); + priv->sub_bitmaps_func = avx2 ? ass_sub_bitmaps_avx2 : + (sse2 ? ass_sub_bitmaps_sse2 : ass_sub_bitmaps_x86); + #else + priv->be_blur_func = be_blur_c; + priv->mul_bitmaps_func = mul_bitmaps_c; + priv->sub_bitmaps_func = ass_sub_bitmaps_x86; + #endif + #else + priv->add_bitmaps_func = add_bitmaps_c; + priv->sub_bitmaps_func = sub_bitmaps_c; + priv->mul_bitmaps_func = mul_bitmaps_c; + priv->be_blur_func = be_blur_c; + #endif priv->restride_bitmap_func = restride_bitmap_c; priv->cache.font_cache = ass_font_cache_create(); diff --git a/libass/ass_utils.c b/libass/ass_utils.c index 2a95ddc..549eb9b 100644 --- a/libass/ass_utils.c +++ b/libass/ass_utils.c @@ -29,6 +29,42 @@ #include "ass.h" #include "ass_utils.h" +#if (defined(__i386__) || defined(__x86_64__)) && CONFIG_ASM + +#include "x86/cpuid.h" + +int has_sse2() +{ + uint32_t eax = 1, ebx, ecx, edx; + ass_get_cpuid(&eax, &ebx, &ecx, &edx); + return (!!(edx & (1 << 26))); +} + +int has_avx() +{ + uint32_t eax = 1, ebx, ecx, edx; + ass_get_cpuid(&eax, &ebx, &ecx, &edx); + if(!(ecx & (1 << 27))){ + return 0; + } + uint32_t misc = ecx; + eax = 0; + ass_get_cpuid(&eax, &ebx, &ecx, &edx); + if((ecx & (0x2 | 0x4)) != (0x2 | 0x4)){ + return 0; + } + return (!!(misc & (1 << 28))); +} + +int has_avx2() +{ + uint32_t eax = 7, ebx, ecx, edx; + ass_get_cpuid(&eax, &ebx, &ecx, &edx); + return (!!(ebx & (1 << 5))) && has_avx(); +} + +#endif // ASM + int mystrtoi(char **p, int *res) { double temp_res; diff --git a/libass/ass_utils.h b/libass/ass_utils.h index 39a9da0..6d795f0 100644 --- a/libass/ass_utils.h +++ b/libass/ass_utils.h @@ -43,6 +43,12 @@ #define FFMIN(a,b) ((a) > (b) ? (b) : (a)) #define FFMINMAX(c,a,b) FFMIN(FFMAX(c, a), b) +#if (defined(__i386__) || defined(__x86_64__)) && CONFIG_ASM +int has_sse2(void); +int has_avx(void); +int has_avx2(void); +#endif + int mystrtoi(char **p, int *res); int mystrtoll(char **p, long long *res); int mystrtou32(char **p, int base, uint32_t *res); diff --git a/libass/x86/be_blur.asm b/libass/x86/be_blur.asm new file mode 100644 index 0000000..8acf409 --- /dev/null +++ b/libass/x86/be_blur.asm @@ -0,0 +1,239 @@ +;****************************************************************************** +;* be_blur.asm: SSE2 \be blur +;****************************************************************************** + +%include "x86inc.asm" + +SECTION_RODATA 32 +low_word_zero: dd 0xFFFF0000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF + +SECTION .text + +;------------------------------------------------------------------------------ +; void be_blur_pass( uint8_t *buf, unsigned width, +; unsigned height, unsigned stride, +; uint16_t *tmp); +;------------------------------------------------------------------------------ + +INIT_XMM sse2 +cglobal be_blur, 5,15 +.skip_prologue: + xor r5, r5 ; int y = 0; + mov r6, 2 ; int x = 2; + pxor xmm6, xmm6 ; __m128i temp3 = 0; + mov r7, r0 ; unsigned char *src=buf; + movzx r8, byte [r7 + 1] ; int old_pix = src[1]; + movzx r9, byte [r7] ; int old_sum = src[0]; + add r9, r8 ; old_sum += old_pix; + lea r12, [r4 + r3 * 2] ; unsigned char *col_sum_buf = tmp + stride * 2; + lea r14, [r1 - 2] ; tmpreg = (stride-2); + and r14, -8 ; tmpreg &= (~7); +.first_loop + movzx r10, byte [r7 + r6] ; int temp1 = src[x]; + lea r11, [r8 + r10] ; int temp2 = old_pix + temp1; + mov r8, r10 ; old_pix = temp1; + lea r10, [r9 + r11] ; temp1 = old_sum + temp2; + mov r9, r11 ; old_sum = temp2; + mov word [r4 + r6 * 2], r10w ; col_pix_buf[x] = temp1; + inc r6 ; x++ + cmp r6, r1 ; x < w + jl .first_loop + mov r5, 1 ; int y = 1; + mov r6, 2 ; int x = 2; + lea r7, [r0 + r3] ; unsigned char *src=buf+stride; + movzx r8, byte [r7 + 1] ; int old_pix = src[1]; + movzx r9, byte [r7] ; int old_sum = src[0]; + add r9, r8 ; old_sum += old_pix +.second_loop + movzx r10, byte [r7 + r6] ; int temp1 = src[x]; + lea r11, [r8 + r10] ; int temp2 = old_pix + temp1; + mov r8, r10 ; old_pix = temp1; + lea r10, [r9 + r11] ; temp1 = old_sum + temp2; + mov r9, r11 ; old_sum = temp2; + movzx r11, word [r4 + r6 * 2] ; temp2 = col_pix_buf[x]; + add r11, r10 ; temp2 += temp1; + mov word [r4 + r6 * 2], r10w ; col_pix_buf[x] = temp1; + mov word [r12 + r6 * 2], r11w ; col_sum_buf[x] = temp2; + inc r6 ; x++ + cmp r6, r1 ; x < w + jl .second_loop +.height_loop + mov r10, r5; int tmpreg = y; + imul r10, r3; tmpreg *= stride; + lea r7, [r0 + r10] ; unsigned char *src=buf+y*stride; + sub r10, r3 ; tmpreg -= stride; + lea r13, [r0 + r10]; unsigned char *dst=buf+(y-1)*stride; + mov r6, 2 ; int x = 2; + movzx r10, byte [r7] ; temp1 = src[0]; + movzx r11, byte [r7 + 1] ; temp2 = src[1]; + add r10, r11; temp1 += temp2 + movd xmm0, r10; __m128i old_pix_128 = temp2; + movd xmm1, r11; __m128i old_sum_128 = temp1; +.width_loop + movq xmm2, [r7 + r6]; __m128i new_pix = (src+x); + punpcklbw xmm2, xmm6 ; new_pix = _mm_unpacklo_epi8(new_pix, temp3); + movdqa xmm3, xmm2 ; __m128i temp = new_pix; + pslldq xmm3, 2 ; temp = temp << 2 * 8; + paddw xmm3, xmm0 ; temp = _mm_add_epi16(temp, old_pix_128); + paddw xmm3, xmm2 ; temp = _mm_add_epi16(temp, new_pix); + movdqa xmm0, xmm2 ; old_pix_128 = new_pix; + psrldq xmm0, 14 ; old_pix_128 = old_pix_128 >> 14 * 8; + movdqa xmm2, xmm3 ; new_pix = temp; + pslldq xmm2, 2 ; new_pix = new_pix << 2 * 8; + paddw xmm2, xmm1 ; new_pix = _mm_add_epi16(new_pix, old_sum_128); + paddw xmm2, xmm3 ; new_pix = _mm_add_epi16(new_pix, temp); + movdqa xmm1, xmm3 ; old_sum_128 = temp; + psrldq xmm1, 14 ; old_sum_128 = old_sum_128 >> 14 * 8; + movdqu xmm4, [r4 + r6 * 2] ; __m128i old_col_pix = *(col_pix_buf+x); + movdqu [r4 + r6 * 2], xmm2 ; *(col_pix_buf+x) = new_pix ; + movdqu xmm5, [r12 + r6 * 2] ; __m128i old_col_sum = *(col_pix_sum+x); + movdqa xmm3, xmm2 ; temp = new_pix; + paddw xmm3, xmm4 ; temp = _mm_add_epi16(temp, old_col_pix); + movdqu [r12 + r6 * 2], xmm3 ; *(col_sum_buf+x) = temp; + paddw xmm5, xmm3 ; old_col_sum = _mm_add_epi16(old_col_sum, temp); + psrlw xmm5, 4 ; old_col_sum = old_col_sum >> 4; + packuswb xmm5, xmm5 ; old_col_sum = _mm_packus_epi16(old_col_sum, old_col_sum); + movq qword [r13 + r6 - 1], xmm5 ; *(dst+x-1) = old_col_sum; + add r6, 8; x += 8; + cmp r6, r14; x < ((w - 2) & (~7)); + jl .width_loop + movzx r8, byte [r7 + r6 - 1] ; old_pix = src[x-1]; + movzx r9, byte [r7 + r6 - 2] ; old_sum = old_pix + src[x-2]; + add r9, r8 +.final_width_loop + movzx r10, byte [r7 + r6] ; temp1 = src[x]; + lea r11, [r8 + r10] ; temp2 = old_pix + temp1; + mov r8, r10 ; old_pix = temp1; + lea r10, [r9 + r11] ; temp1 = old_sum + temp2; + mov r9, r11 ; old_sum = temp2; + movzx r11, word [r4 + r6 * 2] ; temp2 = col_pix_buf[x]; + add r11, r10 ; temp2 += temp1; + mov word [r4 + r6 * 2], r10w ; col_pix_buf[x] = temp1; + movzx r10, word [r12 + r6 * 2] ; temp1 = col_sum_buf[x]; + add r10, r11 ; temp1 += temp2; + shr r10, 4 ; temp1 >>= 4; + mov byte [r13 + r6 - 1], r10b ; dst[x-1] = temp1 + mov [r12 + r6 * 2], r11w ; col_sum_buf[x] = temp2; + inc r6 ; x++ + cmp r6, r1 ; x < w + jl .final_width_loop + inc r5 ; y++; + cmp r5, r2 ; y < h; + jl .height_loop + RET + +INIT_YMM avx2 +cglobal be_blur, 5,15 + %if mmsize == 32 + vzeroupper + %endif + cmp r1, 32 + jl be_blur_sse2.skip_prologue + xor r5, r5 ; int y = 0; + mov r6, 2 ; int x = 2; + vpxor ymm6, ymm6 ; __m128i temp3 = 0; + mov r7, r0 ; unsigned char *src=buf; + movzx r8, byte [r7 + 1] ; int old_pix = src[1]; + movzx r9, byte [r7] ; int old_sum = src[0]; + add r9, r8 ; old_sum += old_pix; + lea r12, [r4 + r3 * 2] ; unsigned char *col_sum_buf = tmp + stride * 2; + lea r14, [r1 - 2] ; tmpreg = (stride-2); + and r14, -16 ; tmpreg &= (~15); + vmovdqa ymm8, [low_word_zero wrt rip] +.first_loop + movzx r10, byte [r7 + r6] ; int temp1 = src[x]; + lea r11, [r8 + r10] ; int temp2 = old_pix + temp1; + mov r8, r10 ; old_pix = temp1; + lea r10, [r9 + r11] ; temp1 = old_sum + temp2; + mov r9, r11 ; old_sum = temp2; + mov word [r4 + r6 * 2], r10w ; col_pix_buf[x] = temp1; + inc r6 ; x++ + cmp r6, r1 ; x < w + jl .first_loop + mov r5, 1 ; int y = 1; + mov r6, 2 ; int x = 2; + lea r7, [r0 + r3] ; unsigned char *src=buf+stride; + movzx r8, byte [r7 + 1] ; int old_pix = src[1]; + movzx r9, byte [r7] ; int old_sum = src[0]; + add r9, r8 ; old_sum += old_pix +.second_loop + movzx r10, byte [r7 + r6] ; int temp1 = src[x]; + lea r11, [r8 + r10] ; int temp2 = old_pix + temp1; + mov r8, r10 ; old_pix = temp1; + lea r10, [r9 + r11] ; temp1 = old_sum + temp2; + mov r9, r11 ; old_sum = temp2; + movzx r11, word [r4 + r6 * 2] ; temp2 = col_pix_buf[x]; + add r11, r10 ; temp2 += temp1; + mov word [r4 + r6 * 2], r10w ; col_pix_buf[x] = temp1; + mov word [r12 + r6 * 2], r11w ; col_sum_buf[x] = temp2; + inc r6 ; x++ + cmp r6, r1 ; x < w + jl .second_loop +.height_loop + mov r10, r5; int tmpreg = y; + imul r10, r3; tmpreg *= stride; + lea r7, [r0 + r10] ; unsigned char *src=buf+y*stride; + sub r10, r3 ; tmpreg -= stride; + lea r13, [r0 + r10]; unsigned char *dst=buf+(y-1)*stride; + mov r6, 2 ; int x = 2; + movzx r10, byte [r7] ; temp1 = src[0]; + movzx r11, byte [r7 + 1] ; temp2 = src[1]; + add r10, r11; temp1 += temp2 + vmovd xmm0, r10d; __m128i old_pix_128 = temp2; + vmovd xmm1, r11d; __m128i old_sum_128 = temp1; +.width_loop + vpermq ymm2, [r7 + r6], 0x10 + vpunpcklbw ymm2, ymm2, ymm6 ; new_pix = _mm_unpacklo_epi8(new_pix, temp3); + vpermq ymm11, ymm2, 0x4e + vpalignr ymm3, ymm2, ymm11, 14 + vpand ymm3, ymm3, ymm8 + vpaddw ymm3, ymm0 ; temp = _mm_add_epi16(temp, old_pix_128); + vpaddw ymm3, ymm2 ; temp = _mm_add_epi16(temp, new_pix); + vperm2i128 ymm0, ymm2, ymm6, 0x21 + vpsrldq ymm0, ymm0, 14; temp = temp >> 14 * 8; + vpermq ymm11, ymm3, 0x4e + vpand ymm11, ymm11, ymm8; + vpalignr ymm2, ymm3, ymm11, 14 + vpand ymm2, ymm2, ymm8 + vpaddw ymm2, ymm1 ; new_pix = _mm_add_epi16(new_pix, old_sum_128); + vpaddw ymm2, ymm3 ; new_pix = _mm_add_epi16(new_pix, temp); + vperm2i128 ymm1, ymm3, ymm6, 0x21 + vpsrldq ymm1, ymm1, 14; temp = temp << 2 * 8; + vmovdqu ymm4, [r4 + r6 * 2] ; __m128i old_col_pix = *(col_pix_buf+x); + vmovdqu [r4 + r6 * 2], ymm2 ; *(col_pix_buf+x) = new_pix ; + vmovdqu ymm5, [r12 + r6 * 2] ; __m128i old_col_sum = *(col_pix_sum+x); + vpaddw ymm3, ymm2, ymm4 + vmovdqu [r12 + r6 * 2], ymm3 ; *(col_sum_buf+x) = temp; + vpaddw ymm5, ymm3 ; old_col_sum = _mm_add_epi16(old_col_sum, temp); + vpsrlw ymm5, 4 ; old_col_sum = old_col_sum >> 4; + vpackuswb ymm5, ymm5 ; old_col_sum = _mm_packus_epi16(old_col_sum, old_col_sum); + vpermq ymm5, ymm5, 11_01_10_00b + vmovdqu [r13 + r6 - 1], xmm5 ; *(dst+x-1) = old_col_sum; + add r6, 16; x += 16; + cmp r6, r14; x < ((w - 2) & (~15)); + jl .width_loop + movzx r8, byte [r7 + r6 - 1] ; old_pix = src[x-1]; + movzx r9, byte [r7 + r6 - 2] ; old_sum = old_pix + src[x-2]; + add r9, r8 +.final_width_loop + movzx r10, byte [r7 + r6] ; temp1 = src[x]; + lea r11, [r8 + r10] ; temp2 = old_pix + temp1; + mov r8, r10 ; old_pix = temp1; + lea r10, [r9 + r11] ; temp1 = old_sum + temp2; + mov r9, r11 ; old_sum = temp2; + movzx r11, word [r4 + r6 * 2] ; temp2 = col_pix_buf[x]; + add r11, r10 ; temp2 += temp1; + mov word [r4 + r6 * 2], r10w ; col_pix_buf[x] = temp1; + movzx r10, word [r12 + r6 * 2] ; temp1 = col_sum_buf[x]; + add r10, r11 ; temp1 += temp2; + shr r10, 4 ; temp1 >>= 4; + mov byte [r13 + r6 - 1], r10b ; dst[x-1] = temp1 + mov [r12 + r6 * 2], r11w ; col_sum_buf[x] = temp2; + inc r6 ; x++ + cmp r6, r1 ; x < w + jl .final_width_loop + inc r5 ; y++; + cmp r5, r2 ; y < h; + jl .height_loop + RET + diff --git a/libass/x86/be_blur.h b/libass/x86/be_blur.h new file mode 100644 index 0000000..2f53ec9 --- /dev/null +++ b/libass/x86/be_blur.h @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2013 Rodger Combs + * + * This file is part of libass. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifndef INTEL_BE_BLUR_H +#define INTEL_BE_BLUR_H + +void ass_be_blur_sse2( uint8_t *buf, intptr_t width, + intptr_t height, intptr_t stride, + uint16_t *tmp); + +void ass_be_blur_avx2( uint8_t *buf, intptr_t width, + intptr_t height, intptr_t stride, + uint16_t *tmp); + +#endif diff --git a/libass/x86/blend_bitmaps.asm b/libass/x86/blend_bitmaps.asm new file mode 100644 index 0000000..bd8358a --- /dev/null +++ b/libass/x86/blend_bitmaps.asm @@ -0,0 +1,290 @@ +;****************************************************************************** +;* add_bitmaps.asm: SSE2 and x86 add_bitmaps +;****************************************************************************** + +%define HAVE_ALIGNED_STACK 1 +%include "x86inc.asm" + +SECTION_RODATA 32 + +words_255: dw 0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF,0xFF + +SECTION .text + +;------------------------------------------------------------------------------ +; void add_bitmaps( uint8_t *dst, intptr_t dst_stride, +; uint8_t *src, intptr_t src_stride, +; intptr_t height, intptr_t width ); +;------------------------------------------------------------------------------ + +INIT_XMM +cglobal add_bitmaps_x86, 6,7 +.skip_prologue: + imul r4, r3 + add r4, r2 + PUSH r4 + mov r4, r3 +.height_loop: + xor r6, r6 ; x offset +.stride_loop: + movzx r3, byte [r0 + r6] + add r3b, byte [r2 + r6] + jnc .continue + mov r3b, 0xff +.continue: + mov byte [r0 + r6], r3b + inc r6 + cmp r6, r5 + jl .stride_loop ; still in scan line + add r0, r1 + add r2, r4 + cmp r2, [rsp] + jl .height_loop + ADD rsp, gprsize + RET + +%macro ADD_BITMAPS 0 + cglobal add_bitmaps, 6,7 + .skip_prologue: + cmp r5, mmsize + %if mmsize == 16 + jl add_bitmaps_x86.skip_prologue + %else + jl add_bitmaps_sse2.skip_prologue + %endif + %if mmsize == 32 + vzeroupper + %endif + imul r4, r3 + add r4, r2 ; last address + .height_loop: + xor r6, r6 ; x offset + .stride_loop: + movu m0, [r0 + r6] + paddusb m0, [r2 + r6] + movu [r0 + r6], m0 + add r6, mmsize + cmp r6, r5 + jl .stride_loop ; still in scan line + add r0, r1 + add r2, r3 + cmp r2, r4 + jl .height_loop + RET +%endmacro + +INIT_XMM sse2 +ADD_BITMAPS +INIT_YMM avx2 +ADD_BITMAPS + +;------------------------------------------------------------------------------ +; void sub_bitmaps( uint8_t *dst, intptr_t dst_stride, +; uint8_t *src, intptr_t src_stride, +; intptr_t height, intptr_t width ); +;------------------------------------------------------------------------------ + +INIT_XMM +cglobal sub_bitmaps_x86, 6,10 +.skip_prologue: + imul r4, r3 + add r4, r2 ; last address + PUSH r4 + mov r4, r3 +.height_loop: + xor r6, r6 ; x offset +.stride_loop: + mov r3b, byte [r0 + r6] + sub r3b, byte [r2 + r6] + jnc .continue + mov r3b, 0x0 +.continue: + mov byte [r0 + r6], r3b + inc r6 + cmp r6, r5 + jl .stride_loop ; still in scan line + add r0, r1 + add r2, r4 + cmp r2, [rsp] + jl .height_loop + ADD rsp, gprsize + RET + +%if ARCH_X86_64 + +%macro SUB_BITMAPS 0 + cglobal sub_bitmaps, 6,10 + .skip_prologue: + cmp r5, mmsize + %if mmsize == 16 + jl sub_bitmaps_x86.skip_prologue + %else + jl sub_bitmaps_sse2.skip_prologue + %endif + %if mmsize == 32 + vzeroupper + %endif + imul r4, r3 + add r4, r2 ; last address + mov r7, r5 + and r7, -mmsize ; &= (16); + xor r9, r9 + .height_loop: + xor r6, r6 ; x offset + .stride_loop: + movu m0, [r0 + r6] + movu m1, [r2 + r6] + psubusb m0, m1 + movu [r0 + r6], m0 + add r6, mmsize + cmp r6, r7 + jl .stride_loop ; still in scan line + .stride_loop2 + cmp r6, r5 + jge .finish + movzx r8, byte [r0 + r6] + sub r8b, byte [r2 + r6] + cmovc r8, r9 + mov byte [r0 + r6], r8b + inc r6 + jmp .stride_loop2 + .finish + add r0, r1 + add r2, r3 + cmp r2, r4 + jl .height_loop + RET +%endmacro + +INIT_XMM sse2 +SUB_BITMAPS +INIT_YMM avx2 +SUB_BITMAPS + +;------------------------------------------------------------------------------ +; void mul_bitmaps( uint8_t *dst, intptr_t dst_stride, +; uint8_t *src1, intptr_t src1_stride, +; uint8_t *src2, intptr_t src2_stride, +; intptr_t width, intptr_t height ); +;------------------------------------------------------------------------------ + +INIT_XMM +cglobal mul_bitmaps_x86, 8,12 +.skip_prologue: + imul r7, r3 + add r7, r2 ; last address +.height_loop: + xor r8, r8 ; x offset +.stride_loop: + movzx r9, byte [r2 + r8] + movzx r10, byte [r4 + r8] + imul r9, r10 + add r9, 255 + shr r9, 8 + mov byte [r0 + r8], r9b + inc r8 + cmp r8, r6 + jl .stride_loop ; still in scan line + add r0, r1 + add r2, r3 + add r4, r5 + cmp r2, r7 + jl .height_loop + RET + +INIT_XMM sse2 +cglobal mul_bitmaps, 8,12 +.skip_prologue: + cmp r6, 8 + jl mul_bitmaps_x86.skip_prologue + imul r7, r3 + add r7, r2 ; last address + pxor xmm2, xmm2 + movdqa xmm3, [words_255 wrt rip] + mov r9, r6 + and r9, -8 ; &= (~8); +.height_loop: + xor r8, r8 ; x offset +.stride_loop: + movq xmm0, [r2 + r8] + movq xmm1, [r4 + r8] + punpcklbw xmm0, xmm2 + punpcklbw xmm1, xmm2 + pmullw xmm0, xmm1 + paddw xmm0, xmm3 + psrlw xmm0, 0x08 + packuswb xmm0, xmm0 + movq [r0 + r8], xmm0 + add r8, 8 + cmp r8, r9 + jl .stride_loop ; still in scan line +.stride_loop2 + cmp r8, r6 + jge .finish + movzx r10, byte [r2 + r8] + movzx r11, byte [r4 + r8] + imul r10, r11 + add r10, 255 + shr r10, 8 + mov byte [r0 + r8], r10b + inc r8 + jmp .stride_loop2 +.finish: + add r0, r1 + add r2, r3 + add r4, r5 + cmp r2, r7 + jl .height_loop + RET + +INIT_YMM avx2 +cglobal mul_bitmaps, 8,12 + cmp r6, 16 + jl mul_bitmaps_sse2.skip_prologue + %if mmsize == 32 + vzeroupper + %endif + imul r7, r3 + add r7, r2 ; last address + vpxor ymm2, ymm2 + vmovdqa ymm3, [words_255 wrt rip] + mov r9, r6 + and r9, -16 ; &= (~16); +.height_loop: + xor r8, r8 ; x offset +.stride_loop: + vmovdqu xmm0, [r2 + r8] + vpermq ymm0, ymm0, 0x10 + vmovdqu xmm1, [r4 + r8] + vpermq ymm1, ymm1, 0x10 + vpunpcklbw ymm0, ymm0, ymm2 + vpunpcklbw ymm1, ymm1, ymm2 + vpmullw ymm0, ymm0, ymm1 + vpaddw ymm0, ymm0, ymm3 + vpsrlw ymm0, ymm0, 0x08 + vextracti128 xmm4, ymm0, 0x1 + vpackuswb ymm0, ymm0, ymm4 + vmovdqa [r0 + r8], xmm0 + add r8, 16 + cmp r8, r9 + jl .stride_loop ; still in scan line +.stride_loop2 + cmp r8, r6 + jge .finish + movzx r10, byte [r2 + r8] + movzx r11, byte [r4 + r8] + imul r10, r11 + add r10, 255 + shr r10, 8 + mov byte [r0 + r8], r10b + inc r8 + jmp .stride_loop2 +.finish: + add r0, r1 + add r2, r3 + add r4, r5 + cmp r2, r7 + jl .height_loop + RET + +%endif diff --git a/libass/x86/blend_bitmaps.h b/libass/x86/blend_bitmaps.h new file mode 100644 index 0000000..30058ed --- /dev/null +++ b/libass/x86/blend_bitmaps.h @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2013 Rodger Combs + * + * This file is part of libass. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifndef INTEL_BLEND_BITMAPS_H +#define INTEL_BLEND_BITMAPS_H + +void ass_add_bitmaps_avx2( uint8_t *dst, intptr_t dst_stride, + uint8_t *src, intptr_t src_stride, + intptr_t height, intptr_t width ); + +void ass_add_bitmaps_sse2( uint8_t *dst, intptr_t dst_stride, + uint8_t *src, intptr_t src_stride, + intptr_t height, intptr_t width ); + +void ass_add_bitmaps_x86( uint8_t *dst, intptr_t dst_stride, + uint8_t *src, intptr_t src_stride, + intptr_t height, intptr_t width ); + +void ass_sub_bitmaps_avx2( uint8_t *dst, intptr_t dst_stride, + uint8_t *src, intptr_t src_stride, + intptr_t height, intptr_t width ); + +void ass_sub_bitmaps_sse2( uint8_t *dst, intptr_t dst_stride, + uint8_t *src, intptr_t src_stride, + intptr_t height, intptr_t width ); + +void ass_sub_bitmaps_x86( uint8_t *dst, intptr_t dst_stride, + uint8_t *src, intptr_t src_stride, + intptr_t height, intptr_t width ); + +void ass_mul_bitmaps_avx2( uint8_t *dst, intptr_t dst_stride, + uint8_t *src1, intptr_t src1_stride, + uint8_t *src2, intptr_t src2_stride, + intptr_t width, intptr_t height ); + +void ass_mul_bitmaps_sse2( uint8_t *dst, intptr_t dst_stride, + uint8_t *src1, intptr_t src1_stride, + uint8_t *src2, intptr_t src2_stride, + intptr_t width, intptr_t height ); + +#endif diff --git a/libass/x86/cpuid.asm b/libass/x86/cpuid.asm new file mode 100644 index 0000000..ca0792c --- /dev/null +++ b/libass/x86/cpuid.asm @@ -0,0 +1,32 @@ +;****************************************************************************** +;* add_bitmaps.asm: SSE2 and x86 add_bitmaps +;****************************************************************************** + +%include "x86inc.asm" + +SECTION .text + +;------------------------------------------------------------------------------ +; void get_cpuid( uint32_t *eax, uint32_t *ebx, uint32_t *ecx, uint32_t *edx); +;------------------------------------------------------------------------------ + +INIT_XMM +cglobal get_cpuid, 4, 5, 0 + push rbx + push r3 + push r2 + push r1 + push r0 + mov eax, [r0] + xor ecx, ecx + cpuid + pop r4 + mov [r4], eax + pop r4 + mov [r4], ebx + pop r4 + mov [r4], ecx + pop r4 + mov [r4], edx + pop rbx + RET diff --git a/libass/x86/cpuid.h b/libass/x86/cpuid.h new file mode 100644 index 0000000..34e4b19 --- /dev/null +++ b/libass/x86/cpuid.h @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2013 Rodger Combs + * + * This file is part of libass. + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifndef INTEL_CPUID_H +#define INTEL_CPUID_H + +void ass_get_cpuid( uint32_t *eax, uint32_t *ebx, uint32_t *ecx, uint32_t *edx); + +#endif diff --git a/libass/x86/x86inc.asm b/libass/x86/x86inc.asm new file mode 100644 index 0000000..53e104d --- /dev/null +++ b/libass/x86/x86inc.asm @@ -0,0 +1,1450 @@ +;***************************************************************************** +;* x86inc.asm: x264asm abstraction layer +;***************************************************************************** +;* Copyright (C) 2005-2013 x264 project +;* +;* Authors: Loren Merritt +;* Anton Mitrofanov +;* Jason Garrett-Glaser +;* Henrik Gramner +;* +;* Permission to use, copy, modify, and/or distribute this software for any +;* purpose with or without fee is hereby granted, provided that the above +;* copyright notice and this permission notice appear in all copies. +;* +;* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +;* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +;* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +;* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +;* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +;* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +;* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +;***************************************************************************** + +; This is a header file for the x264ASM assembly language, which uses +; NASM/YASM syntax combined with a large number of macros to provide easy +; abstraction between different calling conventions (x86_32, win64, linux64). +; It also has various other useful features to simplify writing the kind of +; DSP functions that are most often used in x264. + +; Unlike the rest of x264, this file is available under an ISC license, as it +; has significant usefulness outside of x264 and we want it to be available +; to the largest audience possible. Of course, if you modify it for your own +; purposes to add a new feature, we strongly encourage contributing a patch +; as this feature might be useful for others as well. Send patches or ideas +; to x264-devel@videolan.org . + +%ifndef private_prefix + %define private_prefix ass +%endif + +%ifndef public_prefix + %define public_prefix private_prefix +%endif + +%define WIN64 0 +%define UNIX64 0 +%if ARCH_X86_64 + %ifidn __OUTPUT_FORMAT__,win32 + %define WIN64 1 + %elifidn __OUTPUT_FORMAT__,win64 + %define WIN64 1 + %elifidn __OUTPUT_FORMAT__,x64 + %define WIN64 1 + %else + %define UNIX64 1 + %endif +%endif + +%ifdef PREFIX + %define mangle(x) _ %+ x +%else + %define mangle(x) x +%endif + +%macro SECTION_RODATA 0-1 16 + SECTION .rodata align=%1 +%endmacro + +%macro SECTION_TEXT 0-1 16 + SECTION .text align=%1 +%endmacro + +%if WIN64 + %define PIC +%elif ARCH_X86_64 == 0 +; x86_32 doesn't require PIC. +; Some distros prefer shared objects to be PIC, but nothing breaks if +; the code contains a few textrels, so we'll skip that complexity. + %undef PIC +%endif +%ifdef PIC + default rel +%endif + +; Always use long nops (reduces 0x90 spam in disassembly on x86_32) +CPU amdnop + +; Macros to eliminate most code duplication between x86_32 and x86_64: +; Currently this works only for leaf functions which load all their arguments +; into registers at the start, and make no other use of the stack. Luckily that +; covers most of x264's asm. + +; PROLOGUE: +; %1 = number of arguments. loads them from stack if needed. +; %2 = number of registers used. pushes callee-saved regs if needed. +; %3 = number of xmm registers used. pushes callee-saved xmm regs if needed. +; %4 = (optional) stack size to be allocated. If not aligned (x86-32 ICC 10.x, +; MSVC or YMM), the stack will be manually aligned (to 16 or 32 bytes), +; and an extra register will be allocated to hold the original stack +; pointer (to not invalidate r0m etc.). To prevent the use of an extra +; register as stack pointer, request a negative stack size. +; %4+/%5+ = list of names to define to registers +; PROLOGUE can also be invoked by adding the same options to cglobal + +; e.g. +; cglobal foo, 2,3,0, dst, src, tmp +; declares a function (foo), taking two args (dst and src) and one local variable (tmp) + +; TODO Some functions can use some args directly from the stack. If they're the +; last args then you can just not declare them, but if they're in the middle +; we need more flexible macro. + +; RET: +; Pops anything that was pushed by PROLOGUE, and returns. + +; REP_RET: +; Use this instead of RET if it's a branch target. + +; registers: +; rN and rNq are the native-size register holding function argument N +; rNd, rNw, rNb are dword, word, and byte size +; rNh is the high 8 bits of the word size +; rNm is the original location of arg N (a register or on the stack), dword +; rNmp is native size + +%macro DECLARE_REG 2-3 + %define r%1q %2 + %define r%1d %2d + %define r%1w %2w + %define r%1b %2b + %define r%1h %2h + %if %0 == 2 + %define r%1m %2d + %define r%1mp %2 + %elif ARCH_X86_64 ; memory + %define r%1m [rstk + stack_offset + %3] + %define r%1mp qword r %+ %1 %+ m + %else + %define r%1m [rstk + stack_offset + %3] + %define r%1mp dword r %+ %1 %+ m + %endif + %define r%1 %2 +%endmacro + +%macro DECLARE_REG_SIZE 3 + %define r%1q r%1 + %define e%1q r%1 + %define r%1d e%1 + %define e%1d e%1 + %define r%1w %1 + %define e%1w %1 + %define r%1h %3 + %define e%1h %3 + %define r%1b %2 + %define e%1b %2 +%if ARCH_X86_64 == 0 + %define r%1 e%1 +%endif +%endmacro + +DECLARE_REG_SIZE ax, al, ah +DECLARE_REG_SIZE bx, bl, bh +DECLARE_REG_SIZE cx, cl, ch +DECLARE_REG_SIZE dx, dl, dh +DECLARE_REG_SIZE si, sil, null +DECLARE_REG_SIZE di, dil, null +DECLARE_REG_SIZE bp, bpl, null + +; t# defines for when per-arch register allocation is more complex than just function arguments + +%macro DECLARE_REG_TMP 1-* + %assign %%i 0 + %rep %0 + CAT_XDEFINE t, %%i, r%1 + %assign %%i %%i+1 + %rotate 1 + %endrep +%endmacro + +%macro DECLARE_REG_TMP_SIZE 0-* + %rep %0 + %define t%1q t%1 %+ q + %define t%1d t%1 %+ d + %define t%1w t%1 %+ w + %define t%1h t%1 %+ h + %define t%1b t%1 %+ b + %rotate 1 + %endrep +%endmacro + +DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 + +%if ARCH_X86_64 + %define gprsize 8 +%else + %define gprsize 4 +%endif + +%macro PUSH 1 + push %1 + %ifidn rstk, rsp + %assign stack_offset stack_offset+gprsize + %endif +%endmacro + +%macro POP 1 + pop %1 + %ifidn rstk, rsp + %assign stack_offset stack_offset-gprsize + %endif +%endmacro + +%macro PUSH_IF_USED 1-* + %rep %0 + %if %1 < regs_used + PUSH r%1 + %endif + %rotate 1 + %endrep +%endmacro + +%macro POP_IF_USED 1-* + %rep %0 + %if %1 < regs_used + pop r%1 + %endif + %rotate 1 + %endrep +%endmacro + +%macro LOAD_IF_USED 1-* + %rep %0 + %if %1 < num_args + mov r%1, r %+ %1 %+ mp + %endif + %rotate 1 + %endrep +%endmacro + +%macro SUB 2 + sub %1, %2 + %ifidn %1, rstk + %assign stack_offset stack_offset+(%2) + %endif +%endmacro + +%macro ADD 2 + add %1, %2 + %ifidn %1, rstk + %assign stack_offset stack_offset-(%2) + %endif +%endmacro + +%macro movifnidn 2 + %ifnidn %1, %2 + mov %1, %2 + %endif +%endmacro + +%macro movsxdifnidn 2 + %ifnidn %1, %2 + movsxd %1, %2 + %endif +%endmacro + +%macro ASSERT 1 + %if (%1) == 0 + %error assert failed + %endif +%endmacro + +%macro DEFINE_ARGS 0-* + %ifdef n_arg_names + %assign %%i 0 + %rep n_arg_names + CAT_UNDEF arg_name %+ %%i, q + CAT_UNDEF arg_name %+ %%i, d + CAT_UNDEF arg_name %+ %%i, w + CAT_UNDEF arg_name %+ %%i, h + CAT_UNDEF arg_name %+ %%i, b + CAT_UNDEF arg_name %+ %%i, m + CAT_UNDEF arg_name %+ %%i, mp + CAT_UNDEF arg_name, %%i + %assign %%i %%i+1 + %endrep + %endif + + %xdefine %%stack_offset stack_offset + %undef stack_offset ; so that the current value of stack_offset doesn't get baked in by xdefine + %assign %%i 0 + %rep %0 + %xdefine %1q r %+ %%i %+ q + %xdefine %1d r %+ %%i %+ d + %xdefine %1w r %+ %%i %+ w + %xdefine %1h r %+ %%i %+ h + %xdefine %1b r %+ %%i %+ b + %xdefine %1m r %+ %%i %+ m + %xdefine %1mp r %+ %%i %+ mp + CAT_XDEFINE arg_name, %%i, %1 + %assign %%i %%i+1 + %rotate 1 + %endrep + %xdefine stack_offset %%stack_offset + %assign n_arg_names %0 +%endmacro + +%macro ALLOC_STACK 1-2 0 ; stack_size, n_xmm_regs (for win64 only) + %ifnum %1 + %if %1 != 0 + %assign %%stack_alignment ((mmsize + 15) & ~15) + %assign stack_size %1 + %if stack_size < 0 + %assign stack_size -stack_size + %endif + %assign stack_size_padded stack_size + %if WIN64 + %assign stack_size_padded stack_size_padded + 32 ; reserve 32 bytes for shadow space + %if mmsize != 8 + %assign xmm_regs_used %2 + %if xmm_regs_used > 8 + %assign stack_size_padded stack_size_padded + (xmm_regs_used-8)*16 + %endif + %endif + %endif + %if mmsize <= 16 && HAVE_ALIGNED_STACK + %assign stack_size_padded stack_size_padded + %%stack_alignment - gprsize - (stack_offset & (%%stack_alignment - 1)) + SUB rsp, stack_size_padded + %else + %assign %%reg_num (regs_used - 1) + %xdefine rstk r %+ %%reg_num + ; align stack, and save original stack location directly above + ; it, i.e. in [rsp+stack_size_padded], so we can restore the + ; stack in a single instruction (i.e. mov rsp, rstk or mov + ; rsp, [rsp+stack_size_padded]) + mov rstk, rsp + %if %1 < 0 ; need to store rsp on stack + sub rsp, gprsize+stack_size_padded + and rsp, ~(%%stack_alignment-1) + %xdefine rstkm [rsp+stack_size_padded] + mov rstkm, rstk + %else ; can keep rsp in rstk during whole function + sub rsp, stack_size_padded + and rsp, ~(%%stack_alignment-1) + %xdefine rstkm rstk + %endif + %endif + WIN64_PUSH_XMM + %endif + %endif +%endmacro + +%macro SETUP_STACK_POINTER 1 + %ifnum %1 + %if %1 != 0 && (HAVE_ALIGNED_STACK == 0 || mmsize == 32) + %if %1 > 0 + %assign regs_used (regs_used + 1) + %elif ARCH_X86_64 && regs_used == num_args && num_args <= 4 + UNIX64 * 2 + %warning "Stack pointer will overwrite register argument" + %endif + %endif + %endif +%endmacro + +%macro DEFINE_ARGS_INTERNAL 3+ + %ifnum %2 + DEFINE_ARGS %3 + %elif %1 == 4 + DEFINE_ARGS %2 + %elif %1 > 4 + DEFINE_ARGS %2, %3 + %endif +%endmacro + +%if WIN64 ; Windows x64 ;================================================= + +DECLARE_REG 0, rcx +DECLARE_REG 1, rdx +DECLARE_REG 2, R8 +DECLARE_REG 3, R9 +DECLARE_REG 4, R10, 40 +DECLARE_REG 5, R11, 48 +DECLARE_REG 6, rax, 56 +DECLARE_REG 7, rdi, 64 +DECLARE_REG 8, rsi, 72 +DECLARE_REG 9, rbx, 80 +DECLARE_REG 10, rbp, 88 +DECLARE_REG 11, R12, 96 +DECLARE_REG 12, R13, 104 +DECLARE_REG 13, R14, 112 +DECLARE_REG 14, R15, 120 + +%macro PROLOGUE 2-5+ 0 ; #args, #regs, #xmm_regs, [stack_size,] arg_names... + %assign num_args %1 + %assign regs_used %2 + ASSERT regs_used >= num_args + SETUP_STACK_POINTER %4 + ASSERT regs_used <= 15 + PUSH_IF_USED 7, 8, 9, 10, 11, 12, 13, 14 + ALLOC_STACK %4, %3 + %if mmsize != 8 && stack_size == 0 + WIN64_SPILL_XMM %3 + %endif + LOAD_IF_USED 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 + DEFINE_ARGS_INTERNAL %0, %4, %5 +%endmacro + +%macro WIN64_PUSH_XMM 0 + ; Use the shadow space to store XMM6 and XMM7, the rest needs stack space allocated. + %if xmm_regs_used > 6 + movaps [rstk + stack_offset + 8], xmm6 + %endif + %if xmm_regs_used > 7 + movaps [rstk + stack_offset + 24], xmm7 + %endif + %if xmm_regs_used > 8 + %assign %%i 8 + %rep xmm_regs_used-8 + movaps [rsp + (%%i-8)*16 + stack_size + 32], xmm %+ %%i + %assign %%i %%i+1 + %endrep + %endif +%endmacro + +%macro WIN64_SPILL_XMM 1 + %assign xmm_regs_used %1 + ASSERT xmm_regs_used <= 16 + %if xmm_regs_used > 8 + %assign stack_size_padded (xmm_regs_used-8)*16 + (~stack_offset&8) + 32 + SUB rsp, stack_size_padded + %endif + WIN64_PUSH_XMM +%endmacro + +%macro WIN64_RESTORE_XMM_INTERNAL 1 + %assign %%pad_size 0 + %if xmm_regs_used > 8 + %assign %%i xmm_regs_used + %rep xmm_regs_used-8 + %assign %%i %%i-1 + movaps xmm %+ %%i, [%1 + (%%i-8)*16 + stack_size + 32] + %endrep + %endif + %if stack_size_padded > 0 + %if stack_size > 0 && (mmsize == 32 || HAVE_ALIGNED_STACK == 0) + mov rsp, rstkm + %else + add %1, stack_size_padded + %assign %%pad_size stack_size_padded + %endif + %endif + %if xmm_regs_used > 7 + movaps xmm7, [%1 + stack_offset - %%pad_size + 24] + %endif + %if xmm_regs_used > 6 + movaps xmm6, [%1 + stack_offset - %%pad_size + 8] + %endif +%endmacro + +%macro WIN64_RESTORE_XMM 1 + WIN64_RESTORE_XMM_INTERNAL %1 + %assign stack_offset (stack_offset-stack_size_padded) + %assign xmm_regs_used 0 +%endmacro + +%define has_epilogue regs_used > 7 || xmm_regs_used > 6 || mmsize == 32 || stack_size > 0 + +%macro RET 0 + WIN64_RESTORE_XMM_INTERNAL rsp + POP_IF_USED 14, 13, 12, 11, 10, 9, 8, 7 +%if mmsize == 32 + vzeroupper +%endif + AUTO_REP_RET +%endmacro + +%elif ARCH_X86_64 ; *nix x64 ;============================================= + +DECLARE_REG 0, rdi +DECLARE_REG 1, rsi +DECLARE_REG 2, rdx +DECLARE_REG 3, rcx +DECLARE_REG 4, R8 +DECLARE_REG 5, R9 +DECLARE_REG 6, rax, 8 +DECLARE_REG 7, R10, 16 +DECLARE_REG 8, R11, 24 +DECLARE_REG 9, rbx, 32 +DECLARE_REG 10, rbp, 40 +DECLARE_REG 11, R12, 48 +DECLARE_REG 12, R13, 56 +DECLARE_REG 13, R14, 64 +DECLARE_REG 14, R15, 72 + +%macro PROLOGUE 2-5+ ; #args, #regs, #xmm_regs, [stack_size,] arg_names... + %assign num_args %1 + %assign regs_used %2 + ASSERT regs_used >= num_args + SETUP_STACK_POINTER %4 + ASSERT regs_used <= 15 + PUSH_IF_USED 9, 10, 11, 12, 13, 14 + ALLOC_STACK %4 + LOAD_IF_USED 6, 7, 8, 9, 10, 11, 12, 13, 14 + DEFINE_ARGS_INTERNAL %0, %4, %5 +%endmacro + +%define has_epilogue regs_used > 9 || mmsize == 32 || stack_size > 0 + +%macro RET 0 +%if stack_size_padded > 0 +%if mmsize == 32 || HAVE_ALIGNED_STACK == 0 + mov rsp, rstkm +%else + add rsp, stack_size_padded +%endif +%endif + POP_IF_USED 14, 13, 12, 11, 10, 9 +%if mmsize == 32 + vzeroupper +%endif + AUTO_REP_RET +%endmacro + +%else ; X86_32 ;============================================================== + +DECLARE_REG 0, eax, 4 +DECLARE_REG 1, ecx, 8 +DECLARE_REG 2, edx, 12 +DECLARE_REG 3, ebx, 16 +DECLARE_REG 4, esi, 20 +DECLARE_REG 5, edi, 24 +DECLARE_REG 6, ebp, 28 +%define rsp esp + +%macro DECLARE_ARG 1-* + %rep %0 + %define r%1m [rstk + stack_offset + 4*%1 + 4] + %define r%1mp dword r%1m + %rotate 1 + %endrep +%endmacro + +DECLARE_ARG 7, 8, 9, 10, 11, 12, 13, 14 + +%macro PROLOGUE 2-5+ ; #args, #regs, #xmm_regs, [stack_size,] arg_names... + %assign num_args %1 + %assign regs_used %2 + ASSERT regs_used >= num_args + %if num_args > 7 + %assign num_args 7 + %endif + %if regs_used > 7 + %assign regs_used 7 + %endif + SETUP_STACK_POINTER %4 + ASSERT regs_used <= 7 + PUSH_IF_USED 3, 4, 5, 6 + ALLOC_STACK %4 + LOAD_IF_USED 0, 1, 2, 3, 4, 5, 6 + DEFINE_ARGS_INTERNAL %0, %4, %5 +%endmacro + +%define has_epilogue regs_used > 3 || mmsize == 32 || stack_size > 0 + +%macro RET 0 +%if stack_size_padded > 0 +%if mmsize == 32 || HAVE_ALIGNED_STACK == 0 + mov rsp, rstkm +%else + add rsp, stack_size_padded +%endif +%endif + POP_IF_USED 6, 5, 4, 3 +%if mmsize == 32 + vzeroupper +%endif + AUTO_REP_RET +%endmacro + +%endif ;====================================================================== + +%if WIN64 == 0 +%macro WIN64_SPILL_XMM 1 +%endmacro +%macro WIN64_RESTORE_XMM 1 +%endmacro +%macro WIN64_PUSH_XMM 0 +%endmacro +%endif + +; On AMD cpus <=K10, an ordinary ret is slow if it immediately follows either +; a branch or a branch target. So switch to a 2-byte form of ret in that case. +; We can automatically detect "follows a branch", but not a branch target. +; (SSSE3 is a sufficient condition to know that your cpu doesn't have this problem.) +%macro REP_RET 0 + %if has_epilogue + RET + %else + rep ret + %endif +%endmacro + +%define last_branch_adr $$ +%macro AUTO_REP_RET 0 + %ifndef cpuflags + times ((last_branch_adr-$)>>31)+1 rep ; times 1 iff $ != last_branch_adr. + %elif notcpuflag(ssse3) + times ((last_branch_adr-$)>>31)+1 rep + %endif + ret +%endmacro + +%macro BRANCH_INSTR 0-* + %rep %0 + %macro %1 1-2 %1 + %2 %1 + %%branch_instr: + %xdefine last_branch_adr %%branch_instr + %endmacro + %rotate 1 + %endrep +%endmacro + +BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, jna, jnae, jb, jbe, jnb, jnbe, jc, jnc, js, jns, jo, jno, jp, jnp + +%macro TAIL_CALL 2 ; callee, is_nonadjacent + %if has_epilogue + call %1 + RET + %elif %2 + jmp %1 + %endif +%endmacro + +;============================================================================= +; arch-independent part +;============================================================================= + +%assign function_align 16 + +; Begin a function. +; Applies any symbol mangling needed for C linkage, and sets up a define such that +; subsequent uses of the function name automatically refer to the mangled version. +; Appends cpuflags to the function name if cpuflags has been specified. +; The "" empty default parameter is a workaround for nasm, which fails if SUFFIX +; is empty and we call cglobal_internal with just %1 %+ SUFFIX (without %2). +%macro cglobal 1-2+ "" ; name, [PROLOGUE args] + cglobal_internal 1, %1 %+ SUFFIX, %2 +%endmacro +%macro cvisible 1-2+ "" ; name, [PROLOGUE args] + cglobal_internal 0, %1 %+ SUFFIX, %2 +%endmacro +%macro cglobal_internal 2-3+ + %if %1 + %xdefine %%FUNCTION_PREFIX private_prefix + %xdefine %%VISIBILITY hidden + %else + %xdefine %%FUNCTION_PREFIX public_prefix + %xdefine %%VISIBILITY + %endif + %ifndef cglobaled_%2 + %xdefine %2 mangle(%%FUNCTION_PREFIX %+ _ %+ %2) + %xdefine %2.skip_prologue %2 %+ .skip_prologue + CAT_XDEFINE cglobaled_, %2, 1 + %endif + %xdefine current_function %2 + %ifidn __OUTPUT_FORMAT__,elf + global %2:function %%VISIBILITY + %else + global %2 + %endif + align function_align + %2: + RESET_MM_PERMUTATION ; needed for x86-64, also makes disassembly somewhat nicer + %xdefine rstk rsp ; copy of the original stack pointer, used when greater alignment than the known stack alignment is required + %assign stack_offset 0 ; stack pointer offset relative to the return address + %assign stack_size 0 ; amount of stack space that can be freely used inside a function + %assign stack_size_padded 0 ; total amount of allocated stack space, including space for callee-saved xmm registers on WIN64 and alignment padding + %assign xmm_regs_used 0 ; number of XMM registers requested, used for dealing with callee-saved registers on WIN64 + %ifnidn %3, "" + PROLOGUE %3 + %endif +%endmacro + +%macro cextern 1 + %xdefine %1 mangle(private_prefix %+ _ %+ %1) + CAT_XDEFINE cglobaled_, %1, 1 + extern %1 +%endmacro + +; like cextern, but without the prefix +%macro cextern_naked 1 + %xdefine %1 mangle(%1) + CAT_XDEFINE cglobaled_, %1, 1 + extern %1 +%endmacro + +%macro const 1-2+ + %xdefine %1 mangle(private_prefix %+ _ %+ %1) + %ifidn __OUTPUT_FORMAT__,elf + global %1:data hidden + %else + global %1 + %endif + %1: %2 +%endmacro + +; This is needed for ELF, otherwise the GNU linker assumes the stack is +; executable by default. +%ifidn __OUTPUT_FORMAT__,elf +SECTION .note.GNU-stack noalloc noexec nowrite progbits +%endif + +; cpuflags + +%assign cpuflags_mmx (1<<0) +%assign cpuflags_mmx2 (1<<1) | cpuflags_mmx +%assign cpuflags_3dnow (1<<2) | cpuflags_mmx +%assign cpuflags_3dnowext (1<<3) | cpuflags_3dnow +%assign cpuflags_sse (1<<4) | cpuflags_mmx2 +%assign cpuflags_sse2 (1<<5) | cpuflags_sse +%assign cpuflags_sse2slow (1<<6) | cpuflags_sse2 +%assign cpuflags_sse3 (1<<7) | cpuflags_sse2 +%assign cpuflags_ssse3 (1<<8) | cpuflags_sse3 +%assign cpuflags_sse4 (1<<9) | cpuflags_ssse3 +%assign cpuflags_sse42 (1<<10)| cpuflags_sse4 +%assign cpuflags_avx (1<<11)| cpuflags_sse42 +%assign cpuflags_xop (1<<12)| cpuflags_avx +%assign cpuflags_fma4 (1<<13)| cpuflags_avx +%assign cpuflags_avx2 (1<<14)| cpuflags_avx +%assign cpuflags_fma3 (1<<15)| cpuflags_avx + +%assign cpuflags_cache32 (1<<16) +%assign cpuflags_cache64 (1<<17) +%assign cpuflags_slowctz (1<<18) +%assign cpuflags_lzcnt (1<<19) +%assign cpuflags_aligned (1<<20) ; not a cpu feature, but a function variant +%assign cpuflags_atom (1<<21) +%assign cpuflags_bmi1 (1<<22)|cpuflags_lzcnt +%assign cpuflags_bmi2 (1<<23)|cpuflags_bmi1 + +%define cpuflag(x) ((cpuflags & (cpuflags_ %+ x)) == (cpuflags_ %+ x)) +%define notcpuflag(x) ((cpuflags & (cpuflags_ %+ x)) != (cpuflags_ %+ x)) + +; Takes up to 2 cpuflags from the above list. +; All subsequent functions (up to the next INIT_CPUFLAGS) is built for the specified cpu. +; You shouldn't need to invoke this macro directly, it's a subroutine for INIT_MMX &co. +%macro INIT_CPUFLAGS 0-2 + CPU amdnop + %if %0 >= 1 + %xdefine cpuname %1 + %assign cpuflags cpuflags_%1 + %if %0 >= 2 + %xdefine cpuname %1_%2 + %assign cpuflags cpuflags | cpuflags_%2 + %endif + %xdefine SUFFIX _ %+ cpuname + %if cpuflag(avx) + %assign avx_enabled 1 + %endif + %if (mmsize == 16 && notcpuflag(sse2)) || (mmsize == 32 && notcpuflag(avx2)) + %define mova movaps + %define movu movups + %define movnta movntps + %endif + %if cpuflag(aligned) + %define movu mova + %elifidn %1, sse3 + %define movu lddqu + %endif + %if ARCH_X86_64 == 0 && notcpuflag(sse2) + CPU basicnop + %endif + %else + %xdefine SUFFIX + %undef cpuname + %undef cpuflags + %endif +%endmacro + +; Merge mmx and sse* +; m# is a simd register of the currently selected size +; xm# is the corresponding xmm register if mmsize >= 16, otherwise the same as m# +; ym# is the corresponding ymm register if mmsize >= 32, otherwise the same as m# +; (All 3 remain in sync through SWAP.) + +%macro CAT_XDEFINE 3 + %xdefine %1%2 %3 +%endmacro + +%macro CAT_UNDEF 2 + %undef %1%2 +%endmacro + +%macro INIT_MMX 0-1+ + %assign avx_enabled 0 + %define RESET_MM_PERMUTATION INIT_MMX %1 + %define mmsize 8 + %define num_mmregs 8 + %define mova movq + %define movu movq + %define movh movd + %define movnta movntq + %assign %%i 0 + %rep 8 + CAT_XDEFINE m, %%i, mm %+ %%i + CAT_XDEFINE nmm, %%i, %%i + %assign %%i %%i+1 + %endrep + %rep 8 + CAT_UNDEF m, %%i + CAT_UNDEF nmm, %%i + %assign %%i %%i+1 + %endrep + INIT_CPUFLAGS %1 +%endmacro + +%macro INIT_XMM 0-1+ + %assign avx_enabled 0 + %define RESET_MM_PERMUTATION INIT_XMM %1 + %define mmsize 16 + %define num_mmregs 8 + %if ARCH_X86_64 + %define num_mmregs 16 + %endif + %define mova movdqa + %define movu movdqu + %define movh movq + %define movnta movntdq + %assign %%i 0 + %rep num_mmregs + CAT_XDEFINE m, %%i, xmm %+ %%i + CAT_XDEFINE nxmm, %%i, %%i + %assign %%i %%i+1 + %endrep + INIT_CPUFLAGS %1 +%endmacro + +%macro INIT_YMM 0-1+ + %assign avx_enabled 1 + %define RESET_MM_PERMUTATION INIT_YMM %1 + %define mmsize 32 + %define num_mmregs 8 + %if ARCH_X86_64 + %define num_mmregs 16 + %endif + %define mova movdqa + %define movu movdqu + %undef movh + %define movnta movntdq + %assign %%i 0 + %rep num_mmregs + CAT_XDEFINE m, %%i, ymm %+ %%i + CAT_XDEFINE nymm, %%i, %%i + %assign %%i %%i+1 + %endrep + INIT_CPUFLAGS %1 +%endmacro + +INIT_XMM + +%macro DECLARE_MMCAST 1 + %define mmmm%1 mm%1 + %define mmxmm%1 mm%1 + %define mmymm%1 mm%1 + %define xmmmm%1 mm%1 + %define xmmxmm%1 xmm%1 + %define xmmymm%1 xmm%1 + %define ymmmm%1 mm%1 + %define ymmxmm%1 xmm%1 + %define ymmymm%1 ymm%1 + %define xm%1 xmm %+ m%1 + %define ym%1 ymm %+ m%1 +%endmacro + +%assign i 0 +%rep 16 + DECLARE_MMCAST i +%assign i i+1 +%endrep + +; I often want to use macros that permute their arguments. e.g. there's no +; efficient way to implement butterfly or transpose or dct without swapping some +; arguments. +; +; I would like to not have to manually keep track of the permutations: +; If I insert a permutation in the middle of a function, it should automatically +; change everything that follows. For more complex macros I may also have multiple +; implementations, e.g. the SSE2 and SSSE3 versions may have different permutations. +; +; Hence these macros. Insert a PERMUTE or some SWAPs at the end of a macro that +; permutes its arguments. It's equivalent to exchanging the contents of the +; registers, except that this way you exchange the register names instead, so it +; doesn't cost any cycles. + +%macro PERMUTE 2-* ; takes a list of pairs to swap +%rep %0/2 + %xdefine %%tmp%2 m%2 + %rotate 2 +%endrep +%rep %0/2 + %xdefine m%1 %%tmp%2 + CAT_XDEFINE n, m%1, %1 + %rotate 2 +%endrep +%endmacro + +%macro SWAP 2+ ; swaps a single chain (sometimes more concise than pairs) +%ifnum %1 ; SWAP 0, 1, ... + SWAP_INTERNAL_NUM %1, %2 +%else ; SWAP m0, m1, ... + SWAP_INTERNAL_NAME %1, %2 +%endif +%endmacro + +%macro SWAP_INTERNAL_NUM 2-* + %rep %0-1 + %xdefine %%tmp m%1 + %xdefine m%1 m%2 + %xdefine m%2 %%tmp + CAT_XDEFINE n, m%1, %1 + CAT_XDEFINE n, m%2, %2 + %rotate 1 + %endrep +%endmacro + +%macro SWAP_INTERNAL_NAME 2-* + %xdefine %%args n %+ %1 + %rep %0-1 + %xdefine %%args %%args, n %+ %2 + %rotate 1 + %endrep + SWAP_INTERNAL_NUM %%args +%endmacro + +; If SAVE_MM_PERMUTATION is placed at the end of a function, then any later +; calls to that function will automatically load the permutation, so values can +; be returned in mmregs. +%macro SAVE_MM_PERMUTATION 0-1 + %if %0 + %xdefine %%f %1_m + %else + %xdefine %%f current_function %+ _m + %endif + %assign %%i 0 + %rep num_mmregs + CAT_XDEFINE %%f, %%i, m %+ %%i + %assign %%i %%i+1 + %endrep +%endmacro + +%macro LOAD_MM_PERMUTATION 1 ; name to load from + %ifdef %1_m0 + %assign %%i 0 + %rep num_mmregs + CAT_XDEFINE m, %%i, %1_m %+ %%i + CAT_XDEFINE n, m %+ %%i, %%i + %assign %%i %%i+1 + %endrep + %endif +%endmacro + +; Append cpuflags to the callee's name iff the appended name is known and the plain name isn't +%macro call 1 + call_internal %1, %1 %+ SUFFIX +%endmacro +%macro call_internal 2 + %xdefine %%i %1 + %ifndef cglobaled_%1 + %ifdef cglobaled_%2 + %xdefine %%i %2 + %endif + %endif + call %%i + LOAD_MM_PERMUTATION %%i +%endmacro + +; Substitutions that reduce instruction size but are functionally equivalent +%macro add 2 + %ifnum %2 + %if %2==128 + sub %1, -128 + %else + add %1, %2 + %endif + %else + add %1, %2 + %endif +%endmacro + +%macro sub 2 + %ifnum %2 + %if %2==128 + add %1, -128 + %else + sub %1, %2 + %endif + %else + sub %1, %2 + %endif +%endmacro + +;============================================================================= +; AVX abstraction layer +;============================================================================= + +%assign i 0 +%rep 16 + %if i < 8 + CAT_XDEFINE sizeofmm, i, 8 + %endif + CAT_XDEFINE sizeofxmm, i, 16 + CAT_XDEFINE sizeofymm, i, 32 +%assign i i+1 +%endrep +%undef i + +%macro CHECK_AVX_INSTR_EMU 3-* + %xdefine %%opcode %1 + %xdefine %%dst %2 + %rep %0-2 + %ifidn %%dst, %3 + %error non-avx emulation of ``%%opcode'' is not supported + %endif + %rotate 1 + %endrep +%endmacro + +;%1 == instruction +;%2 == 1 if float, 0 if int +;%3 == 1 if non-destructive or 4-operand (xmm, xmm, xmm, imm), 0 otherwise +;%4 == 1 if commutative (i.e. doesn't matter which src arg is which), 0 if not +;%5+: operands +%macro RUN_AVX_INSTR 5-8+ + %ifnum sizeof%6 + %assign %%sizeofreg sizeof%6 + %elifnum sizeof%5 + %assign %%sizeofreg sizeof%5 + %else + %assign %%sizeofreg mmsize + %endif + %assign %%emulate_avx 0 + %if avx_enabled && %%sizeofreg >= 16 + %xdefine %%instr v%1 + %else + %xdefine %%instr %1 + %if %0 >= 7+%3 + %assign %%emulate_avx 1 + %endif + %endif + + %if %%emulate_avx + %xdefine %%src1 %6 + %xdefine %%src2 %7 + %ifnidn %5, %6 + %if %0 >= 8 + CHECK_AVX_INSTR_EMU {%1 %5, %6, %7, %8}, %5, %7, %8 + %else + CHECK_AVX_INSTR_EMU {%1 %5, %6, %7}, %5, %7 + %endif + %if %4 && %3 == 0 + %ifnid %7 + ; 3-operand AVX instructions with a memory arg can only have it in src2, + ; whereas SSE emulation prefers to have it in src1 (i.e. the mov). + ; So, if the instruction is commutative with a memory arg, swap them. + %xdefine %%src1 %7 + %xdefine %%src2 %6 + %endif + %endif + %if %%sizeofreg == 8 + MOVQ %5, %%src1 + %elif %2 + MOVAPS %5, %%src1 + %else + MOVDQA %5, %%src1 + %endif + %endif + %if %0 >= 8 + %1 %5, %%src2, %8 + %else + %1 %5, %%src2 + %endif + %elif %0 >= 8 + %%instr %5, %6, %7, %8 + %elif %0 == 7 + %%instr %5, %6, %7 + %elif %0 == 6 + %%instr %5, %6 + %else + %%instr %5 + %endif +%endmacro + +;%1 == instruction +;%2 == 1 if float, 0 if int +;%3 == 1 if non-destructive or 4-operand (xmm, xmm, xmm, imm), 0 otherwise +;%4 == 1 if commutative (i.e. doesn't matter which src arg is which), 0 if not +%macro AVX_INSTR 1-4 0, 1, 0 + %macro %1 1-9 fnord, fnord, fnord, fnord, %1, %2, %3, %4 + %ifidn %2, fnord + RUN_AVX_INSTR %6, %7, %8, %9, %1 + %elifidn %3, fnord + RUN_AVX_INSTR %6, %7, %8, %9, %1, %2 + %elifidn %4, fnord + RUN_AVX_INSTR %6, %7, %8, %9, %1, %2, %3 + %elifidn %5, fnord + RUN_AVX_INSTR %6, %7, %8, %9, %1, %2, %3, %4 + %else + RUN_AVX_INSTR %6, %7, %8, %9, %1, %2, %3, %4, %5 + %endif + %endmacro +%endmacro + +; Instructions with both VEX and non-VEX encodings +; Non-destructive instructions are written without parameters +AVX_INSTR addpd, 1, 0, 1 +AVX_INSTR addps, 1, 0, 1 +AVX_INSTR addsd, 1, 0, 1 +AVX_INSTR addss, 1, 0, 1 +AVX_INSTR addsubpd, 1, 0, 0 +AVX_INSTR addsubps, 1, 0, 0 +AVX_INSTR aesdec, 0, 0, 0 +AVX_INSTR aesdeclast, 0, 0, 0 +AVX_INSTR aesenc, 0, 0, 0 +AVX_INSTR aesenclast, 0, 0, 0 +AVX_INSTR aesimc +AVX_INSTR aeskeygenassist +AVX_INSTR andnpd, 1, 0, 0 +AVX_INSTR andnps, 1, 0, 0 +AVX_INSTR andpd, 1, 0, 1 +AVX_INSTR andps, 1, 0, 1 +AVX_INSTR blendpd, 1, 0, 0 +AVX_INSTR blendps, 1, 0, 0 +AVX_INSTR blendvpd, 1, 0, 0 +AVX_INSTR blendvps, 1, 0, 0 +AVX_INSTR cmppd, 1, 1, 0 +AVX_INSTR cmpps, 1, 1, 0 +AVX_INSTR cmpsd, 1, 1, 0 +AVX_INSTR cmpss, 1, 1, 0 +AVX_INSTR comisd +AVX_INSTR comiss +AVX_INSTR cvtdq2pd +AVX_INSTR cvtdq2ps +AVX_INSTR cvtpd2dq +AVX_INSTR cvtpd2ps +AVX_INSTR cvtps2dq +AVX_INSTR cvtps2pd +AVX_INSTR cvtsd2si +AVX_INSTR cvtsd2ss +AVX_INSTR cvtsi2sd +AVX_INSTR cvtsi2ss +AVX_INSTR cvtss2sd +AVX_INSTR cvtss2si +AVX_INSTR cvttpd2dq +AVX_INSTR cvttps2dq +AVX_INSTR cvttsd2si +AVX_INSTR cvttss2si +AVX_INSTR divpd, 1, 0, 0 +AVX_INSTR divps, 1, 0, 0 +AVX_INSTR divsd, 1, 0, 0 +AVX_INSTR divss, 1, 0, 0 +AVX_INSTR dppd, 1, 1, 0 +AVX_INSTR dpps, 1, 1, 0 +AVX_INSTR extractps +AVX_INSTR haddpd, 1, 0, 0 +AVX_INSTR haddps, 1, 0, 0 +AVX_INSTR hsubpd, 1, 0, 0 +AVX_INSTR hsubps, 1, 0, 0 +AVX_INSTR insertps, 1, 1, 0 +AVX_INSTR lddqu +AVX_INSTR ldmxcsr +AVX_INSTR maskmovdqu +AVX_INSTR maxpd, 1, 0, 1 +AVX_INSTR maxps, 1, 0, 1 +AVX_INSTR maxsd, 1, 0, 1 +AVX_INSTR maxss, 1, 0, 1 +AVX_INSTR minpd, 1, 0, 1 +AVX_INSTR minps, 1, 0, 1 +AVX_INSTR minsd, 1, 0, 1 +AVX_INSTR minss, 1, 0, 1 +AVX_INSTR movapd +AVX_INSTR movaps +AVX_INSTR movd +AVX_INSTR movddup +AVX_INSTR movdqa +AVX_INSTR movdqu +AVX_INSTR movhlps, 1, 0, 0 +AVX_INSTR movhpd, 1, 0, 0 +AVX_INSTR movhps, 1, 0, 0 +AVX_INSTR movlhps, 1, 0, 0 +AVX_INSTR movlpd, 1, 0, 0 +AVX_INSTR movlps, 1, 0, 0 +AVX_INSTR movmskpd +AVX_INSTR movmskps +AVX_INSTR movntdq +AVX_INSTR movntdqa +AVX_INSTR movntpd +AVX_INSTR movntps +AVX_INSTR movq +AVX_INSTR movsd, 1, 0, 0 +AVX_INSTR movshdup +AVX_INSTR movsldup +AVX_INSTR movss, 1, 0, 0 +AVX_INSTR movupd +AVX_INSTR movups +AVX_INSTR mpsadbw, 0, 1, 0 +AVX_INSTR mulpd, 1, 0, 1 +AVX_INSTR mulps, 1, 0, 1 +AVX_INSTR mulsd, 1, 0, 1 +AVX_INSTR mulss, 1, 0, 1 +AVX_INSTR orpd, 1, 0, 1 +AVX_INSTR orps, 1, 0, 1 +AVX_INSTR pabsb +AVX_INSTR pabsd +AVX_INSTR pabsw +AVX_INSTR packsswb, 0, 0, 0 +AVX_INSTR packssdw, 0, 0, 0 +AVX_INSTR packuswb, 0, 0, 0 +AVX_INSTR packusdw, 0, 0, 0 +AVX_INSTR paddb, 0, 0, 1 +AVX_INSTR paddw, 0, 0, 1 +AVX_INSTR paddd, 0, 0, 1 +AVX_INSTR paddq, 0, 0, 1 +AVX_INSTR paddsb, 0, 0, 1 +AVX_INSTR paddsw, 0, 0, 1 +AVX_INSTR paddusb, 0, 0, 1 +AVX_INSTR paddusw, 0, 0, 1 +AVX_INSTR palignr, 0, 1, 0 +AVX_INSTR pand, 0, 0, 1 +AVX_INSTR pandn, 0, 0, 0 +AVX_INSTR pavgb, 0, 0, 1 +AVX_INSTR pavgw, 0, 0, 1 +AVX_INSTR pblendvb, 0, 0, 0 +AVX_INSTR pblendw, 0, 1, 0 +AVX_INSTR pclmulqdq, 0, 1, 0 +AVX_INSTR pc