Since psadbw only exists for 8-bits, we have to emulate it for 16-bit inputs. The simplest sequence is to use a normal subtraction, which is safe as long as the inputs do not exceed 32767 - so limit this implementation to 15-bit inputs and below. For 16-bit inputs, we could in theory instead use a pminw / pmaxw to ensure the resulting difference does not overflow, but this is slower, and also breaks the subsequent use of pmaddwd, so I opted to skip 16-bit SIMD for now. scene_sad10_c: 114175.6 ( 1.00x) scene_sad10_avx2: 9617.7 (11.87x) scene_sad10_avx512: 5208.8 (21.92x) scene_sad12_c: 114537.8 ( 1.00x) scene_sad12_avx2: 9614.0 (11.91x) scene_sad12_avx512: 5186.3 (22.08x) scene_sad14_c: 114113.9 ( 1.00x) scene_sad14_avx2: 9612.9 (11.87x) scene_sad14_avx512: 5186.0 (22.00x) scene_sad15_c: 114108.9 ( 1.00x) scene_sad15_avx2: 9612.3 (11.87x) scene_sad15_avx512: 5186.4 (22.00x) scene_sad16_c: 114136.0 ( 1.00x)
Unified · Split
| @@ -24,25 +24,47 @@ · 23 unmodified lines | ||
| 24 | 24 | |
| 25 | 25 | %include "libavutil/x86/x86util.asm" |
| 26 | 26 | |
| 27 | +SECTION_RODATA | |
| 28 | + | |
| 29 | +pw_1: times 32 dw 1 | |
| 30 | + | |
| 27 | 31 | SECTION .text |
| 28 | 32 | |
| 29 | 33 | |
| 30 | −%macro SAD_INIT 0 | |
| 31 | −cglobal scene_sad, 6, 7, 2, src1, stride1, src2, stride2, width, end, x | |
| 34 | +%macro SAD_INIT 1 ; depth | |
| 35 | +cglobal scene_sad%1, 6, 7, 3, src1, stride1, src2, stride2, width, end, x | |
| 32 | 36 | add src1q, widthq |
| 33 | 37 | add src2q, widthq |
| 34 | 38 | neg widthq |
| 35 | 39 | pxor m1, m1 |
| 36 | 40 | %endmacro |
| 37 | 41 | |
| 42 | +%macro PSADQ 4 ; depth, dst, [src2], tmp | |
| 43 | +%if %1 == 8 | |
| 44 | + psadbw %2, %3 | |
| 45 | +%else | |
| 46 | + psubw %2, %3 | |
| 47 | + pabsw %2, %2 | |
| 48 | + pmaddwd %2, [pw_1] | |
| 49 | + %if mmsize == 64 | |
| 50 | + vextracti32x8 ymm%4, %2, 1 | |
| 51 | + paddd ymm%2, ymm%4 | |
| 52 | + pmovzxdq %2, ymm%2 | |
| 53 | + %else | |
| 54 | + vextracti128 xmm%4, %2, 1 | |
| 55 | + paddd xmm%2, xmm%4 | |
| 56 | + pmovzxdq %2, xmm%2 | |
| 57 | + %endif | |
| 58 | +%endif | |
| 59 | +%endmacro | |
| 38 | 60 | |
| 39 | −%macro SAD_LOOP 0 | |
| 61 | +%macro SAD_LOOP 1 ; depth | |
| 40 | 62 | .nextrow: |
| 41 | 63 | mov xq, widthq |
| 42 | 64 | |
| 43 | 65 | .loop: |
| 44 | 66 | movu m0, [src1q + xq] |
| 45 | − psadbw m0, [src2q + xq] | |
| 67 | + PSADQ %1, m0, [src2q + xq], m2 | |
| 46 | 68 | paddq m1, m0 |
| 47 | 69 | add xq, mmsize |
| 48 | 70 | jl .loop |
| @@ -57,25 +79,27 @@ RET · 8 unmodified lines | ||
| 57 | 79 | %endmacro |
| 58 | 80 | |
| 59 | 81 | |
| 60 | −%macro SAD_FRAMES 0 | |
| 61 | − SAD_INIT | |
| 62 | − SAD_LOOP | |
| 82 | +%macro SAD_FRAMES 1 ; depth | |
| 83 | + SAD_INIT %1 | |
| 84 | + SAD_LOOP %1 | |
| 63 | 85 | %endmacro |
| 64 | 86 | |
| 65 | 87 | |
| 66 | 88 | INIT_XMM sse2 |
| 67 | −SAD_FRAMES | |
| 89 | +SAD_FRAMES 8 | |
| 68 | 90 | |
| 69 | 91 | %if HAVE_AVX2_EXTERNAL |
| 70 | 92 | |
| 71 | 93 | INIT_YMM avx2 |
| 72 | −SAD_FRAMES | |
| 94 | +SAD_FRAMES 8 | |
| 95 | +SAD_FRAMES 16 | |
| 73 | 96 | |
| 74 | 97 | %endif |
| 75 | 98 | |
| 76 | 99 | %if HAVE_AVX512_EXTERNAL |
| 77 | 100 | |
| 78 | 101 | INIT_ZMM avx512 |
| 79 | −SAD_FRAMES | |
| 102 | +SAD_FRAMES 8 | |
| 103 | +SAD_FRAMES 16 | |
| 80 | 104 | |
| 81 | 105 | %endif |
| @@ -36,13 +36,31 @@ static void FUNC_NAME(SCENE_SAD_PARAMS) { \ · 35 unmodified lines | ||
| 36 | 36 | *sum += sad[0]; \ |
| 37 | 37 | } |
| 38 | 38 | |
| 39 | +#define SCENE_SAD16_FUNC(FUNC_NAME, ASM_FUNC_NAME, MMSIZE) \ | |
| 40 | +void ASM_FUNC_NAME(SCENE_SAD_PARAMS); \ | |
| 41 | + \ | |
| 42 | +static void FUNC_NAME(SCENE_SAD_PARAMS) { \ | |
| 43 | + uint64_t sad[MMSIZE / 8] = {0}; \ | |
| 44 | + ptrdiff_t bytes = (width << 1) & ~(MMSIZE - 1); \ | |
| 45 | + *sum = 0; \ | |
| 46 | + ASM_FUNC_NAME(src1, stride1, src2, stride2, bytes, height, sad); \ | |
| 47 | + for (int i = 0; i < MMSIZE / 8; i++) \ | |
| 48 | + *sum += sad[i]; \ | |
| 49 | + ff_scene_sad16_c(src1 + bytes, stride1, \ | |
| 50 | + src2 + bytes, stride2, \ | |
| 51 | + width - (bytes >> 1), height, sad); \ | |
| 52 | + *sum += sad[0]; \ | |
| 53 | +} | |
| 54 | + | |
| 39 | 55 | #if HAVE_X86ASM |
| 40 | −SCENE_SAD_FUNC(scene_sad_sse2, ff_scene_sad_sse2, 16) | |
| 56 | +SCENE_SAD_FUNC(scene_sad_sse2, ff_scene_sad8_sse2, 16) | |
| 41 | 57 | #if HAVE_AVX2_EXTERNAL |
| 42 | −SCENE_SAD_FUNC(scene_sad_avx2, ff_scene_sad_avx2, 32) | |
| 58 | +SCENE_SAD_FUNC(scene_sad_avx2, ff_scene_sad8_avx2, 32) | |
| 59 | +SCENE_SAD16_FUNC(scene_sad16_avx2, ff_scene_sad16_avx2, 32) | |
| 43 | 60 | #endif |
| 44 | 61 | #if HAVE_AVX512_EXTERNAL |
| 45 | −SCENE_SAD_FUNC(scene_sad_avx512, ff_scene_sad_avx512, 64) | |
| 62 | +SCENE_SAD_FUNC(scene_sad_avx512, ff_scene_sad8_avx512, 64) | |
| 63 | +SCENE_SAD16_FUNC(scene_sad16_avx512, ff_scene_sad16_avx512, 64) | |
| 46 | 64 | #endif |
| 47 | 65 | #endif |
| 48 | 66 | |
| @@ -61,6 +79,15 @@ ff_scene_sad_fn ff_scene_sad_get_fn_x86(int depth) · 12 unmodified lines | ||
| 61 | 79 | #endif |
| 62 | 80 | if (EXTERNAL_SSE2(cpu_flags)) |
| 63 | 81 | return scene_sad_sse2; |
| 82 | + } else if (depth < 16) { /* this routine is only safe up to 15 bits */ | |
| 83 | +#if HAVE_AVX512_EXTERNAL | |
| 84 | + if (EXTERNAL_AVX512(cpu_flags)) | |
| 85 | + return scene_sad16_avx512; | |
| 86 | +#endif | |
| 87 | +#if HAVE_AVX2_EXTERNAL | |
| 88 | + if (EXTERNAL_AVX2_FAST(cpu_flags)) | |
| 89 | + return scene_sad16_avx2; | |
| 90 | +#endif | |
| 64 | 91 | } |
| 65 | 92 | #endif |
| 66 | 93 | return NULL; |