Since psadbw only exists for 8-bits, we have to emulate it for 16-bit inputs. The simplest sequence is to use a normal subtraction, which is safe as long as the inputs do not exceed 32767 - so limit this implementation to 15-bit inputs and below. For 16-bit inputs, we could in theory instead use a pminw / pmaxw to ensure the resulting difference does not overflow, but this is slower, and also breaks the subsequent use of pmaddwd, so I opted to skip 16-bit SIMD for now. scene_sad10_c: 114175.6 ( 1.00x) scene_sad10_avx2: 9617.7 (11.87x) scene_sad10_avx512: 5208.8 (21.92x) scene_sad12_c: 114537.8 ( 1.00x) scene_sad12_avx2: 9614.0 (11.91x) scene_sad12_avx512: 5186.3 (22.08x) scene_sad14_c: 114113.9 ( 1.00x) scene_sad14_avx2: 9612.9 (11.87x) scene_sad14_avx512: 5186.0 (22.00x) scene_sad15_c: 114108.9 ( 1.00x) scene_sad15_avx2: 9612.3 (11.87x) scene_sad15_avx512: 5186.4 (22.00x) scene_sad16_c: 114136.0 ( 1.00x)
Unified · Split
| @@ -24,25 +24,47 @@ · 23 unmodified lines | |||
| 24 | 24 | ||
| 25 | %include "libavutil/x86/x86util.asm" | 25 | %include "libavutil/x86/x86util.asm" |
| 26 | 26 | ||
| 27 | SECTION_RODATA | ||
| 28 | |||
| 29 | pw_1: times 32 dw 1 | ||
| 30 | |||
| 27 | SECTION .text | 31 | SECTION .text |
| 28 | 32 | ||
| 29 | 33 | ||
| 30 | %macro SAD_INIT 0 | 34 | %macro SAD_INIT 1 ; depth |
| 31 | cglobal scene_sad, 6, 7, 2, src1, stride1, src2, stride2, width, end, x | 35 | cglobal scene_sad%1, 6, 7, 3, src1, stride1, src2, stride2, width, end, x |
| 32 | add src1q, widthq | 36 | add src1q, widthq |
| 33 | add src2q, widthq | 37 | add src2q, widthq |
| 34 | neg widthq | 38 | neg widthq |
| 35 | pxor m1, m1 | 39 | pxor m1, m1 |
| 36 | %endmacro | 40 | %endmacro |
| 37 | 41 | ||
| 42 | %macro PSADQ 4 ; depth, dst, [src2], tmp | ||
| 43 | %if %1 == 8 | ||
| 44 | psadbw %2, %3 | ||
| 45 | %else | ||
| 46 | psubw %2, %3 | ||
| 47 | pabsw %2, %2 | ||
| 48 | pmaddwd %2, [pw_1] | ||
| 49 | %if mmsize == 64 | ||
| 50 | vextracti32x8 ymm%4, %2, 1 | ||
| 51 | paddd ymm%2, ymm%4 | ||
| 52 | pmovzxdq %2, ymm%2 | ||
| 53 | %else | ||
| 54 | vextracti128 xmm%4, %2, 1 | ||
| 55 | paddd xmm%2, xmm%4 | ||
| 56 | pmovzxdq %2, xmm%2 | ||
| 57 | %endif | ||
| 58 | %endif | ||
| 59 | %endmacro | ||
| 38 | 60 | ||
| 39 | %macro SAD_LOOP 0 | 61 | %macro SAD_LOOP 1 ; depth |
| 40 | .nextrow: | 62 | .nextrow: |
| 41 | mov xq, widthq | 63 | mov xq, widthq |
| 42 | 64 | ||
| 43 | .loop: | 65 | .loop: |
| 44 | movu m0, [src1q + xq] | 66 | movu m0, [src1q + xq] |
| 45 | psadbw m0, [src2q + xq] | 67 | PSADQ %1, m0, [src2q + xq], m2 |
| 46 | paddq m1, m0 | 68 | paddq m1, m0 |
| 47 | add xq, mmsize | 69 | add xq, mmsize |
| 48 | jl .loop | 70 | jl .loop |
| @@ -57,25 +79,27 @@ RET · 8 unmodified lines | |||
| 57 | %endmacro | 79 | %endmacro |
| 58 | 80 | ||
| 59 | 81 | ||
| 60 | %macro SAD_FRAMES 0 | 82 | %macro SAD_FRAMES 1 ; depth |
| 61 | SAD_INIT | 83 | SAD_INIT %1 |
| 62 | SAD_LOOP | 84 | SAD_LOOP %1 |
| 63 | %endmacro | 85 | %endmacro |
| 64 | 86 | ||
| 65 | 87 | ||
| 66 | INIT_XMM sse2 | 88 | INIT_XMM sse2 |
| 67 | SAD_FRAMES | 89 | SAD_FRAMES 8 |
| 68 | 90 | ||
| 69 | %if HAVE_AVX2_EXTERNAL | 91 | %if HAVE_AVX2_EXTERNAL |
| 70 | 92 | ||
| 71 | INIT_YMM avx2 | 93 | INIT_YMM avx2 |
| 72 | SAD_FRAMES | 94 | SAD_FRAMES 8 |
| 95 | SAD_FRAMES 16 | ||
| 73 | 96 | ||
| 74 | %endif | 97 | %endif |
| 75 | 98 | ||
| 76 | %if HAVE_AVX512_EXTERNAL | 99 | %if HAVE_AVX512_EXTERNAL |
| 77 | 100 | ||
| 78 | INIT_ZMM avx512 | 101 | INIT_ZMM avx512 |
| 79 | SAD_FRAMES | 102 | SAD_FRAMES 8 |
| 103 | SAD_FRAMES 16 | ||
| 80 | 104 | ||
| 81 | %endif | 105 | %endif |
| @@ -36,13 +36,31 @@ static void FUNC_NAME(SCENE_SAD_PARAMS) { \ · 35 unmodified lines | |||
| 36 | *sum += sad[0]; \ | 36 | *sum += sad[0]; \ |
| 37 | } | 37 | } |
| 38 | 38 | ||
| 39 | #define SCENE_SAD16_FUNC(FUNC_NAME, ASM_FUNC_NAME, MMSIZE) \ | ||
| 40 | void ASM_FUNC_NAME(SCENE_SAD_PARAMS); \ | ||
| 41 | \ | ||
| 42 | static void FUNC_NAME(SCENE_SAD_PARAMS) { \ | ||
| 43 | uint64_t sad[MMSIZE / 8] = {0}; \ | ||
| 44 | ptrdiff_t bytes = (width << 1) & ~(MMSIZE - 1); \ | ||
| 45 | *sum = 0; \ | ||
| 46 | ASM_FUNC_NAME(src1, stride1, src2, stride2, bytes, height, sad); \ | ||
| 47 | for (int i = 0; i < MMSIZE / 8; i++) \ | ||
| 48 | *sum += sad[i]; \ | ||
| 49 | ff_scene_sad16_c(src1 + bytes, stride1, \ | ||
| 50 | src2 + bytes, stride2, \ | ||
| 51 | width - (bytes >> 1), height, sad); \ | ||
| 52 | *sum += sad[0]; \ | ||
| 53 | } | ||
| 54 | |||
| 39 | #if HAVE_X86ASM | 55 | #if HAVE_X86ASM |
| 40 | SCENE_SAD_FUNC(scene_sad_sse2, ff_scene_sad_sse2, 16) | 56 | SCENE_SAD_FUNC(scene_sad_sse2, ff_scene_sad8_sse2, 16) |
| 41 | #if HAVE_AVX2_EXTERNAL | 57 | #if HAVE_AVX2_EXTERNAL |
| 42 | SCENE_SAD_FUNC(scene_sad_avx2, ff_scene_sad_avx2, 32) | 58 | SCENE_SAD_FUNC(scene_sad_avx2, ff_scene_sad8_avx2, 32) |
| 59 | SCENE_SAD16_FUNC(scene_sad16_avx2, ff_scene_sad16_avx2, 32) | ||
| 43 | #endif | 60 | #endif |
| 44 | #if HAVE_AVX512_EXTERNAL | 61 | #if HAVE_AVX512_EXTERNAL |
| 45 | SCENE_SAD_FUNC(scene_sad_avx512, ff_scene_sad_avx512, 64) | 62 | SCENE_SAD_FUNC(scene_sad_avx512, ff_scene_sad8_avx512, 64) |
| 63 | SCENE_SAD16_FUNC(scene_sad16_avx512, ff_scene_sad16_avx512, 64) | ||
| 46 | #endif | 64 | #endif |
| 47 | #endif | 65 | #endif |
| 48 | 66 | ||
| @@ -61,6 +79,15 @@ ff_scene_sad_fn ff_scene_sad_get_fn_x86(int depth) · 12 unmodified lines | |||
| 61 | #endif | 79 | #endif |
| 62 | if (EXTERNAL_SSE2(cpu_flags)) | 80 | if (EXTERNAL_SSE2(cpu_flags)) |
| 63 | return scene_sad_sse2; | 81 | return scene_sad_sse2; |
| 82 | } else if (depth < 16) { /* this routine is only safe up to 15 bits */ | ||
| 83 | #if HAVE_AVX512_EXTERNAL | ||
| 84 | if (EXTERNAL_AVX512(cpu_flags)) | ||
| 85 | return scene_sad16_avx512; | ||
| 86 | #endif | ||
| 87 | #if HAVE_AVX2_EXTERNAL | ||
| 88 | if (EXTERNAL_AVX2_FAST(cpu_flags)) | ||
| 89 | return scene_sad16_avx2; | ||
| 90 | #endif | ||
| 64 | } | 91 | } |
| 65 | #endif | 92 | #endif |
| 66 | return NULL; | 93 | return NULL; |