History

[Processing] Add AVX2 VAA routines

Process 8 lines at a time rather than 16 lines at a time because this appears to give more reliable memory subsystem performance on Haswell. Speedup is > 2x as compared to SSE2 when not memory-bound on Haswell. On my Haswell MBP, VAACalcSadSsdBgd is about ~3x faster when uncached, which appears to be related to processing 8 lines at a time as opposed to 16 lines at a time. The other routines are also faster as compared to the SSE2 routines in this case but to a lesser extent.

Sindre Aamås committed 10 years ago

Commit 57fc3e91 parent eb9f5654 files · +1547 −0Browse files

Showing the changes to codec/processing/src (3 of 4 files). Show all

Unified · Split

3 files
  1. codec/processing/src/vaacalc/vaacalculation.cpp +7 −0
  2. codec/processing/src/vaacalc/vaacalculation.h +5 −0
  3. codec/processing/src/x86/vaa.asm +1529 −0
codec/processing/src/vaacalc/vaacalculation.cpp+7−0
@@ -64,6 +64,13 @@ void CVAACalculation::InitVaaFuncs (SVaaFuncs& sVaaFuncs, int32_t iCpuFlag) { · 63 unmodified lines
64 sVaaFuncs.pfVAACalcSadSsdBgd = VAACalcSadSsdBgd_sse2;64 sVaaFuncs.pfVAACalcSadSsdBgd = VAACalcSadSsdBgd_sse2;
65 sVaaFuncs.pfVAACalcSadVar = VAACalcSadVar_sse2;65 sVaaFuncs.pfVAACalcSadVar = VAACalcSadVar_sse2;
66 }66 }
67 if (iCpuFlag & WELS_CPU_AVX2) {
68 sVaaFuncs.pfVAACalcSad = VAACalcSad_avx2;
69 sVaaFuncs.pfVAACalcSadBgd = VAACalcSadBgd_avx2;
70 sVaaFuncs.pfVAACalcSadSsd = VAACalcSadSsd_avx2;
71 sVaaFuncs.pfVAACalcSadSsdBgd = VAACalcSadSsdBgd_avx2;
72 sVaaFuncs.pfVAACalcSadVar = VAACalcSadVar_avx2;
73 }
67#endif//X86_ASM74#endif//X86_ASM
68#ifdef HAVE_NEON75#ifdef HAVE_NEON
69 if ((iCpuFlag & WELS_CPU_NEON) == WELS_CPU_NEON) {76 if ((iCpuFlag & WELS_CPU_NEON) == WELS_CPU_NEON) {
codec/processing/src/vaacalc/vaacalculation.h+5−0
@@ -104,6 +104,11 @@ VAACalcSadSsdBgdFunc VAACalcSadSsdBgd_sse2; · 103 unmodified lines
104VAACalcSadFunc VAACalcSad_sse2;104VAACalcSadFunc VAACalcSad_sse2;
105VAACalcSadVarFunc VAACalcSadVar_sse2;105VAACalcSadVarFunc VAACalcSadVar_sse2;
106VAACalcSadSsdFunc VAACalcSadSsd_sse2;106VAACalcSadSsdFunc VAACalcSadSsd_sse2;
107VAACalcSadBgdFunc VAACalcSadBgd_avx2;
108VAACalcSadSsdBgdFunc VAACalcSadSsdBgd_avx2;
109VAACalcSadFunc VAACalcSad_avx2;
110VAACalcSadVarFunc VAACalcSadVar_avx2;
111VAACalcSadSsdFunc VAACalcSadSsd_avx2;
107WELSVP_EXTERN_C_END112WELSVP_EXTERN_C_END
108#endif113#endif
109 114
codec/processing/src/x86/vaa.asm+1529−0
@@ -2028,3 +2028,1532 @@ sqdiff_bgd_width_loop: · 2027 unmodified lines
2028%undef localsize2028%undef localsize
2029 ret2029 ret
2030%endif2030%endif
2031
2032%ifdef X86_32
2033%define ptrword dword
2034%else
2035%define ptrword qword
2036%endif
2037
2038%define xmm_width 16
2039%define ymm_width 32
2040
2041%macro PUSHM 1-*
2042 %rep %0
2043 push %1
2044 %rotate 1
2045 %endrep
2046 %assign push_num push_num + %0
2047%endmacro
2048
2049%macro POPM 1-*
2050 %rep %0
2051 %rotate -1
2052 pop %1
2053 %endrep
2054 %assign push_num push_num - %0
2055%endmacro
2056
2057%ifdef X86_32
2058%define stack_alloc_min 4
2059%else
2060%define stack_alloc_min 8
2061%endif
2062
2063; Allocate aligned stack space.
2064; address_out=%1 size=%2 alignment=%3
2065%macro STACK_ALLOC 3
2066%if (%3) & ((%3) - 1)
2067 %error non-power-of-2 alignment requested.
2068%endif
2069%if (%3) > 0
2070 %assign stack_alloc_align ((%3) + stack_alloc_min - 1) / stack_alloc_min
2071%else
2072 %assign stack_alloc_align 1
2073%endif
2074 %assign stack_alloc_num ((%2) + stack_alloc_min - 1) / stack_alloc_min + stack_alloc_align - 1
2075 %assign push_num push_num + stack_alloc_num
2076 sub r7, stack_alloc_min * stack_alloc_num
2077%if stack_alloc_align == 1
2078 mov %1, r7
2079%else
2080 lea %1, [r7 + stack_alloc_min * (stack_alloc_align - 1)]
2081 and %1, -(stack_alloc_min * stack_alloc_align)
2082%endif
2083%endmacro
2084
2085; Deallocate stack space allocated with STACK_ALLOC.
2086%macro STACK_DEALLOC 0
2087 add r7, stack_alloc_min * stack_alloc_num
2088 %assign push_num push_num - stack_alloc_num
2089%endmacro
2090
2091; Max unsigned byte per quadword
2092; out=%1 in=%2 tmp=%3
2093%macro AVX2_Maxubq 3
2094 vpsrlq %3, %2, 32
2095 vpmaxub %1, %2, %3
2096 vpsrlq %3, %1, 16
2097 vpmaxub %1, %1, %3
2098 vpsrlq %3, %1, 8
2099 vpmaxub %1, %1, %3
2100%endmacro
2101
2102; Max unsigned byte per quadword. 2 register input.
2103; Results interleaved as least significant byte of even/odd doublewords.
2104; out=%1 in_a=%2 in_b=%3 tmp=%4
2105%macro AVX2_Maxubq2 4
2106 vpblendd %4, %2, %3, 10101010b
2107 vpshufd %4, %4, 10110001b
2108 vpblendd %1, %2, %3, 01010101b
2109 vpmaxub %1, %4, %1
2110 vpsrld %4, %1, 16
2111 vpmaxub %1, %1, %4
2112 vpsrld %4, %1, 8
2113 vpmaxub %1, %1, %4
2114%endmacro
2115
2116; res=%1 src=%2 zero=%3 tmp=%4 add_to_res=%5
2117%macro AVX2_Sqsumbdw 5
2118 vpunpcklbw %4, %2, %3
2119%if %5
2120 vpmaddwd %4, %4, %4
2121 vpaddd %1, %1, %4
2122%else
2123 vpmaddwd %1, %4, %4
2124%endif
2125 vpunpckhbw %4, %2, %3
2126 vpmaddwd %4, %4, %4
2127 vpaddd %1, %1, %4
2128%endmacro
2129
2130; res=%1 src=%2 zero=%3 tmp=%4 add_to_res=%5
2131%macro AVX2_Sumbdw 5
2132%if %5
2133 vpsadbw %4, %2, %3
2134 vpaddd %1, %1, %4
2135%else
2136 vpsadbw %1, %2, %3
2137%endif
2138%endmacro
2139
2140; res=%1 a=%2 b=%3 a=%4 tmp=%5
2141%macro AVX2_AbsDiffub 5
2142 vpsubusb %5, %2, %3
2143 vpsubusb %1, %3, %4
2144 vpor %1, %5, %1
2145%endmacro
2146
2147; sad=%1 cur_data=%2 ref_data=%3 tmp=%4 accumulate_results=%5
2148%macro AVX2_Sadbdw 5
2149%if %5
2150 vpsadbw %4, %2, %3
2151 vpaddd %1, %1, %4
2152%else
2153 vpsadbw %1, %2, %3
2154%endif
2155%endmacro
2156
2157; sad=%1 sum_cur=%2 sqsum_cur=%3 cur_data=%4 ref_data=%5 zero=%6 tmp=%7 accumulate_results=%8
2158%macro AVX2_SadSumSqsumbdw 8
2159 AVX2_Sadbdw %1, %4, %5, %7, %8
2160 AVX2_Sumbdw %2, %4, %6, %7, %8
2161 AVX2_Sqsumbdw %3, %4, %6, %7, %8
2162%endmacro
2163
2164; sad=%1 pCur=%2 pRef=%3 tmp=%4 accumulate_results=%5
2165%macro AVX2_Sad 5
2166 vmovdqu %4, [%2]
2167 AVX2_Sadbdw %1, %4, [%3], %4, %5
2168%endmacro
2169
2170; sad=%1 sum_cur=%2 sqsum_cur=%3 pCur=%4 pRef=%5 zero=%6 tmp=%7,%8 accumulate_results=%9
2171%macro AVX2_SadSumSqsum 9
2172 vmovdqu %7, [%4]
2173 AVX2_SadSumSqsumbdw %1, %2, %3, %7, [%5], %6, %8, %9
2174%endmacro
2175
2176; sad=%1 sum_cur=%2 sqsum_cur=%3 sqdiff=%4 pCur=%5 pRef=%6 zero=%7 tmp=%8,%9,%10 accumulate_results=%11
2177%macro AVX2_SadSumSqsumSqdiff 11
2178 vmovdqu %8, [%5]
2179 vmovdqu %9, [%6]
2180 AVX2_SadSumSqsumbdw %1, %2, %3, %8, %9, %7, %10, %11
2181 AVX2_AbsDiffub %9, %8, %9, %8, %10
2182 AVX2_Sqsumbdw %4, %9, %7, %10, %11
2183%endmacro
2184
2185; sad=%1 sum_cur=%2 sum_ref=%3 mad=%4 pCur=%5 pRef=%6 zero=%7 tmp=%8,%9,%10 accumulate_results=%11
2186%macro AVX2_SadSdMad 11
2187 vmovdqu %8, [%5]
2188 vmovdqu %9, [%6]
2189 AVX2_Sumbdw %2, %8, %7, %10, %11
2190 AVX2_Sumbdw %3, %9, %7, %10, %11
2191 AVX2_Sadbdw %1, %8, %9, %10, %11
2192%if %11
2193 AVX2_AbsDiffub %9, %8, %9, %8, %10
2194 vpmaxub %4, %4, %9
2195%else
2196 AVX2_AbsDiffub %4, %8, %9, %8, %10
2197%endif
2198%endmacro
2199
2200; sad=%1 sum_cur=%2 sum_ref=%3 mad=%4 sqdiff=%5 sqsum_cur=%6 pCur=%7 pRef=%8 zero=%9 tmp=%10,%11,%12 accumulate_results=%13
2201%macro AVX2_SadBgdSqdiff 13
2202%ifidn %12, 0
2203 vmovdqu %10, [%7]
2204 AVX2_Sumbdw %2, %10, %9, %11, %13
2205 AVX2_Sqsumbdw %6, %10, %9, %11, %13
2206 vmovdqu %11, [%8]
2207 AVX2_Sadbdw %1, %10, %11, %10, %13
2208 AVX2_Sumbdw %3, %11, %9, %10, %13
2209 vmovdqu %10, [%7]
2210%if %13
2211 AVX2_AbsDiffub %11, %10, %11, [%7], %10
2212 vpmaxub %4, %4, %11
2213 AVX2_Sqsumbdw %5, %11, %9, %10, %13
2214%else
2215 AVX2_AbsDiffub %4, %10, %11, [%7], %10
2216 AVX2_Sqsumbdw %5, %4, %9, %10, %13
2217%endif
2218%else
2219 vmovdqu %10, [%7]
2220 vmovdqu %11, [%8]
2221 AVX2_Sadbdw %1, %10, %11, %12, %13
2222 AVX2_Sumbdw %2, %10, %9, %12, %13
2223 AVX2_Sumbdw %3, %11, %9, %12, %13
2224 AVX2_Sqsumbdw %6, %10, %9, %12, %13
2225%if %13
2226 AVX2_AbsDiffub %11, %10, %11, %10, %12
2227 vpmaxub %4, %4, %11
2228 AVX2_Sqsumbdw %5, %11, %9, %10, %13
2229%else
2230 AVX2_AbsDiffub %4, %10, %11, %10, %12
2231 AVX2_Sqsumbdw %5, %4, %9, %10, %13
2232%endif
2233%endif
2234%endmacro
2235
2236; p_dst=%1 mmreg_prefix=%2 data=%3 tmp=%4 second_blocks=%5
2237%macro AVX2_Store8x8Accdw 5
2238 vpshufd %2%4, %2%3, 1000b
2239%ifidni %2, x
2240 vmovlps [%1 + 8 * %5], x%4
2241%elif %5 == 0
2242 vmovdqu [%1], %2%4
2243%else
2244 vmovlps [%1 + 8], x%4
2245 vextracti128 x%4, %2%4, 1
2246 vmovlps [%1 + 24], x%4
2247%endif
2248%endmacro
2249
2250; p_dst=%1 mmreg_prefix=%2 data=%3 tmp=%4 second_blocks=%5
2251%macro AVX2_Store8x8Accb 5
2252 vpunpckhqdq %2%4, %2%3, %2%3
2253 vpunpcklbw %2%4, %2%3, %2%4
2254%if %5 == 0
2255 vmovd [%1 + 0], x%4
2256%ifidni %2, y
2257 vextracti128 x%4, %2%4, 1
2258 vmovd [%1 + 4], x%4
2259%endif
2260%else
2261 vpextrw [%1 + 2], x%4, 0
2262%ifidni %2, y
2263 vextracti128 x%4, %2%4, 1
2264 vpextrw [%1 + 6], x%4, 0
2265%endif
2266%endif
2267%endmacro
2268
2269; p_dst=%1 data=%2 tmp=%3,%4 second_blocks=%5
2270%macro AVX2_Store2x8x8Accb 5
2271 vpunpckhqdq y%3, y%2, y%2
2272 vpunpcklbw y%3, y%2, y%3
2273 vextracti128 x%4, y%3, 1
2274 vpsllq x%4, x%4, 32
2275 vpblendd x%4, x%3, x%4, 1010b
2276%if %5
2277 vpslld x%4, x%4, 16
2278 vpblendw x%4, x%4, [%1], 01010101b
2279%endif
2280 vmovdqu [%1], x%4
2281%endmacro
2282
2283; p_dst=%1 mmreg_prefix=%2 data=%3 tmp=%4 add_to_dst=%5
2284%macro AVX2_Store16x16Accdw 5
2285%ifidni %2, x
2286%if %5
2287 vmovd x%4, [%1 + 0]
2288 vpaddd x%3, x%4, x%3
2289%endif
2290 vmovd [%1 + 0], x%3
2291%elif %5 == 0
2292 vmovd [%1 + 0], x%3
2293 vextracti128 x%3, %2%3, 1
2294 vmovd [%1 + 4], x%3
2295%else
2296 vextracti128 x%4, %2%3, 1
2297 vpunpckldq x%4, x%3, x%4
2298 vmovq x%3, [%1 + 0]
2299 vpaddd x%3, x%3, x%4
2300 vmovlps [%1 + 0], x%3
2301%endif
2302%endmacro
2303
2304; p_dst1=%1 p_dst2=%2 i_dst_offset=%3 gpr_tmp=%4 mmreg_prefix=%5 data=%6 mm_tmp=%7 add_to_dst=%8
2305%macro AVX2_Store2x16x16Accdw 8
2306%ifidni %5, x
2307 mov %4, %1
2308%if %8 == 0
2309 vmovd [%4 + %3], x%6
2310 mov %4, %2
2311 vpextrd [%4 + %3], x%6, 2
2312%else
2313 vmovd x%7, [%4 + %3]
2314 vpaddd x%7, x%7, x%6
2315 vmovd [%4 + %3], x%7
2316 mov %4, %2
2317 vpbroadcastd x%7, [%4 + %3]
2318 vpaddd x%7, x%7, x%6
2319 vpextrd [%4 + %3], x%7, 2
2320%endif
2321%else
2322 vextracti128 x%7, %5%6, 1
2323 vpblendd x%6, x%6, x%7, 1010b
2324 mov %4, %1
2325%if %8 == 0
2326 vmovlps [%4 + %3], x%6
2327 mov %4, %2
2328 vmovhps [%4 + %3], x%6
2329%else
2330 vmovq x%7, [%4 + %3]
2331 vpaddd x%7, x%7, x%6
2332 vmovlps [%4 + %3], x%7
2333 mov %4, %2
2334 vpbroadcastq x%7, [%4 + %3]
2335 vpaddd x%7, x%7, x%6
2336 vmovhps [%4 + %3], x%7
2337%endif
2338%endif
2339%endmacro
2340
2341
2342; x/y-mm_prefix=%1 mm_clobber=%2,%3,%4,%5,%6 b_second_blocks=%7
2343%macro AVX2_CalcSad_8Lines 7
2344%define mm_tmp0 %2
2345%define mm_sad %3
2346%define mm_sad2 %4
2347%define mm_sad3 %5
2348%define mm_sad4 %6
2349%define b_second_blocks %7
2350%ifdef i_stride5
2351 %define i_stride5_ i_stride5
2352%else
2353 lea r_tmp, [5 * i_stride]
2354 %define i_stride5_ r_tmp
2355%endif
2356 ; Use multiple accumulators to shorten dependency chains and enable more parallelism.
2357 AVX2_Sad %1 %+ mm_sad, p_cur, p_ref, %1 %+ mm_tmp0, 0
2358 AVX2_Sad %1 %+ mm_sad2, p_cur + 1 * i_stride, p_ref + 1 * i_stride, %1 %+ mm_tmp0, 0
2359 AVX2_Sad %1 %+ mm_sad3, p_cur + 2 * i_stride, p_ref + 2 * i_stride, %1 %+ mm_tmp0, 0
2360 AVX2_Sad %1 %+ mm_sad4, p_cur + 1 * i_stride3, p_ref + 1 * i_stride3, %1 %+ mm_tmp0, 0
2361 AVX2_Sad %1 %+ mm_sad, p_cur + 4 * i_stride, p_ref + 4 * i_stride, %1 %+ mm_tmp0, 1
2362 AVX2_Sad %1 %+ mm_sad2, p_cur + 1 * i_stride5_, p_ref + 1 * i_stride5_, %1 %+ mm_tmp0, 1
2363%ifdef i_stride7
2364 %define i_stride7_ i_stride7
2365%else
2366 lea r_tmp, [i_stride + 2 * i_stride3]
2367 %define i_stride7_ r_tmp
2368%endif
2369 AVX2_Sad %1 %+ mm_sad3, p_cur + 2 * i_stride3, p_ref + 2 * i_stride3, %1 %+ mm_tmp0, 1
2370 AVX2_Sad %1 %+ mm_sad4, p_cur + 1 * i_stride7_, p_ref + 1 * i_stride7_, %1 %+ mm_tmp0, 1
2371%undef i_stride5_
2372%undef i_stride7_
2373 ; Increment addresses for the next iteration. Doing this early is beneficial on Haswell.
2374 add p_cur, %1 %+ mm_width
2375 add p_ref, %1 %+ mm_width
2376 ; Collapse accumulators.
2377 vpaddd %1 %+ mm_sad, %1 %+ mm_sad, %1 %+ mm_sad2
2378 vpaddd %1 %+ mm_sad3, %1 %+ mm_sad3, %1 %+ mm_sad4
2379 vpaddd %1 %+ mm_sad, %1 %+ mm_sad, %1 %+ mm_sad3
2380 AVX2_Store8x8Accdw p_sad8x8 + xcnt_unit * i_xcnt, %1, mm_sad, mm_tmp0, b_second_blocks
2381 vpaddd y %+ mm_sadframe, y %+ mm_sadframe, y %+ mm_sad
2382%undef mm_tmp0
2383%undef mm_sad
2384%undef mm_sad2
2385%undef mm_sad3
2386%undef mm_sad4
2387%undef b_second_blocks
2388%endmacro
2389
2390;*************************************************************************************************************
2391;void VAACalcSad_avx2( const uint8_t *cur_data, const uint8_t *ref_data, int32_t iPicWidth, int32_t iPicHeight
2392; int32_t iPicStride, int32_t *psadframe, int32_t *psad8x8)
2393;*************************************************************************************************************
2394
2395WELS_EXTERN VAACalcSad_avx2
2396%define p_sadframe ptrword arg6
2397%define p_sad8x8 ptrword arg7
2398%ifdef X86_32
2399%define saveregs r5, r6
2400%else
2401%define saveregs rbx, rbp, r12
2402%endif
2403
2404%assign push_num 0
2405 LOAD_5_PARA
2406 PUSH_XMM 7
2407 SIGN_EXTENSION r2, r2d
2408 SIGN_EXTENSION r3, r3d
2409 SIGN_EXTENSION r4, r4d
2410 PUSHM saveregs
2411
2412%define mm_zero mm0
2413%define mm_sadframe mm6
2414 vpxor x %+ mm_zero, x %+ mm_zero, x %+ mm_zero
2415 vmovdqa y %+ mm_sadframe, y %+ mm_zero
2416
2417 and r2, -16 ; iPicWidth &= -16
2418 jle .done ; bail if iPicWidth < 16
2419 sar r3, 4 ; iPicHeight / 16
2420 jle .done ; bail if iPicHeight < 16
2421 shr r2, 2 ; iPicWidth / 4
2422
2423%define p_cur r0
2424%define p_ref r1
2425%define i_xcnt r2
2426%define i_ycnt ptrword arg4
2427%define i_stride r4
2428%define xcnt_unit 4
2429%ifdef X86_32
2430 mov i_ycnt, r3
2431 mov r5, p_sad8x8
2432 %define i_stride3 r3
2433 %undef p_sad8x8
2434 %define p_sad8x8 r5
2435 %define r_tmp r6
2436 lea i_stride3, [3 * i_stride]
2437%else
2438 mov rbp, p_sad8x8
2439 %define i_stride3 rbx
2440 %define i_stride5 r12
2441 %define i_stride7 r6
2442 %undef p_sad8x8
2443 %define p_sad8x8 rbp
2444 lea i_stride3, [3 * i_stride]
2445 lea i_stride5, [5 * i_stride]
2446 lea i_stride7, [i_stride + 2 * i_stride3]
2447%endif
2448
2449 ; offset pointer so as to compensate for the i_xcnt offset below.
2450 sub p_sad8x8, 4 * 16 / xcnt_unit
2451
2452 push i_xcnt
2453%assign push_num push_num + 1
2454%define i_xcnt_load ptrword [r7]
2455
2456.height_loop:
2457 ; use end-of-line pointers so as to enable use of a negative counter as index.
2458 lea p_sad8x8, [p_sad8x8 + xcnt_unit * i_xcnt]
2459 ; use a negative loop counter so as to enable counting toward zero and indexing with the same counter.
2460 neg i_xcnt
2461 add i_xcnt, 16 / xcnt_unit
2462 jz .width_loop_upper8_remaining16
2463.width_loop_upper8:
2464 AVX2_CalcSad_8Lines y, mm1, mm2, mm3, mm4, mm5, 0
2465 add i_xcnt, 32 / xcnt_unit
2466 jl .width_loop_upper8
2467 jg .width_loop_upper8_end
2468.width_loop_upper8_remaining16:
2469 AVX2_CalcSad_8Lines x, mm1, mm2, mm3, mm4, mm5, 0
2470.width_loop_upper8_end:
2471 lea p_cur, [p_cur + 8 * i_stride]
2472 lea p_ref, [p_ref + 8 * i_stride]
2473 xor i_xcnt, i_xcnt
2474 sub i_xcnt, i_xcnt_load
2475 lea p_cur, [p_cur + xcnt_unit * i_xcnt]
2476 lea p_ref, [p_ref + xcnt_unit * i_xcnt]
2477 add i_xcnt, 16 / xcnt_unit
2478 jz .width_loop_lower8_remaining16
2479.width_loop_lower8:
2480 AVX2_CalcSad_8Lines y, mm1, mm2, mm3, mm4, mm5, 1
2481 add i_xcnt, 32 / xcnt_unit
2482 jl .width_loop_lower8
2483 jg .width_loop_lower8_end
2484.width_loop_lower8_remaining16:
2485 AVX2_CalcSad_8Lines x, mm1, mm2, mm3, mm4, mm5, 1
2486.width_loop_lower8_end:
2487 lea p_cur, [p_cur + 8 * i_stride]
2488 lea p_ref, [p_ref + 8 * i_stride]
2489 xor i_xcnt, i_xcnt
2490 sub i_xcnt, i_xcnt_load
2491 lea p_cur, [p_cur + xcnt_unit * i_xcnt]
2492 lea p_ref, [p_ref + xcnt_unit * i_xcnt]
2493 neg i_xcnt
2494 sub i_ycnt, 1
2495 jnz .height_loop
2496
2497 pop i_xcnt
2498%assign push_num push_num - 1
2499%undef i_xcnt_load
2500
2501.done:
2502 mov r6, p_sadframe
2503 vextracti128 xmm2, y %+ mm_sadframe, 1
2504 vpaddd xmm2, x %+ mm_sadframe, xmm2
2505 vpunpckhqdq xmm1, xmm2, xmm2
2506 vpaddd xmm2, xmm2, xmm1
2507 vmovd [r6], xmm2
2508 vzeroupper
2509
2510 POPM saveregs
2511 POP_XMM
2512 LOAD_5_PARA_POP
2513%undef p_cur
2514%undef p_ref
2515%undef i_xcnt
2516%undef i_ycnt
2517%undef i_stride
2518%undef r_tmp
2519%undef xcnt_unit
2520%undef i_stride3
2521%undef i_stride5
2522%undef i_stride7
2523%undef mm_sadframe
2524%undef mm_zero
2525%undef saveregs
2526%undef p_sadframe
2527%undef p_sad8x8
2528 ret
2529
2530
2531; x/y-mm_prefix=%1 mm_clobber=%2,%3,%4,%5,%6 b_second_blocks=%7
2532%macro AVX2_CalcSadVar_8Lines 7
2533%define mm_tmp0 %2
2534%define mm_tmp1 %3
2535%define mm_sad %4
2536%define mm_sum %5
2537%define mm_sqsum %6
2538%define b_second_blocks %7
2539 ; Unroll for better performance on Haswell.
2540 ; Avoid unrolling for the 16 px case so as to reduce the code footprint.
2541%ifidni %1, y
2542 lea r_tmp, [5 * i_stride]
2543 AVX2_SadSumSqsum %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, p_cur, p_ref, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, 0
2544 AVX2_SadSumSqsum %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, p_cur + 1 * i_stride, p_ref + 1 * i_stride, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, 1
2545 AVX2_SadSumSqsum %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, p_cur + 2 * i_stride, p_ref + 2 * i_stride, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, 1
2546 AVX2_SadSumSqsum %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, p_cur + 1 * i_stride3, p_ref + 1 * i_stride3, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, 1
2547 AVX2_SadSumSqsum %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, p_cur + 4 * i_stride, p_ref + 4 * i_stride, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, 1
2548 AVX2_SadSumSqsum %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, p_cur + r_tmp, p_ref + r_tmp, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, 1
2549 lea r_tmp, [i_stride + 2 * i_stride3]
2550 AVX2_SadSumSqsum %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, p_cur + 2 * i_stride3, p_ref + 2 * i_stride3, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, 1
2551 AVX2_SadSumSqsum %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, p_cur + r_tmp, p_ref + r_tmp, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, 1
2552 ; Increment addresses for the next iteration. Doing this early is beneficial on Haswell.
2553 add p_cur, %1 %+ mm_width
2554 add p_ref, %1 %+ mm_width
2555%else
2556 vpxor x %+ mm_sad, x %+ mm_sad, x %+ mm_sad
2557 vpxor x %+ mm_sum, x %+ mm_sum, x %+ mm_sum
2558 vpxor x %+ mm_sqsum, x %+ mm_sqsum, x %+ mm_sqsum
2559 lea r_tmp, [8 * i_stride]
2560 add p_cur, r_tmp
2561 add p_ref, r_tmp
2562 neg r_tmp
2563%%loop:
2564 AVX2_SadSumSqsum %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, p_cur + r_tmp, p_ref + r_tmp, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, 1
2565 add r_tmp, i_stride
2566 jl %%loop
2567 ; Increment addresses for the next iteration. Doing this early is beneficial on Haswell.
2568 lea r_tmp, [8 * i_stride - %1 %+ mm_width]
2569 sub p_cur, r_tmp
2570 sub p_ref, r_tmp
2571%endif
2572 AVX2_Store8x8Accdw p_sad8x8 + 4 * i_xcnt, %1, mm_sad, mm_tmp1, b_second_blocks
2573 vpaddd y %+ mm_sadframe, y %+ mm_sadframe, y %+ mm_sad
2574 vpunpcklqdq %1 %+ mm_tmp0, %1 %+ mm_sum, %1 %+ mm_sqsum
2575 vpunpckhqdq %1 %+ mm_tmp1, %1 %+ mm_sum, %1 %+ mm_sqsum
2576 vpaddd %1 %+ mm_tmp0, %1 %+ mm_tmp0, %1 %+ mm_tmp1
2577 vpshufd %1 %+ mm_tmp1, %1 %+ mm_tmp0, 10110001b
2578 vpaddd %1 %+ mm_tmp0, %1 %+ mm_tmp0, %1 %+ mm_tmp1
2579 AVX2_Store2x16x16Accdw p_sum16x16, p_sqsum16x16, i_xcnt, r_tmp, %1, mm_tmp0, mm_tmp1, b_second_blocks
2580%undef mm_tmp0
2581%undef mm_tmp1
2582%undef mm_sad
2583%undef mm_sum
2584%undef mm_sqsum
2585%undef b_second_blocks
2586%endmacro
2587
2588;*************************************************************************************************************
2589;void VAACalcSadVar_avx2( const uint8_t *cur_data, const uint8_t *ref_data, int32_t iPicWidth, int32_t iPicHeight
2590; int32_t iPicStride, int32_t *psadframe, int32_t *psad8x8, int32_t *psum16x16, int32_t *psqsum16x16)
2591;*************************************************************************************************************
2592
2593WELS_EXTERN VAACalcSadVar_avx2
2594%define p_sadframe ptrword arg6
2595%define p_sad8x8 ptrword arg7
2596%define p_sum16x16 ptrword arg8
2597%define p_sqsum16x16 ptrword arg9
2598%ifdef X86_32
2599%define saveregs r5, r6
2600%else
2601%define saveregs rbx, rbp, r12, r13
2602%endif
2603
2604%assign push_num 0
2605 LOAD_5_PARA
2606 PUSH_XMM 7
2607 SIGN_EXTENSION r2, r2d
2608 SIGN_EXTENSION r3, r3d
2609 SIGN_EXTENSION r4, r4d
2610 PUSHM saveregs
2611
2612%define mm_zero mm0
2613%define mm_sadframe mm6
2614 vpxor x %+ mm_zero, x %+ mm_zero, x %+ mm_zero
2615 vmovdqa y %+ mm_sadframe, y %+ mm_zero
2616
2617 and r2, -16 ; iPicWidth &= -16
2618 jle .done ; bail if iPicWidth < 16
2619 sar r3, 4 ; iPicHeight / 16
2620 jle .done ; bail if iPicHeight < 16
2621 shr r2, 2 ; iPicWidth / 4
2622
2623%define p_cur r0
2624%define p_ref r1
2625%define i_xcnt r2
2626%define i_ycnt ptrword arg4
2627%define i_stride r4
2628%define r_tmp r6
2629%define xcnt_unit 4
2630%ifdef X86_32
2631 mov i_ycnt, r3
2632 mov r3, p_sad8x8
2633 %undef p_sad8x8
2634 %define p_sad8x8 r3
2635 %define i_stride3 r5
2636%else
2637 mov rbp, p_sad8x8
2638 mov r12, p_sum16x16
2639 mov r13, p_sqsum16x16
2640 %undef p_sad8x8
2641 %undef p_sum16x16
2642 %undef p_sqsum16x16
2643 %define p_sad8x8 rbp
2644 %define p_sum16x16 r12
2645 %define p_sqsum16x16 r13
2646 %define i_stride3 rbx
2647%endif
2648 lea i_stride3, [3 * i_stride]
2649
2650 ; offset pointers so as to compensate for the i_xcnt offset below.
2651 sub p_sad8x8, 4 * 16 / xcnt_unit
2652 sub p_sum16x16, 1 * 16 / xcnt_unit
2653 sub p_sqsum16x16, 1 * 16 / xcnt_unit
2654
2655 ; use a negative loop counter so as to enable counting toward zero and indexing with the same counter.
2656 neg i_xcnt
2657
2658.height_loop:
2659 push i_xcnt
2660%assign push_num push_num + 1
2661%define i_xcnt_load ptrword [r7]
2662 ; use end-of-line pointers so as to enable use of a negative counter as index.
2663 lea r_tmp, [xcnt_unit * i_xcnt]
2664 sub p_sad8x8, r_tmp
2665 sub p_sum16x16, i_xcnt
2666 sub p_sqsum16x16, i_xcnt
2667 add i_xcnt, 16 / xcnt_unit
2668 jz .width_loop_upper8_remaining16
2669.width_loop_upper8:
2670 AVX2_CalcSadVar_8Lines y, mm1, mm2, mm3, mm4, mm5, 0
2671 add i_xcnt, 32 / xcnt_unit
2672 jl .width_loop_upper8
2673 jg .width_loop_upper8_end
2674.width_loop_upper8_remaining16:
2675 AVX2_CalcSadVar_8Lines x, mm1, mm2, mm3, mm4, mm5, 0
2676.width_loop_upper8_end:
2677 lea p_cur, [p_cur + 8 * i_stride]
2678 lea p_ref, [p_ref + 8 * i_stride]
2679 mov i_xcnt, i_xcnt_load
2680 lea p_cur, [p_cur + xcnt_unit * i_xcnt]
2681 lea p_ref, [p_ref + xcnt_unit * i_xcnt]
2682 add i_xcnt, 16 / xcnt_unit
2683 jz .width_loop_lower8_remaining16
2684.width_loop_lower8:
2685 AVX2_CalcSadVar_8Lines y, mm1, mm2, mm3, mm4, mm5, 1
2686 add i_xcnt, 32 / xcnt_unit
2687 jl .width_loop_lower8
2688 jg .width_loop_lower8_end
2689.width_loop_lower8_remaining16:
2690 AVX2_CalcSadVar_8Lines x, mm1, mm2, mm3, mm4, mm5, 1
2691.width_loop_lower8_end:
2692 lea p_cur, [p_cur + 8 * i_stride]
2693 lea p_ref, [p_ref + 8 * i_stride]
2694%undef i_xcnt_load
2695 pop i_xcnt
2696 %assign push_num push_num - 1
2697 lea p_cur, [p_cur + xcnt_unit * i_xcnt]
2698 lea p_ref, [p_ref + xcnt_unit * i_xcnt]
2699 sub i_ycnt, 1
2700 jnz .height_loop
2701
2702.done:
2703 mov r_tmp, p_sadframe
2704 vextracti128 xmm2, y %+ mm_sadframe, 1
2705 vpaddd xmm2, x %+ mm_sadframe, xmm2
2706 vpunpckhqdq xmm1, xmm2, xmm2
2707 vpaddd xmm2, xmm2, xmm1
2708 vmovd [r_tmp], xmm2
2709 vzeroupper
2710
2711 POPM saveregs
2712 POP_XMM
2713 LOAD_5_PARA_POP
2714%undef p_cur
2715%undef p_ref
2716%undef i_xcnt
2717%undef i_ycnt
2718%undef i_stride
2719%undef i_stride3
2720%undef r_tmp
2721%undef xcnt_unit
2722%undef mm_sadframe
2723%undef mm_zero
2724%undef saveregs
2725%undef p_sadframe
2726%undef p_sad8x8
2727%undef p_sum16x16
2728%undef p_sqsum16x16
2729 ret
2730
2731
2732; x/y-mm_prefix=%1 mm_clobber=%2,%3,%4,%5,%6,%7,%8 b_second_blocks=%9
2733%macro AVX2_CalcSadSsd_8Lines 9
2734%define mm_tmp0 %2
2735%define mm_tmp1 %3
2736%define mm_tmp2 %4
2737%define mm_sad %5
2738%define mm_sum %6
2739%define mm_sqsum %7
2740%define mm_sqdiff %8
2741%define b_second_blocks %9
2742 ; Unroll for better performance on Haswell.
2743 ; Avoid unrolling for the 16 px case so as to reduce the code footprint.
2744%ifidni %1, y
2745%ifdef i_stride5
2746 lea r_tmp, [i_stride + 2 * i_stride3]
2747 %define i_stride5_ i_stride5
2748%else
2749 lea r_tmp, [5 * i_stride]
2750 %define i_stride5_ r_tmp
2751%endif
2752 AVX2_SadSumSqsumSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, %1 %+ mm_sqdiff, p_cur, p_ref, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 0
2753 AVX2_SadSumSqsumSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, %1 %+ mm_sqdiff, p_cur + 1 * i_stride, p_ref + 1 * i_stride, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
2754 AVX2_SadSumSqsumSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, %1 %+ mm_sqdiff, p_cur + 2 * i_stride, p_ref + 2 * i_stride, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
2755 AVX2_SadSumSqsumSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, %1 %+ mm_sqdiff, p_cur + 1 * i_stride3, p_ref + 1 * i_stride3, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
2756 AVX2_SadSumSqsumSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, %1 %+ mm_sqdiff, p_cur + 4 * i_stride, p_ref + 4 * i_stride, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
2757 AVX2_SadSumSqsumSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, %1 %+ mm_sqdiff, p_cur + 1 * i_stride5_, p_ref + 1 * i_stride5_, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
2758%ifndef i_stride5
2759 lea r_tmp, [i_stride + 2 * i_stride3]
2760%endif
2761%undef i_stride5_
2762 AVX2_SadSumSqsumSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, %1 %+ mm_sqdiff, p_cur + 2 * i_stride3, p_ref + 2 * i_stride3, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
2763 AVX2_SadSumSqsumSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, %1 %+ mm_sqdiff, p_cur + r_tmp, p_ref + r_tmp, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
2764 ; Increment addresses for the next iteration. Doing this early is beneficial on Haswell.
2765 add p_cur, %1 %+ mm_width
2766 add p_ref, %1 %+ mm_width
2767%else
2768 vpxor x %+ mm_sad, x %+ mm_sad, x %+ mm_sad
2769 vpxor x %+ mm_sum, x %+ mm_sum, x %+ mm_sum
2770 vpxor x %+ mm_sqsum, x %+ mm_sqsum, x %+ mm_sqsum
2771 vpxor x %+ mm_sqdiff, x %+ mm_sqdiff, x %+ mm_sqdiff
2772 lea r_tmp, [8 * i_stride]
2773 add p_cur, r_tmp
2774 add p_ref, r_tmp
2775 neg r_tmp
2776%%loop:
2777 AVX2_SadSumSqsumSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, %1 %+ mm_sqdiff, p_cur + r_tmp, p_ref + r_tmp, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
2778 add r_tmp, i_stride
2779 jl %%loop
2780 ; Increment addresses for the next iteration. Doing this early is beneficial on Haswell.
2781 lea r_tmp, [8 * i_stride - %1 %+ mm_width]
2782 sub p_cur, r_tmp
2783 sub p_ref, r_tmp
2784%endif
2785 mov r_tmp, p_sad8x8
2786 AVX2_Store8x8Accdw r_tmp + 4 * i_xcnt, %1, mm_sad, mm_tmp1, b_second_blocks
2787%ifdef X86_32
2788 vpaddd y %+ mm_tmp1, y %+ mm_sad, sadframe_acc
2789 vmovdqa sadframe_acc, y %+ mm_tmp1
2790%else
2791 vpaddd sadframe_acc, sadframe_acc, y %+ mm_sad
2792%endif
2793 mov r_tmp, i_xcnt
2794 add r_tmp, p_sum16x16
2795 vpunpckhqdq %1 %+ mm_tmp1, %1 %+ mm_sum, %1 %+ mm_sum
2796 vpaddd %1 %+ mm_tmp0, %1 %+ mm_sum, %1 %+ mm_tmp1
2797 AVX2_Store16x16Accdw r_tmp, %1, mm_tmp0, mm_tmp1, b_second_blocks
2798 vpunpcklqdq %1 %+ mm_tmp0, %1 %+ mm_sqsum, %1 %+ mm_sqdiff
2799 vpunpckhqdq %1 %+ mm_tmp1, %1 %+ mm_sqsum, %1 %+ mm_sqdiff
2800 vpaddd %1 %+ mm_tmp0, %1 %+ mm_tmp0, %1 %+ mm_tmp1
2801 vpshufd %1 %+ mm_tmp1, %1 %+ mm_tmp0, 10110001b
2802 vpaddd %1 %+ mm_tmp0, %1 %+ mm_tmp0, %1 %+ mm_tmp1
2803 AVX2_Store2x16x16Accdw p_sqsum16x16, p_sqdiff16x16, i_xcnt, r_tmp, %1, mm_tmp0, mm_tmp1, b_second_blocks
2804%undef mm_tmp0
2805%undef mm_tmp1
2806%undef mm_tmp2
2807%undef mm_sad
2808%undef mm_sum
2809%undef mm_sqsum
2810%undef mm_sqdiff
2811%undef b_second_blocks
2812%endmacro
2813
2814;*************************************************************************************************************
2815;void VAACalcSadSsd_avx2(const uint8_t *cur_data, const uint8_t *ref_data, int32_t iPicWidth, int32_t iPicHeight,
2816; int32_t iPicStride,int32_t *psadframe, int32_t *psad8x8, int32_t *psum16x16, int32_t *psqsum16x16, int32_t *psqdiff16x16)
2817;*************************************************************************************************************
2818
2819WELS_EXTERN VAACalcSadSsd_avx2
2820%define p_sadframe ptrword arg6
2821%define p_sad8x8 ptrword arg7
2822%define p_sum16x16 ptrword arg8
2823%define p_sqsum16x16 ptrword arg9
2824%define p_sqdiff16x16 ptrword arg10
2825%ifdef X86_32
2826%define saveregs r5, r6
2827%else
2828%define saveregs rbx, rbp, r12, r13, r14, r15
2829%endif
2830
2831%assign push_num 0
2832 LOAD_5_PARA
2833 PUSH_XMM 9
2834 SIGN_EXTENSION r2, r2d
2835 SIGN_EXTENSION r3, r3d
2836 SIGN_EXTENSION r4, r4d
2837 PUSHM saveregs
2838
2839%define mm_zero mm0
2840 vpxor x %+ mm_zero, x %+ mm_zero, x %+ mm_zero
2841
2842%ifdef X86_32
2843 STACK_ALLOC r5, ymm_width, ymm_width
2844 %define sadframe_acc_addr r5
2845 %define sadframe_acc [sadframe_acc_addr]
2846%else
2847 %define sadframe_acc ymm8
2848 %define xsadframe_acc xmm8
2849%endif
2850 vmovdqa sadframe_acc, y %+ mm_zero
2851
2852 and r2, -16 ; iPicWidth &= -16
2853 jle .done ; bail if iPicWidth < 16
2854 sar r3, 4 ; iPicHeight / 16
2855 jle .done ; bail if iPicHeight < 16
2856 shr r2, 2 ; iPicWidth / 4
2857
2858%define p_cur r0
2859%define p_ref r1
2860%define i_xcnt r2
2861%define i_ycnt ptrword arg4
2862%define i_stride r4
2863%define r_tmp r6
2864%define xcnt_unit 4
2865%ifdef X86_32
2866 mov i_ycnt, r3
2867 %define i_stride3 r3
2868%else
2869 mov r12, p_sad8x8
2870 mov r13, p_sum16x16
2871 mov r14, p_sqsum16x16
2872 mov r15, p_sqdiff16x16
2873 %undef p_sad8x8
2874 %undef p_sum16x16
2875 %undef p_sqsum16x16
2876 %undef p_sqdiff16x16
2877 %define p_sad8x8 r12
2878 %define p_sum16x16 r13
2879 %define p_sqsum16x16 r14
2880 %define p_sqdiff16x16 r15
2881 %define i_stride3 rbx
2882 %define i_stride5 rbp
2883 lea i_stride5, [5 * i_stride]
2884%endif
2885 lea i_stride3, [3 * i_stride]
2886
2887 ; offset pointers so as to compensate for i_xcnt offset below.
2888 sub p_sad8x8, 4 * 16 / xcnt_unit
2889 sub p_sum16x16, 1 * 16 / xcnt_unit
2890 sub p_sqsum16x16, 1 * 16 / xcnt_unit
2891 sub p_sqdiff16x16, 1 * 16 / xcnt_unit
2892
2893 ; use a negative loop counter so as to enable counting toward zero and indexing with the same counter.
2894 neg i_xcnt
2895
2896.height_loop:
2897 push i_xcnt
2898%assign push_num push_num + 1
2899%define i_xcnt_load ptrword [r7]
2900 ; use end-of-line pointers so as to enable use of a negative counter as index.
2901 lea r_tmp, [xcnt_unit * i_xcnt]
2902 sub p_sad8x8, r_tmp
2903 sub p_sum16x16, i_xcnt
2904 sub p_sqsum16x16, i_xcnt
2905 sub p_sqdiff16x16, i_xcnt
2906 add i_xcnt, 16 / xcnt_unit
2907 jz .width_loop_upper8_remaining16
2908.width_loop_upper8:
2909 AVX2_CalcSadSsd_8Lines y, mm1, mm2, mm3, mm4, mm5, mm6, mm7, 0
2910 add i_xcnt, 32 / xcnt_unit
2911 jl .width_loop_upper8
2912 jg .width_loop_upper8_end
2913.width_loop_upper8_remaining16:
2914 AVX2_CalcSadSsd_8Lines x, mm1, mm2, mm3, mm4, mm5, mm6, mm7, 0
2915.width_loop_upper8_end:
2916 lea p_cur, [p_cur + 8 * i_stride]
2917 lea p_ref, [p_ref + 8 * i_stride]
2918 mov i_xcnt, i_xcnt_load
2919 lea p_cur, [p_cur + xcnt_unit * i_xcnt]
2920 lea p_ref, [p_ref + xcnt_unit * i_xcnt]
2921 add i_xcnt, 16 / xcnt_unit
2922 jz .width_loop_lower8_remaining16
2923.width_loop_lower8:
2924 AVX2_CalcSadSsd_8Lines y, mm1, mm2, mm3, mm4, mm5, mm6, mm7, 1
2925 add i_xcnt, 32 / xcnt_unit
2926 jl .width_loop_lower8
2927 jg .width_loop_lower8_end
2928.width_loop_lower8_remaining16:
2929 AVX2_CalcSadSsd_8Lines x, mm1, mm2, mm3, mm4, mm5, mm6, mm7, 1
2930.width_loop_lower8_end:
2931 lea p_cur, [p_cur + 8 * i_stride]
2932 lea p_ref, [p_ref + 8 * i_stride]
2933%undef i_xcnt_load
2934 pop i_xcnt
2935 %assign push_num push_num - 1
2936 lea p_cur, [p_cur + xcnt_unit * i_xcnt]
2937 lea p_ref, [p_ref + xcnt_unit * i_xcnt]
2938 sub i_ycnt, 1
2939 jnz .height_loop
2940
2941.done:
2942 mov r_tmp, p_sadframe
2943%ifdef X86_32
2944 vmovdqa xmm2, sadframe_acc
2945 vpaddd xmm2, xmm2, [sadframe_acc_addr + xmm_width]
2946%else
2947 vextracti128 xmm2, sadframe_acc, 1
2948 vpaddd xmm2, xsadframe_acc, xmm2
2949%endif
2950 vpunpckhqdq xmm1, xmm2, xmm2
2951 vpaddd xmm2, xmm2, xmm1
2952 vmovd [r_tmp], xmm2
2953 vzeroupper
2954%ifdef X86_32
2955 STACK_DEALLOC
2956%endif
2957 POPM saveregs
2958 POP_XMM
2959 LOAD_5_PARA_POP
2960%undef p_cur
2961%undef p_ref
2962%undef i_xcnt
2963%undef i_ycnt
2964%undef i_stride
2965%undef i_stride3
2966%undef i_stride5
2967%undef r_tmp
2968%undef xcnt_unit
2969%undef sadframe_acc
2970%undef sadframe_acc_addr
2971%undef xsadframe_acc
2972%undef mm_zero
2973%undef saveregs
2974%undef p_sadframe
2975%undef p_sad8x8
2976%undef p_sum16x16
2977%undef p_sqsum16x16
2978%undef p_sqdiff16x16
2979 ret
2980
2981
2982; x/y-mm_prefix=%1 mm_clobber=%2,%3,%4,%5,%6,%7,%8 b_second_blocks=%9
2983%macro AVX2_CalcSadBgd_8Lines 9
2984%define mm_tmp0 %2
2985%define mm_tmp1 %3
2986%define mm_tmp2 %8
2987%define mm_mad %4
2988%define mm_sumcur %5
2989%define mm_sumref %6
2990%define mm_sad %7
2991%define b_second_blocks %9
2992 ; Unroll for better performance on Haswell.
2993 ; Avoid unrolling for the 16 px case so as to reduce the code footprint.
2994%ifidni %1, y
2995 lea r_tmp, [5 * i_stride]
2996 AVX2_SadSdMad %1 %+ mm_sad, %1 %+ mm_sumcur, %1 %+ mm_sumref, %1 %+ mm_mad, p_cur, p_ref, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 0
2997 AVX2_SadSdMad %1 %+ mm_sad, %1 %+ mm_sumcur, %1 %+ mm_sumref, %1 %+ mm_mad, p_cur + 1 * i_stride, p_ref + 1 * i_stride, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
2998 AVX2_SadSdMad %1 %+ mm_sad, %1 %+ mm_sumcur, %1 %+ mm_sumref, %1 %+ mm_mad, p_cur + 2 * i_stride, p_ref + 2 * i_stride, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
2999 AVX2_SadSdMad %1 %+ mm_sad, %1 %+ mm_sumcur, %1 %+ mm_sumref, %1 %+ mm_mad, p_cur + 1 * i_stride3, p_ref + 1 * i_stride3, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
3000 AVX2_SadSdMad %1 %+ mm_sad, %1 %+ mm_sumcur, %1 %+ mm_sumref, %1 %+ mm_mad, p_cur + 4 * i_stride, p_ref + 4 * i_stride, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
3001 AVX2_SadSdMad %1 %+ mm_sad, %1 %+ mm_sumcur, %1 %+ mm_sumref, %1 %+ mm_mad, p_cur + r_tmp, p_ref + r_tmp, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
3002 lea r_tmp, [i_stride + 2 * i_stride3]
3003 AVX2_SadSdMad %1 %+ mm_sad, %1 %+ mm_sumcur, %1 %+ mm_sumref, %1 %+ mm_mad, p_cur + 2 * i_stride3, p_ref + 2 * i_stride3, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
3004 AVX2_SadSdMad %1 %+ mm_sad, %1 %+ mm_sumcur, %1 %+ mm_sumref, %1 %+ mm_mad, p_cur + r_tmp, p_ref + r_tmp, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
3005 ; Increment addresses for the next iteration. Doing this early is beneficial on Haswell.
3006 add p_cur, %1 %+ mm_width
3007 add p_ref, %1 %+ mm_width
3008%else
3009 vpxor x %+ mm_sad, x %+ mm_sad, x %+ mm_sad
3010 vpxor x %+ mm_sumcur, x %+ mm_sumcur, x %+ mm_sumcur
3011 vpxor x %+ mm_sumref, x %+ mm_sumref, x %+ mm_sumref
3012 vpxor x %+ mm_mad, x %+ mm_mad, x %+ mm_mad
3013 lea r_tmp, [8 * i_stride]
3014 add p_cur, r_tmp
3015 add p_ref, r_tmp
3016 neg r_tmp
3017%%loop:
3018 AVX2_SadSdMad %1 %+ mm_sad, %1 %+ mm_sumcur, %1 %+ mm_sumref, %1 %+ mm_mad, p_cur + r_tmp, p_ref + r_tmp, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
3019 add r_tmp, i_stride
3020 jl %%loop
3021 ; Increment addresses for the next iteration. Doing this early is beneficial on Haswell.
3022 lea r_tmp, [8 * i_stride - %1 %+ mm_width]
3023 sub p_cur, r_tmp
3024 sub p_ref, r_tmp
3025%endif
3026 mov r_tmp, p_sad8x8
3027 AVX2_Store8x8Accdw r_tmp + 4 * i_xcnt, %1, mm_sad, mm_tmp1, b_second_blocks
3028%ifdef X86_32
3029 vpaddd y %+ mm_tmp1, y %+ mm_sad, sadframe_acc
3030 vmovdqa sadframe_acc, y %+ mm_tmp1
3031%else
3032 vpaddd sadframe_acc, sadframe_acc, y %+ mm_sad
3033%endif
3034 mov r_tmp, p_sd8x8
3035 vpsubd %1 %+ mm_tmp0, %1 %+ mm_sumcur, %1 %+ mm_sumref
3036 AVX2_Store8x8Accdw r_tmp + 4 * i_xcnt, %1, mm_tmp0, mm_tmp1, b_second_blocks
3037 ; Coalesce store and horizontal reduction of MAD accumulator for even and
3038 ; odd iterations so as to enable more parallelism.
3039%ifidni %1, y
3040 test i_xcnt, 32 / xcnt_unit
3041 jz %%preserve_mad
3042 mov r_tmp, p_mad8x8
3043 AVX2_Maxubq2 y %+ mm_mad, y %+ mm_mad, prev_mad, y %+ mm_tmp0
3044 AVX2_Store2x8x8Accb r_tmp + i_xcnt - 8, mm_mad, mm_tmp0, mm_tmp1, b_second_blocks
3045%%preserve_mad:
3046 vmovdqa prev_mad, y %+ mm_mad
3047%else
3048 mov r_tmp, p_mad8x8
3049 AVX2_Maxubq %1 %+ mm_mad, %1 %+ mm_mad, %1 %+ mm_tmp0
3050 AVX2_Store8x8Accb r_tmp + i_xcnt, %1, mm_mad, mm_tmp0, b_second_blocks
3051%endif
3052%undef mm_tmp0
3053%undef mm_tmp1
3054%undef mm_tmp2
3055%undef mm_mad
3056%undef mm_sumcur
3057%undef mm_sumref
3058%undef mm_sad
3059%undef b_second_blocks
3060%endmacro
3061
3062; Store remaining MAD accumulator for width & 32 cases.
3063; width/xcnt_unit=%1 mm_tmp=%2,%3 b_second_blocks=%4
3064%macro AVX2_StoreRemainingSingleMad 4
3065 test %1, 32 / xcnt_unit
3066 jz %%skip
3067 mov r_tmp, p_mad8x8
3068 vmovdqa y%2, prev_mad
3069 AVX2_Maxubq y%2, y%2, y%3
3070 AVX2_Store8x8Accb r_tmp + i_xcnt - 8, y, %2, %3, %4
3071%%skip:
3072%endmacro
3073
3074;*************************************************************************************************************
3075;void VAACalcSadBgd_avx2(const uint8_t *cur_data, const uint8_t *ref_data, int32_t iPicWidth, int32_t iPicHeight,
3076; int32_t iPicStride, int32_t *psadframe, int32_t *psad8x8, int32_t *p_sd8x8, uint8_t *p_mad8x8)
3077;*************************************************************************************************************
3078
3079WELS_EXTERN VAACalcSadBgd_avx2
3080%define p_sadframe arg6
3081%define p_sad8x8 arg7
3082%define p_sd8x8 arg8
3083%define p_mad8x8 arg9
3084%ifdef X86_32
3085%define saveregs r5, r6
3086%else
3087%define saveregs rbx, rbp, r12, r13
3088%endif
3089
3090%assign push_num 0
3091 LOAD_5_PARA
3092 PUSH_XMM 10
3093 SIGN_EXTENSION r2, r2d
3094 SIGN_EXTENSION r3, r3d
3095 SIGN_EXTENSION r4, r4d
3096 PUSHM saveregs
3097
3098%define mm_zero mm0
3099 vpxor x %+ mm_zero, x %+ mm_zero, x %+ mm_zero
3100
3101%ifdef X86_32
3102 STACK_ALLOC r5, 2 * ymm_width, ymm_width
3103 %define sadframe_acc_addr r5
3104 %define sadframe_acc [sadframe_acc_addr]
3105 %define prev_mad [r5 + ymm_width]
3106%else
3107 %define sadframe_acc ymm8
3108 %define xsadframe_acc xmm8
3109 %define prev_mad ymm9
3110%endif
3111 vmovdqa sadframe_acc, y %+ mm_zero
3112
3113 and r2, -16 ; iPicWidth &= -16
3114 jle .done ; bail if iPicWidth < 16
3115 sar r3, 4 ; iPicHeight / 16
3116 jle .done ; bail if iPicHeight < 16
3117 shr r2, 2 ; iPicWidth / 4
3118
3119%define p_cur r0
3120%define p_ref r1
3121%define i_xcnt r2
3122%define i_ycnt ptrword arg4
3123%define i_stride r4
3124%define r_tmp r6
3125%define xcnt_unit 4
3126%ifdef X86_32
3127 mov i_ycnt, r3
3128 %define i_stride3 r3
3129%else
3130 mov rbp, p_sad8x8
3131 mov r12, p_sd8x8
3132 mov r13, p_mad8x8
3133 %undef p_sad8x8
3134 %undef p_sd8x8
3135 %undef p_mad8x8
3136 %define p_sad8x8 rbp
3137 %define p_sd8x8 r12
3138 %define p_mad8x8 r13
3139 %define i_stride3 rbx
3140%endif
3141 lea i_stride3, [3 * i_stride]
3142
3143 ; offset pointers to compensate for the i_xcnt offset below.
3144 mov r_tmp, i_xcnt
3145 and r_tmp, 64 / xcnt_unit - 1
3146 sub p_mad8x8, r_tmp
3147 shl r_tmp, 2
3148 sub p_sad8x8, r_tmp
3149 sub p_sd8x8, r_tmp
3150
3151.height_loop:
3152 push i_xcnt
3153%assign push_num push_num + 1
3154%define i_xcnt_load ptrword [r7]
3155 ; use end-of-line pointers so as to enable use of a negative counter as index.
3156 lea r_tmp, [xcnt_unit * i_xcnt]
3157 add p_sad8x8, r_tmp
3158 add p_sd8x8, r_tmp
3159 add p_mad8x8, i_xcnt
3160 and i_xcnt, -(64 / xcnt_unit)
3161 jz .width_loop_upper8_64x_end
3162 ; use a negative loop counter to enable counting toward zero and indexing with the same counter.
3163 neg i_xcnt
3164.width_loop_upper8:
3165 AVX2_CalcSadBgd_8Lines y, mm1, mm2, mm3, mm4, mm5, mm6, mm7, 0
3166 add i_xcnt, 32 / xcnt_unit
3167 jl .width_loop_upper8
3168 jg .width_loop_upper8_32x_end
3169.width_loop_upper8_64x_end:
3170 test i_xcnt_load, 32 / xcnt_unit
3171 jnz .width_loop_upper8
3172.width_loop_upper8_32x_end:
3173 AVX2_StoreRemainingSingleMad i_xcnt_load, mm1, mm2, 0
3174 test i_xcnt_load, 16 / xcnt_unit
3175 jz .width_loop_upper8_end
3176 ; remaining 16.
3177 AVX2_CalcSadBgd_8Lines x, mm1, mm2, mm3, mm4, mm5, mm6, mm7, 0
3178.width_loop_upper8_end:
3179 lea p_cur, [p_cur + 8 * i_stride]
3180 lea p_ref, [p_ref + 8 * i_stride]
3181 mov i_xcnt, i_xcnt_load
3182 lea r_tmp, [xcnt_unit * i_xcnt]
3183 sub p_cur, r_tmp
3184 sub p_ref, r_tmp
3185 and i_xcnt, -(64 / xcnt_unit)
3186 jz .width_loop_lower8_64x_end
3187 neg i_xcnt
3188.width_loop_lower8:
3189 AVX2_CalcSadBgd_8Lines y, mm1, mm2, mm3, mm4, mm5, mm6, mm7, 1
3190 add i_xcnt, 32 / xcnt_unit
3191 jl .width_loop_lower8
3192 jg .width_loop_lower8_32x_end
3193.width_loop_lower8_64x_end:
3194 test i_xcnt_load, 32 / xcnt_unit
3195 jnz .width_loop_lower8
3196.width_loop_lower8_32x_end:
3197 AVX2_StoreRemainingSingleMad i_xcnt_load, mm1, mm2, 1
3198 test i_xcnt_load, 16 / xcnt_unit
3199 jz .width_loop_lower8_end
3200 ; remaining 16.
3201 AVX2_CalcSadBgd_8Lines x, mm1, mm2, mm3, mm4, mm5, mm6, mm7, 1
3202.width_loop_lower8_end:
3203 lea p_cur, [p_cur + 8 * i_stride]
3204 lea p_ref, [p_ref + 8 * i_stride]
3205 pop i_xcnt
3206%undef i_xcnt_load
3207 %assign push_num push_num - 1
3208 lea r_tmp, [xcnt_unit * i_xcnt]
3209 sub p_cur, r_tmp
3210 sub p_ref, r_tmp
3211 sub i_ycnt, 1
3212 jnz .height_loop
3213
3214.done:
3215 mov r_tmp, p_sadframe
3216%ifdef X86_32
3217 vmovdqa xmm2, sadframe_acc
3218 vpaddd xmm2, xmm2, [sadframe_acc_addr + xmm_width]
3219%else
3220 vextracti128 xmm2, sadframe_acc, 1
3221 vpaddd xmm2, xsadframe_acc, xmm2
3222%endif
3223 vpunpckhqdq xmm1, xmm2, xmm2
3224 vpaddd xmm2, xmm2, xmm1
3225 vmovd [r_tmp], xmm2
3226 vzeroupper
3227%ifdef X86_32
3228 STACK_DEALLOC
3229%endif
3230 POPM saveregs
3231 POP_XMM
3232 LOAD_5_PARA_POP
3233%undef p_cur
3234%undef p_ref
3235%undef i_xcnt
3236%undef i_ycnt
3237%undef i_stride
3238%undef i_stride3
3239%undef r_tmp
3240%undef xcnt_unit
3241%undef sadframe_acc
3242%undef sadframe_acc_addr
3243%undef xsadframe_acc
3244%undef prev_mad
3245%undef mm_zero
3246%undef saveregs
3247%undef p_sadframe
3248%undef p_sad8x8
3249%undef p_sd8x8
3250%undef p_mad8x8
3251 ret
3252
3253
3254; x/y-mm_prefix=%1 mm_clobber=%2,%3,%4,%5,%6,%7,%8,%9,%10 b_second_blocks=%11
3255%macro AVX2_CalcSadSsdBgd_8Lines 11
3256%define mm_tmp0 %2
3257%define mm_tmp1 %3
3258%define mm_sad %4
3259%define mm_sum %5
3260%define mm_sumref %6
3261%define mm_mad %7
3262%define mm_sqsum %8
3263%define mm_sqdiff %9
3264%ifidn %10, 0
3265%define tmp2 0
3266%else
3267%define tmp2 %1 %+ %10
3268%endif
3269%define b_second_blocks %11
3270 ; Unroll for better performance on Haswell.
3271 ; Avoid unrolling for the 16 px case so as to reduce the code footprint.
3272%ifidni %1, y
3273 lea r_tmp, [5 * i_stride]
3274 AVX2_SadBgdSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sumref, %1 %+ mm_mad, %1 %+ mm_sqdiff, %1 %+ mm_sqsum, p_cur, p_ref, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, tmp2, 0
3275 AVX2_SadBgdSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sumref, %1 %+ mm_mad, %1 %+ mm_sqdiff, %1 %+ mm_sqsum, p_cur + 1 * i_stride, p_ref + 1 * i_stride, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, tmp2, 1
3276 AVX2_SadBgdSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sumref, %1 %+ mm_mad, %1 %+ mm_sqdiff, %1 %+ mm_sqsum, p_cur + 2 * i_stride, p_ref + 2 * i_stride, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, tmp2, 1
3277 AVX2_SadBgdSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sumref, %1 %+ mm_mad, %1 %+ mm_sqdiff, %1 %+ mm_sqsum, p_cur + 1 * i_stride3, p_ref + 1 * i_stride3, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, tmp2, 1
3278 AVX2_SadBgdSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sumref, %1 %+ mm_mad, %1 %+ mm_sqdiff, %1 %+ mm_sqsum, p_cur + 4 * i_stride, p_ref + 4 * i_stride, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, tmp2, 1
3279 AVX2_SadBgdSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sumref, %1 %+ mm_mad, %1 %+ mm_sqdiff, %1 %+ mm_sqsum, p_cur + r_tmp, p_ref + r_tmp, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, tmp2, 1
3280 lea r_tmp, [i_stride + 2 * i_stride3]
3281 AVX2_SadBgdSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sumref, %1 %+ mm_mad, %1 %+ mm_sqdiff, %1 %+ mm_sqsum, p_cur + 2 * i_stride3, p_ref + 2 * i_stride3, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, tmp2, 1
3282 AVX2_SadBgdSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sumref, %1 %+ mm_mad, %1 %+ mm_sqdiff, %1 %+ mm_sqsum, p_cur + r_tmp, p_ref + r_tmp, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, tmp2, 1
3283 ; Increment addresses for the next iteration. Doing this early is beneficial on Haswell.
3284 add p_cur, %1 %+ mm_width
3285 add p_ref, %1 %+ mm_width
3286%else
3287 vpxor x %+ mm_sad, x %+ mm_sad, x %+ mm_sad
3288 vpxor x %+ mm_sum, x %+ mm_sum, x %+ mm_sum
3289 vpxor x %+ mm_sumref, x %+ mm_sumref, x %+ mm_sumref
3290 vpxor x %+ mm_mad, x %+ mm_mad, x %+ mm_mad
3291 vpxor x %+ mm_sqsum, x %+ mm_sqsum, x %+ mm_sqsum
3292 vpxor x %+ mm_sqdiff, x %+ mm_sqdiff, x %+ mm_sqdiff
3293 lea r_tmp, [8 * i_stride]
3294 add p_cur, r_tmp
3295 add p_ref, r_tmp
3296 neg r_tmp
3297%%loop:
3298 AVX2_SadBgdSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sumref, %1 %+ mm_mad, %1 %+ mm_sqdiff, %1 %+ mm_sqsum, p_cur + r_tmp, p_ref + r_tmp, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, tmp2, 1
3299 add r_tmp, i_stride
3300 jl %%loop
3301 ; Increment addresses for the next iteration. Doing this early is beneficial on Haswell.
3302 lea r_tmp, [8 * i_stride - %1 %+ mm_width]
3303 sub p_cur, r_tmp
3304 sub p_ref, r_tmp
3305%endif
3306 mov r_tmp, p_sad8x8
3307 AVX2_Store8x8Accdw r_tmp + 4 * i_xcnt, %1, mm_sad, mm_tmp1, b_second_blocks
3308%ifdef X86_32
3309 vpaddd y %+ mm_tmp1, y %+ mm_sad, sadframe_acc
3310 vmovdqa sadframe_acc, y %+ mm_tmp1
3311%else
3312 vpaddd sadframe_acc, sadframe_acc, y %+ mm_sad
3313%endif
3314 mov r_tmp, i_xcnt
3315 add r_tmp, p_sum16x16
3316 vpunpckhqdq %1 %+ mm_tmp1, %1 %+ mm_sum, %1 %+ mm_sum
3317 vpaddd %1 %+ mm_tmp0, %1 %+ mm_sum, %1 %+ mm_tmp1
3318 AVX2_Store16x16Accdw r_tmp, %1, mm_tmp0, mm_tmp1, b_second_blocks
3319 mov r_tmp, p_sd8x8
3320 vpsubd %1 %+ mm_sum, %1 %+ mm_sum, %1 %+ mm_sumref
3321 AVX2_Store8x8Accdw r_tmp + 4 * i_xcnt, %1, mm_sum, mm_tmp0, b_second_blocks
3322 ; Coalesce store and horizontal reduction of MAD accumulator for even and
3323 ; odd iterations so as to enable more parallelism.
3324%ifidni %1, y
3325 test i_xcnt, 32 / xcnt_unit
3326 jz %%preserve_mad
3327 mov r_tmp, p_mad8x8
3328 AVX2_Maxubq2 y %+ mm_mad, y %+ mm_mad, prev_mad, y %+ mm_tmp0
3329 AVX2_Store2x8x8Accb r_tmp + i_xcnt - 8, mm_mad, mm_tmp0, mm_tmp1, b_second_blocks
3330%%preserve_mad:
3331 vmovdqa prev_mad, y %+ mm_mad
3332%else
3333 mov r_tmp, p_mad8x8
3334 AVX2_Maxubq %1 %+ mm_mad, %1 %+ mm_mad, %1 %+ mm_tmp0
3335 AVX2_Store8x8Accb r_tmp + i_xcnt, %1, mm_mad, mm_tmp0, b_second_blocks
3336%endif
3337 vpunpcklqdq %1 %+ mm_tmp0, %1 %+ mm_sqsum, %1 %+ mm_sqdiff
3338 vpunpckhqdq %1 %+ mm_tmp1, %1 %+ mm_sqsum, %1 %+ mm_sqdiff
3339 vpaddd %1 %+ mm_tmp0, %1 %+ mm_tmp0, %1 %+ mm_tmp1
3340 vpshufd %1 %+ mm_tmp1, %1 %+ mm_tmp0, 10110001b
3341 vpaddd %1 %+ mm_tmp0, %1 %+ mm_tmp0, %1 %+ mm_tmp1
3342 AVX2_Store2x16x16Accdw p_sqsum16x16, p_sqdiff16x16, i_xcnt, r_tmp, %1, mm_tmp0, mm_tmp1, b_second_blocks
3343%undef mm_tmp0
3344%undef mm_tmp1
3345%undef mm_sqsum
3346%undef mm_sqdiff
3347%undef mm_mad
3348%undef mm_sum
3349%undef mm_sumref
3350%undef mm_sad
3351%undef tmp2
3352%undef b_second_blocks
3353%endmacro
3354
3355;*************************************************************************************************************
3356;void VAACalcSadSsdBgd_avx2(const uint8_t *cur_data, const uint8_t *ref_data, int32_t iPicWidth, int32_t iPicHeight,
3357; int32_t iPicStride, int32_t *psadframe, int32_t *psad8x8, int32_t *psum16x16, int32_t *psqsum16x16,
3358; int32_t *psqdiff16x16, int32_t *p_sd8x8, uint8_t *p_mad8x8)
3359;*************************************************************************************************************
3360
3361WELS_EXTERN VAACalcSadSsdBgd_avx2
3362%define p_sadframe arg6
3363%define p_sad8x8 arg7
3364%define p_sum16x16 arg8
3365%define p_sqsum16x16 arg9
3366%define p_sqdiff16x16 arg10
3367%define p_sd8x8 arg11
3368%define p_mad8x8 arg12
3369%ifdef X86_32
3370%define saveregs r5, r6
3371%else
3372%define saveregs rbx, rbp, r12, r13, r14, r15
3373%endif
3374
3375%assign push_num 0
3376 LOAD_5_PARA
3377 PUSH_XMM 12
3378 SIGN_EXTENSION r2, r2d
3379 SIGN_EXTENSION r3, r3d
3380 SIGN_EXTENSION r4, r4d
3381 PUSHM saveregs
3382
3383%ifdef X86_32
3384 STACK_ALLOC r5, 3 * ymm_width, ymm_width
3385 %define mm8 0
3386 %define sadframe_acc_addr r5
3387 %define sadframe_acc [sadframe_acc_addr]
3388 %define prev_mad [r5 + ymm_width]
3389 %define ymm_zero [r5 + 2 * ymm_width]
3390 %define xmm_zero ymm_zero
3391 vpxor xmm0, xmm0, xmm0
3392 vmovdqa sadframe_acc, ymm0
3393 vmovdqa ymm_zero, ymm0
3394%else
3395 %define sadframe_acc ymm9
3396 %define xsadframe_acc xmm9
3397 %define prev_mad ymm10
3398 %define ymm_zero ymm11
3399 %define xmm_zero xmm11
3400 vpxor xmm_zero, xmm_zero, xmm_zero
3401 vpxor xsadframe_acc, xsadframe_acc, xsadframe_acc
3402%endif
3403
3404 and r2, -16 ; iPicWidth &= -16
3405 jle .done ; bail if iPicWidth < 16
3406 sar r3, 4 ; iPicHeight / 16
3407 jle .done ; bail if iPicHeight < 16
3408 shr r2, 2 ; iPicWidth / 4
3409
3410%define p_cur r0
3411%define p_ref r1
3412%define i_xcnt r2
3413%define i_ycnt ptrword arg4
3414%define i_stride r4
3415%define r_tmp r6
3416%define xcnt_unit 4
3417%ifdef X86_32
3418 mov i_ycnt, r3
3419 %define i_stride3 r3
3420%else
3421 mov rbp, p_sad8x8
3422 mov r12, p_sum16x16
3423 mov r13, p_sqsum16x16
3424 mov r14, p_sqdiff16x16
3425 mov r15, p_sd8x8
3426 %undef p_sad8x8
3427 %undef p_sum16x16
3428 %undef p_sqsum16x16
3429 %undef p_sqdiff16x16
3430 %undef p_sd8x8
3431 %define p_sad8x8 rbp
3432 %define p_sum16x16 r12
3433 %define p_sqsum16x16 r13
3434 %define p_sqdiff16x16 r14
3435 %define p_sd8x8 r15
3436 %define i_stride3 rbx
3437%endif
3438 lea i_stride3, [3 * i_stride]
3439
3440 ; offset pointers so as to compensate for the i_xcnt offset below.
3441 mov r_tmp, i_xcnt
3442 and r_tmp, 64 / xcnt_unit - 1
3443 sub p_sum16x16, r_tmp
3444 sub p_sqsum16x16, r_tmp
3445 sub p_sqdiff16x16, r_tmp
3446 sub p_mad8x8, r_tmp
3447 shl r_tmp, 2
3448 sub p_sad8x8, r_tmp
3449 sub p_sd8x8, r_tmp
3450
3451.height_loop:
3452 push i_xcnt
3453%assign push_num push_num + 1
3454%define i_xcnt_load ptrword [r7]
3455 ; use end-of-line pointers so as to enable use of a negative counter as index.
3456 lea r_tmp, [xcnt_unit * i_xcnt]
3457 add p_sad8x8, r_tmp
3458 add p_sum16x16, i_xcnt
3459 add p_sqsum16x16, i_xcnt
3460 add p_sqdiff16x16, i_xcnt
3461 add p_sd8x8, r_tmp
3462 add p_mad8x8, i_xcnt
3463 and i_xcnt, -(64 / xcnt_unit)
3464 jz .width_loop_upper8_64x_end
3465 ; use a negative loop counter to enable counting toward zero and indexing with the same counter.
3466 neg i_xcnt
3467.width_loop_upper8:
3468 AVX2_CalcSadSsdBgd_8Lines y, mm0, mm1, mm2, mm3, mm4, mm5, mm6, mm7, mm8, 0
3469 add i_xcnt, 32 / xcnt_unit
3470 jl .width_loop_upper8
3471 jg .width_loop_upper8_32x_end
3472.width_loop_upper8_64x_end:
3473 test i_xcnt_load, 32 / xcnt_unit
3474 jnz .width_loop_upper8
3475.width_loop_upper8_32x_end:
3476 AVX2_StoreRemainingSingleMad i_xcnt_load, mm1, mm2, 0
3477 test i_xcnt_load, 16 / xcnt_unit
3478 jz .width_loop_upper8_end
3479 ; remaining 16.
3480 AVX2_CalcSadSsdBgd_8Lines x, mm0, mm1, mm2, mm3, mm4, mm5, mm6, mm7, mm8, 0
3481.width_loop_upper8_end:
3482 lea p_cur, [p_cur + 8 * i_stride]
3483 lea p_ref, [p_ref + 8 * i_stride]
3484 mov i_xcnt, i_xcnt_load
3485 lea r_tmp, [xcnt_unit * i_xcnt]
3486 sub p_cur, r_tmp
3487 sub p_ref, r_tmp
3488 and i_xcnt, -(64 / xcnt_unit)
3489 jz .width_loop_lower8_64x_end
3490 neg i_xcnt
3491.width_loop_lower8:
3492 AVX2_CalcSadSsdBgd_8Lines y, mm0, mm1, mm2, mm3, mm4, mm5, mm6, mm7, mm8, 1
3493 add i_xcnt, 32 / xcnt_unit
3494 jl .width_loop_lower8
3495 jg .width_loop_lower8_32x_end
3496.width_loop_lower8_64x_end:
3497 test i_xcnt_load, 32 / xcnt_unit
3498 jnz .width_loop_lower8
3499.width_loop_lower8_32x_end:
3500 AVX2_StoreRemainingSingleMad i_xcnt_load, mm1, mm2, 1
3501 test i_xcnt_load, 16 / xcnt_unit
3502 jz .width_loop_lower8_end
3503 ; remaining 16.
3504 AVX2_CalcSadSsdBgd_8Lines x, mm0, mm1, mm2, mm3, mm4, mm5, mm6, mm7, mm8, 1
3505.width_loop_lower8_end:
3506 lea p_cur, [p_cur + 8 * i_stride]
3507 lea p_ref, [p_ref + 8 * i_stride]
3508 pop i_xcnt
3509%undef i_xcnt_load
3510 %assign push_num push_num - 1
3511 lea r_tmp, [xcnt_unit * i_xcnt]
3512 sub p_cur, r_tmp
3513 sub p_ref, r_tmp
3514 sub i_ycnt, 1
3515 jnz .height_loop
3516
3517.done:
3518 mov r_tmp, p_sadframe
3519%ifdef X86_32
3520 vmovdqa xmm2, sadframe_acc
3521 vpaddd xmm2, xmm2, [sadframe_acc_addr + xmm_width]
3522%else
3523 vextracti128 xmm2, sadframe_acc, 1
3524 vpaddd xmm2, xsadframe_acc, xmm2
3525%endif
3526 vpunpckhqdq xmm1, xmm2, xmm2
3527 vpaddd xmm2, xmm2, xmm1
3528 vmovd [r_tmp], xmm2
3529 vzeroupper
3530%ifdef X86_32
3531 STACK_DEALLOC
3532%endif
3533 POPM saveregs
3534 POP_XMM
3535 LOAD_5_PARA_POP
3536%undef p_cur
3537%undef p_ref
3538%undef i_xcnt
3539%undef i_ycnt
3540%undef i_stride
3541%undef i_stride3
3542%undef r_tmp
3543%undef xcnt_unit
3544%undef mm8
3545%undef sadframe_acc
3546%undef sadframe_acc_addr
3547%undef xsadframe_acc
3548%undef prev_mad
3549%undef ymm_zero
3550%undef xmm_zero
3551%undef saveregs
3552%undef p_sadframe
3553%undef p_sad8x8
3554%undef p_sum16x16
3555%undef p_sqsum16x16
3556%undef p_sqdiff16x16
3557%undef p_sd8x8
3558%undef p_mad8x8
3559 ret

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t