History

[Processing] Add AVX2 VAA routines

Process 8 lines at a time rather than 16 lines at a time because this appears to give more reliable memory subsystem performance on Haswell. Speedup is > 2x as compared to SSE2 when not memory-bound on Haswell. On my Haswell MBP, VAACalcSadSsdBgd is about ~3x faster when uncached, which appears to be related to processing 8 lines at a time as opposed to 16 lines at a time. The other routines are also faster as compared to the SSE2 routines in this case but to a lesser extent.

Sindre Aamås committed 10 years ago

Commit 57fc3e91 parent eb9f5654 files · +1547 −0Browse files

Showing the changes to codec/processing/src (3 of 4 files). Show all

Unified · Split

3 files
  1. codec/processing/src/vaacalc/vaacalculation.cpp +7 −0
  2. codec/processing/src/vaacalc/vaacalculation.h +5 −0
  3. codec/processing/src/x86/vaa.asm +1529 −0
codec/processing/src/vaacalc/vaacalculation.cpp+7−0
@@ -64,6 +64,13 @@ void CVAACalculation::InitVaaFuncs (SVaaFuncs& sVaaFuncs, int32_t iCpuFlag) { · 63 unmodified lines
6464 sVaaFuncs.pfVAACalcSadSsdBgd = VAACalcSadSsdBgd_sse2;
6565 sVaaFuncs.pfVAACalcSadVar = VAACalcSadVar_sse2;
6666 }
67+ if (iCpuFlag & WELS_CPU_AVX2) {
68+ sVaaFuncs.pfVAACalcSad = VAACalcSad_avx2;
69+ sVaaFuncs.pfVAACalcSadBgd = VAACalcSadBgd_avx2;
70+ sVaaFuncs.pfVAACalcSadSsd = VAACalcSadSsd_avx2;
71+ sVaaFuncs.pfVAACalcSadSsdBgd = VAACalcSadSsdBgd_avx2;
72+ sVaaFuncs.pfVAACalcSadVar = VAACalcSadVar_avx2;
73+ }
6774 #endif//X86_ASM
6875 #ifdef HAVE_NEON
6976 if ((iCpuFlag & WELS_CPU_NEON) == WELS_CPU_NEON) {
codec/processing/src/vaacalc/vaacalculation.h+5−0
@@ -104,6 +104,11 @@ VAACalcSadSsdBgdFunc VAACalcSadSsdBgd_sse2; · 103 unmodified lines
104104 VAACalcSadFunc VAACalcSad_sse2;
105105 VAACalcSadVarFunc VAACalcSadVar_sse2;
106106 VAACalcSadSsdFunc VAACalcSadSsd_sse2;
107+VAACalcSadBgdFunc VAACalcSadBgd_avx2;
108+VAACalcSadSsdBgdFunc VAACalcSadSsdBgd_avx2;
109+VAACalcSadFunc VAACalcSad_avx2;
110+VAACalcSadVarFunc VAACalcSadVar_avx2;
111+VAACalcSadSsdFunc VAACalcSadSsd_avx2;
107112 WELSVP_EXTERN_C_END
108113 #endif
109114
codec/processing/src/x86/vaa.asm+1529−0
@@ -2028,3 +2028,1532 @@ sqdiff_bgd_width_loop: · 2027 unmodified lines
20282028 %undef localsize
20292029 ret
20302030 %endif
2031+
2032+%ifdef X86_32
2033+%define ptrword dword
2034+%else
2035+%define ptrword qword
2036+%endif
2037+
2038+%define xmm_width 16
2039+%define ymm_width 32
2040+
2041+%macro PUSHM 1-*
2042+ %rep %0
2043+ push %1
2044+ %rotate 1
2045+ %endrep
2046+ %assign push_num push_num + %0
2047+%endmacro
2048+
2049+%macro POPM 1-*
2050+ %rep %0
2051+ %rotate -1
2052+ pop %1
2053+ %endrep
2054+ %assign push_num push_num - %0
2055+%endmacro
2056+
2057+%ifdef X86_32
2058+%define stack_alloc_min 4
2059+%else
2060+%define stack_alloc_min 8
2061+%endif
2062+
2063+; Allocate aligned stack space.
2064+; address_out=%1 size=%2 alignment=%3
2065+%macro STACK_ALLOC 3
2066+%if (%3) & ((%3) - 1)
2067+ %error non-power-of-2 alignment requested.
2068+%endif
2069+%if (%3) > 0
2070+ %assign stack_alloc_align ((%3) + stack_alloc_min - 1) / stack_alloc_min
2071+%else
2072+ %assign stack_alloc_align 1
2073+%endif
2074+ %assign stack_alloc_num ((%2) + stack_alloc_min - 1) / stack_alloc_min + stack_alloc_align - 1
2075+ %assign push_num push_num + stack_alloc_num
2076+ sub r7, stack_alloc_min * stack_alloc_num
2077+%if stack_alloc_align == 1
2078+ mov %1, r7
2079+%else
2080+ lea %1, [r7 + stack_alloc_min * (stack_alloc_align - 1)]
2081+ and %1, -(stack_alloc_min * stack_alloc_align)
2082+%endif
2083+%endmacro
2084+
2085+; Deallocate stack space allocated with STACK_ALLOC.
2086+%macro STACK_DEALLOC 0
2087+ add r7, stack_alloc_min * stack_alloc_num
2088+ %assign push_num push_num - stack_alloc_num
2089+%endmacro
2090+
2091+; Max unsigned byte per quadword
2092+; out=%1 in=%2 tmp=%3
2093+%macro AVX2_Maxubq 3
2094+ vpsrlq %3, %2, 32
2095+ vpmaxub %1, %2, %3
2096+ vpsrlq %3, %1, 16
2097+ vpmaxub %1, %1, %3
2098+ vpsrlq %3, %1, 8
2099+ vpmaxub %1, %1, %3
2100+%endmacro
2101+
2102+; Max unsigned byte per quadword. 2 register input.
2103+; Results interleaved as least significant byte of even/odd doublewords.
2104+; out=%1 in_a=%2 in_b=%3 tmp=%4
2105+%macro AVX2_Maxubq2 4
2106+ vpblendd %4, %2, %3, 10101010b
2107+ vpshufd %4, %4, 10110001b
2108+ vpblendd %1, %2, %3, 01010101b
2109+ vpmaxub %1, %4, %1
2110+ vpsrld %4, %1, 16
2111+ vpmaxub %1, %1, %4
2112+ vpsrld %4, %1, 8
2113+ vpmaxub %1, %1, %4
2114+%endmacro
2115+
2116+; res=%1 src=%2 zero=%3 tmp=%4 add_to_res=%5
2117+%macro AVX2_Sqsumbdw 5
2118+ vpunpcklbw %4, %2, %3
2119+%if %5
2120+ vpmaddwd %4, %4, %4
2121+ vpaddd %1, %1, %4
2122+%else
2123+ vpmaddwd %1, %4, %4
2124+%endif
2125+ vpunpckhbw %4, %2, %3
2126+ vpmaddwd %4, %4, %4
2127+ vpaddd %1, %1, %4
2128+%endmacro
2129+
2130+; res=%1 src=%2 zero=%3 tmp=%4 add_to_res=%5
2131+%macro AVX2_Sumbdw 5
2132+%if %5
2133+ vpsadbw %4, %2, %3
2134+ vpaddd %1, %1, %4
2135+%else
2136+ vpsadbw %1, %2, %3
2137+%endif
2138+%endmacro
2139+
2140+; res=%1 a=%2 b=%3 a=%4 tmp=%5
2141+%macro AVX2_AbsDiffub 5
2142+ vpsubusb %5, %2, %3
2143+ vpsubusb %1, %3, %4
2144+ vpor %1, %5, %1
2145+%endmacro
2146+
2147+; sad=%1 cur_data=%2 ref_data=%3 tmp=%4 accumulate_results=%5
2148+%macro AVX2_Sadbdw 5
2149+%if %5
2150+ vpsadbw %4, %2, %3
2151+ vpaddd %1, %1, %4
2152+%else
2153+ vpsadbw %1, %2, %3
2154+%endif
2155+%endmacro
2156+
2157+; sad=%1 sum_cur=%2 sqsum_cur=%3 cur_data=%4 ref_data=%5 zero=%6 tmp=%7 accumulate_results=%8
2158+%macro AVX2_SadSumSqsumbdw 8
2159+ AVX2_Sadbdw %1, %4, %5, %7, %8
2160+ AVX2_Sumbdw %2, %4, %6, %7, %8
2161+ AVX2_Sqsumbdw %3, %4, %6, %7, %8
2162+%endmacro
2163+
2164+; sad=%1 pCur=%2 pRef=%3 tmp=%4 accumulate_results=%5
2165+%macro AVX2_Sad 5
2166+ vmovdqu %4, [%2]
2167+ AVX2_Sadbdw %1, %4, [%3], %4, %5
2168+%endmacro
2169+
2170+; sad=%1 sum_cur=%2 sqsum_cur=%3 pCur=%4 pRef=%5 zero=%6 tmp=%7,%8 accumulate_results=%9
2171+%macro AVX2_SadSumSqsum 9
2172+ vmovdqu %7, [%4]
2173+ AVX2_SadSumSqsumbdw %1, %2, %3, %7, [%5], %6, %8, %9
2174+%endmacro
2175+
2176+; sad=%1 sum_cur=%2 sqsum_cur=%3 sqdiff=%4 pCur=%5 pRef=%6 zero=%7 tmp=%8,%9,%10 accumulate_results=%11
2177+%macro AVX2_SadSumSqsumSqdiff 11
2178+ vmovdqu %8, [%5]
2179+ vmovdqu %9, [%6]
2180+ AVX2_SadSumSqsumbdw %1, %2, %3, %8, %9, %7, %10, %11
2181+ AVX2_AbsDiffub %9, %8, %9, %8, %10
2182+ AVX2_Sqsumbdw %4, %9, %7, %10, %11
2183+%endmacro
2184+
2185+; sad=%1 sum_cur=%2 sum_ref=%3 mad=%4 pCur=%5 pRef=%6 zero=%7 tmp=%8,%9,%10 accumulate_results=%11
2186+%macro AVX2_SadSdMad 11
2187+ vmovdqu %8, [%5]
2188+ vmovdqu %9, [%6]
2189+ AVX2_Sumbdw %2, %8, %7, %10, %11
2190+ AVX2_Sumbdw %3, %9, %7, %10, %11
2191+ AVX2_Sadbdw %1, %8, %9, %10, %11
2192+%if %11
2193+ AVX2_AbsDiffub %9, %8, %9, %8, %10
2194+ vpmaxub %4, %4, %9
2195+%else
2196+ AVX2_AbsDiffub %4, %8, %9, %8, %10
2197+%endif
2198+%endmacro
2199+
2200+; sad=%1 sum_cur=%2 sum_ref=%3 mad=%4 sqdiff=%5 sqsum_cur=%6 pCur=%7 pRef=%8 zero=%9 tmp=%10,%11,%12 accumulate_results=%13
2201+%macro AVX2_SadBgdSqdiff 13
2202+%ifidn %12, 0
2203+ vmovdqu %10, [%7]
2204+ AVX2_Sumbdw %2, %10, %9, %11, %13
2205+ AVX2_Sqsumbdw %6, %10, %9, %11, %13
2206+ vmovdqu %11, [%8]
2207+ AVX2_Sadbdw %1, %10, %11, %10, %13
2208+ AVX2_Sumbdw %3, %11, %9, %10, %13
2209+ vmovdqu %10, [%7]
2210+%if %13
2211+ AVX2_AbsDiffub %11, %10, %11, [%7], %10
2212+ vpmaxub %4, %4, %11
2213+ AVX2_Sqsumbdw %5, %11, %9, %10, %13
2214+%else
2215+ AVX2_AbsDiffub %4, %10, %11, [%7], %10
2216+ AVX2_Sqsumbdw %5, %4, %9, %10, %13
2217+%endif
2218+%else
2219+ vmovdqu %10, [%7]
2220+ vmovdqu %11, [%8]
2221+ AVX2_Sadbdw %1, %10, %11, %12, %13
2222+ AVX2_Sumbdw %2, %10, %9, %12, %13
2223+ AVX2_Sumbdw %3, %11, %9, %12, %13
2224+ AVX2_Sqsumbdw %6, %10, %9, %12, %13
2225+%if %13
2226+ AVX2_AbsDiffub %11, %10, %11, %10, %12
2227+ vpmaxub %4, %4, %11
2228+ AVX2_Sqsumbdw %5, %11, %9, %10, %13
2229+%else
2230+ AVX2_AbsDiffub %4, %10, %11, %10, %12
2231+ AVX2_Sqsumbdw %5, %4, %9, %10, %13
2232+%endif
2233+%endif
2234+%endmacro
2235+
2236+; p_dst=%1 mmreg_prefix=%2 data=%3 tmp=%4 second_blocks=%5
2237+%macro AVX2_Store8x8Accdw 5
2238+ vpshufd %2%4, %2%3, 1000b
2239+%ifidni %2, x
2240+ vmovlps [%1 + 8 * %5], x%4
2241+%elif %5 == 0
2242+ vmovdqu [%1], %2%4
2243+%else
2244+ vmovlps [%1 + 8], x%4
2245+ vextracti128 x%4, %2%4, 1
2246+ vmovlps [%1 + 24], x%4
2247+%endif
2248+%endmacro
2249+
2250+; p_dst=%1 mmreg_prefix=%2 data=%3 tmp=%4 second_blocks=%5
2251+%macro AVX2_Store8x8Accb 5
2252+ vpunpckhqdq %2%4, %2%3, %2%3
2253+ vpunpcklbw %2%4, %2%3, %2%4
2254+%if %5 == 0
2255+ vmovd [%1 + 0], x%4
2256+%ifidni %2, y
2257+ vextracti128 x%4, %2%4, 1
2258+ vmovd [%1 + 4], x%4
2259+%endif
2260+%else
2261+ vpextrw [%1 + 2], x%4, 0
2262+%ifidni %2, y
2263+ vextracti128 x%4, %2%4, 1
2264+ vpextrw [%1 + 6], x%4, 0
2265+%endif
2266+%endif
2267+%endmacro
2268+
2269+; p_dst=%1 data=%2 tmp=%3,%4 second_blocks=%5
2270+%macro AVX2_Store2x8x8Accb 5
2271+ vpunpckhqdq y%3, y%2, y%2
2272+ vpunpcklbw y%3, y%2, y%3
2273+ vextracti128 x%4, y%3, 1
2274+ vpsllq x%4, x%4, 32
2275+ vpblendd x%4, x%3, x%4, 1010b
2276+%if %5
2277+ vpslld x%4, x%4, 16
2278+ vpblendw x%4, x%4, [%1], 01010101b
2279+%endif
2280+ vmovdqu [%1], x%4
2281+%endmacro
2282+
2283+; p_dst=%1 mmreg_prefix=%2 data=%3 tmp=%4 add_to_dst=%5
2284+%macro AVX2_Store16x16Accdw 5
2285+%ifidni %2, x
2286+%if %5
2287+ vmovd x%4, [%1 + 0]
2288+ vpaddd x%3, x%4, x%3
2289+%endif
2290+ vmovd [%1 + 0], x%3
2291+%elif %5 == 0
2292+ vmovd [%1 + 0], x%3
2293+ vextracti128 x%3, %2%3, 1
2294+ vmovd [%1 + 4], x%3
2295+%else
2296+ vextracti128 x%4, %2%3, 1
2297+ vpunpckldq x%4, x%3, x%4
2298+ vmovq x%3, [%1 + 0]
2299+ vpaddd x%3, x%3, x%4
2300+ vmovlps [%1 + 0], x%3
2301+%endif
2302+%endmacro
2303+
2304+; p_dst1=%1 p_dst2=%2 i_dst_offset=%3 gpr_tmp=%4 mmreg_prefix=%5 data=%6 mm_tmp=%7 add_to_dst=%8
2305+%macro AVX2_Store2x16x16Accdw 8
2306+%ifidni %5, x
2307+ mov %4, %1
2308+%if %8 == 0
2309+ vmovd [%4 + %3], x%6
2310+ mov %4, %2
2311+ vpextrd [%4 + %3], x%6, 2
2312+%else
2313+ vmovd x%7, [%4 + %3]
2314+ vpaddd x%7, x%7, x%6
2315+ vmovd [%4 + %3], x%7
2316+ mov %4, %2
2317+ vpbroadcastd x%7, [%4 + %3]
2318+ vpaddd x%7, x%7, x%6
2319+ vpextrd [%4 + %3], x%7, 2
2320+%endif
2321+%else
2322+ vextracti128 x%7, %5%6, 1
2323+ vpblendd x%6, x%6, x%7, 1010b
2324+ mov %4, %1
2325+%if %8 == 0
2326+ vmovlps [%4 + %3], x%6
2327+ mov %4, %2
2328+ vmovhps [%4 + %3], x%6
2329+%else
2330+ vmovq x%7, [%4 + %3]
2331+ vpaddd x%7, x%7, x%6
2332+ vmovlps [%4 + %3], x%7
2333+ mov %4, %2
2334+ vpbroadcastq x%7, [%4 + %3]
2335+ vpaddd x%7, x%7, x%6
2336+ vmovhps [%4 + %3], x%7
2337+%endif
2338+%endif
2339+%endmacro
2340+
2341+
2342+; x/y-mm_prefix=%1 mm_clobber=%2,%3,%4,%5,%6 b_second_blocks=%7
2343+%macro AVX2_CalcSad_8Lines 7
2344+%define mm_tmp0 %2
2345+%define mm_sad %3
2346+%define mm_sad2 %4
2347+%define mm_sad3 %5
2348+%define mm_sad4 %6
2349+%define b_second_blocks %7
2350+%ifdef i_stride5
2351+ %define i_stride5_ i_stride5
2352+%else
2353+ lea r_tmp, [5 * i_stride]
2354+ %define i_stride5_ r_tmp
2355+%endif
2356+ ; Use multiple accumulators to shorten dependency chains and enable more parallelism.
2357+ AVX2_Sad %1 %+ mm_sad, p_cur, p_ref, %1 %+ mm_tmp0, 0
2358+ AVX2_Sad %1 %+ mm_sad2, p_cur + 1 * i_stride, p_ref + 1 * i_stride, %1 %+ mm_tmp0, 0
2359+ AVX2_Sad %1 %+ mm_sad3, p_cur + 2 * i_stride, p_ref + 2 * i_stride, %1 %+ mm_tmp0, 0
2360+ AVX2_Sad %1 %+ mm_sad4, p_cur + 1 * i_stride3, p_ref + 1 * i_stride3, %1 %+ mm_tmp0, 0
2361+ AVX2_Sad %1 %+ mm_sad, p_cur + 4 * i_stride, p_ref + 4 * i_stride, %1 %+ mm_tmp0, 1
2362+ AVX2_Sad %1 %+ mm_sad2, p_cur + 1 * i_stride5_, p_ref + 1 * i_stride5_, %1 %+ mm_tmp0, 1
2363+%ifdef i_stride7
2364+ %define i_stride7_ i_stride7
2365+%else
2366+ lea r_tmp, [i_stride + 2 * i_stride3]
2367+ %define i_stride7_ r_tmp
2368+%endif
2369+ AVX2_Sad %1 %+ mm_sad3, p_cur + 2 * i_stride3, p_ref + 2 * i_stride3, %1 %+ mm_tmp0, 1
2370+ AVX2_Sad %1 %+ mm_sad4, p_cur + 1 * i_stride7_, p_ref + 1 * i_stride7_, %1 %+ mm_tmp0, 1
2371+%undef i_stride5_
2372+%undef i_stride7_
2373+ ; Increment addresses for the next iteration. Doing this early is beneficial on Haswell.
2374+ add p_cur, %1 %+ mm_width
2375+ add p_ref, %1 %+ mm_width
2376+ ; Collapse accumulators.
2377+ vpaddd %1 %+ mm_sad, %1 %+ mm_sad, %1 %+ mm_sad2
2378+ vpaddd %1 %+ mm_sad3, %1 %+ mm_sad3, %1 %+ mm_sad4
2379+ vpaddd %1 %+ mm_sad, %1 %+ mm_sad, %1 %+ mm_sad3
2380+ AVX2_Store8x8Accdw p_sad8x8 + xcnt_unit * i_xcnt, %1, mm_sad, mm_tmp0, b_second_blocks
2381+ vpaddd y %+ mm_sadframe, y %+ mm_sadframe, y %+ mm_sad
2382+%undef mm_tmp0
2383+%undef mm_sad
2384+%undef mm_sad2
2385+%undef mm_sad3
2386+%undef mm_sad4
2387+%undef b_second_blocks
2388+%endmacro
2389+
2390+;*************************************************************************************************************
2391+;void VAACalcSad_avx2( const uint8_t *cur_data, const uint8_t *ref_data, int32_t iPicWidth, int32_t iPicHeight
2392+; int32_t iPicStride, int32_t *psadframe, int32_t *psad8x8)
2393+;*************************************************************************************************************
2394+
2395+WELS_EXTERN VAACalcSad_avx2
2396+%define p_sadframe ptrword arg6
2397+%define p_sad8x8 ptrword arg7
2398+%ifdef X86_32
2399+%define saveregs r5, r6
2400+%else
2401+%define saveregs rbx, rbp, r12
2402+%endif
2403+
2404+%assign push_num 0
2405+ LOAD_5_PARA
2406+ PUSH_XMM 7
2407+ SIGN_EXTENSION r2, r2d
2408+ SIGN_EXTENSION r3, r3d
2409+ SIGN_EXTENSION r4, r4d
2410+ PUSHM saveregs
2411+
2412+%define mm_zero mm0
2413+%define mm_sadframe mm6
2414+ vpxor x %+ mm_zero, x %+ mm_zero, x %+ mm_zero
2415+ vmovdqa y %+ mm_sadframe, y %+ mm_zero
2416+
2417+ and r2, -16 ; iPicWidth &= -16
2418+ jle .done ; bail if iPicWidth < 16
2419+ sar r3, 4 ; iPicHeight / 16
2420+ jle .done ; bail if iPicHeight < 16
2421+ shr r2, 2 ; iPicWidth / 4
2422+
2423+%define p_cur r0
2424+%define p_ref r1
2425+%define i_xcnt r2
2426+%define i_ycnt ptrword arg4
2427+%define i_stride r4
2428+%define xcnt_unit 4
2429+%ifdef X86_32
2430+ mov i_ycnt, r3
2431+ mov r5, p_sad8x8
2432+ %define i_stride3 r3
2433+ %undef p_sad8x8
2434+ %define p_sad8x8 r5
2435+ %define r_tmp r6
2436+ lea i_stride3, [3 * i_stride]
2437+%else
2438+ mov rbp, p_sad8x8
2439+ %define i_stride3 rbx
2440+ %define i_stride5 r12
2441+ %define i_stride7 r6
2442+ %undef p_sad8x8
2443+ %define p_sad8x8 rbp
2444+ lea i_stride3, [3 * i_stride]
2445+ lea i_stride5, [5 * i_stride]
2446+ lea i_stride7, [i_stride + 2 * i_stride3]
2447+%endif
2448+
2449+ ; offset pointer so as to compensate for the i_xcnt offset below.
2450+ sub p_sad8x8, 4 * 16 / xcnt_unit
2451+
2452+ push i_xcnt
2453+%assign push_num push_num + 1
2454+%define i_xcnt_load ptrword [r7]
2455+
2456+.height_loop:
2457+ ; use end-of-line pointers so as to enable use of a negative counter as index.
2458+ lea p_sad8x8, [p_sad8x8 + xcnt_unit * i_xcnt]
2459+ ; use a negative loop counter so as to enable counting toward zero and indexing with the same counter.
2460+ neg i_xcnt
2461+ add i_xcnt, 16 / xcnt_unit
2462+ jz .width_loop_upper8_remaining16
2463+.width_loop_upper8:
2464+ AVX2_CalcSad_8Lines y, mm1, mm2, mm3, mm4, mm5, 0
2465+ add i_xcnt, 32 / xcnt_unit
2466+ jl .width_loop_upper8
2467+ jg .width_loop_upper8_end
2468+.width_loop_upper8_remaining16:
2469+ AVX2_CalcSad_8Lines x, mm1, mm2, mm3, mm4, mm5, 0
2470+.width_loop_upper8_end:
2471+ lea p_cur, [p_cur + 8 * i_stride]
2472+ lea p_ref, [p_ref + 8 * i_stride]
2473+ xor i_xcnt, i_xcnt
2474+ sub i_xcnt, i_xcnt_load
2475+ lea p_cur, [p_cur + xcnt_unit * i_xcnt]
2476+ lea p_ref, [p_ref + xcnt_unit * i_xcnt]
2477+ add i_xcnt, 16 / xcnt_unit
2478+ jz .width_loop_lower8_remaining16
2479+.width_loop_lower8:
2480+ AVX2_CalcSad_8Lines y, mm1, mm2, mm3, mm4, mm5, 1
2481+ add i_xcnt, 32 / xcnt_unit
2482+ jl .width_loop_lower8
2483+ jg .width_loop_lower8_end
2484+.width_loop_lower8_remaining16:
2485+ AVX2_CalcSad_8Lines x, mm1, mm2, mm3, mm4, mm5, 1
2486+.width_loop_lower8_end:
2487+ lea p_cur, [p_cur + 8 * i_stride]
2488+ lea p_ref, [p_ref + 8 * i_stride]
2489+ xor i_xcnt, i_xcnt
2490+ sub i_xcnt, i_xcnt_load
2491+ lea p_cur, [p_cur + xcnt_unit * i_xcnt]
2492+ lea p_ref, [p_ref + xcnt_unit * i_xcnt]
2493+ neg i_xcnt
2494+ sub i_ycnt, 1
2495+ jnz .height_loop
2496+
2497+ pop i_xcnt
2498+%assign push_num push_num - 1
2499+%undef i_xcnt_load
2500+
2501+.done:
2502+ mov r6, p_sadframe
2503+ vextracti128 xmm2, y %+ mm_sadframe, 1
2504+ vpaddd xmm2, x %+ mm_sadframe, xmm2
2505+ vpunpckhqdq xmm1, xmm2, xmm2
2506+ vpaddd xmm2, xmm2, xmm1
2507+ vmovd [r6], xmm2
2508+ vzeroupper
2509+
2510+ POPM saveregs
2511+ POP_XMM
2512+ LOAD_5_PARA_POP
2513+%undef p_cur
2514+%undef p_ref
2515+%undef i_xcnt
2516+%undef i_ycnt
2517+%undef i_stride
2518+%undef r_tmp
2519+%undef xcnt_unit
2520+%undef i_stride3
2521+%undef i_stride5
2522+%undef i_stride7
2523+%undef mm_sadframe
2524+%undef mm_zero
2525+%undef saveregs
2526+%undef p_sadframe
2527+%undef p_sad8x8
2528+ ret
2529+
2530+
2531+; x/y-mm_prefix=%1 mm_clobber=%2,%3,%4,%5,%6 b_second_blocks=%7
2532+%macro AVX2_CalcSadVar_8Lines 7
2533+%define mm_tmp0 %2
2534+%define mm_tmp1 %3
2535+%define mm_sad %4
2536+%define mm_sum %5
2537+%define mm_sqsum %6
2538+%define b_second_blocks %7
2539+ ; Unroll for better performance on Haswell.
2540+ ; Avoid unrolling for the 16 px case so as to reduce the code footprint.
2541+%ifidni %1, y
2542+ lea r_tmp, [5 * i_stride]
2543+ AVX2_SadSumSqsum %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, p_cur, p_ref, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, 0
2544+ AVX2_SadSumSqsum %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, p_cur + 1 * i_stride, p_ref + 1 * i_stride, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, 1
2545+ AVX2_SadSumSqsum %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, p_cur + 2 * i_stride, p_ref + 2 * i_stride, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, 1
2546+ AVX2_SadSumSqsum %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, p_cur + 1 * i_stride3, p_ref + 1 * i_stride3, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, 1
2547+ AVX2_SadSumSqsum %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, p_cur + 4 * i_stride, p_ref + 4 * i_stride, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, 1
2548+ AVX2_SadSumSqsum %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, p_cur + r_tmp, p_ref + r_tmp, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, 1
2549+ lea r_tmp, [i_stride + 2 * i_stride3]
2550+ AVX2_SadSumSqsum %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, p_cur + 2 * i_stride3, p_ref + 2 * i_stride3, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, 1
2551+ AVX2_SadSumSqsum %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, p_cur + r_tmp, p_ref + r_tmp, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, 1
2552+ ; Increment addresses for the next iteration. Doing this early is beneficial on Haswell.
2553+ add p_cur, %1 %+ mm_width
2554+ add p_ref, %1 %+ mm_width
2555+%else
2556+ vpxor x %+ mm_sad, x %+ mm_sad, x %+ mm_sad
2557+ vpxor x %+ mm_sum, x %+ mm_sum, x %+ mm_sum
2558+ vpxor x %+ mm_sqsum, x %+ mm_sqsum, x %+ mm_sqsum
2559+ lea r_tmp, [8 * i_stride]
2560+ add p_cur, r_tmp
2561+ add p_ref, r_tmp
2562+ neg r_tmp
2563+%%loop:
2564+ AVX2_SadSumSqsum %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, p_cur + r_tmp, p_ref + r_tmp, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, 1
2565+ add r_tmp, i_stride
2566+ jl %%loop
2567+ ; Increment addresses for the next iteration. Doing this early is beneficial on Haswell.
2568+ lea r_tmp, [8 * i_stride - %1 %+ mm_width]
2569+ sub p_cur, r_tmp
2570+ sub p_ref, r_tmp
2571+%endif
2572+ AVX2_Store8x8Accdw p_sad8x8 + 4 * i_xcnt, %1, mm_sad, mm_tmp1, b_second_blocks
2573+ vpaddd y %+ mm_sadframe, y %+ mm_sadframe, y %+ mm_sad
2574+ vpunpcklqdq %1 %+ mm_tmp0, %1 %+ mm_sum, %1 %+ mm_sqsum
2575+ vpunpckhqdq %1 %+ mm_tmp1, %1 %+ mm_sum, %1 %+ mm_sqsum
2576+ vpaddd %1 %+ mm_tmp0, %1 %+ mm_tmp0, %1 %+ mm_tmp1
2577+ vpshufd %1 %+ mm_tmp1, %1 %+ mm_tmp0, 10110001b
2578+ vpaddd %1 %+ mm_tmp0, %1 %+ mm_tmp0, %1 %+ mm_tmp1
2579+ AVX2_Store2x16x16Accdw p_sum16x16, p_sqsum16x16, i_xcnt, r_tmp, %1, mm_tmp0, mm_tmp1, b_second_blocks
2580+%undef mm_tmp0
2581+%undef mm_tmp1
2582+%undef mm_sad
2583+%undef mm_sum
2584+%undef mm_sqsum
2585+%undef b_second_blocks
2586+%endmacro
2587+
2588+;*************************************************************************************************************
2589+;void VAACalcSadVar_avx2( const uint8_t *cur_data, const uint8_t *ref_data, int32_t iPicWidth, int32_t iPicHeight
2590+; int32_t iPicStride, int32_t *psadframe, int32_t *psad8x8, int32_t *psum16x16, int32_t *psqsum16x16)
2591+;*************************************************************************************************************
2592+
2593+WELS_EXTERN VAACalcSadVar_avx2
2594+%define p_sadframe ptrword arg6
2595+%define p_sad8x8 ptrword arg7
2596+%define p_sum16x16 ptrword arg8
2597+%define p_sqsum16x16 ptrword arg9
2598+%ifdef X86_32
2599+%define saveregs r5, r6
2600+%else
2601+%define saveregs rbx, rbp, r12, r13
2602+%endif
2603+
2604+%assign push_num 0
2605+ LOAD_5_PARA
2606+ PUSH_XMM 7
2607+ SIGN_EXTENSION r2, r2d
2608+ SIGN_EXTENSION r3, r3d
2609+ SIGN_EXTENSION r4, r4d
2610+ PUSHM saveregs
2611+
2612+%define mm_zero mm0
2613+%define mm_sadframe mm6
2614+ vpxor x %+ mm_zero, x %+ mm_zero, x %+ mm_zero
2615+ vmovdqa y %+ mm_sadframe, y %+ mm_zero
2616+
2617+ and r2, -16 ; iPicWidth &= -16
2618+ jle .done ; bail if iPicWidth < 16
2619+ sar r3, 4 ; iPicHeight / 16
2620+ jle .done ; bail if iPicHeight < 16
2621+ shr r2, 2 ; iPicWidth / 4
2622+
2623+%define p_cur r0
2624+%define p_ref r1
2625+%define i_xcnt r2
2626+%define i_ycnt ptrword arg4
2627+%define i_stride r4
2628+%define r_tmp r6
2629+%define xcnt_unit 4
2630+%ifdef X86_32
2631+ mov i_ycnt, r3
2632+ mov r3, p_sad8x8
2633+ %undef p_sad8x8
2634+ %define p_sad8x8 r3
2635+ %define i_stride3 r5
2636+%else
2637+ mov rbp, p_sad8x8
2638+ mov r12, p_sum16x16
2639+ mov r13, p_sqsum16x16
2640+ %undef p_sad8x8
2641+ %undef p_sum16x16
2642+ %undef p_sqsum16x16
2643+ %define p_sad8x8 rbp
2644+ %define p_sum16x16 r12
2645+ %define p_sqsum16x16 r13
2646+ %define i_stride3 rbx
2647+%endif
2648+ lea i_stride3, [3 * i_stride]
2649+
2650+ ; offset pointers so as to compensate for the i_xcnt offset below.
2651+ sub p_sad8x8, 4 * 16 / xcnt_unit
2652+ sub p_sum16x16, 1 * 16 / xcnt_unit
2653+ sub p_sqsum16x16, 1 * 16 / xcnt_unit
2654+
2655+ ; use a negative loop counter so as to enable counting toward zero and indexing with the same counter.
2656+ neg i_xcnt
2657+
2658+.height_loop:
2659+ push i_xcnt
2660+%assign push_num push_num + 1
2661+%define i_xcnt_load ptrword [r7]
2662+ ; use end-of-line pointers so as to enable use of a negative counter as index.
2663+ lea r_tmp, [xcnt_unit * i_xcnt]
2664+ sub p_sad8x8, r_tmp
2665+ sub p_sum16x16, i_xcnt
2666+ sub p_sqsum16x16, i_xcnt
2667+ add i_xcnt, 16 / xcnt_unit
2668+ jz .width_loop_upper8_remaining16
2669+.width_loop_upper8:
2670+ AVX2_CalcSadVar_8Lines y, mm1, mm2, mm3, mm4, mm5, 0
2671+ add i_xcnt, 32 / xcnt_unit
2672+ jl .width_loop_upper8
2673+ jg .width_loop_upper8_end
2674+.width_loop_upper8_remaining16:
2675+ AVX2_CalcSadVar_8Lines x, mm1, mm2, mm3, mm4, mm5, 0
2676+.width_loop_upper8_end:
2677+ lea p_cur, [p_cur + 8 * i_stride]
2678+ lea p_ref, [p_ref + 8 * i_stride]
2679+ mov i_xcnt, i_xcnt_load
2680+ lea p_cur, [p_cur + xcnt_unit * i_xcnt]
2681+ lea p_ref, [p_ref + xcnt_unit * i_xcnt]
2682+ add i_xcnt, 16 / xcnt_unit
2683+ jz .width_loop_lower8_remaining16
2684+.width_loop_lower8:
2685+ AVX2_CalcSadVar_8Lines y, mm1, mm2, mm3, mm4, mm5, 1
2686+ add i_xcnt, 32 / xcnt_unit
2687+ jl .width_loop_lower8
2688+ jg .width_loop_lower8_end
2689+.width_loop_lower8_remaining16:
2690+ AVX2_CalcSadVar_8Lines x, mm1, mm2, mm3, mm4, mm5, 1
2691+.width_loop_lower8_end:
2692+ lea p_cur, [p_cur + 8 * i_stride]
2693+ lea p_ref, [p_ref + 8 * i_stride]
2694+%undef i_xcnt_load
2695+ pop i_xcnt
2696+ %assign push_num push_num - 1
2697+ lea p_cur, [p_cur + xcnt_unit * i_xcnt]
2698+ lea p_ref, [p_ref + xcnt_unit * i_xcnt]
2699+ sub i_ycnt, 1
2700+ jnz .height_loop
2701+
2702+.done:
2703+ mov r_tmp, p_sadframe
2704+ vextracti128 xmm2, y %+ mm_sadframe, 1
2705+ vpaddd xmm2, x %+ mm_sadframe, xmm2
2706+ vpunpckhqdq xmm1, xmm2, xmm2
2707+ vpaddd xmm2, xmm2, xmm1
2708+ vmovd [r_tmp], xmm2
2709+ vzeroupper
2710+
2711+ POPM saveregs
2712+ POP_XMM
2713+ LOAD_5_PARA_POP
2714+%undef p_cur
2715+%undef p_ref
2716+%undef i_xcnt
2717+%undef i_ycnt
2718+%undef i_stride
2719+%undef i_stride3
2720+%undef r_tmp
2721+%undef xcnt_unit
2722+%undef mm_sadframe
2723+%undef mm_zero
2724+%undef saveregs
2725+%undef p_sadframe
2726+%undef p_sad8x8
2727+%undef p_sum16x16
2728+%undef p_sqsum16x16
2729+ ret
2730+
2731+
2732+; x/y-mm_prefix=%1 mm_clobber=%2,%3,%4,%5,%6,%7,%8 b_second_blocks=%9
2733+%macro AVX2_CalcSadSsd_8Lines 9
2734+%define mm_tmp0 %2
2735+%define mm_tmp1 %3
2736+%define mm_tmp2 %4
2737+%define mm_sad %5
2738+%define mm_sum %6
2739+%define mm_sqsum %7
2740+%define mm_sqdiff %8
2741+%define b_second_blocks %9
2742+ ; Unroll for better performance on Haswell.
2743+ ; Avoid unrolling for the 16 px case so as to reduce the code footprint.
2744+%ifidni %1, y
2745+%ifdef i_stride5
2746+ lea r_tmp, [i_stride + 2 * i_stride3]
2747+ %define i_stride5_ i_stride5
2748+%else
2749+ lea r_tmp, [5 * i_stride]
2750+ %define i_stride5_ r_tmp
2751+%endif
2752+ AVX2_SadSumSqsumSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, %1 %+ mm_sqdiff, p_cur, p_ref, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 0
2753+ AVX2_SadSumSqsumSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, %1 %+ mm_sqdiff, p_cur + 1 * i_stride, p_ref + 1 * i_stride, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
2754+ AVX2_SadSumSqsumSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, %1 %+ mm_sqdiff, p_cur + 2 * i_stride, p_ref + 2 * i_stride, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
2755+ AVX2_SadSumSqsumSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, %1 %+ mm_sqdiff, p_cur + 1 * i_stride3, p_ref + 1 * i_stride3, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
2756+ AVX2_SadSumSqsumSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, %1 %+ mm_sqdiff, p_cur + 4 * i_stride, p_ref + 4 * i_stride, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
2757+ AVX2_SadSumSqsumSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, %1 %+ mm_sqdiff, p_cur + 1 * i_stride5_, p_ref + 1 * i_stride5_, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
2758+%ifndef i_stride5
2759+ lea r_tmp, [i_stride + 2 * i_stride3]
2760+%endif
2761+%undef i_stride5_
2762+ AVX2_SadSumSqsumSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, %1 %+ mm_sqdiff, p_cur + 2 * i_stride3, p_ref + 2 * i_stride3, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
2763+ AVX2_SadSumSqsumSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, %1 %+ mm_sqdiff, p_cur + r_tmp, p_ref + r_tmp, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
2764+ ; Increment addresses for the next iteration. Doing this early is beneficial on Haswell.
2765+ add p_cur, %1 %+ mm_width
2766+ add p_ref, %1 %+ mm_width
2767+%else
2768+ vpxor x %+ mm_sad, x %+ mm_sad, x %+ mm_sad
2769+ vpxor x %+ mm_sum, x %+ mm_sum, x %+ mm_sum
2770+ vpxor x %+ mm_sqsum, x %+ mm_sqsum, x %+ mm_sqsum
2771+ vpxor x %+ mm_sqdiff, x %+ mm_sqdiff, x %+ mm_sqdiff
2772+ lea r_tmp, [8 * i_stride]
2773+ add p_cur, r_tmp
2774+ add p_ref, r_tmp
2775+ neg r_tmp
2776+%%loop:
2777+ AVX2_SadSumSqsumSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sqsum, %1 %+ mm_sqdiff, p_cur + r_tmp, p_ref + r_tmp, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
2778+ add r_tmp, i_stride
2779+ jl %%loop
2780+ ; Increment addresses for the next iteration. Doing this early is beneficial on Haswell.
2781+ lea r_tmp, [8 * i_stride - %1 %+ mm_width]
2782+ sub p_cur, r_tmp
2783+ sub p_ref, r_tmp
2784+%endif
2785+ mov r_tmp, p_sad8x8
2786+ AVX2_Store8x8Accdw r_tmp + 4 * i_xcnt, %1, mm_sad, mm_tmp1, b_second_blocks
2787+%ifdef X86_32
2788+ vpaddd y %+ mm_tmp1, y %+ mm_sad, sadframe_acc
2789+ vmovdqa sadframe_acc, y %+ mm_tmp1
2790+%else
2791+ vpaddd sadframe_acc, sadframe_acc, y %+ mm_sad
2792+%endif
2793+ mov r_tmp, i_xcnt
2794+ add r_tmp, p_sum16x16
2795+ vpunpckhqdq %1 %+ mm_tmp1, %1 %+ mm_sum, %1 %+ mm_sum
2796+ vpaddd %1 %+ mm_tmp0, %1 %+ mm_sum, %1 %+ mm_tmp1
2797+ AVX2_Store16x16Accdw r_tmp, %1, mm_tmp0, mm_tmp1, b_second_blocks
2798+ vpunpcklqdq %1 %+ mm_tmp0, %1 %+ mm_sqsum, %1 %+ mm_sqdiff
2799+ vpunpckhqdq %1 %+ mm_tmp1, %1 %+ mm_sqsum, %1 %+ mm_sqdiff
2800+ vpaddd %1 %+ mm_tmp0, %1 %+ mm_tmp0, %1 %+ mm_tmp1
2801+ vpshufd %1 %+ mm_tmp1, %1 %+ mm_tmp0, 10110001b
2802+ vpaddd %1 %+ mm_tmp0, %1 %+ mm_tmp0, %1 %+ mm_tmp1
2803+ AVX2_Store2x16x16Accdw p_sqsum16x16, p_sqdiff16x16, i_xcnt, r_tmp, %1, mm_tmp0, mm_tmp1, b_second_blocks
2804+%undef mm_tmp0
2805+%undef mm_tmp1
2806+%undef mm_tmp2
2807+%undef mm_sad
2808+%undef mm_sum
2809+%undef mm_sqsum
2810+%undef mm_sqdiff
2811+%undef b_second_blocks
2812+%endmacro
2813+
2814+;*************************************************************************************************************
2815+;void VAACalcSadSsd_avx2(const uint8_t *cur_data, const uint8_t *ref_data, int32_t iPicWidth, int32_t iPicHeight,
2816+; int32_t iPicStride,int32_t *psadframe, int32_t *psad8x8, int32_t *psum16x16, int32_t *psqsum16x16, int32_t *psqdiff16x16)
2817+;*************************************************************************************************************
2818+
2819+WELS_EXTERN VAACalcSadSsd_avx2
2820+%define p_sadframe ptrword arg6
2821+%define p_sad8x8 ptrword arg7
2822+%define p_sum16x16 ptrword arg8
2823+%define p_sqsum16x16 ptrword arg9
2824+%define p_sqdiff16x16 ptrword arg10
2825+%ifdef X86_32
2826+%define saveregs r5, r6
2827+%else
2828+%define saveregs rbx, rbp, r12, r13, r14, r15
2829+%endif
2830+
2831+%assign push_num 0
2832+ LOAD_5_PARA
2833+ PUSH_XMM 9
2834+ SIGN_EXTENSION r2, r2d
2835+ SIGN_EXTENSION r3, r3d
2836+ SIGN_EXTENSION r4, r4d
2837+ PUSHM saveregs
2838+
2839+%define mm_zero mm0
2840+ vpxor x %+ mm_zero, x %+ mm_zero, x %+ mm_zero
2841+
2842+%ifdef X86_32
2843+ STACK_ALLOC r5, ymm_width, ymm_width
2844+ %define sadframe_acc_addr r5
2845+ %define sadframe_acc [sadframe_acc_addr]
2846+%else
2847+ %define sadframe_acc ymm8
2848+ %define xsadframe_acc xmm8
2849+%endif
2850+ vmovdqa sadframe_acc, y %+ mm_zero
2851+
2852+ and r2, -16 ; iPicWidth &= -16
2853+ jle .done ; bail if iPicWidth < 16
2854+ sar r3, 4 ; iPicHeight / 16
2855+ jle .done ; bail if iPicHeight < 16
2856+ shr r2, 2 ; iPicWidth / 4
2857+
2858+%define p_cur r0
2859+%define p_ref r1
2860+%define i_xcnt r2
2861+%define i_ycnt ptrword arg4
2862+%define i_stride r4
2863+%define r_tmp r6
2864+%define xcnt_unit 4
2865+%ifdef X86_32
2866+ mov i_ycnt, r3
2867+ %define i_stride3 r3
2868+%else
2869+ mov r12, p_sad8x8
2870+ mov r13, p_sum16x16
2871+ mov r14, p_sqsum16x16
2872+ mov r15, p_sqdiff16x16
2873+ %undef p_sad8x8
2874+ %undef p_sum16x16
2875+ %undef p_sqsum16x16
2876+ %undef p_sqdiff16x16
2877+ %define p_sad8x8 r12
2878+ %define p_sum16x16 r13
2879+ %define p_sqsum16x16 r14
2880+ %define p_sqdiff16x16 r15
2881+ %define i_stride3 rbx
2882+ %define i_stride5 rbp
2883+ lea i_stride5, [5 * i_stride]
2884+%endif
2885+ lea i_stride3, [3 * i_stride]
2886+
2887+ ; offset pointers so as to compensate for i_xcnt offset below.
2888+ sub p_sad8x8, 4 * 16 / xcnt_unit
2889+ sub p_sum16x16, 1 * 16 / xcnt_unit
2890+ sub p_sqsum16x16, 1 * 16 / xcnt_unit
2891+ sub p_sqdiff16x16, 1 * 16 / xcnt_unit
2892+
2893+ ; use a negative loop counter so as to enable counting toward zero and indexing with the same counter.
2894+ neg i_xcnt
2895+
2896+.height_loop:
2897+ push i_xcnt
2898+%assign push_num push_num + 1
2899+%define i_xcnt_load ptrword [r7]
2900+ ; use end-of-line pointers so as to enable use of a negative counter as index.
2901+ lea r_tmp, [xcnt_unit * i_xcnt]
2902+ sub p_sad8x8, r_tmp
2903+ sub p_sum16x16, i_xcnt
2904+ sub p_sqsum16x16, i_xcnt
2905+ sub p_sqdiff16x16, i_xcnt
2906+ add i_xcnt, 16 / xcnt_unit
2907+ jz .width_loop_upper8_remaining16
2908+.width_loop_upper8:
2909+ AVX2_CalcSadSsd_8Lines y, mm1, mm2, mm3, mm4, mm5, mm6, mm7, 0
2910+ add i_xcnt, 32 / xcnt_unit
2911+ jl .width_loop_upper8
2912+ jg .width_loop_upper8_end
2913+.width_loop_upper8_remaining16:
2914+ AVX2_CalcSadSsd_8Lines x, mm1, mm2, mm3, mm4, mm5, mm6, mm7, 0
2915+.width_loop_upper8_end:
2916+ lea p_cur, [p_cur + 8 * i_stride]
2917+ lea p_ref, [p_ref + 8 * i_stride]
2918+ mov i_xcnt, i_xcnt_load
2919+ lea p_cur, [p_cur + xcnt_unit * i_xcnt]
2920+ lea p_ref, [p_ref + xcnt_unit * i_xcnt]
2921+ add i_xcnt, 16 / xcnt_unit
2922+ jz .width_loop_lower8_remaining16
2923+.width_loop_lower8:
2924+ AVX2_CalcSadSsd_8Lines y, mm1, mm2, mm3, mm4, mm5, mm6, mm7, 1
2925+ add i_xcnt, 32 / xcnt_unit
2926+ jl .width_loop_lower8
2927+ jg .width_loop_lower8_end
2928+.width_loop_lower8_remaining16:
2929+ AVX2_CalcSadSsd_8Lines x, mm1, mm2, mm3, mm4, mm5, mm6, mm7, 1
2930+.width_loop_lower8_end:
2931+ lea p_cur, [p_cur + 8 * i_stride]
2932+ lea p_ref, [p_ref + 8 * i_stride]
2933+%undef i_xcnt_load
2934+ pop i_xcnt
2935+ %assign push_num push_num - 1
2936+ lea p_cur, [p_cur + xcnt_unit * i_xcnt]
2937+ lea p_ref, [p_ref + xcnt_unit * i_xcnt]
2938+ sub i_ycnt, 1
2939+ jnz .height_loop
2940+
2941+.done:
2942+ mov r_tmp, p_sadframe
2943+%ifdef X86_32
2944+ vmovdqa xmm2, sadframe_acc
2945+ vpaddd xmm2, xmm2, [sadframe_acc_addr + xmm_width]
2946+%else
2947+ vextracti128 xmm2, sadframe_acc, 1
2948+ vpaddd xmm2, xsadframe_acc, xmm2
2949+%endif
2950+ vpunpckhqdq xmm1, xmm2, xmm2
2951+ vpaddd xmm2, xmm2, xmm1
2952+ vmovd [r_tmp], xmm2
2953+ vzeroupper
2954+%ifdef X86_32
2955+ STACK_DEALLOC
2956+%endif
2957+ POPM saveregs
2958+ POP_XMM
2959+ LOAD_5_PARA_POP
2960+%undef p_cur
2961+%undef p_ref
2962+%undef i_xcnt
2963+%undef i_ycnt
2964+%undef i_stride
2965+%undef i_stride3
2966+%undef i_stride5
2967+%undef r_tmp
2968+%undef xcnt_unit
2969+%undef sadframe_acc
2970+%undef sadframe_acc_addr
2971+%undef xsadframe_acc
2972+%undef mm_zero
2973+%undef saveregs
2974+%undef p_sadframe
2975+%undef p_sad8x8
2976+%undef p_sum16x16
2977+%undef p_sqsum16x16
2978+%undef p_sqdiff16x16
2979+ ret
2980+
2981+
2982+; x/y-mm_prefix=%1 mm_clobber=%2,%3,%4,%5,%6,%7,%8 b_second_blocks=%9
2983+%macro AVX2_CalcSadBgd_8Lines 9
2984+%define mm_tmp0 %2
2985+%define mm_tmp1 %3
2986+%define mm_tmp2 %8
2987+%define mm_mad %4
2988+%define mm_sumcur %5
2989+%define mm_sumref %6
2990+%define mm_sad %7
2991+%define b_second_blocks %9
2992+ ; Unroll for better performance on Haswell.
2993+ ; Avoid unrolling for the 16 px case so as to reduce the code footprint.
2994+%ifidni %1, y
2995+ lea r_tmp, [5 * i_stride]
2996+ AVX2_SadSdMad %1 %+ mm_sad, %1 %+ mm_sumcur, %1 %+ mm_sumref, %1 %+ mm_mad, p_cur, p_ref, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 0
2997+ AVX2_SadSdMad %1 %+ mm_sad, %1 %+ mm_sumcur, %1 %+ mm_sumref, %1 %+ mm_mad, p_cur + 1 * i_stride, p_ref + 1 * i_stride, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
2998+ AVX2_SadSdMad %1 %+ mm_sad, %1 %+ mm_sumcur, %1 %+ mm_sumref, %1 %+ mm_mad, p_cur + 2 * i_stride, p_ref + 2 * i_stride, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
2999+ AVX2_SadSdMad %1 %+ mm_sad, %1 %+ mm_sumcur, %1 %+ mm_sumref, %1 %+ mm_mad, p_cur + 1 * i_stride3, p_ref + 1 * i_stride3, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
3000+ AVX2_SadSdMad %1 %+ mm_sad, %1 %+ mm_sumcur, %1 %+ mm_sumref, %1 %+ mm_mad, p_cur + 4 * i_stride, p_ref + 4 * i_stride, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
3001+ AVX2_SadSdMad %1 %+ mm_sad, %1 %+ mm_sumcur, %1 %+ mm_sumref, %1 %+ mm_mad, p_cur + r_tmp, p_ref + r_tmp, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
3002+ lea r_tmp, [i_stride + 2 * i_stride3]
3003+ AVX2_SadSdMad %1 %+ mm_sad, %1 %+ mm_sumcur, %1 %+ mm_sumref, %1 %+ mm_mad, p_cur + 2 * i_stride3, p_ref + 2 * i_stride3, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
3004+ AVX2_SadSdMad %1 %+ mm_sad, %1 %+ mm_sumcur, %1 %+ mm_sumref, %1 %+ mm_mad, p_cur + r_tmp, p_ref + r_tmp, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
3005+ ; Increment addresses for the next iteration. Doing this early is beneficial on Haswell.
3006+ add p_cur, %1 %+ mm_width
3007+ add p_ref, %1 %+ mm_width
3008+%else
3009+ vpxor x %+ mm_sad, x %+ mm_sad, x %+ mm_sad
3010+ vpxor x %+ mm_sumcur, x %+ mm_sumcur, x %+ mm_sumcur
3011+ vpxor x %+ mm_sumref, x %+ mm_sumref, x %+ mm_sumref
3012+ vpxor x %+ mm_mad, x %+ mm_mad, x %+ mm_mad
3013+ lea r_tmp, [8 * i_stride]
3014+ add p_cur, r_tmp
3015+ add p_ref, r_tmp
3016+ neg r_tmp
3017+%%loop:
3018+ AVX2_SadSdMad %1 %+ mm_sad, %1 %+ mm_sumcur, %1 %+ mm_sumref, %1 %+ mm_mad, p_cur + r_tmp, p_ref + r_tmp, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, %1 %+ mm_tmp2, 1
3019+ add r_tmp, i_stride
3020+ jl %%loop
3021+ ; Increment addresses for the next iteration. Doing this early is beneficial on Haswell.
3022+ lea r_tmp, [8 * i_stride - %1 %+ mm_width]
3023+ sub p_cur, r_tmp
3024+ sub p_ref, r_tmp
3025+%endif
3026+ mov r_tmp, p_sad8x8
3027+ AVX2_Store8x8Accdw r_tmp + 4 * i_xcnt, %1, mm_sad, mm_tmp1, b_second_blocks
3028+%ifdef X86_32
3029+ vpaddd y %+ mm_tmp1, y %+ mm_sad, sadframe_acc
3030+ vmovdqa sadframe_acc, y %+ mm_tmp1
3031+%else
3032+ vpaddd sadframe_acc, sadframe_acc, y %+ mm_sad
3033+%endif
3034+ mov r_tmp, p_sd8x8
3035+ vpsubd %1 %+ mm_tmp0, %1 %+ mm_sumcur, %1 %+ mm_sumref
3036+ AVX2_Store8x8Accdw r_tmp + 4 * i_xcnt, %1, mm_tmp0, mm_tmp1, b_second_blocks
3037+ ; Coalesce store and horizontal reduction of MAD accumulator for even and
3038+ ; odd iterations so as to enable more parallelism.
3039+%ifidni %1, y
3040+ test i_xcnt, 32 / xcnt_unit
3041+ jz %%preserve_mad
3042+ mov r_tmp, p_mad8x8
3043+ AVX2_Maxubq2 y %+ mm_mad, y %+ mm_mad, prev_mad, y %+ mm_tmp0
3044+ AVX2_Store2x8x8Accb r_tmp + i_xcnt - 8, mm_mad, mm_tmp0, mm_tmp1, b_second_blocks
3045+%%preserve_mad:
3046+ vmovdqa prev_mad, y %+ mm_mad
3047+%else
3048+ mov r_tmp, p_mad8x8
3049+ AVX2_Maxubq %1 %+ mm_mad, %1 %+ mm_mad, %1 %+ mm_tmp0
3050+ AVX2_Store8x8Accb r_tmp + i_xcnt, %1, mm_mad, mm_tmp0, b_second_blocks
3051+%endif
3052+%undef mm_tmp0
3053+%undef mm_tmp1
3054+%undef mm_tmp2
3055+%undef mm_mad
3056+%undef mm_sumcur
3057+%undef mm_sumref
3058+%undef mm_sad
3059+%undef b_second_blocks
3060+%endmacro
3061+
3062+; Store remaining MAD accumulator for width & 32 cases.
3063+; width/xcnt_unit=%1 mm_tmp=%2,%3 b_second_blocks=%4
3064+%macro AVX2_StoreRemainingSingleMad 4
3065+ test %1, 32 / xcnt_unit
3066+ jz %%skip
3067+ mov r_tmp, p_mad8x8
3068+ vmovdqa y%2, prev_mad
3069+ AVX2_Maxubq y%2, y%2, y%3
3070+ AVX2_Store8x8Accb r_tmp + i_xcnt - 8, y, %2, %3, %4
3071+%%skip:
3072+%endmacro
3073+
3074+;*************************************************************************************************************
3075+;void VAACalcSadBgd_avx2(const uint8_t *cur_data, const uint8_t *ref_data, int32_t iPicWidth, int32_t iPicHeight,
3076+; int32_t iPicStride, int32_t *psadframe, int32_t *psad8x8, int32_t *p_sd8x8, uint8_t *p_mad8x8)
3077+;*************************************************************************************************************
3078+
3079+WELS_EXTERN VAACalcSadBgd_avx2
3080+%define p_sadframe arg6
3081+%define p_sad8x8 arg7
3082+%define p_sd8x8 arg8
3083+%define p_mad8x8 arg9
3084+%ifdef X86_32
3085+%define saveregs r5, r6
3086+%else
3087+%define saveregs rbx, rbp, r12, r13
3088+%endif
3089+
3090+%assign push_num 0
3091+ LOAD_5_PARA
3092+ PUSH_XMM 10
3093+ SIGN_EXTENSION r2, r2d
3094+ SIGN_EXTENSION r3, r3d
3095+ SIGN_EXTENSION r4, r4d
3096+ PUSHM saveregs
3097+
3098+%define mm_zero mm0
3099+ vpxor x %+ mm_zero, x %+ mm_zero, x %+ mm_zero
3100+
3101+%ifdef X86_32
3102+ STACK_ALLOC r5, 2 * ymm_width, ymm_width
3103+ %define sadframe_acc_addr r5
3104+ %define sadframe_acc [sadframe_acc_addr]
3105+ %define prev_mad [r5 + ymm_width]
3106+%else
3107+ %define sadframe_acc ymm8
3108+ %define xsadframe_acc xmm8
3109+ %define prev_mad ymm9
3110+%endif
3111+ vmovdqa sadframe_acc, y %+ mm_zero
3112+
3113+ and r2, -16 ; iPicWidth &= -16
3114+ jle .done ; bail if iPicWidth < 16
3115+ sar r3, 4 ; iPicHeight / 16
3116+ jle .done ; bail if iPicHeight < 16
3117+ shr r2, 2 ; iPicWidth / 4
3118+
3119+%define p_cur r0
3120+%define p_ref r1
3121+%define i_xcnt r2
3122+%define i_ycnt ptrword arg4
3123+%define i_stride r4
3124+%define r_tmp r6
3125+%define xcnt_unit 4
3126+%ifdef X86_32
3127+ mov i_ycnt, r3
3128+ %define i_stride3 r3
3129+%else
3130+ mov rbp, p_sad8x8
3131+ mov r12, p_sd8x8
3132+ mov r13, p_mad8x8
3133+ %undef p_sad8x8
3134+ %undef p_sd8x8
3135+ %undef p_mad8x8
3136+ %define p_sad8x8 rbp
3137+ %define p_sd8x8 r12
3138+ %define p_mad8x8 r13
3139+ %define i_stride3 rbx
3140+%endif
3141+ lea i_stride3, [3 * i_stride]
3142+
3143+ ; offset pointers to compensate for the i_xcnt offset below.
3144+ mov r_tmp, i_xcnt
3145+ and r_tmp, 64 / xcnt_unit - 1
3146+ sub p_mad8x8, r_tmp
3147+ shl r_tmp, 2
3148+ sub p_sad8x8, r_tmp
3149+ sub p_sd8x8, r_tmp
3150+
3151+.height_loop:
3152+ push i_xcnt
3153+%assign push_num push_num + 1
3154+%define i_xcnt_load ptrword [r7]
3155+ ; use end-of-line pointers so as to enable use of a negative counter as index.
3156+ lea r_tmp, [xcnt_unit * i_xcnt]
3157+ add p_sad8x8, r_tmp
3158+ add p_sd8x8, r_tmp
3159+ add p_mad8x8, i_xcnt
3160+ and i_xcnt, -(64 / xcnt_unit)
3161+ jz .width_loop_upper8_64x_end
3162+ ; use a negative loop counter to enable counting toward zero and indexing with the same counter.
3163+ neg i_xcnt
3164+.width_loop_upper8:
3165+ AVX2_CalcSadBgd_8Lines y, mm1, mm2, mm3, mm4, mm5, mm6, mm7, 0
3166+ add i_xcnt, 32 / xcnt_unit
3167+ jl .width_loop_upper8
3168+ jg .width_loop_upper8_32x_end
3169+.width_loop_upper8_64x_end:
3170+ test i_xcnt_load, 32 / xcnt_unit
3171+ jnz .width_loop_upper8
3172+.width_loop_upper8_32x_end:
3173+ AVX2_StoreRemainingSingleMad i_xcnt_load, mm1, mm2, 0
3174+ test i_xcnt_load, 16 / xcnt_unit
3175+ jz .width_loop_upper8_end
3176+ ; remaining 16.
3177+ AVX2_CalcSadBgd_8Lines x, mm1, mm2, mm3, mm4, mm5, mm6, mm7, 0
3178+.width_loop_upper8_end:
3179+ lea p_cur, [p_cur + 8 * i_stride]
3180+ lea p_ref, [p_ref + 8 * i_stride]
3181+ mov i_xcnt, i_xcnt_load
3182+ lea r_tmp, [xcnt_unit * i_xcnt]
3183+ sub p_cur, r_tmp
3184+ sub p_ref, r_tmp
3185+ and i_xcnt, -(64 / xcnt_unit)
3186+ jz .width_loop_lower8_64x_end
3187+ neg i_xcnt
3188+.width_loop_lower8:
3189+ AVX2_CalcSadBgd_8Lines y, mm1, mm2, mm3, mm4, mm5, mm6, mm7, 1
3190+ add i_xcnt, 32 / xcnt_unit
3191+ jl .width_loop_lower8
3192+ jg .width_loop_lower8_32x_end
3193+.width_loop_lower8_64x_end:
3194+ test i_xcnt_load, 32 / xcnt_unit
3195+ jnz .width_loop_lower8
3196+.width_loop_lower8_32x_end:
3197+ AVX2_StoreRemainingSingleMad i_xcnt_load, mm1, mm2, 1
3198+ test i_xcnt_load, 16 / xcnt_unit
3199+ jz .width_loop_lower8_end
3200+ ; remaining 16.
3201+ AVX2_CalcSadBgd_8Lines x, mm1, mm2, mm3, mm4, mm5, mm6, mm7, 1
3202+.width_loop_lower8_end:
3203+ lea p_cur, [p_cur + 8 * i_stride]
3204+ lea p_ref, [p_ref + 8 * i_stride]
3205+ pop i_xcnt
3206+%undef i_xcnt_load
3207+ %assign push_num push_num - 1
3208+ lea r_tmp, [xcnt_unit * i_xcnt]
3209+ sub p_cur, r_tmp
3210+ sub p_ref, r_tmp
3211+ sub i_ycnt, 1
3212+ jnz .height_loop
3213+
3214+.done:
3215+ mov r_tmp, p_sadframe
3216+%ifdef X86_32
3217+ vmovdqa xmm2, sadframe_acc
3218+ vpaddd xmm2, xmm2, [sadframe_acc_addr + xmm_width]
3219+%else
3220+ vextracti128 xmm2, sadframe_acc, 1
3221+ vpaddd xmm2, xsadframe_acc, xmm2
3222+%endif
3223+ vpunpckhqdq xmm1, xmm2, xmm2
3224+ vpaddd xmm2, xmm2, xmm1
3225+ vmovd [r_tmp], xmm2
3226+ vzeroupper
3227+%ifdef X86_32
3228+ STACK_DEALLOC
3229+%endif
3230+ POPM saveregs
3231+ POP_XMM
3232+ LOAD_5_PARA_POP
3233+%undef p_cur
3234+%undef p_ref
3235+%undef i_xcnt
3236+%undef i_ycnt
3237+%undef i_stride
3238+%undef i_stride3
3239+%undef r_tmp
3240+%undef xcnt_unit
3241+%undef sadframe_acc
3242+%undef sadframe_acc_addr
3243+%undef xsadframe_acc
3244+%undef prev_mad
3245+%undef mm_zero
3246+%undef saveregs
3247+%undef p_sadframe
3248+%undef p_sad8x8
3249+%undef p_sd8x8
3250+%undef p_mad8x8
3251+ ret
3252+
3253+
3254+; x/y-mm_prefix=%1 mm_clobber=%2,%3,%4,%5,%6,%7,%8,%9,%10 b_second_blocks=%11
3255+%macro AVX2_CalcSadSsdBgd_8Lines 11
3256+%define mm_tmp0 %2
3257+%define mm_tmp1 %3
3258+%define mm_sad %4
3259+%define mm_sum %5
3260+%define mm_sumref %6
3261+%define mm_mad %7
3262+%define mm_sqsum %8
3263+%define mm_sqdiff %9
3264+%ifidn %10, 0
3265+%define tmp2 0
3266+%else
3267+%define tmp2 %1 %+ %10
3268+%endif
3269+%define b_second_blocks %11
3270+ ; Unroll for better performance on Haswell.
3271+ ; Avoid unrolling for the 16 px case so as to reduce the code footprint.
3272+%ifidni %1, y
3273+ lea r_tmp, [5 * i_stride]
3274+ AVX2_SadBgdSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sumref, %1 %+ mm_mad, %1 %+ mm_sqdiff, %1 %+ mm_sqsum, p_cur, p_ref, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, tmp2, 0
3275+ AVX2_SadBgdSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sumref, %1 %+ mm_mad, %1 %+ mm_sqdiff, %1 %+ mm_sqsum, p_cur + 1 * i_stride, p_ref + 1 * i_stride, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, tmp2, 1
3276+ AVX2_SadBgdSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sumref, %1 %+ mm_mad, %1 %+ mm_sqdiff, %1 %+ mm_sqsum, p_cur + 2 * i_stride, p_ref + 2 * i_stride, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, tmp2, 1
3277+ AVX2_SadBgdSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sumref, %1 %+ mm_mad, %1 %+ mm_sqdiff, %1 %+ mm_sqsum, p_cur + 1 * i_stride3, p_ref + 1 * i_stride3, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, tmp2, 1
3278+ AVX2_SadBgdSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sumref, %1 %+ mm_mad, %1 %+ mm_sqdiff, %1 %+ mm_sqsum, p_cur + 4 * i_stride, p_ref + 4 * i_stride, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, tmp2, 1
3279+ AVX2_SadBgdSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sumref, %1 %+ mm_mad, %1 %+ mm_sqdiff, %1 %+ mm_sqsum, p_cur + r_tmp, p_ref + r_tmp, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, tmp2, 1
3280+ lea r_tmp, [i_stride + 2 * i_stride3]
3281+ AVX2_SadBgdSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sumref, %1 %+ mm_mad, %1 %+ mm_sqdiff, %1 %+ mm_sqsum, p_cur + 2 * i_stride3, p_ref + 2 * i_stride3, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, tmp2, 1
3282+ AVX2_SadBgdSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sumref, %1 %+ mm_mad, %1 %+ mm_sqdiff, %1 %+ mm_sqsum, p_cur + r_tmp, p_ref + r_tmp, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, tmp2, 1
3283+ ; Increment addresses for the next iteration. Doing this early is beneficial on Haswell.
3284+ add p_cur, %1 %+ mm_width
3285+ add p_ref, %1 %+ mm_width
3286+%else
3287+ vpxor x %+ mm_sad, x %+ mm_sad, x %+ mm_sad
3288+ vpxor x %+ mm_sum, x %+ mm_sum, x %+ mm_sum
3289+ vpxor x %+ mm_sumref, x %+ mm_sumref, x %+ mm_sumref
3290+ vpxor x %+ mm_mad, x %+ mm_mad, x %+ mm_mad
3291+ vpxor x %+ mm_sqsum, x %+ mm_sqsum, x %+ mm_sqsum
3292+ vpxor x %+ mm_sqdiff, x %+ mm_sqdiff, x %+ mm_sqdiff
3293+ lea r_tmp, [8 * i_stride]
3294+ add p_cur, r_tmp
3295+ add p_ref, r_tmp
3296+ neg r_tmp
3297+%%loop:
3298+ AVX2_SadBgdSqdiff %1 %+ mm_sad, %1 %+ mm_sum, %1 %+ mm_sumref, %1 %+ mm_mad, %1 %+ mm_sqdiff, %1 %+ mm_sqsum, p_cur + r_tmp, p_ref + r_tmp, %1 %+ mm_zero, %1 %+ mm_tmp0, %1 %+ mm_tmp1, tmp2, 1
3299+ add r_tmp, i_stride
3300+ jl %%loop
3301+ ; Increment addresses for the next iteration. Doing this early is beneficial on Haswell.
3302+ lea r_tmp, [8 * i_stride - %1 %+ mm_width]
3303+ sub p_cur, r_tmp
3304+ sub p_ref, r_tmp
3305+%endif
3306+ mov r_tmp, p_sad8x8
3307+ AVX2_Store8x8Accdw r_tmp + 4 * i_xcnt, %1, mm_sad, mm_tmp1, b_second_blocks
3308+%ifdef X86_32
3309+ vpaddd y %+ mm_tmp1, y %+ mm_sad, sadframe_acc
3310+ vmovdqa sadframe_acc, y %+ mm_tmp1
3311+%else
3312+ vpaddd sadframe_acc, sadframe_acc, y %+ mm_sad
3313+%endif
3314+ mov r_tmp, i_xcnt
3315+ add r_tmp, p_sum16x16
3316+ vpunpckhqdq %1 %+ mm_tmp1, %1 %+ mm_sum, %1 %+ mm_sum
3317+ vpaddd %1 %+ mm_tmp0, %1 %+ mm_sum, %1 %+ mm_tmp1
3318+ AVX2_Store16x16Accdw r_tmp, %1, mm_tmp0, mm_tmp1, b_second_blocks
3319+ mov r_tmp, p_sd8x8
3320+ vpsubd %1 %+ mm_sum, %1 %+ mm_sum, %1 %+ mm_sumref
3321+ AVX2_Store8x8Accdw r_tmp + 4 * i_xcnt, %1, mm_sum, mm_tmp0, b_second_blocks
3322+ ; Coalesce store and horizontal reduction of MAD accumulator for even and
3323+ ; odd iterations so as to enable more parallelism.
3324+%ifidni %1, y
3325+ test i_xcnt, 32 / xcnt_unit
3326+ jz %%preserve_mad
3327+ mov r_tmp, p_mad8x8
3328+ AVX2_Maxubq2 y %+ mm_mad, y %+ mm_mad, prev_mad, y %+ mm_tmp0
3329+ AVX2_Store2x8x8Accb r_tmp + i_xcnt - 8, mm_mad, mm_tmp0, mm_tmp1, b_second_blocks
3330+%%preserve_mad:
3331+ vmovdqa prev_mad, y %+ mm_mad
3332+%else
3333+ mov r_tmp, p_mad8x8
3334+ AVX2_Maxubq %1 %+ mm_mad, %1 %+ mm_mad, %1 %+ mm_tmp0
3335+ AVX2_Store8x8Accb r_tmp + i_xcnt, %1, mm_mad, mm_tmp0, b_second_blocks
3336+%endif
3337+ vpunpcklqdq %1 %+ mm_tmp0, %1 %+ mm_sqsum, %1 %+ mm_sqdiff
3338+ vpunpckhqdq %1 %+ mm_tmp1, %1 %+ mm_sqsum, %1 %+ mm_sqdiff
3339+ vpaddd %1 %+ mm_tmp0, %1 %+ mm_tmp0, %1 %+ mm_tmp1
3340+ vpshufd %1 %+ mm_tmp1, %1 %+ mm_tmp0, 10110001b
3341+ vpaddd %1 %+ mm_tmp0, %1 %+ mm_tmp0, %1 %+ mm_tmp1
3342+ AVX2_Store2x16x16Accdw p_sqsum16x16, p_sqdiff16x16, i_xcnt, r_tmp, %1, mm_tmp0, mm_tmp1, b_second_blocks
3343+%undef mm_tmp0
3344+%undef mm_tmp1
3345+%undef mm_sqsum
3346+%undef mm_sqdiff
3347+%undef mm_mad
3348+%undef mm_sum
3349+%undef mm_sumref
3350+%undef mm_sad
3351+%undef tmp2
3352+%undef b_second_blocks
3353+%endmacro
3354+
3355+;*************************************************************************************************************
3356+;void VAACalcSadSsdBgd_avx2(const uint8_t *cur_data, const uint8_t *ref_data, int32_t iPicWidth, int32_t iPicHeight,
3357+; int32_t iPicStride, int32_t *psadframe, int32_t *psad8x8, int32_t *psum16x16, int32_t *psqsum16x16,
3358+; int32_t *psqdiff16x16, int32_t *p_sd8x8, uint8_t *p_mad8x8)
3359+;*************************************************************************************************************
3360+
3361+WELS_EXTERN VAACalcSadSsdBgd_avx2
3362+%define p_sadframe arg6
3363+%define p_sad8x8 arg7
3364+%define p_sum16x16 arg8
3365+%define p_sqsum16x16 arg9
3366+%define p_sqdiff16x16 arg10
3367+%define p_sd8x8 arg11
3368+%define p_mad8x8 arg12
3369+%ifdef X86_32
3370+%define saveregs r5, r6
3371+%else
3372+%define saveregs rbx, rbp, r12, r13, r14, r15
3373+%endif
3374+
3375+%assign push_num 0
3376+ LOAD_5_PARA
3377+ PUSH_XMM 12
3378+ SIGN_EXTENSION r2, r2d
3379+ SIGN_EXTENSION r3, r3d
3380+ SIGN_EXTENSION r4, r4d
3381+ PUSHM saveregs
3382+
3383+%ifdef X86_32
3384+ STACK_ALLOC r5, 3 * ymm_width, ymm_width
3385+ %define mm8 0
3386+ %define sadframe_acc_addr r5
3387+ %define sadframe_acc [sadframe_acc_addr]
3388+ %define prev_mad [r5 + ymm_width]
3389+ %define ymm_zero [r5 + 2 * ymm_width]
3390+ %define xmm_zero ymm_zero
3391+ vpxor xmm0, xmm0, xmm0
3392+ vmovdqa sadframe_acc, ymm0
3393+ vmovdqa ymm_zero, ymm0
3394+%else
3395+ %define sadframe_acc ymm9
3396+ %define xsadframe_acc xmm9
3397+ %define prev_mad ymm10
3398+ %define ymm_zero ymm11
3399+ %define xmm_zero xmm11
3400+ vpxor xmm_zero, xmm_zero, xmm_zero
3401+ vpxor xsadframe_acc, xsadframe_acc, xsadframe_acc
3402+%endif
3403+
3404+ and r2, -16 ; iPicWidth &= -16
3405+ jle .done ; bail if iPicWidth < 16
3406+ sar r3, 4 ; iPicHeight / 16
3407+ jle .done ; bail if iPicHeight < 16
3408+ shr r2, 2 ; iPicWidth / 4
3409+
3410+%define p_cur r0
3411+%define p_ref r1
3412+%define i_xcnt r2
3413+%define i_ycnt ptrword arg4
3414+%define i_stride r4
3415+%define r_tmp r6
3416+%define xcnt_unit 4
3417+%ifdef X86_32
3418+ mov i_ycnt, r3
3419+ %define i_stride3 r3
3420+%else
3421+ mov rbp, p_sad8x8
3422+ mov r12, p_sum16x16
3423+ mov r13, p_sqsum16x16
3424+ mov r14, p_sqdiff16x16
3425+ mov r15, p_sd8x8
3426+ %undef p_sad8x8
3427+ %undef p_sum16x16
3428+ %undef p_sqsum16x16
3429+ %undef p_sqdiff16x16
3430+ %undef p_sd8x8
3431+ %define p_sad8x8 rbp
3432+ %define p_sum16x16 r12
3433+ %define p_sqsum16x16 r13
3434+ %define p_sqdiff16x16 r14
3435+ %define p_sd8x8 r15
3436+ %define i_stride3 rbx
3437+%endif
3438+ lea i_stride3, [3 * i_stride]
3439+
3440+ ; offset pointers so as to compensate for the i_xcnt offset below.
3441+ mov r_tmp, i_xcnt
3442+ and r_tmp, 64 / xcnt_unit - 1
3443+ sub p_sum16x16, r_tmp
3444+ sub p_sqsum16x16, r_tmp
3445+ sub p_sqdiff16x16, r_tmp
3446+ sub p_mad8x8, r_tmp
3447+ shl r_tmp, 2
3448+ sub p_sad8x8, r_tmp
3449+ sub p_sd8x8, r_tmp
3450+
3451+.height_loop:
3452+ push i_xcnt
3453+%assign push_num push_num + 1
3454+%define i_xcnt_load ptrword [r7]
3455+ ; use end-of-line pointers so as to enable use of a negative counter as index.
3456+ lea r_tmp, [xcnt_unit * i_xcnt]
3457+ add p_sad8x8, r_tmp
3458+ add p_sum16x16, i_xcnt
3459+ add p_sqsum16x16, i_xcnt
3460+ add p_sqdiff16x16, i_xcnt
3461+ add p_sd8x8, r_tmp
3462+ add p_mad8x8, i_xcnt
3463+ and i_xcnt, -(64 / xcnt_unit)
3464+ jz .width_loop_upper8_64x_end
3465+ ; use a negative loop counter to enable counting toward zero and indexing with the same counter.
3466+ neg i_xcnt
3467+.width_loop_upper8:
3468+ AVX2_CalcSadSsdBgd_8Lines y, mm0, mm1, mm2, mm3, mm4, mm5, mm6, mm7, mm8, 0
3469+ add i_xcnt, 32 / xcnt_unit
3470+ jl .width_loop_upper8
3471+ jg .width_loop_upper8_32x_end
3472+.width_loop_upper8_64x_end:
3473+ test i_xcnt_load, 32 / xcnt_unit
3474+ jnz .width_loop_upper8
3475+.width_loop_upper8_32x_end:
3476+ AVX2_StoreRemainingSingleMad i_xcnt_load, mm1, mm2, 0
3477+ test i_xcnt_load, 16 / xcnt_unit
3478+ jz .width_loop_upper8_end
3479+ ; remaining 16.
3480+ AVX2_CalcSadSsdBgd_8Lines x, mm0, mm1, mm2, mm3, mm4, mm5, mm6, mm7, mm8, 0
3481+.width_loop_upper8_end:
3482+ lea p_cur, [p_cur + 8 * i_stride]
3483+ lea p_ref, [p_ref + 8 * i_stride]
3484+ mov i_xcnt, i_xcnt_load
3485+ lea r_tmp, [xcnt_unit * i_xcnt]
3486+ sub p_cur, r_tmp
3487+ sub p_ref, r_tmp
3488+ and i_xcnt, -(64 / xcnt_unit)
3489+ jz .width_loop_lower8_64x_end
3490+ neg i_xcnt
3491+.width_loop_lower8:
3492+ AVX2_CalcSadSsdBgd_8Lines y, mm0, mm1, mm2, mm3, mm4, mm5, mm6, mm7, mm8, 1
3493+ add i_xcnt, 32 / xcnt_unit
3494+ jl .width_loop_lower8
3495+ jg .width_loop_lower8_32x_end
3496+.width_loop_lower8_64x_end:
3497+ test i_xcnt_load, 32 / xcnt_unit
3498+ jnz .width_loop_lower8
3499+.width_loop_lower8_32x_end:
3500+ AVX2_StoreRemainingSingleMad i_xcnt_load, mm1, mm2, 1
3501+ test i_xcnt_load, 16 / xcnt_unit
3502+ jz .width_loop_lower8_end
3503+ ; remaining 16.
3504+ AVX2_CalcSadSsdBgd_8Lines x, mm0, mm1, mm2, mm3, mm4, mm5, mm6, mm7, mm8, 1
3505+.width_loop_lower8_end:
3506+ lea p_cur, [p_cur + 8 * i_stride]
3507+ lea p_ref, [p_ref + 8 * i_stride]
3508+ pop i_xcnt
3509+%undef i_xcnt_load
3510+ %assign push_num push_num - 1
3511+ lea r_tmp, [xcnt_unit * i_xcnt]
3512+ sub p_cur, r_tmp
3513+ sub p_ref, r_tmp
3514+ sub i_ycnt, 1
3515+ jnz .height_loop
3516+
3517+.done:
3518+ mov r_tmp, p_sadframe
3519+%ifdef X86_32
3520+ vmovdqa xmm2, sadframe_acc
3521+ vpaddd xmm2, xmm2, [sadframe_acc_addr + xmm_width]
3522+%else
3523+ vextracti128 xmm2, sadframe_acc, 1
3524+ vpaddd xmm2, xsadframe_acc, xmm2
3525+%endif
3526+ vpunpckhqdq xmm1, xmm2, xmm2
3527+ vpaddd xmm2, xmm2, xmm1
3528+ vmovd [r_tmp], xmm2
3529+ vzeroupper
3530+%ifdef X86_32
3531+ STACK_DEALLOC
3532+%endif
3533+ POPM saveregs
3534+ POP_XMM
3535+ LOAD_5_PARA_POP
3536+%undef p_cur
3537+%undef p_ref
3538+%undef i_xcnt
3539+%undef i_ycnt
3540+%undef i_stride
3541+%undef i_stride3
3542+%undef r_tmp
3543+%undef xcnt_unit
3544+%undef mm8
3545+%undef sadframe_acc
3546+%undef sadframe_acc_addr
3547+%undef xsadframe_acc
3548+%undef prev_mad
3549+%undef ymm_zero
3550+%undef xmm_zero
3551+%undef saveregs
3552+%undef p_sadframe
3553+%undef p_sad8x8
3554+%undef p_sum16x16
3555+%undef p_sqsum16x16
3556+%undef p_sqdiff16x16
3557+%undef p_sd8x8
3558+%undef p_mad8x8
3559+ ret

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t