The resulting per-block speedup is measured to between ~1.67x and ~2.37x for horizontal filtering, between ~1.24x and ~1.53x for vertical filtering, and between ~1.33x and ~2.37x for 2-dimensional filtering on Haswell, depending on block size, as compared with the existing SSE2 routines. Outliers are 4-wide vertical and 2-dimensional filtering with a speedups of ~5.69x and ~6.84x respectively on Haswell (with clang-703.0.31) due to the C routines used previously.
Showing the changes to test/encoder/EncUT_MotionCompensation.cpp (1 of 5 files). Show all
Unified · Split
| @@ -168,8 +168,8 @@ DEF_MCCOPYTEST (16, 8) · 167 unmodified lines | ||
| 168 | 168 | DEF_MCCOPYTEST (8, 16) |
| 169 | 169 | DEF_MCCOPYTEST (16, 16) |
| 170 | 170 | |
| 171 | −#define DEF_LUMA_MCTEST(iW,iH) \ | |
| 172 | −TEST(McHorVer,iW##x##iH) \ | |
| 171 | +#define DEF_LUMA_MCTEST(iW, iH, cpu_flags, name_suffix) \ | |
| 172 | +TEST(McHorVer, iW##x##iH##_##name_suffix) \ | |
| 173 | 173 | { \ |
| 174 | 174 | for (int32_t a = 0; a < 4; a++) { \ |
| 175 | 175 | for (int32_t b = 0; b < 4; b++) { \ |
| @@ -191,43 +191,38 @@ TEST(McHorVer,iW##x##iH) \ · 15 unmodified lines | ||
| 191 | 191 | uSrcAnchor[0][j][i] = uSrcTest[j][i] = rand()%256; \ |
| 192 | 192 | }\ |
| 193 | 193 | }\ |
| 194 | − int32_t iCpuCores = 1; \ | |
| 195 | − uint32_t uiCpuFlag;\ | |
| 196 | − for(int32_t k =0; k<2; k++)\ | |
| 197 | − {\ | |
| 198 | − if(k==0)\ | |
| 199 | − {\ | |
| 200 | − uiCpuFlag = 0;\ | |
| 201 | − }else \ | |
| 202 | − {\ | |
| 203 | − uiCpuFlag = WelsCPUFeatureDetect (&iCpuCores); \ | |
| 204 | − }\ | |
| 205 | − InitMcFunc(&sMcFunc,uiCpuFlag);\ | |
| 206 | − memset(uDstAnchor,0,sizeof(uint8_t)*MC_BUFF_HEIGHT*MC_BUFF_DST_STRIDE); \ | |
| 207 | − memset(uDstTest,0,sizeof(uint8_t)*MC_BUFF_HEIGHT*MC_BUFF_DST_STRIDE); \ | |
| 208 | − MCHalfPelFilterAnchor(uSrcInputAnchor[1],uSrcInputAnchor[2],uSrcInputAnchor[3],uSrcInputAnchor[0],MC_BUFF_SRC_STRIDE,iW+1,iH+1,pBuf+4); \ | |
| 209 | − MCLumaAnchor(uDstAnchor[0],MC_BUFF_DST_STRIDE,uSrcInputAnchor,MC_BUFF_SRC_STRIDE,a,b,iW,iH); \ | |
| 210 | − sMcFunc.pMcLumaFunc(&uSrcTest[4][4],MC_BUFF_SRC_STRIDE,uDstTest[0],MC_BUFF_DST_STRIDE,a,b,iW,iH);\ | |
| 211 | − for(int32_t j=0;j<MC_BUFF_HEIGHT;j++) \ | |
| 212 | − { \ | |
| 213 | − for(int32_t i=0;i<MC_BUFF_DST_STRIDE;i++) \ | |
| 214 | − { \ | |
| 215 | − ASSERT_EQ(uDstAnchor[j][i],uDstTest[j][i]); \ | |
| 216 | − } \ | |
| 217 | − } \ | |
| 218 | − }\ | |
| 194 | + InitMcFunc(&sMcFunc, WelsCPUFeatureDetect (0) & (cpu_flags)); \ | |
| 195 | + memset(uDstAnchor,0,sizeof(uint8_t)*MC_BUFF_HEIGHT*MC_BUFF_DST_STRIDE); \ | |
| 196 | + memset(uDstTest,0,sizeof(uint8_t)*MC_BUFF_HEIGHT*MC_BUFF_DST_STRIDE); \ | |
| 197 | + MCHalfPelFilterAnchor(uSrcInputAnchor[1],uSrcInputAnchor[2],uSrcInputAnchor[3],uSrcInputAnchor[0],MC_BUFF_SRC_STRIDE,iW+1,iH+1,pBuf+4); \ | |
| 198 | + MCLumaAnchor(uDstAnchor[0],MC_BUFF_DST_STRIDE,uSrcInputAnchor,MC_BUFF_SRC_STRIDE,a,b,iW,iH); \ | |
| 199 | + sMcFunc.pMcLumaFunc(&uSrcTest[4][4],MC_BUFF_SRC_STRIDE,uDstTest[0],MC_BUFF_DST_STRIDE,a,b,iW,iH);\ | |
| 200 | + for(int32_t j=0;j<MC_BUFF_HEIGHT;j++) \ | |
| 201 | + { \ | |
| 202 | + for(int32_t i=0;i<MC_BUFF_DST_STRIDE;i++) \ | |
| 203 | + { \ | |
| 204 | + ASSERT_EQ(uDstAnchor[j][i],uDstTest[j][i]); \ | |
| 205 | + } \ | |
| 206 | + } \ | |
| 219 | 207 | }\ |
| 220 | 208 | }\ |
| 221 | 209 | } |
| 222 | 210 | |
| 211 | +#define DEF_LUMA_MCTESTS(cpu_flags, name_suffix) \ | |
| 212 | + DEF_LUMA_MCTEST ( 4, 4, cpu_flags, name_suffix) \ | |
| 213 | + DEF_LUMA_MCTEST ( 4, 8, cpu_flags, name_suffix) \ | |
| 214 | + DEF_LUMA_MCTEST ( 8, 4, cpu_flags, name_suffix) \ | |
| 215 | + DEF_LUMA_MCTEST ( 8, 8, cpu_flags, name_suffix) \ | |
| 216 | + DEF_LUMA_MCTEST (16, 8, cpu_flags, name_suffix) \ | |
| 217 | + DEF_LUMA_MCTEST ( 8, 16, cpu_flags, name_suffix) \ | |
| 218 | + DEF_LUMA_MCTEST (16, 16, cpu_flags, name_suffix) | |
| 223 | 219 | |
| 224 | −DEF_LUMA_MCTEST (4, 4) | |
| 225 | −DEF_LUMA_MCTEST (4, 8) | |
| 226 | −DEF_LUMA_MCTEST (8, 4) | |
| 227 | −DEF_LUMA_MCTEST (8, 8) | |
| 228 | −DEF_LUMA_MCTEST (16, 8) | |
| 229 | −DEF_LUMA_MCTEST (8, 16) | |
| 230 | −DEF_LUMA_MCTEST (16, 16) | |
| 220 | +DEF_LUMA_MCTESTS(0, c) | |
| 221 | +DEF_LUMA_MCTESTS(~0, native) | |
| 222 | +#ifdef X86_ASM | |
| 223 | +DEF_LUMA_MCTESTS(WELS_CPU_SSE2, sse2) | |
| 224 | +DEF_LUMA_MCTESTS(WELS_CPU_SSE2 | WELS_CPU_SSSE3, ssse3) | |
| 225 | +#endif | |
| 231 | 226 | |
| 232 | 227 | #define DEF_CHROMA_MCTEST(iW,iH) \ |
| 233 | 228 | TEST(McChroma,iW##x##iH) \ |
| @@ -315,81 +310,86 @@ TEST (EncMcAvg, PixelAvg) { · 81 unmodified lines | ||
| 315 | 310 | } |
| 316 | 311 | } |
| 317 | 312 | |
| 318 | −#define DEF_HALFPEL_MCTEST(iW,iH) \ | |
| 319 | −TEST (EncMcHalfpel, iW##x##iH) { \ | |
| 313 | +#define DEF_HALFPEL_MCTEST(iW, iH, cpu_flags, name_suffix) \ | |
| 314 | +TEST (EncMcHalfpel, iW##x##iH##_##name_suffix) { \ | |
| 320 | 315 | SMcFunc sMcFunc; \ |
| 321 | − for (int32_t k = 0; k < 2; k++) { \ | |
| 322 | − for (int32_t w = 0; w < 2; w++) { \ | |
| 323 | − int32_t width = iW ; \ | |
| 324 | − int32_t height = iH; \ | |
| 325 | − uint8_t uAnchor[4][MC_BUFF_HEIGHT][MC_BUFF_SRC_STRIDE]; \ | |
| 326 | − uint8_t uSrcTest[MC_BUFF_HEIGHT][MC_BUFF_SRC_STRIDE]; \ | |
| 327 | − uint8_t uRand[MC_BUFF_HEIGHT][MC_BUFF_DST_STRIDE]; \ | |
| 328 | − ENFORCE_STACK_ALIGN_2D (uint8_t, uDstTest, MC_BUFF_HEIGHT, MC_BUFF_DST_STRIDE, 16); \ | |
| 329 | − uint8_t* uAnchors[4]; \ | |
| 330 | − int16_t pBuf[MC_BUFF_DST_STRIDE]; \ | |
| 331 | − uAnchors[0] = &uAnchor[0][4][4]; \ | |
| 332 | − uAnchors[1] = &uAnchor[1][4][4]; \ | |
| 333 | − uAnchors[2] = &uAnchor[2][4][4]; \ | |
| 334 | − uAnchors[3] = &uAnchor[3][4][4]; \ | |
| 335 | − \ | |
| 336 | − memset (uAnchor, 0, 4 * sizeof (uint8_t)*MC_BUFF_HEIGHT * MC_BUFF_SRC_STRIDE); \ | |
| 337 | − memset (uDstTest, 0, sizeof (uint8_t)*MC_BUFF_HEIGHT * MC_BUFF_DST_STRIDE); \ | |
| 338 | − for (int32_t j = 0; j < MC_BUFF_HEIGHT; j++) { \ | |
| 339 | − for (int32_t i = 0; i < MC_BUFF_SRC_STRIDE; i++) { \ | |
| 340 | − uAnchor[0][j][i] = uSrcTest[j][i] = rand() % 256; \ | |
| 341 | − uRand[j][i] = rand() % 256; \ | |
| 342 | − } \ | |
| 316 | + for (int32_t w = 0; w < 2; w++) { \ | |
| 317 | + int32_t width = iW ; \ | |
| 318 | + int32_t height = iH; \ | |
| 319 | + uint8_t uAnchor[4][MC_BUFF_HEIGHT][MC_BUFF_SRC_STRIDE]; \ | |
| 320 | + uint8_t uSrcTest[MC_BUFF_HEIGHT][MC_BUFF_SRC_STRIDE]; \ | |
| 321 | + uint8_t uRand[MC_BUFF_HEIGHT][MC_BUFF_DST_STRIDE]; \ | |
| 322 | + ENFORCE_STACK_ALIGN_2D (uint8_t, uDstTest, MC_BUFF_HEIGHT, MC_BUFF_DST_STRIDE, 16); \ | |
| 323 | + uint8_t* uAnchors[4]; \ | |
| 324 | + int16_t pBuf[MC_BUFF_DST_STRIDE]; \ | |
| 325 | + uAnchors[0] = &uAnchor[0][4][4]; \ | |
| 326 | + uAnchors[1] = &uAnchor[1][4][4]; \ | |
| 327 | + uAnchors[2] = &uAnchor[2][4][4]; \ | |
| 328 | + uAnchors[3] = &uAnchor[3][4][4]; \ | |
| 329 | + \ | |
| 330 | + memset (uAnchor, 0, 4 * sizeof (uint8_t)*MC_BUFF_HEIGHT * MC_BUFF_SRC_STRIDE); \ | |
| 331 | + memset (uDstTest, 0, sizeof (uint8_t)*MC_BUFF_HEIGHT * MC_BUFF_DST_STRIDE); \ | |
| 332 | + for (int32_t j = 0; j < MC_BUFF_HEIGHT; j++) { \ | |
| 333 | + for (int32_t i = 0; i < MC_BUFF_SRC_STRIDE; i++) { \ | |
| 334 | + uAnchor[0][j][i] = uSrcTest[j][i] = rand() % 256; \ | |
| 335 | + uRand[j][i] = rand() % 256; \ | |
| 343 | 336 | } \ |
| 344 | − \ | |
| 345 | − uint32_t uiCpuFlag = k == 0 ? 0 : WelsCPUFeatureDetect (NULL); \ | |
| 346 | − InitMcFunc (&sMcFunc, uiCpuFlag); \ | |
| 347 | − \ | |
| 348 | − MCHalfPelFilterAnchor (uAnchors[1], uAnchors[2], uAnchors[3], uAnchors[0], MC_BUFF_SRC_STRIDE, width + 1, height + 1, pBuf + 4); \ | |
| 349 | − memcpy (&uDstTest[0][0], &uRand[0][0], sizeof uRand); \ | |
| 350 | − sMcFunc.pfLumaHalfpelHor (&uSrcTest[4][4], MC_BUFF_SRC_STRIDE, uDstTest[0], MC_BUFF_DST_STRIDE, width + 1, height); \ | |
| 351 | − for (int32_t j = 0; j < height; j++) { \ | |
| 352 | − for (int32_t i = 0; i < width + 1; i++) { \ | |
| 353 | − ASSERT_EQ (uAnchor[1][4 + j][4 + i], uDstTest[j][i]); \ | |
| 354 | − } \ | |
| 337 | + } \ | |
| 338 | + \ | |
| 339 | + InitMcFunc (&sMcFunc, WelsCPUFeatureDetect (0) & (cpu_flags)); \ | |
| 340 | + \ | |
| 341 | + MCHalfPelFilterAnchor (uAnchors[1], uAnchors[2], uAnchors[3], uAnchors[0], MC_BUFF_SRC_STRIDE, width + 1, height + 1, pBuf + 4); \ | |
| 342 | + memcpy (&uDstTest[0][0], &uRand[0][0], sizeof uRand); \ | |
| 343 | + sMcFunc.pfLumaHalfpelHor (&uSrcTest[4][4], MC_BUFF_SRC_STRIDE, uDstTest[0], MC_BUFF_DST_STRIDE, width + 1, height); \ | |
| 344 | + for (int32_t j = 0; j < height; j++) { \ | |
| 345 | + for (int32_t i = 0; i < width + 1; i++) { \ | |
| 346 | + ASSERT_EQ (uAnchor[1][4 + j][4 + i], uDstTest[j][i]); \ | |
| 355 | 347 | } \ |
| 356 | − for (int32_t j = 0; j < MC_BUFF_HEIGHT; j++) { \ | |
| 357 | − for (int32_t i = j < height ? width + 1 : 0; i < MC_BUFF_DST_STRIDE; i++) { \ | |
| 358 | − ASSERT_EQ (uRand[j][i], uDstTest[j][i]); \ | |
| 359 | − } \ | |
| 348 | + } \ | |
| 349 | + for (int32_t j = 0; j < MC_BUFF_HEIGHT; j++) { \ | |
| 350 | + for (int32_t i = j < height ? width + 1 : 0; i < MC_BUFF_DST_STRIDE; i++) { \ | |
| 351 | + ASSERT_EQ (uRand[j][i], uDstTest[j][i]); \ | |
| 360 | 352 | } \ |
| 361 | − memcpy (&uDstTest[0][0], &uRand[0][0], sizeof uRand); \ | |
| 362 | − sMcFunc.pfLumaHalfpelVer (&uSrcTest[4][4], MC_BUFF_SRC_STRIDE, uDstTest[0], MC_BUFF_DST_STRIDE, width, height + 1); \ | |
| 363 | − for (int32_t j = 0; j < height + 1; j++) { \ | |
| 364 | − for (int32_t i = 0; i < width; i++) { \ | |
| 365 | − ASSERT_EQ (uAnchor[2][4 + j][4 + i], uDstTest[j][i]); \ | |
| 366 | − } \ | |
| 353 | + } \ | |
| 354 | + memcpy (&uDstTest[0][0], &uRand[0][0], sizeof uRand); \ | |
| 355 | + sMcFunc.pfLumaHalfpelVer (&uSrcTest[4][4], MC_BUFF_SRC_STRIDE, uDstTest[0], MC_BUFF_DST_STRIDE, width, height + 1); \ | |
| 356 | + for (int32_t j = 0; j < height + 1; j++) { \ | |
| 357 | + for (int32_t i = 0; i < width; i++) { \ | |
| 358 | + ASSERT_EQ (uAnchor[2][4 + j][4 + i], uDstTest[j][i]); \ | |
| 367 | 359 | } \ |
| 368 | − for (int32_t j = 0; j < MC_BUFF_HEIGHT; j++) { \ | |
| 369 | − for (int32_t i = j < height + 1 ? width : 0; i < MC_BUFF_DST_STRIDE; i++) { \ | |
| 370 | − ASSERT_EQ (uRand[j][i], uDstTest[j][i]); \ | |
| 371 | − } \ | |
| 360 | + } \ | |
| 361 | + for (int32_t j = 0; j < MC_BUFF_HEIGHT; j++) { \ | |
| 362 | + for (int32_t i = j < height + 1 ? width : 0; i < MC_BUFF_DST_STRIDE; i++) { \ | |
| 363 | + ASSERT_EQ (uRand[j][i], uDstTest[j][i]); \ | |
| 372 | 364 | } \ |
| 373 | − memcpy (&uDstTest[0][0], &uRand[0][0], sizeof uRand); \ | |
| 374 | − sMcFunc.pfLumaHalfpelCen (&uSrcTest[4][4], MC_BUFF_SRC_STRIDE, uDstTest[0], MC_BUFF_DST_STRIDE, width + 1, height + 1); \ | |
| 375 | − for (int32_t j = 0; j < height + 1; j++) { \ | |
| 376 | − for (int32_t i = 0; i < width + 1; i++) { \ | |
| 377 | − ASSERT_EQ (uAnchor[3][4 + j][4 + i], uDstTest[j][i]); \ | |
| 378 | − } \ | |
| 365 | + } \ | |
| 366 | + memcpy (&uDstTest[0][0], &uRand[0][0], sizeof uRand); \ | |
| 367 | + sMcFunc.pfLumaHalfpelCen (&uSrcTest[4][4], MC_BUFF_SRC_STRIDE, uDstTest[0], MC_BUFF_DST_STRIDE, width + 1, height + 1); \ | |
| 368 | + for (int32_t j = 0; j < height + 1; j++) { \ | |
| 369 | + for (int32_t i = 0; i < width + 1; i++) { \ | |
| 370 | + ASSERT_EQ (uAnchor[3][4 + j][4 + i], uDstTest[j][i]); \ | |
| 379 | 371 | } \ |
| 380 | − for (int32_t j = 0; j < MC_BUFF_HEIGHT; j++) { \ | |
| 381 | − for (int32_t i = j < height + 1 ? width + 1 : 0; i < MC_BUFF_DST_STRIDE; i++) { \ | |
| 382 | − ASSERT_EQ (uRand[j][i], uDstTest[j][i]); \ | |
| 383 | − } \ | |
| 372 | + } \ | |
| 373 | + for (int32_t j = 0; j < MC_BUFF_HEIGHT; j++) { \ | |
| 374 | + for (int32_t i = j < height + 1 ? width + 1 : 0; i < MC_BUFF_DST_STRIDE; i++) { \ | |
| 375 | + ASSERT_EQ (uRand[j][i], uDstTest[j][i]); \ | |
| 384 | 376 | } \ |
| 385 | 377 | } \ |
| 386 | 378 | } \ |
| 387 | 379 | } |
| 388 | 380 | |
| 389 | −DEF_HALFPEL_MCTEST(4,4) | |
| 390 | −DEF_HALFPEL_MCTEST(4,8) | |
| 391 | −DEF_HALFPEL_MCTEST(8,4) | |
| 392 | −DEF_HALFPEL_MCTEST(8,8) | |
| 393 | −DEF_HALFPEL_MCTEST(8,16) | |
| 394 | −DEF_HALFPEL_MCTEST(16,8) | |
| 395 | −DEF_HALFPEL_MCTEST(16,16) | |
| 381 | +#define DEF_HALFPEL_MCTESTS(cpu_flags, name_suffix) \ | |
| 382 | + DEF_HALFPEL_MCTEST( 4 , 4, cpu_flags, name_suffix) \ | |
| 383 | + DEF_HALFPEL_MCTEST( 4, 8, cpu_flags, name_suffix) \ | |
| 384 | + DEF_HALFPEL_MCTEST( 8, 4, cpu_flags, name_suffix) \ | |
| 385 | + DEF_HALFPEL_MCTEST( 8, 8, cpu_flags, name_suffix) \ | |
| 386 | + DEF_HALFPEL_MCTEST( 8, 16, cpu_flags, name_suffix) \ | |
| 387 | + DEF_HALFPEL_MCTEST(16, 8, cpu_flags, name_suffix) \ | |
| 388 | + DEF_HALFPEL_MCTEST(16, 16, cpu_flags, name_suffix) | |
| 389 | + | |
| 390 | +DEF_HALFPEL_MCTESTS(0, c) | |
| 391 | +DEF_HALFPEL_MCTESTS(~0, native) | |
| 392 | +#ifdef X86_ASM | |
| 393 | +DEF_HALFPEL_MCTESTS(WELS_CPU_SSE2, sse2) | |
| 394 | +DEF_HALFPEL_MCTESTS(WELS_CPU_SSE2 | WELS_CPU_SSSE3, ssse3) | |
| 395 | +#endif | |