The resulting per-block speedup is measured to between ~1.67x and ~2.37x for horizontal filtering, between ~1.24x and ~1.53x for vertical filtering, and between ~1.33x and ~2.37x for 2-dimensional filtering on Haswell, depending on block size, as compared with the existing SSE2 routines. Outliers are 4-wide vertical and 2-dimensional filtering with a speedups of ~5.69x and ~6.84x respectively on Haswell (with clang-703.0.31) due to the C routines used previously.
Showing the changes to test/encoder/EncUT_MotionCompensation.cpp (1 of 5 files). Show all
Unified · Split
| @@ -168,8 +168,8 @@ DEF_MCCOPYTEST (16, 8) · 167 unmodified lines | |||
| 168 | DEF_MCCOPYTEST (8, 16) | 168 | DEF_MCCOPYTEST (8, 16) |
| 169 | DEF_MCCOPYTEST (16, 16) | 169 | DEF_MCCOPYTEST (16, 16) |
| 170 | 170 | ||
| 171 | #define DEF_LUMA_MCTEST(iW,iH) \ | 171 | #define DEF_LUMA_MCTEST(iW, iH, cpu_flags, name_suffix) \ |
| 172 | TEST(McHorVer,iW##x##iH) \ | 172 | TEST(McHorVer, iW##x##iH##_##name_suffix) \ |
| 173 | { \ | 173 | { \ |
| 174 | for (int32_t a = 0; a < 4; a++) { \ | 174 | for (int32_t a = 0; a < 4; a++) { \ |
| 175 | for (int32_t b = 0; b < 4; b++) { \ | 175 | for (int32_t b = 0; b < 4; b++) { \ |
| @@ -191,43 +191,38 @@ TEST(McHorVer,iW##x##iH) \ · 15 unmodified lines | |||
| 191 | uSrcAnchor[0][j][i] = uSrcTest[j][i] = rand()%256; \ | 191 | uSrcAnchor[0][j][i] = uSrcTest[j][i] = rand()%256; \ |
| 192 | }\ | 192 | }\ |
| 193 | }\ | 193 | }\ |
| 194 | int32_t iCpuCores = 1; \ | 194 | InitMcFunc(&sMcFunc, WelsCPUFeatureDetect (0) & (cpu_flags)); \ |
| 195 | uint32_t uiCpuFlag;\ | 195 | memset(uDstAnchor,0,sizeof(uint8_t)*MC_BUFF_HEIGHT*MC_BUFF_DST_STRIDE); \ |
| 196 | for(int32_t k =0; k<2; k++)\ | 196 | memset(uDstTest,0,sizeof(uint8_t)*MC_BUFF_HEIGHT*MC_BUFF_DST_STRIDE); \ |
| 197 | {\ | 197 | MCHalfPelFilterAnchor(uSrcInputAnchor[1],uSrcInputAnchor[2],uSrcInputAnchor[3],uSrcInputAnchor[0],MC_BUFF_SRC_STRIDE,iW+1,iH+1,pBuf+4); \ |
| 198 | if(k==0)\ | 198 | MCLumaAnchor(uDstAnchor[0],MC_BUFF_DST_STRIDE,uSrcInputAnchor,MC_BUFF_SRC_STRIDE,a,b,iW,iH); \ |
| 199 | {\ | 199 | sMcFunc.pMcLumaFunc(&uSrcTest[4][4],MC_BUFF_SRC_STRIDE,uDstTest[0],MC_BUFF_DST_STRIDE,a,b,iW,iH);\ |
| 200 | uiCpuFlag = 0;\ | 200 | for(int32_t j=0;j<MC_BUFF_HEIGHT;j++) \ |
| 201 | }else \ | 201 | { \ |
| 202 | {\ | 202 | for(int32_t i=0;i<MC_BUFF_DST_STRIDE;i++) \ |
| 203 | uiCpuFlag = WelsCPUFeatureDetect (&iCpuCores); \ | 203 | { \ |
| 204 | }\ | 204 | ASSERT_EQ(uDstAnchor[j][i],uDstTest[j][i]); \ |
| 205 | InitMcFunc(&sMcFunc,uiCpuFlag);\ | 205 | } \ |
| 206 | memset(uDstAnchor,0,sizeof(uint8_t)*MC_BUFF_HEIGHT*MC_BUFF_DST_STRIDE); \ | 206 | } \ |
| 207 | memset(uDstTest,0,sizeof(uint8_t)*MC_BUFF_HEIGHT*MC_BUFF_DST_STRIDE); \ | ||
| 208 | MCHalfPelFilterAnchor(uSrcInputAnchor[1],uSrcInputAnchor[2],uSrcInputAnchor[3],uSrcInputAnchor[0],MC_BUFF_SRC_STRIDE,iW+1,iH+1,pBuf+4); \ | ||
| 209 | MCLumaAnchor(uDstAnchor[0],MC_BUFF_DST_STRIDE,uSrcInputAnchor,MC_BUFF_SRC_STRIDE,a,b,iW,iH); \ | ||
| 210 | sMcFunc.pMcLumaFunc(&uSrcTest[4][4],MC_BUFF_SRC_STRIDE,uDstTest[0],MC_BUFF_DST_STRIDE,a,b,iW,iH);\ | ||
| 211 | for(int32_t j=0;j<MC_BUFF_HEIGHT;j++) \ | ||
| 212 | { \ | ||
| 213 | for(int32_t i=0;i<MC_BUFF_DST_STRIDE;i++) \ | ||
| 214 | { \ | ||
| 215 | ASSERT_EQ(uDstAnchor[j][i],uDstTest[j][i]); \ | ||
| 216 | } \ | ||
| 217 | } \ | ||
| 218 | }\ | ||
| 219 | }\ | 207 | }\ |
| 220 | }\ | 208 | }\ |
| 221 | } | 209 | } |
| 222 | 210 | ||
| 211 | #define DEF_LUMA_MCTESTS(cpu_flags, name_suffix) \ | ||
| 212 | DEF_LUMA_MCTEST ( 4, 4, cpu_flags, name_suffix) \ | ||
| 213 | DEF_LUMA_MCTEST ( 4, 8, cpu_flags, name_suffix) \ | ||
| 214 | DEF_LUMA_MCTEST ( 8, 4, cpu_flags, name_suffix) \ | ||
| 215 | DEF_LUMA_MCTEST ( 8, 8, cpu_flags, name_suffix) \ | ||
| 216 | DEF_LUMA_MCTEST (16, 8, cpu_flags, name_suffix) \ | ||
| 217 | DEF_LUMA_MCTEST ( 8, 16, cpu_flags, name_suffix) \ | ||
| 218 | DEF_LUMA_MCTEST (16, 16, cpu_flags, name_suffix) | ||
| 223 | 219 | ||
| 224 | DEF_LUMA_MCTEST (4, 4) | 220 | DEF_LUMA_MCTESTS(0, c) |
| 225 | DEF_LUMA_MCTEST (4, 8) | 221 | DEF_LUMA_MCTESTS(~0, native) |
| 226 | DEF_LUMA_MCTEST (8, 4) | 222 | #ifdef X86_ASM |
| 227 | DEF_LUMA_MCTEST (8, 8) | 223 | DEF_LUMA_MCTESTS(WELS_CPU_SSE2, sse2) |
| 228 | DEF_LUMA_MCTEST (16, 8) | 224 | DEF_LUMA_MCTESTS(WELS_CPU_SSE2 | WELS_CPU_SSSE3, ssse3) |
| 229 | DEF_LUMA_MCTEST (8, 16) | 225 | #endif |
| 230 | DEF_LUMA_MCTEST (16, 16) | ||
| 231 | 226 | ||
| 232 | #define DEF_CHROMA_MCTEST(iW,iH) \ | 227 | #define DEF_CHROMA_MCTEST(iW,iH) \ |
| 233 | TEST(McChroma,iW##x##iH) \ | 228 | TEST(McChroma,iW##x##iH) \ |
| @@ -315,81 +310,86 @@ TEST (EncMcAvg, PixelAvg) { · 81 unmodified lines | |||
| 315 | } | 310 | } |
| 316 | } | 311 | } |
| 317 | 312 | ||
| 318 | #define DEF_HALFPEL_MCTEST(iW,iH) \ | 313 | #define DEF_HALFPEL_MCTEST(iW, iH, cpu_flags, name_suffix) \ |
| 319 | TEST (EncMcHalfpel, iW##x##iH) { \ | 314 | TEST (EncMcHalfpel, iW##x##iH##_##name_suffix) { \ |
| 320 | SMcFunc sMcFunc; \ | 315 | SMcFunc sMcFunc; \ |
| 321 | for (int32_t k = 0; k < 2; k++) { \ | 316 | for (int32_t w = 0; w < 2; w++) { \ |
| 322 | for (int32_t w = 0; w < 2; w++) { \ | 317 | int32_t width = iW ; \ |
| 323 | int32_t width = iW ; \ | 318 | int32_t height = iH; \ |
| 324 | int32_t height = iH; \ | 319 | uint8_t uAnchor[4][MC_BUFF_HEIGHT][MC_BUFF_SRC_STRIDE]; \ |
| 325 | uint8_t uAnchor[4][MC_BUFF_HEIGHT][MC_BUFF_SRC_STRIDE]; \ | 320 | uint8_t uSrcTest[MC_BUFF_HEIGHT][MC_BUFF_SRC_STRIDE]; \ |
| 326 | uint8_t uSrcTest[MC_BUFF_HEIGHT][MC_BUFF_SRC_STRIDE]; \ | 321 | uint8_t uRand[MC_BUFF_HEIGHT][MC_BUFF_DST_STRIDE]; \ |
| 327 | uint8_t uRand[MC_BUFF_HEIGHT][MC_BUFF_DST_STRIDE]; \ | 322 | ENFORCE_STACK_ALIGN_2D (uint8_t, uDstTest, MC_BUFF_HEIGHT, MC_BUFF_DST_STRIDE, 16); \ |
| 328 | ENFORCE_STACK_ALIGN_2D (uint8_t, uDstTest, MC_BUFF_HEIGHT, MC_BUFF_DST_STRIDE, 16); \ | 323 | uint8_t* uAnchors[4]; \ |
| 329 | uint8_t* uAnchors[4]; \ | 324 | int16_t pBuf[MC_BUFF_DST_STRIDE]; \ |
| 330 | int16_t pBuf[MC_BUFF_DST_STRIDE]; \ | 325 | uAnchors[0] = &uAnchor[0][4][4]; \ |
| 331 | uAnchors[0] = &uAnchor[0][4][4]; \ | 326 | uAnchors[1] = &uAnchor[1][4][4]; \ |
| 332 | uAnchors[1] = &uAnchor[1][4][4]; \ | 327 | uAnchors[2] = &uAnchor[2][4][4]; \ |
| 333 | uAnchors[2] = &uAnchor[2][4][4]; \ | 328 | uAnchors[3] = &uAnchor[3][4][4]; \ |
| 334 | uAnchors[3] = &uAnchor[3][4][4]; \ | 329 | \ |
| 335 | \ | 330 | memset (uAnchor, 0, 4 * sizeof (uint8_t)*MC_BUFF_HEIGHT * MC_BUFF_SRC_STRIDE); \ |
| 336 | memset (uAnchor, 0, 4 * sizeof (uint8_t)*MC_BUFF_HEIGHT * MC_BUFF_SRC_STRIDE); \ | 331 | memset (uDstTest, 0, sizeof (uint8_t)*MC_BUFF_HEIGHT * MC_BUFF_DST_STRIDE); \ |
| 337 | memset (uDstTest, 0, sizeof (uint8_t)*MC_BUFF_HEIGHT * MC_BUFF_DST_STRIDE); \ | 332 | for (int32_t j = 0; j < MC_BUFF_HEIGHT; j++) { \ |
| 338 | for (int32_t j = 0; j < MC_BUFF_HEIGHT; j++) { \ | 333 | for (int32_t i = 0; i < MC_BUFF_SRC_STRIDE; i++) { \ |
| 339 | for (int32_t i = 0; i < MC_BUFF_SRC_STRIDE; i++) { \ | 334 | uAnchor[0][j][i] = uSrcTest[j][i] = rand() % 256; \ |
| 340 | uAnchor[0][j][i] = uSrcTest[j][i] = rand() % 256; \ | 335 | uRand[j][i] = rand() % 256; \ |
| 341 | uRand[j][i] = rand() % 256; \ | ||
| 342 | } \ | ||
| 343 | } \ | 336 | } \ |
| 344 | \ | 337 | } \ |
| 345 | uint32_t uiCpuFlag = k == 0 ? 0 : WelsCPUFeatureDetect (NULL); \ | 338 | \ |
| 346 | InitMcFunc (&sMcFunc, uiCpuFlag); \ | 339 | InitMcFunc (&sMcFunc, WelsCPUFeatureDetect (0) & (cpu_flags)); \ |
| 347 | \ | 340 | \ |
| 348 | MCHalfPelFilterAnchor (uAnchors[1], uAnchors[2], uAnchors[3], uAnchors[0], MC_BUFF_SRC_STRIDE, width + 1, height + 1, pBuf + 4); \ | 341 | MCHalfPelFilterAnchor (uAnchors[1], uAnchors[2], uAnchors[3], uAnchors[0], MC_BUFF_SRC_STRIDE, width + 1, height + 1, pBuf + 4); \ |
| 349 | memcpy (&uDstTest[0][0], &uRand[0][0], sizeof uRand); \ | 342 | memcpy (&uDstTest[0][0], &uRand[0][0], sizeof uRand); \ |
| 350 | sMcFunc.pfLumaHalfpelHor (&uSrcTest[4][4], MC_BUFF_SRC_STRIDE, uDstTest[0], MC_BUFF_DST_STRIDE, width + 1, height); \ | 343 | sMcFunc.pfLumaHalfpelHor (&uSrcTest[4][4], MC_BUFF_SRC_STRIDE, uDstTest[0], MC_BUFF_DST_STRIDE, width + 1, height); \ |
| 351 | for (int32_t j = 0; j < height; j++) { \ | 344 | for (int32_t j = 0; j < height; j++) { \ |
| 352 | for (int32_t i = 0; i < width + 1; i++) { \ | 345 | for (int32_t i = 0; i < width + 1; i++) { \ |
| 353 | ASSERT_EQ (uAnchor[1][4 + j][4 + i], uDstTest[j][i]); \ | 346 | ASSERT_EQ (uAnchor[1][4 + j][4 + i], uDstTest[j][i]); \ |
| 354 | } \ | ||
| 355 | } \ | 347 | } \ |
| 356 | for (int32_t j = 0; j < MC_BUFF_HEIGHT; j++) { \ | 348 | } \ |
| 357 | for (int32_t i = j < height ? width + 1 : 0; i < MC_BUFF_DST_STRIDE; i++) { \ | 349 | for (int32_t j = 0; j < MC_BUFF_HEIGHT; j++) { \ |
| 358 | ASSERT_EQ (uRand[j][i], uDstTest[j][i]); \ | 350 | for (int32_t i = j < height ? width + 1 : 0; i < MC_BUFF_DST_STRIDE; i++) { \ |
| 359 | } \ | 351 | ASSERT_EQ (uRand[j][i], uDstTest[j][i]); \ |
| 360 | } \ | 352 | } \ |
| 361 | memcpy (&uDstTest[0][0], &uRand[0][0], sizeof uRand); \ | 353 | } \ |
| 362 | sMcFunc.pfLumaHalfpelVer (&uSrcTest[4][4], MC_BUFF_SRC_STRIDE, uDstTest[0], MC_BUFF_DST_STRIDE, width, height + 1); \ | 354 | memcpy (&uDstTest[0][0], &uRand[0][0], sizeof uRand); \ |
| 363 | for (int32_t j = 0; j < height + 1; j++) { \ | 355 | sMcFunc.pfLumaHalfpelVer (&uSrcTest[4][4], MC_BUFF_SRC_STRIDE, uDstTest[0], MC_BUFF_DST_STRIDE, width, height + 1); \ |
| 364 | for (int32_t i = 0; i < width; i++) { \ | 356 | for (int32_t j = 0; j < height + 1; j++) { \ |
| 365 | ASSERT_EQ (uAnchor[2][4 + j][4 + i], uDstTest[j][i]); \ | 357 | for (int32_t i = 0; i < width; i++) { \ |
| 366 | } \ | 358 | ASSERT_EQ (uAnchor[2][4 + j][4 + i], uDstTest[j][i]); \ |
| 367 | } \ | 359 | } \ |
| 368 | for (int32_t j = 0; j < MC_BUFF_HEIGHT; j++) { \ | 360 | } \ |
| 369 | for (int32_t i = j < height + 1 ? width : 0; i < MC_BUFF_DST_STRIDE; i++) { \ | 361 | for (int32_t j = 0; j < MC_BUFF_HEIGHT; j++) { \ |
| 370 | ASSERT_EQ (uRand[j][i], uDstTest[j][i]); \ | 362 | for (int32_t i = j < height + 1 ? width : 0; i < MC_BUFF_DST_STRIDE; i++) { \ |
| 371 | } \ | 363 | ASSERT_EQ (uRand[j][i], uDstTest[j][i]); \ |
| 372 | } \ | 364 | } \ |
| 373 | memcpy (&uDstTest[0][0], &uRand[0][0], sizeof uRand); \ | 365 | } \ |
| 374 | sMcFunc.pfLumaHalfpelCen (&uSrcTest[4][4], MC_BUFF_SRC_STRIDE, uDstTest[0], MC_BUFF_DST_STRIDE, width + 1, height + 1); \ | 366 | memcpy (&uDstTest[0][0], &uRand[0][0], sizeof uRand); \ |
| 375 | for (int32_t j = 0; j < height + 1; j++) { \ | 367 | sMcFunc.pfLumaHalfpelCen (&uSrcTest[4][4], MC_BUFF_SRC_STRIDE, uDstTest[0], MC_BUFF_DST_STRIDE, width + 1, height + 1); \ |
| 376 | for (int32_t i = 0; i < width + 1; i++) { \ | 368 | for (int32_t j = 0; j < height + 1; j++) { \ |
| 377 | ASSERT_EQ (uAnchor[3][4 + j][4 + i], uDstTest[j][i]); \ | 369 | for (int32_t i = 0; i < width + 1; i++) { \ |
| 378 | } \ | 370 | ASSERT_EQ (uAnchor[3][4 + j][4 + i], uDstTest[j][i]); \ |
| 379 | } \ | 371 | } \ |
| 380 | for (int32_t j = 0; j < MC_BUFF_HEIGHT; j++) { \ | 372 | } \ |
| 381 | for (int32_t i = j < height + 1 ? width + 1 : 0; i < MC_BUFF_DST_STRIDE; i++) { \ | 373 | for (int32_t j = 0; j < MC_BUFF_HEIGHT; j++) { \ |
| 382 | ASSERT_EQ (uRand[j][i], uDstTest[j][i]); \ | 374 | for (int32_t i = j < height + 1 ? width + 1 : 0; i < MC_BUFF_DST_STRIDE; i++) { \ |
| 383 | } \ | 375 | ASSERT_EQ (uRand[j][i], uDstTest[j][i]); \ |
| 384 | } \ | 376 | } \ |
| 385 | } \ | 377 | } \ |
| 386 | } \ | 378 | } \ |
| 387 | } | 379 | } |
| 388 | 380 | ||
| 389 | DEF_HALFPEL_MCTEST(4,4) | 381 | #define DEF_HALFPEL_MCTESTS(cpu_flags, name_suffix) \ |
| 390 | DEF_HALFPEL_MCTEST(4,8) | 382 | DEF_HALFPEL_MCTEST( 4 , 4, cpu_flags, name_suffix) \ |
| 391 | DEF_HALFPEL_MCTEST(8,4) | 383 | DEF_HALFPEL_MCTEST( 4, 8, cpu_flags, name_suffix) \ |
| 392 | DEF_HALFPEL_MCTEST(8,8) | 384 | DEF_HALFPEL_MCTEST( 8, 4, cpu_flags, name_suffix) \ |
| 393 | DEF_HALFPEL_MCTEST(8,16) | 385 | DEF_HALFPEL_MCTEST( 8, 8, cpu_flags, name_suffix) \ |
| 394 | DEF_HALFPEL_MCTEST(16,8) | 386 | DEF_HALFPEL_MCTEST( 8, 16, cpu_flags, name_suffix) \ |
| 395 | DEF_HALFPEL_MCTEST(16,16) | 387 | DEF_HALFPEL_MCTEST(16, 8, cpu_flags, name_suffix) \ |
| 388 | DEF_HALFPEL_MCTEST(16, 16, cpu_flags, name_suffix) | ||
| 389 | |||
| 390 | DEF_HALFPEL_MCTESTS(0, c) | ||
| 391 | DEF_HALFPEL_MCTESTS(~0, native) | ||
| 392 | #ifdef X86_ASM | ||
| 393 | DEF_HALFPEL_MCTESTS(WELS_CPU_SSE2, sse2) | ||
| 394 | DEF_HALFPEL_MCTESTS(WELS_CPU_SSE2 | WELS_CPU_SSSE3, ssse3) | ||
| 395 | #endif | ||