C vs SSE2 speed gains: _4x4 : ~6.49x _8x8 : ~10.82x _16x16 : ~7.61x _32x32 : ~5.29x BUG=webm:1411
Unified · Split
| @@ -481,28 +481,32 @@ HIGHBD_INTRA_PRED_TEST( · 480 unmodified lines | ||
| 481 | 481 | |
| 482 | 482 | #if HAVE_SSE2 |
| 483 | 483 | HIGHBD_INTRA_PRED_TEST(SSE2, TestHighbdIntraPred4, |
| 484 | − vpx_highbd_dc_predictor_4x4_sse2, NULL, | |
| 484 | + vpx_highbd_dc_predictor_4x4_sse2, | |
| 485 | + vpx_highbd_dc_left_predictor_4x4_sse2, | |
| 485 | 486 | vpx_highbd_dc_top_predictor_4x4_sse2, NULL, |
| 486 | 487 | vpx_highbd_v_predictor_4x4_sse2, |
| 487 | 488 | vpx_highbd_h_predictor_4x4_sse2, NULL, NULL, NULL, NULL, |
| 488 | 489 | NULL, NULL, vpx_highbd_tm_predictor_4x4_c) |
| 489 | 490 | |
| 490 | 491 | HIGHBD_INTRA_PRED_TEST(SSE2, TestHighbdIntraPred8, |
| 491 | − vpx_highbd_dc_predictor_8x8_sse2, NULL, | |
| 492 | + vpx_highbd_dc_predictor_8x8_sse2, | |
| 493 | + vpx_highbd_dc_left_predictor_8x8_sse2, | |
| 492 | 494 | vpx_highbd_dc_top_predictor_8x8_sse2, NULL, |
| 493 | 495 | vpx_highbd_v_predictor_8x8_sse2, |
| 494 | 496 | vpx_highbd_h_predictor_8x8_sse2, NULL, NULL, NULL, NULL, |
| 495 | 497 | NULL, NULL, vpx_highbd_tm_predictor_8x8_sse2) |
| 496 | 498 | |
| 497 | 499 | HIGHBD_INTRA_PRED_TEST(SSE2, TestHighbdIntraPred16, |
| 498 | − vpx_highbd_dc_predictor_16x16_sse2, NULL, | |
| 500 | + vpx_highbd_dc_predictor_16x16_sse2, | |
| 501 | + vpx_highbd_dc_left_predictor_16x16_sse2, | |
| 499 | 502 | vpx_highbd_dc_top_predictor_16x16_sse2, NULL, |
| 500 | 503 | vpx_highbd_v_predictor_16x16_sse2, |
| 501 | 504 | vpx_highbd_h_predictor_16x16_sse2, NULL, NULL, NULL, |
| 502 | 505 | NULL, NULL, NULL, vpx_highbd_tm_predictor_16x16_sse2) |
| 503 | 506 | |
| 504 | 507 | HIGHBD_INTRA_PRED_TEST(SSE2, TestHighbdIntraPred32, |
| 505 | − vpx_highbd_dc_predictor_32x32_sse2, NULL, | |
| 508 | + vpx_highbd_dc_predictor_32x32_sse2, | |
| 509 | + vpx_highbd_dc_left_predictor_32x32_sse2, | |
| 506 | 510 | vpx_highbd_dc_top_predictor_32x32_sse2, NULL, |
| 507 | 511 | vpx_highbd_v_predictor_32x32_sse2, |
| 508 | 512 | vpx_highbd_h_predictor_32x32_sse2, NULL, NULL, NULL, |
| @@ -471,6 +471,14 @@ TEST_P(VP9HighbdIntraPredTest, HighbdIntraPredTests) { · 470 unmodified lines | ||
| 471 | 471 | INSTANTIATE_TEST_CASE_P( |
| 472 | 472 | SSE2_TO_C_8, VP9HighbdIntraPredTest, |
| 473 | 473 | ::testing::Values( |
| 474 | + HighbdIntraPredParam(&vpx_highbd_dc_left_predictor_4x4_sse2, | |
| 475 | + &vpx_highbd_dc_left_predictor_4x4_c, 4, 8), | |
| 476 | + HighbdIntraPredParam(&vpx_highbd_dc_left_predictor_8x8_sse2, | |
| 477 | + &vpx_highbd_dc_left_predictor_8x8_c, 8, 8), | |
| 478 | + HighbdIntraPredParam(&vpx_highbd_dc_left_predictor_16x16_sse2, | |
| 479 | + &vpx_highbd_dc_left_predictor_16x16_c, 16, 8), | |
| 480 | + HighbdIntraPredParam(&vpx_highbd_dc_left_predictor_32x32_sse2, | |
| 481 | + &vpx_highbd_dc_left_predictor_32x32_c, 32, 8), | |
| 474 | 482 | HighbdIntraPredParam(&vpx_highbd_dc_predictor_4x4_sse2, |
| 475 | 483 | &vpx_highbd_dc_predictor_4x4_c, 4, 8), |
| 476 | 484 | HighbdIntraPredParam(&vpx_highbd_dc_predictor_8x8_sse2, |
| @@ -515,6 +523,14 @@ INSTANTIATE_TEST_CASE_P( · 38 unmodified lines | ||
| 515 | 523 | INSTANTIATE_TEST_CASE_P( |
| 516 | 524 | SSE2_TO_C_10, VP9HighbdIntraPredTest, |
| 517 | 525 | ::testing::Values( |
| 526 | + HighbdIntraPredParam(&vpx_highbd_dc_left_predictor_4x4_sse2, | |
| 527 | + &vpx_highbd_dc_left_predictor_4x4_c, 4, 10), | |
| 528 | + HighbdIntraPredParam(&vpx_highbd_dc_left_predictor_8x8_sse2, | |
| 529 | + &vpx_highbd_dc_left_predictor_8x8_c, 8, 10), | |
| 530 | + HighbdIntraPredParam(&vpx_highbd_dc_left_predictor_16x16_sse2, | |
| 531 | + &vpx_highbd_dc_left_predictor_16x16_c, 16, 10), | |
| 532 | + HighbdIntraPredParam(&vpx_highbd_dc_left_predictor_32x32_sse2, | |
| 533 | + &vpx_highbd_dc_left_predictor_32x32_c, 32, 10), | |
| 518 | 534 | HighbdIntraPredParam(&vpx_highbd_dc_predictor_4x4_sse2, |
| 519 | 535 | &vpx_highbd_dc_predictor_4x4_c, 4, 10), |
| 520 | 536 | HighbdIntraPredParam(&vpx_highbd_dc_predictor_8x8_sse2, |
| @@ -559,6 +575,14 @@ INSTANTIATE_TEST_CASE_P( · 38 unmodified lines | ||
| 559 | 575 | INSTANTIATE_TEST_CASE_P( |
| 560 | 576 | SSE2_TO_C_12, VP9HighbdIntraPredTest, |
| 561 | 577 | ::testing::Values( |
| 578 | + HighbdIntraPredParam(&vpx_highbd_dc_left_predictor_4x4_sse2, | |
| 579 | + &vpx_highbd_dc_left_predictor_4x4_c, 4, 12), | |
| 580 | + HighbdIntraPredParam(&vpx_highbd_dc_left_predictor_8x8_sse2, | |
| 581 | + &vpx_highbd_dc_left_predictor_8x8_c, 8, 12), | |
| 582 | + HighbdIntraPredParam(&vpx_highbd_dc_left_predictor_16x16_sse2, | |
| 583 | + &vpx_highbd_dc_left_predictor_16x16_c, 16, 12), | |
| 584 | + HighbdIntraPredParam(&vpx_highbd_dc_left_predictor_32x32_sse2, | |
| 585 | + &vpx_highbd_dc_left_predictor_32x32_c, 32, 12), | |
| 562 | 586 | HighbdIntraPredParam(&vpx_highbd_dc_predictor_4x4_sse2, |
| 563 | 587 | &vpx_highbd_dc_predictor_4x4_c, 4, 12), |
| 564 | 588 | HighbdIntraPredParam(&vpx_highbd_dc_predictor_8x8_sse2, |
| @@ -217,7 +217,7 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { · 216 unmodified lines | ||
| 217 | 217 | specialize qw/vpx_highbd_dc_top_predictor_4x4 neon sse2/; |
| 218 | 218 | |
| 219 | 219 | add_proto qw/void vpx_highbd_dc_left_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; |
| 220 | − specialize qw/vpx_highbd_dc_left_predictor_4x4 neon/; | |
| 220 | + specialize qw/vpx_highbd_dc_left_predictor_4x4 neon sse2/; | |
| 221 | 221 | |
| 222 | 222 | add_proto qw/void vpx_highbd_dc_128_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; |
| 223 | 223 | specialize qw/vpx_highbd_dc_128_predictor_4x4 neon/; |
| @@ -252,7 +252,7 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { · 28 unmodified lines | ||
| 252 | 252 | specialize qw/vpx_highbd_dc_top_predictor_8x8 neon sse2/; |
| 253 | 253 | |
| 254 | 254 | add_proto qw/void vpx_highbd_dc_left_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; |
| 255 | − specialize qw/vpx_highbd_dc_left_predictor_8x8 neon/; | |
| 255 | + specialize qw/vpx_highbd_dc_left_predictor_8x8 neon sse2/; | |
| 256 | 256 | |
| 257 | 257 | add_proto qw/void vpx_highbd_dc_128_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; |
| 258 | 258 | specialize qw/vpx_highbd_dc_128_predictor_8x8 neon/; |
| @@ -287,7 +287,7 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { · 28 unmodified lines | ||
| 287 | 287 | specialize qw/vpx_highbd_dc_top_predictor_16x16 neon sse2/; |
| 288 | 288 | |
| 289 | 289 | add_proto qw/void vpx_highbd_dc_left_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; |
| 290 | − specialize qw/vpx_highbd_dc_left_predictor_16x16 neon/; | |
| 290 | + specialize qw/vpx_highbd_dc_left_predictor_16x16 neon sse2/; | |
| 291 | 291 | |
| 292 | 292 | add_proto qw/void vpx_highbd_dc_128_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; |
| 293 | 293 | specialize qw/vpx_highbd_dc_128_predictor_16x16 neon/; |
| @@ -322,7 +322,7 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { · 28 unmodified lines | ||
| 322 | 322 | specialize qw/vpx_highbd_dc_top_predictor_32x32 neon sse2/; |
| 323 | 323 | |
| 324 | 324 | add_proto qw/void vpx_highbd_dc_left_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; |
| 325 | − specialize qw/vpx_highbd_dc_left_predictor_32x32 neon/; | |
| 325 | + specialize qw/vpx_highbd_dc_left_predictor_32x32 neon sse2/; | |
| 326 | 326 | |
| 327 | 327 | add_proto qw/void vpx_highbd_dc_128_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; |
| 328 | 328 | specialize qw/vpx_highbd_dc_128_predictor_32x32 neon/; |
| @@ -177,6 +177,17 @@ static INLINE void dc_store_4x4(uint16_t *dst, ptrdiff_t stride, · 176 unmodified lines | ||
| 177 | 177 | } |
| 178 | 178 | } |
| 179 | 179 | |
| 180 | +void vpx_highbd_dc_left_predictor_4x4_sse2(uint16_t *dst, ptrdiff_t stride, | |
| 181 | + const uint16_t *above, | |
| 182 | + const uint16_t *left, int bd) { | |
| 183 | + const __m128i two = _mm_cvtsi32_si128(2); | |
| 184 | + const __m128i sum = dc_sum_4(left); | |
| 185 | + const __m128i dc = _mm_srli_epi16(_mm_add_epi16(sum, two), 2); | |
| 186 | + (void)above; | |
| 187 | + (void)bd; | |
| 188 | + dc_store_4x4(dst, stride, &dc); | |
| 189 | +} | |
| 190 | + | |
| 180 | 191 | void vpx_highbd_dc_top_predictor_4x4_sse2(uint16_t *dst, ptrdiff_t stride, |
| 181 | 192 | const uint16_t *above, |
| 182 | 193 | const uint16_t *left, int bd) { |
| @@ -210,6 +221,17 @@ static INLINE void dc_store_8x8(uint16_t *dst, ptrdiff_t stride, · 27 unmodified lines | ||
| 210 | 221 | } |
| 211 | 222 | } |
| 212 | 223 | |
| 224 | +void vpx_highbd_dc_left_predictor_8x8_sse2(uint16_t *dst, ptrdiff_t stride, | |
| 225 | + const uint16_t *above, | |
| 226 | + const uint16_t *left, int bd) { | |
| 227 | + const __m128i four = _mm_cvtsi32_si128(4); | |
| 228 | + const __m128i sum = dc_sum_8(left); | |
| 229 | + const __m128i dc = _mm_srli_epi16(_mm_add_epi16(sum, four), 3); | |
| 230 | + (void)above; | |
| 231 | + (void)bd; | |
| 232 | + dc_store_8x8(dst, stride, &dc); | |
| 233 | +} | |
| 234 | + | |
| 213 | 235 | void vpx_highbd_dc_top_predictor_8x8_sse2(uint16_t *dst, ptrdiff_t stride, |
| 214 | 236 | const uint16_t *above, |
| 215 | 237 | const uint16_t *left, int bd) { |
| @@ -241,6 +263,17 @@ static INLINE void dc_store_16x16(uint16_t *dst, ptrdiff_t stride, · 25 unmodified lines | ||
| 241 | 263 | } |
| 242 | 264 | } |
| 243 | 265 | |
| 266 | +void vpx_highbd_dc_left_predictor_16x16_sse2(uint16_t *dst, ptrdiff_t stride, | |
| 267 | + const uint16_t *above, | |
| 268 | + const uint16_t *left, int bd) { | |
| 269 | + const __m128i eight = _mm_cvtsi32_si128(8); | |
| 270 | + const __m128i sum = dc_sum_16(left); | |
| 271 | + const __m128i dc = _mm_srli_epi16(_mm_add_epi16(sum, eight), 4); | |
| 272 | + (void)above; | |
| 273 | + (void)bd; | |
| 274 | + dc_store_16x16(dst, stride, &dc); | |
| 275 | +} | |
| 276 | + | |
| 244 | 277 | void vpx_highbd_dc_top_predictor_16x16_sse2(uint16_t *dst, ptrdiff_t stride, |
| 245 | 278 | const uint16_t *above, |
| 246 | 279 | const uint16_t *left, int bd) { |
| @@ -277,6 +310,17 @@ static INLINE void dc_store_32x32(uint16_t *dst, ptrdiff_t stride, · 30 unmodified lines | ||
| 277 | 310 | } |
| 278 | 311 | } |
| 279 | 312 | |
| 313 | +void vpx_highbd_dc_left_predictor_32x32_sse2(uint16_t *dst, ptrdiff_t stride, | |
| 314 | + const uint16_t *above, | |
| 315 | + const uint16_t *left, int bd) { | |
| 316 | + const __m128i sixteen = _mm_cvtsi32_si128(16); | |
| 317 | + const __m128i sum = dc_sum_32(left); | |
| 318 | + const __m128i dc = _mm_srli_epi32(_mm_add_epi32(sum, sixteen), 5); | |
| 319 | + (void)above; | |
| 320 | + (void)bd; | |
| 321 | + dc_store_32x32(dst, stride, &dc); | |
| 322 | +} | |
| 323 | + | |
| 280 | 324 | void vpx_highbd_dc_top_predictor_32x32_sse2(uint16_t *dst, ptrdiff_t stride, |
| 281 | 325 | const uint16_t *above, |
| 282 | 326 | const uint16_t *left, int bd) { |