C vs SSE2 speed gains: _4x4 : ~6.49x _8x8 : ~10.82x _16x16 : ~7.61x _32x32 : ~5.29x BUG=webm:1411
Unified · Split
| @@ -481,28 +481,32 @@ HIGHBD_INTRA_PRED_TEST( · 480 unmodified lines | |||
| 481 | 481 | ||
| 482 | #if HAVE_SSE2 | 482 | #if HAVE_SSE2 |
| 483 | HIGHBD_INTRA_PRED_TEST(SSE2, TestHighbdIntraPred4, | 483 | HIGHBD_INTRA_PRED_TEST(SSE2, TestHighbdIntraPred4, |
| 484 | vpx_highbd_dc_predictor_4x4_sse2, NULL, | 484 | vpx_highbd_dc_predictor_4x4_sse2, |
| 485 | vpx_highbd_dc_left_predictor_4x4_sse2, | ||
| 485 | vpx_highbd_dc_top_predictor_4x4_sse2, NULL, | 486 | vpx_highbd_dc_top_predictor_4x4_sse2, NULL, |
| 486 | vpx_highbd_v_predictor_4x4_sse2, | 487 | vpx_highbd_v_predictor_4x4_sse2, |
| 487 | vpx_highbd_h_predictor_4x4_sse2, NULL, NULL, NULL, NULL, | 488 | vpx_highbd_h_predictor_4x4_sse2, NULL, NULL, NULL, NULL, |
| 488 | NULL, NULL, vpx_highbd_tm_predictor_4x4_c) | 489 | NULL, NULL, vpx_highbd_tm_predictor_4x4_c) |
| 489 | 490 | ||
| 490 | HIGHBD_INTRA_PRED_TEST(SSE2, TestHighbdIntraPred8, | 491 | HIGHBD_INTRA_PRED_TEST(SSE2, TestHighbdIntraPred8, |
| 491 | vpx_highbd_dc_predictor_8x8_sse2, NULL, | 492 | vpx_highbd_dc_predictor_8x8_sse2, |
| 493 | vpx_highbd_dc_left_predictor_8x8_sse2, | ||
| 492 | vpx_highbd_dc_top_predictor_8x8_sse2, NULL, | 494 | vpx_highbd_dc_top_predictor_8x8_sse2, NULL, |
| 493 | vpx_highbd_v_predictor_8x8_sse2, | 495 | vpx_highbd_v_predictor_8x8_sse2, |
| 494 | vpx_highbd_h_predictor_8x8_sse2, NULL, NULL, NULL, NULL, | 496 | vpx_highbd_h_predictor_8x8_sse2, NULL, NULL, NULL, NULL, |
| 495 | NULL, NULL, vpx_highbd_tm_predictor_8x8_sse2) | 497 | NULL, NULL, vpx_highbd_tm_predictor_8x8_sse2) |
| 496 | 498 | ||
| 497 | HIGHBD_INTRA_PRED_TEST(SSE2, TestHighbdIntraPred16, | 499 | HIGHBD_INTRA_PRED_TEST(SSE2, TestHighbdIntraPred16, |
| 498 | vpx_highbd_dc_predictor_16x16_sse2, NULL, | 500 | vpx_highbd_dc_predictor_16x16_sse2, |
| 501 | vpx_highbd_dc_left_predictor_16x16_sse2, | ||
| 499 | vpx_highbd_dc_top_predictor_16x16_sse2, NULL, | 502 | vpx_highbd_dc_top_predictor_16x16_sse2, NULL, |
| 500 | vpx_highbd_v_predictor_16x16_sse2, | 503 | vpx_highbd_v_predictor_16x16_sse2, |
| 501 | vpx_highbd_h_predictor_16x16_sse2, NULL, NULL, NULL, | 504 | vpx_highbd_h_predictor_16x16_sse2, NULL, NULL, NULL, |
| 502 | NULL, NULL, NULL, vpx_highbd_tm_predictor_16x16_sse2) | 505 | NULL, NULL, NULL, vpx_highbd_tm_predictor_16x16_sse2) |
| 503 | 506 | ||
| 504 | HIGHBD_INTRA_PRED_TEST(SSE2, TestHighbdIntraPred32, | 507 | HIGHBD_INTRA_PRED_TEST(SSE2, TestHighbdIntraPred32, |
| 505 | vpx_highbd_dc_predictor_32x32_sse2, NULL, | 508 | vpx_highbd_dc_predictor_32x32_sse2, |
| 509 | vpx_highbd_dc_left_predictor_32x32_sse2, | ||
| 506 | vpx_highbd_dc_top_predictor_32x32_sse2, NULL, | 510 | vpx_highbd_dc_top_predictor_32x32_sse2, NULL, |
| 507 | vpx_highbd_v_predictor_32x32_sse2, | 511 | vpx_highbd_v_predictor_32x32_sse2, |
| 508 | vpx_highbd_h_predictor_32x32_sse2, NULL, NULL, NULL, | 512 | vpx_highbd_h_predictor_32x32_sse2, NULL, NULL, NULL, |
| @@ -471,6 +471,14 @@ TEST_P(VP9HighbdIntraPredTest, HighbdIntraPredTests) { · 470 unmodified lines | |||
| 471 | INSTANTIATE_TEST_CASE_P( | 471 | INSTANTIATE_TEST_CASE_P( |
| 472 | SSE2_TO_C_8, VP9HighbdIntraPredTest, | 472 | SSE2_TO_C_8, VP9HighbdIntraPredTest, |
| 473 | ::testing::Values( | 473 | ::testing::Values( |
| 474 | HighbdIntraPredParam(&vpx_highbd_dc_left_predictor_4x4_sse2, | ||
| 475 | &vpx_highbd_dc_left_predictor_4x4_c, 4, 8), | ||
| 476 | HighbdIntraPredParam(&vpx_highbd_dc_left_predictor_8x8_sse2, | ||
| 477 | &vpx_highbd_dc_left_predictor_8x8_c, 8, 8), | ||
| 478 | HighbdIntraPredParam(&vpx_highbd_dc_left_predictor_16x16_sse2, | ||
| 479 | &vpx_highbd_dc_left_predictor_16x16_c, 16, 8), | ||
| 480 | HighbdIntraPredParam(&vpx_highbd_dc_left_predictor_32x32_sse2, | ||
| 481 | &vpx_highbd_dc_left_predictor_32x32_c, 32, 8), | ||
| 474 | HighbdIntraPredParam(&vpx_highbd_dc_predictor_4x4_sse2, | 482 | HighbdIntraPredParam(&vpx_highbd_dc_predictor_4x4_sse2, |
| 475 | &vpx_highbd_dc_predictor_4x4_c, 4, 8), | 483 | &vpx_highbd_dc_predictor_4x4_c, 4, 8), |
| 476 | HighbdIntraPredParam(&vpx_highbd_dc_predictor_8x8_sse2, | 484 | HighbdIntraPredParam(&vpx_highbd_dc_predictor_8x8_sse2, |
| @@ -515,6 +523,14 @@ INSTANTIATE_TEST_CASE_P( · 38 unmodified lines | |||
| 515 | INSTANTIATE_TEST_CASE_P( | 523 | INSTANTIATE_TEST_CASE_P( |
| 516 | SSE2_TO_C_10, VP9HighbdIntraPredTest, | 524 | SSE2_TO_C_10, VP9HighbdIntraPredTest, |
| 517 | ::testing::Values( | 525 | ::testing::Values( |
| 526 | HighbdIntraPredParam(&vpx_highbd_dc_left_predictor_4x4_sse2, | ||
| 527 | &vpx_highbd_dc_left_predictor_4x4_c, 4, 10), | ||
| 528 | HighbdIntraPredParam(&vpx_highbd_dc_left_predictor_8x8_sse2, | ||
| 529 | &vpx_highbd_dc_left_predictor_8x8_c, 8, 10), | ||
| 530 | HighbdIntraPredParam(&vpx_highbd_dc_left_predictor_16x16_sse2, | ||
| 531 | &vpx_highbd_dc_left_predictor_16x16_c, 16, 10), | ||
| 532 | HighbdIntraPredParam(&vpx_highbd_dc_left_predictor_32x32_sse2, | ||
| 533 | &vpx_highbd_dc_left_predictor_32x32_c, 32, 10), | ||
| 518 | HighbdIntraPredParam(&vpx_highbd_dc_predictor_4x4_sse2, | 534 | HighbdIntraPredParam(&vpx_highbd_dc_predictor_4x4_sse2, |
| 519 | &vpx_highbd_dc_predictor_4x4_c, 4, 10), | 535 | &vpx_highbd_dc_predictor_4x4_c, 4, 10), |
| 520 | HighbdIntraPredParam(&vpx_highbd_dc_predictor_8x8_sse2, | 536 | HighbdIntraPredParam(&vpx_highbd_dc_predictor_8x8_sse2, |
| @@ -559,6 +575,14 @@ INSTANTIATE_TEST_CASE_P( · 38 unmodified lines | |||
| 559 | INSTANTIATE_TEST_CASE_P( | 575 | INSTANTIATE_TEST_CASE_P( |
| 560 | SSE2_TO_C_12, VP9HighbdIntraPredTest, | 576 | SSE2_TO_C_12, VP9HighbdIntraPredTest, |
| 561 | ::testing::Values( | 577 | ::testing::Values( |
| 578 | HighbdIntraPredParam(&vpx_highbd_dc_left_predictor_4x4_sse2, | ||
| 579 | &vpx_highbd_dc_left_predictor_4x4_c, 4, 12), | ||
| 580 | HighbdIntraPredParam(&vpx_highbd_dc_left_predictor_8x8_sse2, | ||
| 581 | &vpx_highbd_dc_left_predictor_8x8_c, 8, 12), | ||
| 582 | HighbdIntraPredParam(&vpx_highbd_dc_left_predictor_16x16_sse2, | ||
| 583 | &vpx_highbd_dc_left_predictor_16x16_c, 16, 12), | ||
| 584 | HighbdIntraPredParam(&vpx_highbd_dc_left_predictor_32x32_sse2, | ||
| 585 | &vpx_highbd_dc_left_predictor_32x32_c, 32, 12), | ||
| 562 | HighbdIntraPredParam(&vpx_highbd_dc_predictor_4x4_sse2, | 586 | HighbdIntraPredParam(&vpx_highbd_dc_predictor_4x4_sse2, |
| 563 | &vpx_highbd_dc_predictor_4x4_c, 4, 12), | 587 | &vpx_highbd_dc_predictor_4x4_c, 4, 12), |
| 564 | HighbdIntraPredParam(&vpx_highbd_dc_predictor_8x8_sse2, | 588 | HighbdIntraPredParam(&vpx_highbd_dc_predictor_8x8_sse2, |
| @@ -217,7 +217,7 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { · 216 unmodified lines | |||
| 217 | specialize qw/vpx_highbd_dc_top_predictor_4x4 neon sse2/; | 217 | specialize qw/vpx_highbd_dc_top_predictor_4x4 neon sse2/; |
| 218 | 218 | ||
| 219 | add_proto qw/void vpx_highbd_dc_left_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; | 219 | add_proto qw/void vpx_highbd_dc_left_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; |
| 220 | specialize qw/vpx_highbd_dc_left_predictor_4x4 neon/; | 220 | specialize qw/vpx_highbd_dc_left_predictor_4x4 neon sse2/; |
| 221 | 221 | ||
| 222 | add_proto qw/void vpx_highbd_dc_128_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; | 222 | add_proto qw/void vpx_highbd_dc_128_predictor_4x4/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; |
| 223 | specialize qw/vpx_highbd_dc_128_predictor_4x4 neon/; | 223 | specialize qw/vpx_highbd_dc_128_predictor_4x4 neon/; |
| @@ -252,7 +252,7 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { · 28 unmodified lines | |||
| 252 | specialize qw/vpx_highbd_dc_top_predictor_8x8 neon sse2/; | 252 | specialize qw/vpx_highbd_dc_top_predictor_8x8 neon sse2/; |
| 253 | 253 | ||
| 254 | add_proto qw/void vpx_highbd_dc_left_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; | 254 | add_proto qw/void vpx_highbd_dc_left_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; |
| 255 | specialize qw/vpx_highbd_dc_left_predictor_8x8 neon/; | 255 | specialize qw/vpx_highbd_dc_left_predictor_8x8 neon sse2/; |
| 256 | 256 | ||
| 257 | add_proto qw/void vpx_highbd_dc_128_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; | 257 | add_proto qw/void vpx_highbd_dc_128_predictor_8x8/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; |
| 258 | specialize qw/vpx_highbd_dc_128_predictor_8x8 neon/; | 258 | specialize qw/vpx_highbd_dc_128_predictor_8x8 neon/; |
| @@ -287,7 +287,7 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { · 28 unmodified lines | |||
| 287 | specialize qw/vpx_highbd_dc_top_predictor_16x16 neon sse2/; | 287 | specialize qw/vpx_highbd_dc_top_predictor_16x16 neon sse2/; |
| 288 | 288 | ||
| 289 | add_proto qw/void vpx_highbd_dc_left_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; | 289 | add_proto qw/void vpx_highbd_dc_left_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; |
| 290 | specialize qw/vpx_highbd_dc_left_predictor_16x16 neon/; | 290 | specialize qw/vpx_highbd_dc_left_predictor_16x16 neon sse2/; |
| 291 | 291 | ||
| 292 | add_proto qw/void vpx_highbd_dc_128_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; | 292 | add_proto qw/void vpx_highbd_dc_128_predictor_16x16/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; |
| 293 | specialize qw/vpx_highbd_dc_128_predictor_16x16 neon/; | 293 | specialize qw/vpx_highbd_dc_128_predictor_16x16 neon/; |
| @@ -322,7 +322,7 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { · 28 unmodified lines | |||
| 322 | specialize qw/vpx_highbd_dc_top_predictor_32x32 neon sse2/; | 322 | specialize qw/vpx_highbd_dc_top_predictor_32x32 neon sse2/; |
| 323 | 323 | ||
| 324 | add_proto qw/void vpx_highbd_dc_left_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; | 324 | add_proto qw/void vpx_highbd_dc_left_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; |
| 325 | specialize qw/vpx_highbd_dc_left_predictor_32x32 neon/; | 325 | specialize qw/vpx_highbd_dc_left_predictor_32x32 neon sse2/; |
| 326 | 326 | ||
| 327 | add_proto qw/void vpx_highbd_dc_128_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; | 327 | add_proto qw/void vpx_highbd_dc_128_predictor_32x32/, "uint16_t *dst, ptrdiff_t y_stride, const uint16_t *above, const uint16_t *left, int bd"; |
| 328 | specialize qw/vpx_highbd_dc_128_predictor_32x32 neon/; | 328 | specialize qw/vpx_highbd_dc_128_predictor_32x32 neon/; |
| @@ -177,6 +177,17 @@ static INLINE void dc_store_4x4(uint16_t *dst, ptrdiff_t stride, · 176 unmodified lines | |||
| 177 | } | 177 | } |
| 178 | } | 178 | } |
| 179 | 179 | ||
| 180 | void vpx_highbd_dc_left_predictor_4x4_sse2(uint16_t *dst, ptrdiff_t stride, | ||
| 181 | const uint16_t *above, | ||
| 182 | const uint16_t *left, int bd) { | ||
| 183 | const __m128i two = _mm_cvtsi32_si128(2); | ||
| 184 | const __m128i sum = dc_sum_4(left); | ||
| 185 | const __m128i dc = _mm_srli_epi16(_mm_add_epi16(sum, two), 2); | ||
| 186 | (void)above; | ||
| 187 | (void)bd; | ||
| 188 | dc_store_4x4(dst, stride, &dc); | ||
| 189 | } | ||
| 190 | |||
| 180 | void vpx_highbd_dc_top_predictor_4x4_sse2(uint16_t *dst, ptrdiff_t stride, | 191 | void vpx_highbd_dc_top_predictor_4x4_sse2(uint16_t *dst, ptrdiff_t stride, |
| 181 | const uint16_t *above, | 192 | const uint16_t *above, |
| 182 | const uint16_t *left, int bd) { | 193 | const uint16_t *left, int bd) { |
| @@ -210,6 +221,17 @@ static INLINE void dc_store_8x8(uint16_t *dst, ptrdiff_t stride, · 27 unmodified lines | |||
| 210 | } | 221 | } |
| 211 | } | 222 | } |
| 212 | 223 | ||
| 224 | void vpx_highbd_dc_left_predictor_8x8_sse2(uint16_t *dst, ptrdiff_t stride, | ||
| 225 | const uint16_t *above, | ||
| 226 | const uint16_t *left, int bd) { | ||
| 227 | const __m128i four = _mm_cvtsi32_si128(4); | ||
| 228 | const __m128i sum = dc_sum_8(left); | ||
| 229 | const __m128i dc = _mm_srli_epi16(_mm_add_epi16(sum, four), 3); | ||
| 230 | (void)above; | ||
| 231 | (void)bd; | ||
| 232 | dc_store_8x8(dst, stride, &dc); | ||
| 233 | } | ||
| 234 | |||
| 213 | void vpx_highbd_dc_top_predictor_8x8_sse2(uint16_t *dst, ptrdiff_t stride, | 235 | void vpx_highbd_dc_top_predictor_8x8_sse2(uint16_t *dst, ptrdiff_t stride, |
| 214 | const uint16_t *above, | 236 | const uint16_t *above, |
| 215 | const uint16_t *left, int bd) { | 237 | const uint16_t *left, int bd) { |
| @@ -241,6 +263,17 @@ static INLINE void dc_store_16x16(uint16_t *dst, ptrdiff_t stride, · 25 unmodified lines | |||
| 241 | } | 263 | } |
| 242 | } | 264 | } |
| 243 | 265 | ||
| 266 | void vpx_highbd_dc_left_predictor_16x16_sse2(uint16_t *dst, ptrdiff_t stride, | ||
| 267 | const uint16_t *above, | ||
| 268 | const uint16_t *left, int bd) { | ||
| 269 | const __m128i eight = _mm_cvtsi32_si128(8); | ||
| 270 | const __m128i sum = dc_sum_16(left); | ||
| 271 | const __m128i dc = _mm_srli_epi16(_mm_add_epi16(sum, eight), 4); | ||
| 272 | (void)above; | ||
| 273 | (void)bd; | ||
| 274 | dc_store_16x16(dst, stride, &dc); | ||
| 275 | } | ||
| 276 | |||
| 244 | void vpx_highbd_dc_top_predictor_16x16_sse2(uint16_t *dst, ptrdiff_t stride, | 277 | void vpx_highbd_dc_top_predictor_16x16_sse2(uint16_t *dst, ptrdiff_t stride, |
| 245 | const uint16_t *above, | 278 | const uint16_t *above, |
| 246 | const uint16_t *left, int bd) { | 279 | const uint16_t *left, int bd) { |
| @@ -277,6 +310,17 @@ static INLINE void dc_store_32x32(uint16_t *dst, ptrdiff_t stride, · 30 unmodified lines | |||
| 277 | } | 310 | } |
| 278 | } | 311 | } |
| 279 | 312 | ||
| 313 | void vpx_highbd_dc_left_predictor_32x32_sse2(uint16_t *dst, ptrdiff_t stride, | ||
| 314 | const uint16_t *above, | ||
| 315 | const uint16_t *left, int bd) { | ||
| 316 | const __m128i sixteen = _mm_cvtsi32_si128(16); | ||
| 317 | const __m128i sum = dc_sum_32(left); | ||
| 318 | const __m128i dc = _mm_srli_epi32(_mm_add_epi32(sum, sixteen), 5); | ||
| 319 | (void)above; | ||
| 320 | (void)bd; | ||
| 321 | dc_store_32x32(dst, stride, &dc); | ||
| 322 | } | ||
| 323 | |||
| 280 | void vpx_highbd_dc_top_predictor_32x32_sse2(uint16_t *dst, ptrdiff_t stride, | 324 | void vpx_highbd_dc_top_predictor_32x32_sse2(uint16_t *dst, ptrdiff_t stride, |
| 281 | const uint16_t *above, | 325 | const uint16_t *above, |
| 282 | const uint16_t *left, int bd) { | 326 | const uint16_t *left, int bd) { |