Implementing sad16_neon using ABD, UDOT instead of ABAL, ABAL2 saves a cycle and removes resource contention for a single SIMD pipe on modern out-of-order Arm CPUs. The UDOT accumulation into 32-bit elements also allows for a faster reduction at the end of each SAD function. The existing implementation is retained for CPUs that do not implement the Armv8.4-A UDOT instruction, and CPUs executing in AArch32 mode.
Unified · Split
| @@ -98,6 +98,8 @@ static INLINE void sad_512_pel_final_neon(const uint16x8_t *sum /*[4]*/, · 97 unmodified lines | ||
| 98 | 98 | vst1q_u32(res, r); |
| 99 | 99 | } |
| 100 | 100 | |
| 101 | +#if defined(__arm__) || !defined(__ARM_FEATURE_DOTPROD) | |
| 102 | + | |
| 101 | 103 | // Can handle 1024 pixels' sad sum (such as 32x32) |
| 102 | 104 | static INLINE void sad_1024_pel_final_neon(const uint16x8_t *sum /*[4]*/, |
| 103 | 105 | uint32_t *const res) { |
| @@ -191,6 +193,8 @@ static INLINE void sad_4096_pel_final_neon(const uint16x8_t *sum /*[8]*/, · 87 unmodified lines | ||
| 191 | 193 | #endif |
| 192 | 194 | } |
| 193 | 195 | |
| 196 | +#endif | |
| 197 | + | |
| 194 | 198 | static INLINE void sad8x_4d(const uint8_t *src_ptr, int src_stride, |
| 195 | 199 | const uint8_t *const ref_array[4], int ref_stride, |
| 196 | 200 | uint32_t *res, const int height) { |
| @@ -233,6 +237,41 @@ void vpx_sad8x16x4d_neon(const uint8_t *src_ptr, int src_stride, · 36 unmodified lines | ||
| 233 | 237 | |
| 234 | 238 | //////////////////////////////////////////////////////////////////////////////// |
| 235 | 239 | |
| 240 | +#if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \ | |
| 241 | + (__ARM_FEATURE_DOTPROD == 1) | |
| 242 | + | |
| 243 | +static INLINE void sad16_neon(const uint8_t *ref_ptr, const uint8x16_t src_ptr, | |
| 244 | + uint32x4_t *const sum) { | |
| 245 | + const uint8x16_t r = vld1q_u8(ref_ptr); | |
| 246 | + const uint8x16_t diff = vabdq_u8(src_ptr, r); | |
| 247 | + *sum = vdotq_u32(*sum, diff, vdupq_n_u8(1)); | |
| 248 | +} | |
| 249 | + | |
| 250 | +static INLINE void sad16x_4d(const uint8_t *src_ptr, int src_stride, | |
| 251 | + const uint8_t *const ref_array[4], int ref_stride, | |
| 252 | + uint32_t *res, const int height) { | |
| 253 | + int i; | |
| 254 | + uint32x4_t r0, r1; | |
| 255 | + const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2], | |
| 256 | + ref_array[3] }; | |
| 257 | + uint32x4_t sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0), | |
| 258 | + vdupq_n_u32(0) }; | |
| 259 | + | |
| 260 | + for (i = 0; i < height; ++i) { | |
| 261 | + const uint8x16_t s = vld1q_u8(src_ptr + i * src_stride); | |
| 262 | + sad16_neon(ref_loop[0] + i * ref_stride, s, &sum[0]); | |
| 263 | + sad16_neon(ref_loop[1] + i * ref_stride, s, &sum[1]); | |
| 264 | + sad16_neon(ref_loop[2] + i * ref_stride, s, &sum[2]); | |
| 265 | + sad16_neon(ref_loop[3] + i * ref_stride, s, &sum[3]); | |
| 266 | + } | |
| 267 | + | |
| 268 | + r0 = vpaddq_u32(sum[0], sum[1]); | |
| 269 | + r1 = vpaddq_u32(sum[2], sum[3]); | |
| 270 | + vst1q_u32(res, vpaddq_u32(r0, r1)); | |
| 271 | +} | |
| 272 | + | |
| 273 | +#else | |
| 274 | + | |
| 236 | 275 | static INLINE void sad16_neon(const uint8_t *ref_ptr, const uint8x16_t src_ptr, |
| 237 | 276 | uint16x8_t *const sum) { |
| 238 | 277 | const uint8x16_t r = vld1q_u8(ref_ptr); |
| @@ -266,6 +305,8 @@ static INLINE void sad16x_4d(const uint8_t *src_ptr, int src_stride, · 27 unmodified lines | ||
| 266 | 305 | sad_512_pel_final_neon(sum, res); |
| 267 | 306 | } |
| 268 | 307 | |
| 308 | +#endif | |
| 309 | + | |
| 269 | 310 | void vpx_sad16x8x4d_neon(const uint8_t *src_ptr, int src_stride, |
| 270 | 311 | const uint8_t *const ref_array[4], int ref_stride, |
| 271 | 312 | uint32_t *res) { |
| @@ -286,6 +327,67 @@ void vpx_sad16x32x4d_neon(const uint8_t *src_ptr, int src_stride, · 14 unmodified lines | ||
| 286 | 327 | |
| 287 | 328 | //////////////////////////////////////////////////////////////////////////////// |
| 288 | 329 | |
| 330 | +#if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \ | |
| 331 | + (__ARM_FEATURE_DOTPROD == 1) | |
| 332 | + | |
| 333 | +static INLINE void sad32x_4d(const uint8_t *src_ptr, int src_stride, | |
| 334 | + const uint8_t *const ref_array[4], int ref_stride, | |
| 335 | + uint32_t *res, const int height) { | |
| 336 | + int i; | |
| 337 | + uint32x4_t r0, r1; | |
| 338 | + const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2], | |
| 339 | + ref_array[3] }; | |
| 340 | + | |
| 341 | + uint32x4_t sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0), | |
| 342 | + vdupq_n_u32(0) }; | |
| 343 | + | |
| 344 | + for (i = 0; i < height; ++i) { | |
| 345 | + uint8x16_t s; | |
| 346 | + | |
| 347 | + s = vld1q_u8(src_ptr + 0 * 16); | |
| 348 | + sad16_neon(ref_loop[0] + 0 * 16, s, &sum[0]); | |
| 349 | + sad16_neon(ref_loop[1] + 0 * 16, s, &sum[1]); | |
| 350 | + sad16_neon(ref_loop[2] + 0 * 16, s, &sum[2]); | |
| 351 | + sad16_neon(ref_loop[3] + 0 * 16, s, &sum[3]); | |
| 352 | + | |
| 353 | + s = vld1q_u8(src_ptr + 1 * 16); | |
| 354 | + sad16_neon(ref_loop[0] + 1 * 16, s, &sum[0]); | |
| 355 | + sad16_neon(ref_loop[1] + 1 * 16, s, &sum[1]); | |
| 356 | + sad16_neon(ref_loop[2] + 1 * 16, s, &sum[2]); | |
| 357 | + sad16_neon(ref_loop[3] + 1 * 16, s, &sum[3]); | |
| 358 | + | |
| 359 | + src_ptr += src_stride; | |
| 360 | + ref_loop[0] += ref_stride; | |
| 361 | + ref_loop[1] += ref_stride; | |
| 362 | + ref_loop[2] += ref_stride; | |
| 363 | + ref_loop[3] += ref_stride; | |
| 364 | + } | |
| 365 | + | |
| 366 | + r0 = vpaddq_u32(sum[0], sum[1]); | |
| 367 | + r1 = vpaddq_u32(sum[2], sum[3]); | |
| 368 | + vst1q_u32(res, vpaddq_u32(r0, r1)); | |
| 369 | +} | |
| 370 | + | |
| 371 | +void vpx_sad32x16x4d_neon(const uint8_t *src_ptr, int src_stride, | |
| 372 | + const uint8_t *const ref_array[4], int ref_stride, | |
| 373 | + uint32_t *res) { | |
| 374 | + sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 16); | |
| 375 | +} | |
| 376 | + | |
| 377 | +void vpx_sad32x32x4d_neon(const uint8_t *src_ptr, int src_stride, | |
| 378 | + const uint8_t *const ref_array[4], int ref_stride, | |
| 379 | + uint32_t *res) { | |
| 380 | + sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 32); | |
| 381 | +} | |
| 382 | + | |
| 383 | +void vpx_sad32x64x4d_neon(const uint8_t *src_ptr, int src_stride, | |
| 384 | + const uint8_t *const ref_array[4], int ref_stride, | |
| 385 | + uint32_t *res) { | |
| 386 | + sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 64); | |
| 387 | +} | |
| 388 | + | |
| 389 | +#else | |
| 390 | + | |
| 289 | 391 | static INLINE void sad32x_4d(const uint8_t *src_ptr, int src_stride, |
| 290 | 392 | const uint8_t *const ref_array[4], int ref_stride, |
| 291 | 393 | const int height, uint16x8_t *const sum) { |
| @@ -342,8 +444,118 @@ void vpx_sad32x64x4d_neon(const uint8_t *src_ptr, int src_stride, · 50 unmodified lines | ||
| 342 | 444 | sad_2048_pel_final_neon(sum, res); |
| 343 | 445 | } |
| 344 | 446 | |
| 447 | +#endif | |
| 448 | + | |
| 345 | 449 | //////////////////////////////////////////////////////////////////////////////// |
| 346 | 450 | |
| 451 | +#if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \ | |
| 452 | + (__ARM_FEATURE_DOTPROD == 1) | |
| 453 | + | |
| 454 | +void vpx_sad64x32x4d_neon(const uint8_t *src_ptr, int src_stride, | |
| 455 | + const uint8_t *const ref_array[4], int ref_stride, | |
| 456 | + uint32_t *res) { | |
| 457 | + int i; | |
| 458 | + uint32x4_t r0, r1; | |
| 459 | + const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2], | |
| 460 | + ref_array[3] }; | |
| 461 | + uint32x4_t sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0), | |
| 462 | + vdupq_n_u32(0) }; | |
| 463 | + | |
| 464 | + for (i = 0; i < 32; ++i) { | |
| 465 | + uint8x16_t s; | |
| 466 | + | |
| 467 | + s = vld1q_u8(src_ptr + 0 * 16); | |
| 468 | + sad16_neon(ref_loop[0] + 0 * 16, s, &sum[0]); | |
| 469 | + sad16_neon(ref_loop[1] + 0 * 16, s, &sum[1]); | |
| 470 | + sad16_neon(ref_loop[2] + 0 * 16, s, &sum[2]); | |
| 471 | + sad16_neon(ref_loop[3] + 0 * 16, s, &sum[3]); | |
| 472 | + | |
| 473 | + s = vld1q_u8(src_ptr + 1 * 16); | |
| 474 | + sad16_neon(ref_loop[0] + 1 * 16, s, &sum[0]); | |
| 475 | + sad16_neon(ref_loop[1] + 1 * 16, s, &sum[1]); | |
| 476 | + sad16_neon(ref_loop[2] + 1 * 16, s, &sum[2]); | |
| 477 | + sad16_neon(ref_loop[3] + 1 * 16, s, &sum[3]); | |
| 478 | + | |
| 479 | + s = vld1q_u8(src_ptr + 2 * 16); | |
| 480 | + sad16_neon(ref_loop[0] + 2 * 16, s, &sum[0]); | |
| 481 | + sad16_neon(ref_loop[1] + 2 * 16, s, &sum[1]); | |
| 482 | + sad16_neon(ref_loop[2] + 2 * 16, s, &sum[2]); | |
| 483 | + sad16_neon(ref_loop[3] + 2 * 16, s, &sum[3]); | |
| 484 | + | |
| 485 | + s = vld1q_u8(src_ptr + 3 * 16); | |
| 486 | + sad16_neon(ref_loop[0] + 3 * 16, s, &sum[0]); | |
| 487 | + sad16_neon(ref_loop[1] + 3 * 16, s, &sum[1]); | |
| 488 | + sad16_neon(ref_loop[2] + 3 * 16, s, &sum[2]); | |
| 489 | + sad16_neon(ref_loop[3] + 3 * 16, s, &sum[3]); | |
| 490 | + | |
| 491 | + src_ptr += src_stride; | |
| 492 | + ref_loop[0] += ref_stride; | |
| 493 | + ref_loop[1] += ref_stride; | |
| 494 | + ref_loop[2] += ref_stride; | |
| 495 | + ref_loop[3] += ref_stride; | |
| 496 | + } | |
| 497 | + | |
| 498 | + r0 = vpaddq_u32(sum[0], sum[1]); | |
| 499 | + r1 = vpaddq_u32(sum[2], sum[3]); | |
| 500 | + vst1q_u32(res, vpaddq_u32(r0, r1)); | |
| 501 | +} | |
| 502 | + | |
| 503 | +void vpx_sad64x64x4d_neon(const uint8_t *src_ptr, int src_stride, | |
| 504 | + const uint8_t *const ref_array[4], int ref_stride, | |
| 505 | + uint32_t *res) { | |
| 506 | + int i; | |
| 507 | + uint32x4_t r0, r1, r2, r3; | |
| 508 | + const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2], | |
| 509 | + ref_array[3] }; | |
| 510 | + uint32x4_t sum[8] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0), | |
| 511 | + vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0), | |
| 512 | + vdupq_n_u32(0), vdupq_n_u32(0) }; | |
| 513 | + | |
| 514 | + for (i = 0; i < 64; ++i) { | |
| 515 | + uint8x16_t s; | |
| 516 | + | |
| 517 | + s = vld1q_u8(src_ptr + 0 * 16); | |
| 518 | + sad16_neon(ref_loop[0] + 0 * 16, s, &sum[0]); | |
| 519 | + sad16_neon(ref_loop[1] + 0 * 16, s, &sum[2]); | |
| 520 | + sad16_neon(ref_loop[2] + 0 * 16, s, &sum[4]); | |
| 521 | + sad16_neon(ref_loop[3] + 0 * 16, s, &sum[6]); | |
| 522 | + | |
| 523 | + s = vld1q_u8(src_ptr + 1 * 16); | |
| 524 | + sad16_neon(ref_loop[0] + 1 * 16, s, &sum[0]); | |
| 525 | + sad16_neon(ref_loop[1] + 1 * 16, s, &sum[2]); | |
| 526 | + sad16_neon(ref_loop[2] + 1 * 16, s, &sum[4]); | |
| 527 | + sad16_neon(ref_loop[3] + 1 * 16, s, &sum[6]); | |
| 528 | + | |
| 529 | + s = vld1q_u8(src_ptr + 2 * 16); | |
| 530 | + sad16_neon(ref_loop[0] + 2 * 16, s, &sum[1]); | |
| 531 | + sad16_neon(ref_loop[1] + 2 * 16, s, &sum[3]); | |
| 532 | + sad16_neon(ref_loop[2] + 2 * 16, s, &sum[5]); | |
| 533 | + sad16_neon(ref_loop[3] + 2 * 16, s, &sum[7]); | |
| 534 | + | |
| 535 | + s = vld1q_u8(src_ptr + 3 * 16); | |
| 536 | + sad16_neon(ref_loop[0] + 3 * 16, s, &sum[1]); | |
| 537 | + sad16_neon(ref_loop[1] + 3 * 16, s, &sum[3]); | |
| 538 | + sad16_neon(ref_loop[2] + 3 * 16, s, &sum[5]); | |
| 539 | + sad16_neon(ref_loop[3] + 3 * 16, s, &sum[7]); | |
| 540 | + | |
| 541 | + src_ptr += src_stride; | |
| 542 | + ref_loop[0] += ref_stride; | |
| 543 | + ref_loop[1] += ref_stride; | |
| 544 | + ref_loop[2] += ref_stride; | |
| 545 | + ref_loop[3] += ref_stride; | |
| 546 | + } | |
| 547 | + | |
| 548 | + r0 = vpaddq_u32(sum[0], sum[1]); | |
| 549 | + r1 = vpaddq_u32(sum[2], sum[3]); | |
| 550 | + r2 = vpaddq_u32(sum[4], sum[5]); | |
| 551 | + r3 = vpaddq_u32(sum[6], sum[7]); | |
| 552 | + r0 = vpaddq_u32(r0, r1); | |
| 553 | + r1 = vpaddq_u32(r2, r3); | |
| 554 | + vst1q_u32(res, vpaddq_u32(r0, r1)); | |
| 555 | +} | |
| 556 | + | |
| 557 | +#else | |
| 558 | + | |
| 347 | 559 | void vpx_sad64x32x4d_neon(const uint8_t *src_ptr, int src_stride, |
| 348 | 560 | const uint8_t *const ref_array[4], int ref_stride, |
| 349 | 561 | uint32_t *res) { |
| @@ -436,3 +648,5 @@ void vpx_sad64x64x4d_neon(const uint8_t *src_ptr, int src_stride, · 86 unmodified lines | ||
| 436 | 648 | |
| 437 | 649 | sad_4096_pel_final_neon(sum, res); |
| 438 | 650 | } |
| 651 | + | |
| 652 | +#endif | |