Implementing sad16_neon using ABD, UDOT instead of ABAL, ABAL2 saves a cycle and removes resource contention for a single SIMD pipe on modern out-of-order Arm CPUs. The UDOT accumulation into 32-bit elements also allows for a faster reduction at the end of each SAD function. The existing implementation is retained for CPUs that do not implement the Armv8.4-A UDOT instruction, and CPUs executing in AArch32 mode.
Unified · Split
| @@ -98,6 +98,8 @@ static INLINE void sad_512_pel_final_neon(const uint16x8_t *sum /*[4]*/, · 97 unmodified lines | |||
| 98 | vst1q_u32(res, r); | 98 | vst1q_u32(res, r); |
| 99 | } | 99 | } |
| 100 | 100 | ||
| 101 | #if defined(__arm__) || !defined(__ARM_FEATURE_DOTPROD) | ||
| 102 | |||
| 101 | // Can handle 1024 pixels' sad sum (such as 32x32) | 103 | // Can handle 1024 pixels' sad sum (such as 32x32) |
| 102 | static INLINE void sad_1024_pel_final_neon(const uint16x8_t *sum /*[4]*/, | 104 | static INLINE void sad_1024_pel_final_neon(const uint16x8_t *sum /*[4]*/, |
| 103 | uint32_t *const res) { | 105 | uint32_t *const res) { |
| @@ -191,6 +193,8 @@ static INLINE void sad_4096_pel_final_neon(const uint16x8_t *sum /*[8]*/, · 87 unmodified lines | |||
| 191 | #endif | 193 | #endif |
| 192 | } | 194 | } |
| 193 | 195 | ||
| 196 | #endif | ||
| 197 | |||
| 194 | static INLINE void sad8x_4d(const uint8_t *src_ptr, int src_stride, | 198 | static INLINE void sad8x_4d(const uint8_t *src_ptr, int src_stride, |
| 195 | const uint8_t *const ref_array[4], int ref_stride, | 199 | const uint8_t *const ref_array[4], int ref_stride, |
| 196 | uint32_t *res, const int height) { | 200 | uint32_t *res, const int height) { |
| @@ -233,6 +237,41 @@ void vpx_sad8x16x4d_neon(const uint8_t *src_ptr, int src_stride, · 36 unmodified lines | |||
| 233 | 237 | ||
| 234 | //////////////////////////////////////////////////////////////////////////////// | 238 | //////////////////////////////////////////////////////////////////////////////// |
| 235 | 239 | ||
| 240 | #if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \ | ||
| 241 | (__ARM_FEATURE_DOTPROD == 1) | ||
| 242 | |||
| 243 | static INLINE void sad16_neon(const uint8_t *ref_ptr, const uint8x16_t src_ptr, | ||
| 244 | uint32x4_t *const sum) { | ||
| 245 | const uint8x16_t r = vld1q_u8(ref_ptr); | ||
| 246 | const uint8x16_t diff = vabdq_u8(src_ptr, r); | ||
| 247 | *sum = vdotq_u32(*sum, diff, vdupq_n_u8(1)); | ||
| 248 | } | ||
| 249 | |||
| 250 | static INLINE void sad16x_4d(const uint8_t *src_ptr, int src_stride, | ||
| 251 | const uint8_t *const ref_array[4], int ref_stride, | ||
| 252 | uint32_t *res, const int height) { | ||
| 253 | int i; | ||
| 254 | uint32x4_t r0, r1; | ||
| 255 | const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2], | ||
| 256 | ref_array[3] }; | ||
| 257 | uint32x4_t sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0), | ||
| 258 | vdupq_n_u32(0) }; | ||
| 259 | |||
| 260 | for (i = 0; i < height; ++i) { | ||
| 261 | const uint8x16_t s = vld1q_u8(src_ptr + i * src_stride); | ||
| 262 | sad16_neon(ref_loop[0] + i * ref_stride, s, &sum[0]); | ||
| 263 | sad16_neon(ref_loop[1] + i * ref_stride, s, &sum[1]); | ||
| 264 | sad16_neon(ref_loop[2] + i * ref_stride, s, &sum[2]); | ||
| 265 | sad16_neon(ref_loop[3] + i * ref_stride, s, &sum[3]); | ||
| 266 | } | ||
| 267 | |||
| 268 | r0 = vpaddq_u32(sum[0], sum[1]); | ||
| 269 | r1 = vpaddq_u32(sum[2], sum[3]); | ||
| 270 | vst1q_u32(res, vpaddq_u32(r0, r1)); | ||
| 271 | } | ||
| 272 | |||
| 273 | #else | ||
| 274 | |||
| 236 | static INLINE void sad16_neon(const uint8_t *ref_ptr, const uint8x16_t src_ptr, | 275 | static INLINE void sad16_neon(const uint8_t *ref_ptr, const uint8x16_t src_ptr, |
| 237 | uint16x8_t *const sum) { | 276 | uint16x8_t *const sum) { |
| 238 | const uint8x16_t r = vld1q_u8(ref_ptr); | 277 | const uint8x16_t r = vld1q_u8(ref_ptr); |
| @@ -266,6 +305,8 @@ static INLINE void sad16x_4d(const uint8_t *src_ptr, int src_stride, · 27 unmodified lines | |||
| 266 | sad_512_pel_final_neon(sum, res); | 305 | sad_512_pel_final_neon(sum, res); |
| 267 | } | 306 | } |
| 268 | 307 | ||
| 308 | #endif | ||
| 309 | |||
| 269 | void vpx_sad16x8x4d_neon(const uint8_t *src_ptr, int src_stride, | 310 | void vpx_sad16x8x4d_neon(const uint8_t *src_ptr, int src_stride, |
| 270 | const uint8_t *const ref_array[4], int ref_stride, | 311 | const uint8_t *const ref_array[4], int ref_stride, |
| 271 | uint32_t *res) { | 312 | uint32_t *res) { |
| @@ -286,6 +327,67 @@ void vpx_sad16x32x4d_neon(const uint8_t *src_ptr, int src_stride, · 14 unmodified lines | |||
| 286 | 327 | ||
| 287 | //////////////////////////////////////////////////////////////////////////////// | 328 | //////////////////////////////////////////////////////////////////////////////// |
| 288 | 329 | ||
| 330 | #if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \ | ||
| 331 | (__ARM_FEATURE_DOTPROD == 1) | ||
| 332 | |||
| 333 | static INLINE void sad32x_4d(const uint8_t *src_ptr, int src_stride, | ||
| 334 | const uint8_t *const ref_array[4], int ref_stride, | ||
| 335 | uint32_t *res, const int height) { | ||
| 336 | int i; | ||
| 337 | uint32x4_t r0, r1; | ||
| 338 | const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2], | ||
| 339 | ref_array[3] }; | ||
| 340 | |||
| 341 | uint32x4_t sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0), | ||
| 342 | vdupq_n_u32(0) }; | ||
| 343 | |||
| 344 | for (i = 0; i < height; ++i) { | ||
| 345 | uint8x16_t s; | ||
| 346 | |||
| 347 | s = vld1q_u8(src_ptr + 0 * 16); | ||
| 348 | sad16_neon(ref_loop[0] + 0 * 16, s, &sum[0]); | ||
| 349 | sad16_neon(ref_loop[1] + 0 * 16, s, &sum[1]); | ||
| 350 | sad16_neon(ref_loop[2] + 0 * 16, s, &sum[2]); | ||
| 351 | sad16_neon(ref_loop[3] + 0 * 16, s, &sum[3]); | ||
| 352 | |||
| 353 | s = vld1q_u8(src_ptr + 1 * 16); | ||
| 354 | sad16_neon(ref_loop[0] + 1 * 16, s, &sum[0]); | ||
| 355 | sad16_neon(ref_loop[1] + 1 * 16, s, &sum[1]); | ||
| 356 | sad16_neon(ref_loop[2] + 1 * 16, s, &sum[2]); | ||
| 357 | sad16_neon(ref_loop[3] + 1 * 16, s, &sum[3]); | ||
| 358 | |||
| 359 | src_ptr += src_stride; | ||
| 360 | ref_loop[0] += ref_stride; | ||
| 361 | ref_loop[1] += ref_stride; | ||
| 362 | ref_loop[2] += ref_stride; | ||
| 363 | ref_loop[3] += ref_stride; | ||
| 364 | } | ||
| 365 | |||
| 366 | r0 = vpaddq_u32(sum[0], sum[1]); | ||
| 367 | r1 = vpaddq_u32(sum[2], sum[3]); | ||
| 368 | vst1q_u32(res, vpaddq_u32(r0, r1)); | ||
| 369 | } | ||
| 370 | |||
| 371 | void vpx_sad32x16x4d_neon(const uint8_t *src_ptr, int src_stride, | ||
| 372 | const uint8_t *const ref_array[4], int ref_stride, | ||
| 373 | uint32_t *res) { | ||
| 374 | sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 16); | ||
| 375 | } | ||
| 376 | |||
| 377 | void vpx_sad32x32x4d_neon(const uint8_t *src_ptr, int src_stride, | ||
| 378 | const uint8_t *const ref_array[4], int ref_stride, | ||
| 379 | uint32_t *res) { | ||
| 380 | sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 32); | ||
| 381 | } | ||
| 382 | |||
| 383 | void vpx_sad32x64x4d_neon(const uint8_t *src_ptr, int src_stride, | ||
| 384 | const uint8_t *const ref_array[4], int ref_stride, | ||
| 385 | uint32_t *res) { | ||
| 386 | sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 64); | ||
| 387 | } | ||
| 388 | |||
| 389 | #else | ||
| 390 | |||
| 289 | static INLINE void sad32x_4d(const uint8_t *src_ptr, int src_stride, | 391 | static INLINE void sad32x_4d(const uint8_t *src_ptr, int src_stride, |
| 290 | const uint8_t *const ref_array[4], int ref_stride, | 392 | const uint8_t *const ref_array[4], int ref_stride, |
| 291 | const int height, uint16x8_t *const sum) { | 393 | const int height, uint16x8_t *const sum) { |
| @@ -342,8 +444,118 @@ void vpx_sad32x64x4d_neon(const uint8_t *src_ptr, int src_stride, · 50 unmodified lines | |||
| 342 | sad_2048_pel_final_neon(sum, res); | 444 | sad_2048_pel_final_neon(sum, res); |
| 343 | } | 445 | } |
| 344 | 446 | ||
| 447 | #endif | ||
| 448 | |||
| 345 | //////////////////////////////////////////////////////////////////////////////// | 449 | //////////////////////////////////////////////////////////////////////////////// |
| 346 | 450 | ||
| 451 | #if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \ | ||
| 452 | (__ARM_FEATURE_DOTPROD == 1) | ||
| 453 | |||
| 454 | void vpx_sad64x32x4d_neon(const uint8_t *src_ptr, int src_stride, | ||
| 455 | const uint8_t *const ref_array[4], int ref_stride, | ||
| 456 | uint32_t *res) { | ||
| 457 | int i; | ||
| 458 | uint32x4_t r0, r1; | ||
| 459 | const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2], | ||
| 460 | ref_array[3] }; | ||
| 461 | uint32x4_t sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0), | ||
| 462 | vdupq_n_u32(0) }; | ||
| 463 | |||
| 464 | for (i = 0; i < 32; ++i) { | ||
| 465 | uint8x16_t s; | ||
| 466 | |||
| 467 | s = vld1q_u8(src_ptr + 0 * 16); | ||
| 468 | sad16_neon(ref_loop[0] + 0 * 16, s, &sum[0]); | ||
| 469 | sad16_neon(ref_loop[1] + 0 * 16, s, &sum[1]); | ||
| 470 | sad16_neon(ref_loop[2] + 0 * 16, s, &sum[2]); | ||
| 471 | sad16_neon(ref_loop[3] + 0 * 16, s, &sum[3]); | ||
| 472 | |||
| 473 | s = vld1q_u8(src_ptr + 1 * 16); | ||
| 474 | sad16_neon(ref_loop[0] + 1 * 16, s, &sum[0]); | ||
| 475 | sad16_neon(ref_loop[1] + 1 * 16, s, &sum[1]); | ||
| 476 | sad16_neon(ref_loop[2] + 1 * 16, s, &sum[2]); | ||
| 477 | sad16_neon(ref_loop[3] + 1 * 16, s, &sum[3]); | ||
| 478 | |||
| 479 | s = vld1q_u8(src_ptr + 2 * 16); | ||
| 480 | sad16_neon(ref_loop[0] + 2 * 16, s, &sum[0]); | ||
| 481 | sad16_neon(ref_loop[1] + 2 * 16, s, &sum[1]); | ||
| 482 | sad16_neon(ref_loop[2] + 2 * 16, s, &sum[2]); | ||
| 483 | sad16_neon(ref_loop[3] + 2 * 16, s, &sum[3]); | ||
| 484 | |||
| 485 | s = vld1q_u8(src_ptr + 3 * 16); | ||
| 486 | sad16_neon(ref_loop[0] + 3 * 16, s, &sum[0]); | ||
| 487 | sad16_neon(ref_loop[1] + 3 * 16, s, &sum[1]); | ||
| 488 | sad16_neon(ref_loop[2] + 3 * 16, s, &sum[2]); | ||
| 489 | sad16_neon(ref_loop[3] + 3 * 16, s, &sum[3]); | ||
| 490 | |||
| 491 | src_ptr += src_stride; | ||
| 492 | ref_loop[0] += ref_stride; | ||
| 493 | ref_loop[1] += ref_stride; | ||
| 494 | ref_loop[2] += ref_stride; | ||
| 495 | ref_loop[3] += ref_stride; | ||
| 496 | } | ||
| 497 | |||
| 498 | r0 = vpaddq_u32(sum[0], sum[1]); | ||
| 499 | r1 = vpaddq_u32(sum[2], sum[3]); | ||
| 500 | vst1q_u32(res, vpaddq_u32(r0, r1)); | ||
| 501 | } | ||
| 502 | |||
| 503 | void vpx_sad64x64x4d_neon(const uint8_t *src_ptr, int src_stride, | ||
| 504 | const uint8_t *const ref_array[4], int ref_stride, | ||
| 505 | uint32_t *res) { | ||
| 506 | int i; | ||
| 507 | uint32x4_t r0, r1, r2, r3; | ||
| 508 | const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2], | ||
| 509 | ref_array[3] }; | ||
| 510 | uint32x4_t sum[8] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0), | ||
| 511 | vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0), | ||
| 512 | vdupq_n_u32(0), vdupq_n_u32(0) }; | ||
| 513 | |||
| 514 | for (i = 0; i < 64; ++i) { | ||
| 515 | uint8x16_t s; | ||
| 516 | |||
| 517 | s = vld1q_u8(src_ptr + 0 * 16); | ||
| 518 | sad16_neon(ref_loop[0] + 0 * 16, s, &sum[0]); | ||
| 519 | sad16_neon(ref_loop[1] + 0 * 16, s, &sum[2]); | ||
| 520 | sad16_neon(ref_loop[2] + 0 * 16, s, &sum[4]); | ||
| 521 | sad16_neon(ref_loop[3] + 0 * 16, s, &sum[6]); | ||
| 522 | |||
| 523 | s = vld1q_u8(src_ptr + 1 * 16); | ||
| 524 | sad16_neon(ref_loop[0] + 1 * 16, s, &sum[0]); | ||
| 525 | sad16_neon(ref_loop[1] + 1 * 16, s, &sum[2]); | ||
| 526 | sad16_neon(ref_loop[2] + 1 * 16, s, &sum[4]); | ||
| 527 | sad16_neon(ref_loop[3] + 1 * 16, s, &sum[6]); | ||
| 528 | |||
| 529 | s = vld1q_u8(src_ptr + 2 * 16); | ||
| 530 | sad16_neon(ref_loop[0] + 2 * 16, s, &sum[1]); | ||
| 531 | sad16_neon(ref_loop[1] + 2 * 16, s, &sum[3]); | ||
| 532 | sad16_neon(ref_loop[2] + 2 * 16, s, &sum[5]); | ||
| 533 | sad16_neon(ref_loop[3] + 2 * 16, s, &sum[7]); | ||
| 534 | |||
| 535 | s = vld1q_u8(src_ptr + 3 * 16); | ||
| 536 | sad16_neon(ref_loop[0] + 3 * 16, s, &sum[1]); | ||
| 537 | sad16_neon(ref_loop[1] + 3 * 16, s, &sum[3]); | ||
| 538 | sad16_neon(ref_loop[2] + 3 * 16, s, &sum[5]); | ||
| 539 | sad16_neon(ref_loop[3] + 3 * 16, s, &sum[7]); | ||
| 540 | |||
| 541 | src_ptr += src_stride; | ||
| 542 | ref_loop[0] += ref_stride; | ||
| 543 | ref_loop[1] += ref_stride; | ||
| 544 | ref_loop[2] += ref_stride; | ||
| 545 | ref_loop[3] += ref_stride; | ||
| 546 | } | ||
| 547 | |||
| 548 | r0 = vpaddq_u32(sum[0], sum[1]); | ||
| 549 | r1 = vpaddq_u32(sum[2], sum[3]); | ||
| 550 | r2 = vpaddq_u32(sum[4], sum[5]); | ||
| 551 | r3 = vpaddq_u32(sum[6], sum[7]); | ||
| 552 | r0 = vpaddq_u32(r0, r1); | ||
| 553 | r1 = vpaddq_u32(r2, r3); | ||
| 554 | vst1q_u32(res, vpaddq_u32(r0, r1)); | ||
| 555 | } | ||
| 556 | |||
| 557 | #else | ||
| 558 | |||
| 347 | void vpx_sad64x32x4d_neon(const uint8_t *src_ptr, int src_stride, | 559 | void vpx_sad64x32x4d_neon(const uint8_t *src_ptr, int src_stride, |
| 348 | const uint8_t *const ref_array[4], int ref_stride, | 560 | const uint8_t *const ref_array[4], int ref_stride, |
| 349 | uint32_t *res) { | 561 | uint32_t *res) { |
| @@ -436,3 +648,5 @@ void vpx_sad64x64x4d_neon(const uint8_t *src_ptr, int src_stride, · 86 unmodified lines | |||
| 436 | 648 | ||
| 437 | sad_4096_pel_final_neon(sum, res); | 649 | sad_4096_pel_final_neon(sum, res); |
| 438 | } | 650 | } |
| 651 | |||
| 652 | #endif | ||