Implement sum of squared difference calculations in vpx_mse16x16_neon and vpx_get4x4sse_cs_neon using the ABD and UDOT instructions - instead of widening subtracts followed by a sequence of MLAs. The existing implementation is retained for use on CPUs that do not implement the Armv8.4-A UDOT instruction. This commit also updates the variable names used in the existing implementations to be more descriptive.
Unified · Split
| @@ -357,117 +357,165 @@ unsigned int vpx_variance64x64_neon(const uint8_t *src_ptr, int src_stride, · 356 unmodified lines | |||
| 357 | return *sse - (unsigned int)(((int64_t)sum1 * sum1) >> 12); | 357 | return *sse - (unsigned int)(((int64_t)sum1 * sum1) >> 12); |
| 358 | } | 358 | } |
| 359 | 359 | ||
| 360 | #if defined(__ARM_FEATURE_DOTPROD) && (__ARM_FEATURE_DOTPROD == 1) | ||
| 361 | |||
| 360 | unsigned int vpx_mse16x16_neon(const unsigned char *src_ptr, int src_stride, | 362 | unsigned int vpx_mse16x16_neon(const unsigned char *src_ptr, int src_stride, |
| 361 | const unsigned char *ref_ptr, int ref_stride, | 363 | const unsigned char *ref_ptr, int ref_stride, |
| 362 | unsigned int *sse) { | 364 | unsigned int *sse) { |
| 363 | int i; | 365 | int i; |
| 364 | int16x4_t d22s16, d23s16, d24s16, d25s16, d26s16, d27s16, d28s16, d29s16; | 366 | uint8x16_t a[2], b[2], abs_diff[2]; |
| 365 | int64x1_t d0s64; | 367 | uint32x4_t sse_vec[2] = { vdupq_n_u32(0), vdupq_n_u32(0) }; |
| 366 | uint8x16_t q0u8, q1u8, q2u8, q3u8; | 368 | |
| 367 | int32x4_t q7s32, q8s32, q9s32, q10s32; | 369 | for (i = 0; i < 8; i++) { |
| 368 | uint16x8_t q11u16, q12u16, q13u16, q14u16; | 370 | a[0] = vld1q_u8(src_ptr); |
| 369 | int64x2_t q1s64; | ||
| 370 | |||
| 371 | q7s32 = vdupq_n_s32(0); | ||
| 372 | q8s32 = vdupq_n_s32(0); | ||
| 373 | q9s32 = vdupq_n_s32(0); | ||
| 374 | q10s32 = vdupq_n_s32(0); | ||
| 375 | |||
| 376 | for (i = 0; i < 8; i++) { // mse16x16_neon_loop | ||
| 377 | q0u8 = vld1q_u8(src_ptr); | ||
| 378 | src_ptr += src_stride; | 371 | src_ptr += src_stride; |
| 379 | q1u8 = vld1q_u8(src_ptr); | 372 | a[1] = vld1q_u8(src_ptr); |
| 380 | src_ptr += src_stride; | 373 | src_ptr += src_stride; |
| 381 | q2u8 = vld1q_u8(ref_ptr); | 374 | b[0] = vld1q_u8(ref_ptr); |
| 382 | ref_ptr += ref_stride; | 375 | ref_ptr += ref_stride; |
| 383 | q3u8 = vld1q_u8(ref_ptr); | 376 | b[1] = vld1q_u8(ref_ptr); |
| 384 | ref_ptr += ref_stride; | 377 | ref_ptr += ref_stride; |
| 385 | 378 | ||
| 386 | q11u16 = vsubl_u8(vget_low_u8(q0u8), vget_low_u8(q2u8)); | 379 | abs_diff[0] = vabdq_u8(a[0], b[0]); |
| 387 | q12u16 = vsubl_u8(vget_high_u8(q0u8), vget_high_u8(q2u8)); | 380 | abs_diff[1] = vabdq_u8(a[1], b[1]); |
| 388 | q13u16 = vsubl_u8(vget_low_u8(q1u8), vget_low_u8(q3u8)); | ||
| 389 | q14u16 = vsubl_u8(vget_high_u8(q1u8), vget_high_u8(q3u8)); | ||
| 390 | |||
| 391 | d22s16 = vreinterpret_s16_u16(vget_low_u16(q11u16)); | ||
| 392 | d23s16 = vreinterpret_s16_u16(vget_high_u16(q11u16)); | ||
| 393 | q7s32 = vmlal_s16(q7s32, d22s16, d22s16); | ||
| 394 | q8s32 = vmlal_s16(q8s32, d23s16, d23s16); | ||
| 395 | |||
| 396 | d24s16 = vreinterpret_s16_u16(vget_low_u16(q12u16)); | ||
| 397 | d25s16 = vreinterpret_s16_u16(vget_high_u16(q12u16)); | ||
| 398 | q9s32 = vmlal_s16(q9s32, d24s16, d24s16); | ||
| 399 | q10s32 = vmlal_s16(q10s32, d25s16, d25s16); | ||
| 400 | |||
| 401 | d26s16 = vreinterpret_s16_u16(vget_low_u16(q13u16)); | ||
| 402 | d27s16 = vreinterpret_s16_u16(vget_high_u16(q13u16)); | ||
| 403 | q7s32 = vmlal_s16(q7s32, d26s16, d26s16); | ||
| 404 | q8s32 = vmlal_s16(q8s32, d27s16, d27s16); | ||
| 405 | |||
| 406 | d28s16 = vreinterpret_s16_u16(vget_low_u16(q14u16)); | ||
| 407 | d29s16 = vreinterpret_s16_u16(vget_high_u16(q14u16)); | ||
| 408 | q9s32 = vmlal_s16(q9s32, d28s16, d28s16); | ||
| 409 | q10s32 = vmlal_s16(q10s32, d29s16, d29s16); | ||
| 410 | } | ||
| 411 | |||
| 412 | q7s32 = vaddq_s32(q7s32, q8s32); | ||
| 413 | q9s32 = vaddq_s32(q9s32, q10s32); | ||
| 414 | q10s32 = vaddq_s32(q7s32, q9s32); | ||
| 415 | 381 | ||
| 416 | q1s64 = vpaddlq_s32(q10s32); | 382 | sse_vec[0] = vdotq_u32(sse_vec[0], abs_diff[0], abs_diff[0]); |
| 417 | d0s64 = vadd_s64(vget_low_s64(q1s64), vget_high_s64(q1s64)); | 383 | sse_vec[1] = vdotq_u32(sse_vec[1], abs_diff[1], abs_diff[1]); |
| 384 | } | ||
| 418 | 385 | ||
| 419 | vst1_lane_u32((uint32_t *)sse, vreinterpret_u32_s64(d0s64), 0); | 386 | *sse = horizontal_add_uint32x4(vaddq_u32(sse_vec[0], sse_vec[1])); |
| 420 | return vget_lane_u32(vreinterpret_u32_s64(d0s64), 0); | 387 | return horizontal_add_uint32x4(vaddq_u32(sse_vec[0], sse_vec[1])); |
| 421 | } | 388 | } |
| 422 | 389 | ||
| 423 | unsigned int vpx_get4x4sse_cs_neon(const unsigned char *src_ptr, int src_stride, | 390 | unsigned int vpx_get4x4sse_cs_neon(const unsigned char *src_ptr, int src_stride, |
| 424 | const unsigned char *ref_ptr, | 391 | const unsigned char *ref_ptr, |
| 425 | int ref_stride) { | 392 | int ref_stride) { |
| 426 | int16x4_t d22s16, d24s16, d26s16, d28s16; | 393 | uint8x8_t a[4], b[4], abs_diff[4]; |
| 427 | int64x1_t d0s64; | 394 | uint32x2_t sse = vdup_n_u32(0); |
| 428 | uint8x8_t d0u8, d1u8, d2u8, d3u8, d4u8, d5u8, d6u8, d7u8; | 395 | |
| 429 | int32x4_t q7s32, q8s32, q9s32, q10s32; | 396 | a[0] = vld1_u8(src_ptr); |
| 430 | uint16x8_t q11u16, q12u16, q13u16, q14u16; | ||
| 431 | int64x2_t q1s64; | ||
| 432 | |||
| 433 | d0u8 = vld1_u8(src_ptr); | ||
| 434 | src_ptr += src_stride; | ||
| 435 | d4u8 = vld1_u8(ref_ptr); | ||
| 436 | ref_ptr += ref_stride; | ||
| 437 | d1u8 = vld1_u8(src_ptr); | ||
| 438 | src_ptr += src_stride; | 397 | src_ptr += src_stride; |
| 439 | d5u8 = vld1_u8(ref_ptr); | 398 | b[0] = vld1_u8(ref_ptr); |
| 440 | ref_ptr += ref_stride; | 399 | ref_ptr += ref_stride; |
| 441 | d2u8 = vld1_u8(src_ptr); | 400 | a[1] = vld1_u8(src_ptr); |
| 442 | src_ptr += src_stride; | 401 | src_ptr += src_stride; |
| 443 | d6u8 = vld1_u8(ref_ptr); | 402 | b[1] = vld1_u8(ref_ptr); |
| 444 | ref_ptr += ref_stride; | 403 | ref_ptr += ref_stride; |
| 445 | d3u8 = vld1_u8(src_ptr); | 404 | a[2] = vld1_u8(src_ptr); |
| 446 | src_ptr += src_stride; | 405 | src_ptr += src_stride; |
| 447 | d7u8 = vld1_u8(ref_ptr); | 406 | b[2] = vld1_u8(ref_ptr); |
| 448 | ref_ptr += ref_stride; | 407 | ref_ptr += ref_stride; |
| 408 | a[3] = vld1_u8(src_ptr); | ||
| 409 | b[3] = vld1_u8(ref_ptr); | ||
| 410 | |||
| 411 | abs_diff[0] = vabd_u8(a[0], b[0]); | ||
| 412 | abs_diff[1] = vabd_u8(a[1], b[1]); | ||
| 413 | abs_diff[2] = vabd_u8(a[2], b[2]); | ||
| 414 | abs_diff[3] = vabd_u8(a[3], b[3]); | ||
| 415 | |||
| 416 | sse = vdot_u32(sse, abs_diff[0], abs_diff[0]); | ||
| 417 | sse = vdot_u32(sse, abs_diff[1], abs_diff[1]); | ||
| 418 | sse = vdot_u32(sse, abs_diff[2], abs_diff[2]); | ||
| 419 | sse = vdot_u32(sse, abs_diff[3], abs_diff[3]); | ||
| 420 | |||
| 421 | return vget_lane_u32(sse, 0); | ||
| 422 | } | ||
| 423 | |||
| 424 | #else | ||
| 425 | |||
| 426 | unsigned int vpx_mse16x16_neon(const unsigned char *src_ptr, int src_stride, | ||
| 427 | const unsigned char *ref_ptr, int ref_stride, | ||
| 428 | unsigned int *sse) { | ||
| 429 | int i; | ||
| 430 | uint8x16_t a[2], b[2]; | ||
| 431 | int16x4_t diff_lo[4], diff_hi[4]; | ||
| 432 | uint16x8_t diff[4]; | ||
| 433 | int32x4_t sse_vec[4] = { vdupq_n_s32(0), vdupq_n_s32(0), vdupq_n_s32(0), | ||
| 434 | vdupq_n_s32(0) }; | ||
| 435 | |||
| 436 | for (i = 0; i < 8; i++) { | ||
| 437 | a[0] = vld1q_u8(src_ptr); | ||
| 438 | src_ptr += src_stride; | ||
| 439 | a[1] = vld1q_u8(src_ptr); | ||
| 440 | src_ptr += src_stride; | ||
| 441 | b[0] = vld1q_u8(ref_ptr); | ||
| 442 | ref_ptr += ref_stride; | ||
| 443 | b[1] = vld1q_u8(ref_ptr); | ||
| 444 | ref_ptr += ref_stride; | ||
| 445 | |||
| 446 | diff[0] = vsubl_u8(vget_low_u8(a[0]), vget_low_u8(b[0])); | ||
| 447 | diff[1] = vsubl_u8(vget_high_u8(a[0]), vget_high_u8(b[0])); | ||
| 448 | diff[2] = vsubl_u8(vget_low_u8(a[1]), vget_low_u8(b[1])); | ||
| 449 | diff[3] = vsubl_u8(vget_high_u8(a[1]), vget_high_u8(b[1])); | ||
| 450 | |||
| 451 | diff_lo[0] = vreinterpret_s16_u16(vget_low_u16(diff[0])); | ||
| 452 | diff_lo[1] = vreinterpret_s16_u16(vget_low_u16(diff[1])); | ||
| 453 | sse_vec[0] = vmlal_s16(sse_vec[0], diff_lo[0], diff_lo[0]); | ||
| 454 | sse_vec[1] = vmlal_s16(sse_vec[1], diff_lo[1], diff_lo[1]); | ||
| 455 | |||
| 456 | diff_lo[2] = vreinterpret_s16_u16(vget_low_u16(diff[2])); | ||
| 457 | diff_lo[3] = vreinterpret_s16_u16(vget_low_u16(diff[3])); | ||
| 458 | sse_vec[2] = vmlal_s16(sse_vec[2], diff_lo[2], diff_lo[2]); | ||
| 459 | sse_vec[3] = vmlal_s16(sse_vec[3], diff_lo[3], diff_lo[3]); | ||
| 460 | |||
| 461 | diff_hi[0] = vreinterpret_s16_u16(vget_high_u16(diff[0])); | ||
| 462 | diff_hi[1] = vreinterpret_s16_u16(vget_high_u16(diff[1])); | ||
| 463 | sse_vec[0] = vmlal_s16(sse_vec[0], diff_hi[0], diff_hi[0]); | ||
| 464 | sse_vec[1] = vmlal_s16(sse_vec[1], diff_hi[1], diff_hi[1]); | ||
| 465 | |||
| 466 | diff_hi[2] = vreinterpret_s16_u16(vget_high_u16(diff[2])); | ||
| 467 | diff_hi[3] = vreinterpret_s16_u16(vget_high_u16(diff[3])); | ||
| 468 | sse_vec[2] = vmlal_s16(sse_vec[2], diff_hi[2], diff_hi[2]); | ||
| 469 | sse_vec[3] = vmlal_s16(sse_vec[3], diff_hi[3], diff_hi[3]); | ||
| 470 | } | ||
| 471 | |||
| 472 | sse_vec[0] = vaddq_s32(sse_vec[0], sse_vec[1]); | ||
| 473 | sse_vec[2] = vaddq_s32(sse_vec[2], sse_vec[3]); | ||
| 474 | sse_vec[0] = vaddq_s32(sse_vec[0], sse_vec[2]); | ||
| 449 | 475 | ||
| 450 | q11u16 = vsubl_u8(d0u8, d4u8); | 476 | *sse = horizontal_add_uint32x4(vreinterpretq_u32_s32(sse_vec[0])); |
| 451 | q12u16 = vsubl_u8(d1u8, d5u8); | 477 | return horizontal_add_uint32x4(vreinterpretq_u32_s32(sse_vec[0])); |
| 452 | q13u16 = vsubl_u8(d2u8, d6u8); | 478 | } |
| 453 | q14u16 = vsubl_u8(d3u8, d7u8); | 479 | |
| 480 | unsigned int vpx_get4x4sse_cs_neon(const unsigned char *src_ptr, int src_stride, | ||
| 481 | const unsigned char *ref_ptr, | ||
| 482 | int ref_stride) { | ||
| 483 | uint8x8_t a[4], b[4]; | ||
| 484 | int16x4_t diff_lo[4]; | ||
| 485 | uint16x8_t diff[4]; | ||
| 486 | int32x4_t sse; | ||
| 454 | 487 | ||
| 455 | d22s16 = vget_low_s16(vreinterpretq_s16_u16(q11u16)); | 488 | a[0] = vld1_u8(src_ptr); |
| 456 | d24s16 = vget_low_s16(vreinterpretq_s16_u16(q12u16)); | 489 | src_ptr += src_stride; |
| 457 | d26s16 = vget_low_s16(vreinterpretq_s16_u16(q13u16)); | 490 | b[0] = vld1_u8(ref_ptr); |
| 458 | d28s16 = vget_low_s16(vreinterpretq_s16_u16(q14u16)); | 491 | ref_ptr += ref_stride; |
| 492 | a[1] = vld1_u8(src_ptr); | ||
| 493 | src_ptr += src_stride; | ||
| 494 | b[1] = vld1_u8(ref_ptr); | ||
| 495 | ref_ptr += ref_stride; | ||
| 496 | a[2] = vld1_u8(src_ptr); | ||
| 497 | src_ptr += src_stride; | ||
| 498 | b[2] = vld1_u8(ref_ptr); | ||
| 499 | ref_ptr += ref_stride; | ||
| 500 | a[3] = vld1_u8(src_ptr); | ||
| 501 | b[3] = vld1_u8(ref_ptr); | ||
| 459 | 502 | ||
| 460 | q7s32 = vmull_s16(d22s16, d22s16); | 503 | diff[0] = vsubl_u8(a[0], b[0]); |
| 461 | q8s32 = vmull_s16(d24s16, d24s16); | 504 | diff[1] = vsubl_u8(a[1], b[1]); |
| 462 | q9s32 = vmull_s16(d26s16, d26s16); | 505 | diff[2] = vsubl_u8(a[2], b[2]); |
| 463 | q10s32 = vmull_s16(d28s16, d28s16); | 506 | diff[3] = vsubl_u8(a[3], b[3]); |
| 464 | 507 | ||
| 465 | q7s32 = vaddq_s32(q7s32, q8s32); | 508 | diff_lo[0] = vget_low_s16(vreinterpretq_s16_u16(diff[0])); |
| 466 | q9s32 = vaddq_s32(q9s32, q10s32); | 509 | diff_lo[1] = vget_low_s16(vreinterpretq_s16_u16(diff[1])); |
| 467 | q9s32 = vaddq_s32(q7s32, q9s32); | 510 | diff_lo[2] = vget_low_s16(vreinterpretq_s16_u16(diff[2])); |
| 511 | diff_lo[3] = vget_low_s16(vreinterpretq_s16_u16(diff[3])); | ||
| 468 | 512 | ||
| 469 | q1s64 = vpaddlq_s32(q9s32); | 513 | sse = vmull_s16(diff_lo[0], diff_lo[0]); |
| 470 | d0s64 = vadd_s64(vget_low_s64(q1s64), vget_high_s64(q1s64)); | 514 | sse = vmlal_s16(sse, diff_lo[1], diff_lo[1]); |
| 515 | sse = vmlal_s16(sse, diff_lo[2], diff_lo[2]); | ||
| 516 | sse = vmlal_s16(sse, diff_lo[3], diff_lo[3]); | ||
| 471 | 517 | ||
| 472 | return vget_lane_u32(vreinterpret_u32_s64(d0s64), 0); | 518 | return horizontal_add_uint32x4(vreinterpretq_u32_s32(sse)); |
| 473 | } | 519 | } |
| 520 | |||
| 521 | #endif | ||