Implement sum of squared difference calculations in vpx_mse16x16_neon and vpx_get4x4sse_cs_neon using the ABD and UDOT instructions - instead of widening subtracts followed by a sequence of MLAs. The existing implementation is retained for use on CPUs that do not implement the Armv8.4-A UDOT instruction. This commit also updates the variable names used in the existing implementations to be more descriptive.
Unified · Split
| @@ -357,117 +357,165 @@ unsigned int vpx_variance64x64_neon(const uint8_t *src_ptr, int src_stride, · 356 unmodified lines | ||
| 357 | 357 | return *sse - (unsigned int)(((int64_t)sum1 * sum1) >> 12); |
| 358 | 358 | } |
| 359 | 359 | |
| 360 | +#if defined(__ARM_FEATURE_DOTPROD) && (__ARM_FEATURE_DOTPROD == 1) | |
| 361 | + | |
| 360 | 362 | unsigned int vpx_mse16x16_neon(const unsigned char *src_ptr, int src_stride, |
| 361 | 363 | const unsigned char *ref_ptr, int ref_stride, |
| 362 | 364 | unsigned int *sse) { |
| 363 | 365 | int i; |
| 364 | − int16x4_t d22s16, d23s16, d24s16, d25s16, d26s16, d27s16, d28s16, d29s16; | |
| 365 | − int64x1_t d0s64; | |
| 366 | − uint8x16_t q0u8, q1u8, q2u8, q3u8; | |
| 367 | − int32x4_t q7s32, q8s32, q9s32, q10s32; | |
| 368 | − uint16x8_t q11u16, q12u16, q13u16, q14u16; | |
| 369 | − int64x2_t q1s64; | |
| 370 | − | |
| 371 | − q7s32 = vdupq_n_s32(0); | |
| 372 | − q8s32 = vdupq_n_s32(0); | |
| 373 | − q9s32 = vdupq_n_s32(0); | |
| 374 | − q10s32 = vdupq_n_s32(0); | |
| 375 | − | |
| 376 | − for (i = 0; i < 8; i++) { // mse16x16_neon_loop | |
| 377 | − q0u8 = vld1q_u8(src_ptr); | |
| 366 | + uint8x16_t a[2], b[2], abs_diff[2]; | |
| 367 | + uint32x4_t sse_vec[2] = { vdupq_n_u32(0), vdupq_n_u32(0) }; | |
| 368 | + | |
| 369 | + for (i = 0; i < 8; i++) { | |
| 370 | + a[0] = vld1q_u8(src_ptr); | |
| 378 | 371 | src_ptr += src_stride; |
| 379 | − q1u8 = vld1q_u8(src_ptr); | |
| 372 | + a[1] = vld1q_u8(src_ptr); | |
| 380 | 373 | src_ptr += src_stride; |
| 381 | − q2u8 = vld1q_u8(ref_ptr); | |
| 374 | + b[0] = vld1q_u8(ref_ptr); | |
| 382 | 375 | ref_ptr += ref_stride; |
| 383 | − q3u8 = vld1q_u8(ref_ptr); | |
| 376 | + b[1] = vld1q_u8(ref_ptr); | |
| 384 | 377 | ref_ptr += ref_stride; |
| 385 | 378 | |
| 386 | − q11u16 = vsubl_u8(vget_low_u8(q0u8), vget_low_u8(q2u8)); | |
| 387 | − q12u16 = vsubl_u8(vget_high_u8(q0u8), vget_high_u8(q2u8)); | |
| 388 | − q13u16 = vsubl_u8(vget_low_u8(q1u8), vget_low_u8(q3u8)); | |
| 389 | − q14u16 = vsubl_u8(vget_high_u8(q1u8), vget_high_u8(q3u8)); | |
| 390 | − | |
| 391 | − d22s16 = vreinterpret_s16_u16(vget_low_u16(q11u16)); | |
| 392 | − d23s16 = vreinterpret_s16_u16(vget_high_u16(q11u16)); | |
| 393 | − q7s32 = vmlal_s16(q7s32, d22s16, d22s16); | |
| 394 | − q8s32 = vmlal_s16(q8s32, d23s16, d23s16); | |
| 395 | − | |
| 396 | − d24s16 = vreinterpret_s16_u16(vget_low_u16(q12u16)); | |
| 397 | − d25s16 = vreinterpret_s16_u16(vget_high_u16(q12u16)); | |
| 398 | − q9s32 = vmlal_s16(q9s32, d24s16, d24s16); | |
| 399 | − q10s32 = vmlal_s16(q10s32, d25s16, d25s16); | |
| 400 | − | |
| 401 | − d26s16 = vreinterpret_s16_u16(vget_low_u16(q13u16)); | |
| 402 | − d27s16 = vreinterpret_s16_u16(vget_high_u16(q13u16)); | |
| 403 | − q7s32 = vmlal_s16(q7s32, d26s16, d26s16); | |
| 404 | − q8s32 = vmlal_s16(q8s32, d27s16, d27s16); | |
| 405 | − | |
| 406 | − d28s16 = vreinterpret_s16_u16(vget_low_u16(q14u16)); | |
| 407 | − d29s16 = vreinterpret_s16_u16(vget_high_u16(q14u16)); | |
| 408 | − q9s32 = vmlal_s16(q9s32, d28s16, d28s16); | |
| 409 | − q10s32 = vmlal_s16(q10s32, d29s16, d29s16); | |
| 410 | − } | |
| 411 | − | |
| 412 | − q7s32 = vaddq_s32(q7s32, q8s32); | |
| 413 | − q9s32 = vaddq_s32(q9s32, q10s32); | |
| 414 | − q10s32 = vaddq_s32(q7s32, q9s32); | |
| 379 | + abs_diff[0] = vabdq_u8(a[0], b[0]); | |
| 380 | + abs_diff[1] = vabdq_u8(a[1], b[1]); | |
| 415 | 381 | |
| 416 | − q1s64 = vpaddlq_s32(q10s32); | |
| 417 | − d0s64 = vadd_s64(vget_low_s64(q1s64), vget_high_s64(q1s64)); | |
| 382 | + sse_vec[0] = vdotq_u32(sse_vec[0], abs_diff[0], abs_diff[0]); | |
| 383 | + sse_vec[1] = vdotq_u32(sse_vec[1], abs_diff[1], abs_diff[1]); | |
| 384 | + } | |
| 418 | 385 | |
| 419 | − vst1_lane_u32((uint32_t *)sse, vreinterpret_u32_s64(d0s64), 0); | |
| 420 | − return vget_lane_u32(vreinterpret_u32_s64(d0s64), 0); | |
| 386 | + *sse = horizontal_add_uint32x4(vaddq_u32(sse_vec[0], sse_vec[1])); | |
| 387 | + return horizontal_add_uint32x4(vaddq_u32(sse_vec[0], sse_vec[1])); | |
| 421 | 388 | } |
| 422 | 389 | |
| 423 | 390 | unsigned int vpx_get4x4sse_cs_neon(const unsigned char *src_ptr, int src_stride, |
| 424 | 391 | const unsigned char *ref_ptr, |
| 425 | 392 | int ref_stride) { |
| 426 | − int16x4_t d22s16, d24s16, d26s16, d28s16; | |
| 427 | − int64x1_t d0s64; | |
| 428 | − uint8x8_t d0u8, d1u8, d2u8, d3u8, d4u8, d5u8, d6u8, d7u8; | |
| 429 | − int32x4_t q7s32, q8s32, q9s32, q10s32; | |
| 430 | − uint16x8_t q11u16, q12u16, q13u16, q14u16; | |
| 431 | − int64x2_t q1s64; | |
| 432 | − | |
| 433 | − d0u8 = vld1_u8(src_ptr); | |
| 434 | − src_ptr += src_stride; | |
| 435 | − d4u8 = vld1_u8(ref_ptr); | |
| 436 | − ref_ptr += ref_stride; | |
| 437 | − d1u8 = vld1_u8(src_ptr); | |
| 393 | + uint8x8_t a[4], b[4], abs_diff[4]; | |
| 394 | + uint32x2_t sse = vdup_n_u32(0); | |
| 395 | + | |
| 396 | + a[0] = vld1_u8(src_ptr); | |
| 438 | 397 | src_ptr += src_stride; |
| 439 | − d5u8 = vld1_u8(ref_ptr); | |
| 398 | + b[0] = vld1_u8(ref_ptr); | |
| 440 | 399 | ref_ptr += ref_stride; |
| 441 | − d2u8 = vld1_u8(src_ptr); | |
| 400 | + a[1] = vld1_u8(src_ptr); | |
| 442 | 401 | src_ptr += src_stride; |
| 443 | − d6u8 = vld1_u8(ref_ptr); | |
| 402 | + b[1] = vld1_u8(ref_ptr); | |
| 444 | 403 | ref_ptr += ref_stride; |
| 445 | − d3u8 = vld1_u8(src_ptr); | |
| 404 | + a[2] = vld1_u8(src_ptr); | |
| 446 | 405 | src_ptr += src_stride; |
| 447 | − d7u8 = vld1_u8(ref_ptr); | |
| 406 | + b[2] = vld1_u8(ref_ptr); | |
| 448 | 407 | ref_ptr += ref_stride; |
| 408 | + a[3] = vld1_u8(src_ptr); | |
| 409 | + b[3] = vld1_u8(ref_ptr); | |
| 410 | + | |
| 411 | + abs_diff[0] = vabd_u8(a[0], b[0]); | |
| 412 | + abs_diff[1] = vabd_u8(a[1], b[1]); | |
| 413 | + abs_diff[2] = vabd_u8(a[2], b[2]); | |
| 414 | + abs_diff[3] = vabd_u8(a[3], b[3]); | |
| 415 | + | |
| 416 | + sse = vdot_u32(sse, abs_diff[0], abs_diff[0]); | |
| 417 | + sse = vdot_u32(sse, abs_diff[1], abs_diff[1]); | |
| 418 | + sse = vdot_u32(sse, abs_diff[2], abs_diff[2]); | |
| 419 | + sse = vdot_u32(sse, abs_diff[3], abs_diff[3]); | |
| 420 | + | |
| 421 | + return vget_lane_u32(sse, 0); | |
| 422 | +} | |
| 423 | + | |
| 424 | +#else | |
| 425 | + | |
| 426 | +unsigned int vpx_mse16x16_neon(const unsigned char *src_ptr, int src_stride, | |
| 427 | + const unsigned char *ref_ptr, int ref_stride, | |
| 428 | + unsigned int *sse) { | |
| 429 | + int i; | |
| 430 | + uint8x16_t a[2], b[2]; | |
| 431 | + int16x4_t diff_lo[4], diff_hi[4]; | |
| 432 | + uint16x8_t diff[4]; | |
| 433 | + int32x4_t sse_vec[4] = { vdupq_n_s32(0), vdupq_n_s32(0), vdupq_n_s32(0), | |
| 434 | + vdupq_n_s32(0) }; | |
| 435 | + | |
| 436 | + for (i = 0; i < 8; i++) { | |
| 437 | + a[0] = vld1q_u8(src_ptr); | |
| 438 | + src_ptr += src_stride; | |
| 439 | + a[1] = vld1q_u8(src_ptr); | |
| 440 | + src_ptr += src_stride; | |
| 441 | + b[0] = vld1q_u8(ref_ptr); | |
| 442 | + ref_ptr += ref_stride; | |
| 443 | + b[1] = vld1q_u8(ref_ptr); | |
| 444 | + ref_ptr += ref_stride; | |
| 445 | + | |
| 446 | + diff[0] = vsubl_u8(vget_low_u8(a[0]), vget_low_u8(b[0])); | |
| 447 | + diff[1] = vsubl_u8(vget_high_u8(a[0]), vget_high_u8(b[0])); | |
| 448 | + diff[2] = vsubl_u8(vget_low_u8(a[1]), vget_low_u8(b[1])); | |
| 449 | + diff[3] = vsubl_u8(vget_high_u8(a[1]), vget_high_u8(b[1])); | |
| 450 | + | |
| 451 | + diff_lo[0] = vreinterpret_s16_u16(vget_low_u16(diff[0])); | |
| 452 | + diff_lo[1] = vreinterpret_s16_u16(vget_low_u16(diff[1])); | |
| 453 | + sse_vec[0] = vmlal_s16(sse_vec[0], diff_lo[0], diff_lo[0]); | |
| 454 | + sse_vec[1] = vmlal_s16(sse_vec[1], diff_lo[1], diff_lo[1]); | |
| 455 | + | |
| 456 | + diff_lo[2] = vreinterpret_s16_u16(vget_low_u16(diff[2])); | |
| 457 | + diff_lo[3] = vreinterpret_s16_u16(vget_low_u16(diff[3])); | |
| 458 | + sse_vec[2] = vmlal_s16(sse_vec[2], diff_lo[2], diff_lo[2]); | |
| 459 | + sse_vec[3] = vmlal_s16(sse_vec[3], diff_lo[3], diff_lo[3]); | |
| 460 | + | |
| 461 | + diff_hi[0] = vreinterpret_s16_u16(vget_high_u16(diff[0])); | |
| 462 | + diff_hi[1] = vreinterpret_s16_u16(vget_high_u16(diff[1])); | |
| 463 | + sse_vec[0] = vmlal_s16(sse_vec[0], diff_hi[0], diff_hi[0]); | |
| 464 | + sse_vec[1] = vmlal_s16(sse_vec[1], diff_hi[1], diff_hi[1]); | |
| 465 | + | |
| 466 | + diff_hi[2] = vreinterpret_s16_u16(vget_high_u16(diff[2])); | |
| 467 | + diff_hi[3] = vreinterpret_s16_u16(vget_high_u16(diff[3])); | |
| 468 | + sse_vec[2] = vmlal_s16(sse_vec[2], diff_hi[2], diff_hi[2]); | |
| 469 | + sse_vec[3] = vmlal_s16(sse_vec[3], diff_hi[3], diff_hi[3]); | |
| 470 | + } | |
| 471 | + | |
| 472 | + sse_vec[0] = vaddq_s32(sse_vec[0], sse_vec[1]); | |
| 473 | + sse_vec[2] = vaddq_s32(sse_vec[2], sse_vec[3]); | |
| 474 | + sse_vec[0] = vaddq_s32(sse_vec[0], sse_vec[2]); | |
| 449 | 475 | |
| 450 | − q11u16 = vsubl_u8(d0u8, d4u8); | |
| 451 | − q12u16 = vsubl_u8(d1u8, d5u8); | |
| 452 | − q13u16 = vsubl_u8(d2u8, d6u8); | |
| 453 | − q14u16 = vsubl_u8(d3u8, d7u8); | |
| 476 | + *sse = horizontal_add_uint32x4(vreinterpretq_u32_s32(sse_vec[0])); | |
| 477 | + return horizontal_add_uint32x4(vreinterpretq_u32_s32(sse_vec[0])); | |
| 478 | +} | |
| 479 | + | |
| 480 | +unsigned int vpx_get4x4sse_cs_neon(const unsigned char *src_ptr, int src_stride, | |
| 481 | + const unsigned char *ref_ptr, | |
| 482 | + int ref_stride) { | |
| 483 | + uint8x8_t a[4], b[4]; | |
| 484 | + int16x4_t diff_lo[4]; | |
| 485 | + uint16x8_t diff[4]; | |
| 486 | + int32x4_t sse; | |
| 454 | 487 | |
| 455 | − d22s16 = vget_low_s16(vreinterpretq_s16_u16(q11u16)); | |
| 456 | − d24s16 = vget_low_s16(vreinterpretq_s16_u16(q12u16)); | |
| 457 | − d26s16 = vget_low_s16(vreinterpretq_s16_u16(q13u16)); | |
| 458 | − d28s16 = vget_low_s16(vreinterpretq_s16_u16(q14u16)); | |
| 488 | + a[0] = vld1_u8(src_ptr); | |
| 489 | + src_ptr += src_stride; | |
| 490 | + b[0] = vld1_u8(ref_ptr); | |
| 491 | + ref_ptr += ref_stride; | |
| 492 | + a[1] = vld1_u8(src_ptr); | |
| 493 | + src_ptr += src_stride; | |
| 494 | + b[1] = vld1_u8(ref_ptr); | |
| 495 | + ref_ptr += ref_stride; | |
| 496 | + a[2] = vld1_u8(src_ptr); | |
| 497 | + src_ptr += src_stride; | |
| 498 | + b[2] = vld1_u8(ref_ptr); | |
| 499 | + ref_ptr += ref_stride; | |
| 500 | + a[3] = vld1_u8(src_ptr); | |
| 501 | + b[3] = vld1_u8(ref_ptr); | |
| 459 | 502 | |
| 460 | − q7s32 = vmull_s16(d22s16, d22s16); | |
| 461 | − q8s32 = vmull_s16(d24s16, d24s16); | |
| 462 | − q9s32 = vmull_s16(d26s16, d26s16); | |
| 463 | − q10s32 = vmull_s16(d28s16, d28s16); | |
| 503 | + diff[0] = vsubl_u8(a[0], b[0]); | |
| 504 | + diff[1] = vsubl_u8(a[1], b[1]); | |
| 505 | + diff[2] = vsubl_u8(a[2], b[2]); | |
| 506 | + diff[3] = vsubl_u8(a[3], b[3]); | |
| 464 | 507 | |
| 465 | − q7s32 = vaddq_s32(q7s32, q8s32); | |
| 466 | − q9s32 = vaddq_s32(q9s32, q10s32); | |
| 467 | − q9s32 = vaddq_s32(q7s32, q9s32); | |
| 508 | + diff_lo[0] = vget_low_s16(vreinterpretq_s16_u16(diff[0])); | |
| 509 | + diff_lo[1] = vget_low_s16(vreinterpretq_s16_u16(diff[1])); | |
| 510 | + diff_lo[2] = vget_low_s16(vreinterpretq_s16_u16(diff[2])); | |
| 511 | + diff_lo[3] = vget_low_s16(vreinterpretq_s16_u16(diff[3])); | |
| 468 | 512 | |
| 469 | − q1s64 = vpaddlq_s32(q9s32); | |
| 470 | − d0s64 = vadd_s64(vget_low_s64(q1s64), vget_high_s64(q1s64)); | |
| 513 | + sse = vmull_s16(diff_lo[0], diff_lo[0]); | |
| 514 | + sse = vmlal_s16(sse, diff_lo[1], diff_lo[1]); | |
| 515 | + sse = vmlal_s16(sse, diff_lo[2], diff_lo[2]); | |
| 516 | + sse = vmlal_s16(sse, diff_lo[3], diff_lo[3]); | |
| 471 | 517 | |
| 472 | − return vget_lane_u32(vreinterpret_u32_s64(d0s64), 0); | |
| 518 | + return horizontal_add_uint32x4(vreinterpretq_u32_s32(sse)); | |
| 473 | 519 | } |
| 520 | + | |
| 521 | +#endif | |