History

Merge "Fold adds in 16->32-bit converts in SSE2/AVX2 fDCT"

Kyle Siefring committed 8 years ago

Commit dccb8b42 parents c1435e3 811b2e42 files · +189 −332Browse files

Unified · Split

2 files
  1. vpx_dsp/x86/fwd_dct32x32_impl_avx2.h +94 −166
  2. vpx_dsp/x86/fwd_dct32x32_impl_sse2.h +95 −166
vpx_dsp/x86/fwd_dct32x32_impl_avx2.h+94−166
@@ -1374,59 +1374,37 @@ void FDCT32x32_2D_AVX2(const int16_t *input, int16_t *output_org, int stride) { · 1373 unmodified lines
1374 __m256i lstep1[64], lstep2[64], lstep3[64];1374 __m256i lstep1[64], lstep2[64], lstep3[64];
1375 __m256i u[32], v[32], sign[16];1375 __m256i u[32], v[32], sign[16];
1376 const __m256i K32One = _mm256_set_epi32(1, 1, 1, 1, 1, 1, 1, 1);1376 const __m256i K32One = _mm256_set_epi32(1, 1, 1, 1, 1, 1, 1, 1);
1377 const __m256i k__pOne_mOne = pair256_set_epi16(1, -1);
1377 // start using 32-bit operations1378 // start using 32-bit operations
1378 // stage 31379 // stage 3
1379 {1380 {
1380 // expanding to 32-bit length priori to addition operations1381 // expanding to 32-bit length while adding and subtracting
1381 lstep2[0] = _mm256_unpacklo_epi16(step2[0], kZero);1382 lstep2[0] = _mm256_unpacklo_epi16(step2[0], step2[7]);
1382 lstep2[1] = _mm256_unpackhi_epi16(step2[0], kZero);1383 lstep2[1] = _mm256_unpackhi_epi16(step2[0], step2[7]);
1383 lstep2[2] = _mm256_unpacklo_epi16(step2[1], kZero);1384 lstep2[2] = _mm256_unpacklo_epi16(step2[1], step2[6]);
1384 lstep2[3] = _mm256_unpackhi_epi16(step2[1], kZero);1385 lstep2[3] = _mm256_unpackhi_epi16(step2[1], step2[6]);
1385 lstep2[4] = _mm256_unpacklo_epi16(step2[2], kZero);1386 lstep2[4] = _mm256_unpacklo_epi16(step2[2], step2[5]);
1386 lstep2[5] = _mm256_unpackhi_epi16(step2[2], kZero);1387 lstep2[5] = _mm256_unpackhi_epi16(step2[2], step2[5]);
1387 lstep2[6] = _mm256_unpacklo_epi16(step2[3], kZero);1388 lstep2[6] = _mm256_unpacklo_epi16(step2[3], step2[4]);
1388 lstep2[7] = _mm256_unpackhi_epi16(step2[3], kZero);1389 lstep2[7] = _mm256_unpackhi_epi16(step2[3], step2[4]);
1389 lstep2[8] = _mm256_unpacklo_epi16(step2[4], kZero);1390
1390 lstep2[9] = _mm256_unpackhi_epi16(step2[4], kZero);1391 lstep3[0] = _mm256_madd_epi16(lstep2[0], kOne);
1391 lstep2[10] = _mm256_unpacklo_epi16(step2[5], kZero);1392 lstep3[1] = _mm256_madd_epi16(lstep2[1], kOne);
1392 lstep2[11] = _mm256_unpackhi_epi16(step2[5], kZero);1393 lstep3[2] = _mm256_madd_epi16(lstep2[2], kOne);
1393 lstep2[12] = _mm256_unpacklo_epi16(step2[6], kZero);1394 lstep3[3] = _mm256_madd_epi16(lstep2[3], kOne);
1394 lstep2[13] = _mm256_unpackhi_epi16(step2[6], kZero);1395 lstep3[4] = _mm256_madd_epi16(lstep2[4], kOne);
1395 lstep2[14] = _mm256_unpacklo_epi16(step2[7], kZero);1396 lstep3[5] = _mm256_madd_epi16(lstep2[5], kOne);
1396 lstep2[15] = _mm256_unpackhi_epi16(step2[7], kZero);1397 lstep3[6] = _mm256_madd_epi16(lstep2[6], kOne);
1397 lstep2[0] = _mm256_madd_epi16(lstep2[0], kOne);1398 lstep3[7] = _mm256_madd_epi16(lstep2[7], kOne);
1398 lstep2[1] = _mm256_madd_epi16(lstep2[1], kOne);1399
1399 lstep2[2] = _mm256_madd_epi16(lstep2[2], kOne);1400 lstep3[8] = _mm256_madd_epi16(lstep2[6], k__pOne_mOne);
1400 lstep2[3] = _mm256_madd_epi16(lstep2[3], kOne);1401 lstep3[9] = _mm256_madd_epi16(lstep2[7], k__pOne_mOne);
1401 lstep2[4] = _mm256_madd_epi16(lstep2[4], kOne);1402 lstep3[10] = _mm256_madd_epi16(lstep2[4], k__pOne_mOne);
1402 lstep2[5] = _mm256_madd_epi16(lstep2[5], kOne);1403 lstep3[11] = _mm256_madd_epi16(lstep2[5], k__pOne_mOne);
1403 lstep2[6] = _mm256_madd_epi16(lstep2[6], kOne);1404 lstep3[12] = _mm256_madd_epi16(lstep2[2], k__pOne_mOne);
1404 lstep2[7] = _mm256_madd_epi16(lstep2[7], kOne);1405 lstep3[13] = _mm256_madd_epi16(lstep2[3], k__pOne_mOne);
1405 lstep2[8] = _mm256_madd_epi16(lstep2[8], kOne);1406 lstep3[14] = _mm256_madd_epi16(lstep2[0], k__pOne_mOne);
1406 lstep2[9] = _mm256_madd_epi16(lstep2[9], kOne);1407 lstep3[15] = _mm256_madd_epi16(lstep2[1], k__pOne_mOne);
1407 lstep2[10] = _mm256_madd_epi16(lstep2[10], kOne);
1408 lstep2[11] = _mm256_madd_epi16(lstep2[11], kOne);
1409 lstep2[12] = _mm256_madd_epi16(lstep2[12], kOne);
1410 lstep2[13] = _mm256_madd_epi16(lstep2[13], kOne);
1411 lstep2[14] = _mm256_madd_epi16(lstep2[14], kOne);
1412 lstep2[15] = _mm256_madd_epi16(lstep2[15], kOne);
1413
1414 lstep3[0] = _mm256_add_epi32(lstep2[14], lstep2[0]);
1415 lstep3[1] = _mm256_add_epi32(lstep2[15], lstep2[1]);
1416 lstep3[2] = _mm256_add_epi32(lstep2[12], lstep2[2]);
1417 lstep3[3] = _mm256_add_epi32(lstep2[13], lstep2[3]);
1418 lstep3[4] = _mm256_add_epi32(lstep2[10], lstep2[4]);
1419 lstep3[5] = _mm256_add_epi32(lstep2[11], lstep2[5]);
1420 lstep3[6] = _mm256_add_epi32(lstep2[8], lstep2[6]);
1421 lstep3[7] = _mm256_add_epi32(lstep2[9], lstep2[7]);
1422 lstep3[8] = _mm256_sub_epi32(lstep2[6], lstep2[8]);
1423 lstep3[9] = _mm256_sub_epi32(lstep2[7], lstep2[9]);
1424 lstep3[10] = _mm256_sub_epi32(lstep2[4], lstep2[10]);
1425 lstep3[11] = _mm256_sub_epi32(lstep2[5], lstep2[11]);
1426 lstep3[12] = _mm256_sub_epi32(lstep2[2], lstep2[12]);
1427 lstep3[13] = _mm256_sub_epi32(lstep2[3], lstep2[13]);
1428 lstep3[14] = _mm256_sub_epi32(lstep2[0], lstep2[14]);
1429 lstep3[15] = _mm256_sub_epi32(lstep2[1], lstep2[15]);
1430 }1408 }
1431 {1409 {
1432 const __m256i s3_10_0 = _mm256_unpacklo_epi16(step2[13], step2[10]);1410 const __m256i s3_10_0 = _mm256_unpacklo_epi16(step2[13], step2[10]);
@@ -1468,126 +1446,76 @@ void FDCT32x32_2D_AVX2(const int16_t *input, int16_t *output_org, int stride) { · 35 unmodified lines
1468 lstep3[27] = _mm256_srai_epi32(s3_13_5, DCT_CONST_BITS);1446 lstep3[27] = _mm256_srai_epi32(s3_13_5, DCT_CONST_BITS);
1469 }1447 }
1470 {1448 {
1471 lstep2[40] = _mm256_unpacklo_epi16(step2[20], kZero);1449 lstep1[32] = _mm256_unpacklo_epi16(step1[16], step2[23]);
1472 lstep2[41] = _mm256_unpackhi_epi16(step2[20], kZero);1450 lstep1[33] = _mm256_unpackhi_epi16(step1[16], step2[23]);
1473 lstep2[42] = _mm256_unpacklo_epi16(step2[21], kZero);1451 lstep1[34] = _mm256_unpacklo_epi16(step1[17], step2[22]);
1474 lstep2[43] = _mm256_unpackhi_epi16(step2[21], kZero);1452 lstep1[35] = _mm256_unpackhi_epi16(step1[17], step2[22]);
1475 lstep2[44] = _mm256_unpacklo_epi16(step2[22], kZero);1453 lstep1[36] = _mm256_unpacklo_epi16(step1[18], step2[21]);
1476 lstep2[45] = _mm256_unpackhi_epi16(step2[22], kZero);1454 lstep1[37] = _mm256_unpackhi_epi16(step1[18], step2[21]);
1477 lstep2[46] = _mm256_unpacklo_epi16(step2[23], kZero);1455 lstep1[38] = _mm256_unpacklo_epi16(step1[19], step2[20]);
1478 lstep2[47] = _mm256_unpackhi_epi16(step2[23], kZero);1456 lstep1[39] = _mm256_unpackhi_epi16(step1[19], step2[20]);
1479 lstep2[48] = _mm256_unpacklo_epi16(step2[24], kZero);1457
1480 lstep2[49] = _mm256_unpackhi_epi16(step2[24], kZero);1458 lstep1[56] = _mm256_unpacklo_epi16(step1[28], step2[27]);
1481 lstep2[50] = _mm256_unpacklo_epi16(step2[25], kZero);1459 lstep1[57] = _mm256_unpackhi_epi16(step1[28], step2[27]);
1482 lstep2[51] = _mm256_unpackhi_epi16(step2[25], kZero);1460 lstep1[58] = _mm256_unpacklo_epi16(step1[29], step2[26]);
1483 lstep2[52] = _mm256_unpacklo_epi16(step2[26], kZero);1461 lstep1[59] = _mm256_unpackhi_epi16(step1[29], step2[26]);
1484 lstep2[53] = _mm256_unpackhi_epi16(step2[26], kZero);1462 lstep1[60] = _mm256_unpacklo_epi16(step1[30], step2[25]);
1485 lstep2[54] = _mm256_unpacklo_epi16(step2[27], kZero);1463 lstep1[61] = _mm256_unpackhi_epi16(step1[30], step2[25]);
1486 lstep2[55] = _mm256_unpackhi_epi16(step2[27], kZero);1464 lstep1[62] = _mm256_unpacklo_epi16(step1[31], step2[24]);
1487 lstep2[40] = _mm256_madd_epi16(lstep2[40], kOne);1465 lstep1[63] = _mm256_unpackhi_epi16(step1[31], step2[24]);
1488 lstep2[41] = _mm256_madd_epi16(lstep2[41], kOne);1466
1489 lstep2[42] = _mm256_madd_epi16(lstep2[42], kOne);1467 lstep3[32] = _mm256_madd_epi16(lstep1[32], kOne);
1490 lstep2[43] = _mm256_madd_epi16(lstep2[43], kOne);1468 lstep3[33] = _mm256_madd_epi16(lstep1[33], kOne);
1491 lstep2[44] = _mm256_madd_epi16(lstep2[44], kOne);1469 lstep3[34] = _mm256_madd_epi16(lstep1[34], kOne);
1492 lstep2[45] = _mm256_madd_epi16(lstep2[45], kOne);1470 lstep3[35] = _mm256_madd_epi16(lstep1[35], kOne);
1493 lstep2[46] = _mm256_madd_epi16(lstep2[46], kOne);1471 lstep3[36] = _mm256_madd_epi16(lstep1[36], kOne);
1494 lstep2[47] = _mm256_madd_epi16(lstep2[47], kOne);1472 lstep3[37] = _mm256_madd_epi16(lstep1[37], kOne);
1495 lstep2[48] = _mm256_madd_epi16(lstep2[48], kOne);1473 lstep3[38] = _mm256_madd_epi16(lstep1[38], kOne);
1496 lstep2[49] = _mm256_madd_epi16(lstep2[49], kOne);1474 lstep3[39] = _mm256_madd_epi16(lstep1[39], kOne);
1497 lstep2[50] = _mm256_madd_epi16(lstep2[50], kOne);1475
1498 lstep2[51] = _mm256_madd_epi16(lstep2[51], kOne);1476 lstep3[40] = _mm256_madd_epi16(lstep1[38], k__pOne_mOne);
1499 lstep2[52] = _mm256_madd_epi16(lstep2[52], kOne);1477 lstep3[41] = _mm256_madd_epi16(lstep1[39], k__pOne_mOne);
1500 lstep2[53] = _mm256_madd_epi16(lstep2[53], kOne);1478 lstep3[42] = _mm256_madd_epi16(lstep1[36], k__pOne_mOne);
1501 lstep2[54] = _mm256_madd_epi16(lstep2[54], kOne);1479 lstep3[43] = _mm256_madd_epi16(lstep1[37], k__pOne_mOne);
1502 lstep2[55] = _mm256_madd_epi16(lstep2[55], kOne);1480 lstep3[44] = _mm256_madd_epi16(lstep1[34], k__pOne_mOne);
1503 1481 lstep3[45] = _mm256_madd_epi16(lstep1[35], k__pOne_mOne);
1504 lstep1[32] = _mm256_unpacklo_epi16(step1[16], kZero);1482 lstep3[46] = _mm256_madd_epi16(lstep1[32], k__pOne_mOne);
1505 lstep1[33] = _mm256_unpackhi_epi16(step1[16], kZero);1483 lstep3[47] = _mm256_madd_epi16(lstep1[33], k__pOne_mOne);
1506 lstep1[34] = _mm256_unpacklo_epi16(step1[17], kZero);1484
1507 lstep1[35] = _mm256_unpackhi_epi16(step1[17], kZero);1485 lstep3[48] = _mm256_madd_epi16(lstep1[62], k__pOne_mOne);
1508 lstep1[36] = _mm256_unpacklo_epi16(step1[18], kZero);1486 lstep3[49] = _mm256_madd_epi16(lstep1[63], k__pOne_mOne);
1509 lstep1[37] = _mm256_unpackhi_epi16(step1[18], kZero);1487 lstep3[50] = _mm256_madd_epi16(lstep1[60], k__pOne_mOne);
1510 lstep1[38] = _mm256_unpacklo_epi16(step1[19], kZero);1488 lstep3[51] = _mm256_madd_epi16(lstep1[61], k__pOne_mOne);
1511 lstep1[39] = _mm256_unpackhi_epi16(step1[19], kZero);1489 lstep3[52] = _mm256_madd_epi16(lstep1[58], k__pOne_mOne);
1512 lstep1[56] = _mm256_unpacklo_epi16(step1[28], kZero);1490 lstep3[53] = _mm256_madd_epi16(lstep1[59], k__pOne_mOne);
1513 lstep1[57] = _mm256_unpackhi_epi16(step1[28], kZero);1491 lstep3[54] = _mm256_madd_epi16(lstep1[56], k__pOne_mOne);
1514 lstep1[58] = _mm256_unpacklo_epi16(step1[29], kZero);1492 lstep3[55] = _mm256_madd_epi16(lstep1[57], k__pOne_mOne);
1515 lstep1[59] = _mm256_unpackhi_epi16(step1[29], kZero);1493
1516 lstep1[60] = _mm256_unpacklo_epi16(step1[30], kZero);1494 lstep3[56] = _mm256_madd_epi16(lstep1[56], kOne);
1517 lstep1[61] = _mm256_unpackhi_epi16(step1[30], kZero);1495 lstep3[57] = _mm256_madd_epi16(lstep1[57], kOne);
1518 lstep1[62] = _mm256_unpacklo_epi16(step1[31], kZero);1496 lstep3[58] = _mm256_madd_epi16(lstep1[58], kOne);
1519 lstep1[63] = _mm256_unpackhi_epi16(step1[31], kZero);1497 lstep3[59] = _mm256_madd_epi16(lstep1[59], kOne);
1520 lstep1[32] = _mm256_madd_epi16(lstep1[32], kOne);1498 lstep3[60] = _mm256_madd_epi16(lstep1[60], kOne);
1521 lstep1[33] = _mm256_madd_epi16(lstep1[33], kOne);1499 lstep3[61] = _mm256_madd_epi16(lstep1[61], kOne);
1522 lstep1[34] = _mm256_madd_epi16(lstep1[34], kOne);1500 lstep3[62] = _mm256_madd_epi16(lstep1[62], kOne);
1523 lstep1[35] = _mm256_madd_epi16(lstep1[35], kOne);1501 lstep3[63] = _mm256_madd_epi16(lstep1[63], kOne);
1524 lstep1[36] = _mm256_madd_epi16(lstep1[36], kOne);
1525 lstep1[37] = _mm256_madd_epi16(lstep1[37], kOne);
1526 lstep1[38] = _mm256_madd_epi16(lstep1[38], kOne);
1527 lstep1[39] = _mm256_madd_epi16(lstep1[39], kOne);
1528 lstep1[56] = _mm256_madd_epi16(lstep1[56], kOne);
1529 lstep1[57] = _mm256_madd_epi16(lstep1[57], kOne);
1530 lstep1[58] = _mm256_madd_epi16(lstep1[58], kOne);
1531 lstep1[59] = _mm256_madd_epi16(lstep1[59], kOne);
1532 lstep1[60] = _mm256_madd_epi16(lstep1[60], kOne);
1533 lstep1[61] = _mm256_madd_epi16(lstep1[61], kOne);
1534 lstep1[62] = _mm256_madd_epi16(lstep1[62], kOne);
1535 lstep1[63] = _mm256_madd_epi16(lstep1[63], kOne);
1536
1537 lstep3[32] = _mm256_add_epi32(lstep2[46], lstep1[32]);
1538 lstep3[33] = _mm256_add_epi32(lstep2[47], lstep1[33]);
1539
1540 lstep3[34] = _mm256_add_epi32(lstep2[44], lstep1[34]);
1541 lstep3[35] = _mm256_add_epi32(lstep2[45], lstep1[35]);
1542 lstep3[36] = _mm256_add_epi32(lstep2[42], lstep1[36]);
1543 lstep3[37] = _mm256_add_epi32(lstep2[43], lstep1[37]);
1544 lstep3[38] = _mm256_add_epi32(lstep2[40], lstep1[38]);
1545 lstep3[39] = _mm256_add_epi32(lstep2[41], lstep1[39]);
1546 lstep3[40] = _mm256_sub_epi32(lstep1[38], lstep2[40]);
1547 lstep3[41] = _mm256_sub_epi32(lstep1[39], lstep2[41]);
1548 lstep3[42] = _mm256_sub_epi32(lstep1[36], lstep2[42]);
1549 lstep3[43] = _mm256_sub_epi32(lstep1[37], lstep2[43]);
1550 lstep3[44] = _mm256_sub_epi32(lstep1[34], lstep2[44]);
1551 lstep3[45] = _mm256_sub_epi32(lstep1[35], lstep2[45]);
1552 lstep3[46] = _mm256_sub_epi32(lstep1[32], lstep2[46]);
1553 lstep3[47] = _mm256_sub_epi32(lstep1[33], lstep2[47]);
1554 lstep3[48] = _mm256_sub_epi32(lstep1[62], lstep2[48]);
1555 lstep3[49] = _mm256_sub_epi32(lstep1[63], lstep2[49]);
1556 lstep3[50] = _mm256_sub_epi32(lstep1[60], lstep2[50]);
1557 lstep3[51] = _mm256_sub_epi32(lstep1[61], lstep2[51]);
1558 lstep3[52] = _mm256_sub_epi32(lstep1[58], lstep2[52]);
1559 lstep3[53] = _mm256_sub_epi32(lstep1[59], lstep2[53]);
1560 lstep3[54] = _mm256_sub_epi32(lstep1[56], lstep2[54]);
1561 lstep3[55] = _mm256_sub_epi32(lstep1[57], lstep2[55]);
1562 lstep3[56] = _mm256_add_epi32(lstep2[54], lstep1[56]);
1563 lstep3[57] = _mm256_add_epi32(lstep2[55], lstep1[57]);
1564 lstep3[58] = _mm256_add_epi32(lstep2[52], lstep1[58]);
1565 lstep3[59] = _mm256_add_epi32(lstep2[53], lstep1[59]);
1566 lstep3[60] = _mm256_add_epi32(lstep2[50], lstep1[60]);
1567 lstep3[61] = _mm256_add_epi32(lstep2[51], lstep1[61]);
1568 lstep3[62] = _mm256_add_epi32(lstep2[48], lstep1[62]);
1569 lstep3[63] = _mm256_add_epi32(lstep2[49], lstep1[63]);
1570 }1502 }
1571 1503
1572 // stage 41504 // stage 4
1573 {1505 {
1574 // expanding to 32-bit length priori to addition operations1506 // expanding to 32-bit length prior to addition operations
1575 lstep2[16] = _mm256_unpacklo_epi16(step2[8], kZero);1507 sign[0] = _mm256_cmpgt_epi16(kZero, step2[8]);
1576 lstep2[17] = _mm256_unpackhi_epi16(step2[8], kZero);1508 sign[1] = _mm256_cmpgt_epi16(kZero, step2[9]);
1577 lstep2[18] = _mm256_unpacklo_epi16(step2[9], kZero);1509 sign[2] = _mm256_cmpgt_epi16(kZero, step2[14]);
1578 lstep2[19] = _mm256_unpackhi_epi16(step2[9], kZero);1510 sign[3] = _mm256_cmpgt_epi16(kZero, step2[15]);
1579 lstep2[28] = _mm256_unpacklo_epi16(step2[14], kZero);1511 lstep2[16] = _mm256_unpacklo_epi16(step2[8], sign[0]);
1580 lstep2[29] = _mm256_unpackhi_epi16(step2[14], kZero);1512 lstep2[17] = _mm256_unpackhi_epi16(step2[8], sign[0]);
1581 lstep2[30] = _mm256_unpacklo_epi16(step2[15], kZero);1513 lstep2[18] = _mm256_unpacklo_epi16(step2[9], sign[1]);
1582 lstep2[31] = _mm256_unpackhi_epi16(step2[15], kZero);1514 lstep2[19] = _mm256_unpackhi_epi16(step2[9], sign[1]);
1583 lstep2[16] = _mm256_madd_epi16(lstep2[16], kOne);1515 lstep2[28] = _mm256_unpacklo_epi16(step2[14], sign[2]);
1584 lstep2[17] = _mm256_madd_epi16(lstep2[17], kOne);1516 lstep2[29] = _mm256_unpackhi_epi16(step2[14], sign[2]);
1585 lstep2[18] = _mm256_madd_epi16(lstep2[18], kOne);1517 lstep2[30] = _mm256_unpacklo_epi16(step2[15], sign[3]);
1586 lstep2[19] = _mm256_madd_epi16(lstep2[19], kOne);1518 lstep2[31] = _mm256_unpackhi_epi16(step2[15], sign[3]);
1587 lstep2[28] = _mm256_madd_epi16(lstep2[28], kOne);
1588 lstep2[29] = _mm256_madd_epi16(lstep2[29], kOne);
1589 lstep2[30] = _mm256_madd_epi16(lstep2[30], kOne);
1590 lstep2[31] = _mm256_madd_epi16(lstep2[31], kOne);
1591 1519
1592 lstep1[0] = _mm256_add_epi32(lstep3[6], lstep3[0]);1520 lstep1[0] = _mm256_add_epi32(lstep3[6], lstep3[0]);
1593 lstep1[1] = _mm256_add_epi32(lstep3[7], lstep3[1]);1521 lstep1[1] = _mm256_add_epi32(lstep3[7], lstep3[1]);
vpx_dsp/x86/fwd_dct32x32_impl_sse2.h+95−166
@@ -101,6 +101,7 @@ void FDCT32x32_2D(const int16_t *input, tran_low_t *output_org, int stride) { · 100 unmodified lines
101 const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING);101 const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING);
102 const __m128i kZero = _mm_set1_epi16(0);102 const __m128i kZero = _mm_set1_epi16(0);
103 const __m128i kOne = _mm_set1_epi16(1);103 const __m128i kOne = _mm_set1_epi16(1);
104
104 // Do the two transform/transpose passes105 // Do the two transform/transpose passes
105 int pass;106 int pass;
106#if DCT_HIGH_BIT_DEPTH107#if DCT_HIGH_BIT_DEPTH
@@ -1508,59 +1509,37 @@ void FDCT32x32_2D(const int16_t *input, tran_low_t *output_org, int stride) { · 1401 unmodified lines
1508 __m128i lstep1[64], lstep2[64], lstep3[64];1509 __m128i lstep1[64], lstep2[64], lstep3[64];
1509 __m128i u[32], v[32], sign[16];1510 __m128i u[32], v[32], sign[16];
1510 const __m128i K32One = _mm_set_epi32(1, 1, 1, 1);1511 const __m128i K32One = _mm_set_epi32(1, 1, 1, 1);
1512 const __m128i k__pOne_mOne = pair_set_epi16(1, -1);
1511 // start using 32-bit operations1513 // start using 32-bit operations
1512 // stage 31514 // stage 3
1513 {1515 {
1514 // expanding to 32-bit length priori to addition operations1516 // expanding to 32-bit length while adding and subtracting
1515 lstep2[0] = _mm_unpacklo_epi16(step2[0], kZero);1517 lstep2[0] = _mm_unpacklo_epi16(step2[0], step2[7]);
1516 lstep2[1] = _mm_unpackhi_epi16(step2[0], kZero);1518 lstep2[1] = _mm_unpackhi_epi16(step2[0], step2[7]);
1517 lstep2[2] = _mm_unpacklo_epi16(step2[1], kZero);1519 lstep2[2] = _mm_unpacklo_epi16(step2[1], step2[6]);
1518 lstep2[3] = _mm_unpackhi_epi16(step2[1], kZero);1520 lstep2[3] = _mm_unpackhi_epi16(step2[1], step2[6]);
1519 lstep2[4] = _mm_unpacklo_epi16(step2[2], kZero);1521 lstep2[4] = _mm_unpacklo_epi16(step2[2], step2[5]);
1520 lstep2[5] = _mm_unpackhi_epi16(step2[2], kZero);1522 lstep2[5] = _mm_unpackhi_epi16(step2[2], step2[5]);
1521 lstep2[6] = _mm_unpacklo_epi16(step2[3], kZero);1523 lstep2[6] = _mm_unpacklo_epi16(step2[3], step2[4]);
1522 lstep2[7] = _mm_unpackhi_epi16(step2[3], kZero);1524 lstep2[7] = _mm_unpackhi_epi16(step2[3], step2[4]);
1523 lstep2[8] = _mm_unpacklo_epi16(step2[4], kZero);1525
1524 lstep2[9] = _mm_unpackhi_epi16(step2[4], kZero);1526 lstep3[0] = _mm_madd_epi16(lstep2[0], kOne);
1525 lstep2[10] = _mm_unpacklo_epi16(step2[5], kZero);1527 lstep3[1] = _mm_madd_epi16(lstep2[1], kOne);
1526 lstep2[11] = _mm_unpackhi_epi16(step2[5], kZero);1528 lstep3[2] = _mm_madd_epi16(lstep2[2], kOne);
1527 lstep2[12] = _mm_unpacklo_epi16(step2[6], kZero);1529 lstep3[3] = _mm_madd_epi16(lstep2[3], kOne);
1528 lstep2[13] = _mm_unpackhi_epi16(step2[6], kZero);1530 lstep3[4] = _mm_madd_epi16(lstep2[4], kOne);
1529 lstep2[14] = _mm_unpacklo_epi16(step2[7], kZero);1531 lstep3[5] = _mm_madd_epi16(lstep2[5], kOne);
1530 lstep2[15] = _mm_unpackhi_epi16(step2[7], kZero);1532 lstep3[6] = _mm_madd_epi16(lstep2[6], kOne);
1531 lstep2[0] = _mm_madd_epi16(lstep2[0], kOne);1533 lstep3[7] = _mm_madd_epi16(lstep2[7], kOne);
1532 lstep2[1] = _mm_madd_epi16(lstep2[1], kOne);1534
1533 lstep2[2] = _mm_madd_epi16(lstep2[2], kOne);1535 lstep3[8] = _mm_madd_epi16(lstep2[6], k__pOne_mOne);
1534 lstep2[3] = _mm_madd_epi16(lstep2[3], kOne);1536 lstep3[9] = _mm_madd_epi16(lstep2[7], k__pOne_mOne);
1535 lstep2[4] = _mm_madd_epi16(lstep2[4], kOne);1537 lstep3[10] = _mm_madd_epi16(lstep2[4], k__pOne_mOne);
1536 lstep2[5] = _mm_madd_epi16(lstep2[5], kOne);1538 lstep3[11] = _mm_madd_epi16(lstep2[5], k__pOne_mOne);
1537 lstep2[6] = _mm_madd_epi16(lstep2[6], kOne);1539 lstep3[12] = _mm_madd_epi16(lstep2[2], k__pOne_mOne);
1538 lstep2[7] = _mm_madd_epi16(lstep2[7], kOne);1540 lstep3[13] = _mm_madd_epi16(lstep2[3], k__pOne_mOne);
1539 lstep2[8] = _mm_madd_epi16(lstep2[8], kOne);1541 lstep3[14] = _mm_madd_epi16(lstep2[0], k__pOne_mOne);
1540 lstep2[9] = _mm_madd_epi16(lstep2[9], kOne);1542 lstep3[15] = _mm_madd_epi16(lstep2[1], k__pOne_mOne);
1541 lstep2[10] = _mm_madd_epi16(lstep2[10], kOne);
1542 lstep2[11] = _mm_madd_epi16(lstep2[11], kOne);
1543 lstep2[12] = _mm_madd_epi16(lstep2[12], kOne);
1544 lstep2[13] = _mm_madd_epi16(lstep2[13], kOne);
1545 lstep2[14] = _mm_madd_epi16(lstep2[14], kOne);
1546 lstep2[15] = _mm_madd_epi16(lstep2[15], kOne);
1547
1548 lstep3[0] = _mm_add_epi32(lstep2[14], lstep2[0]);
1549 lstep3[1] = _mm_add_epi32(lstep2[15], lstep2[1]);
1550 lstep3[2] = _mm_add_epi32(lstep2[12], lstep2[2]);
1551 lstep3[3] = _mm_add_epi32(lstep2[13], lstep2[3]);
1552 lstep3[4] = _mm_add_epi32(lstep2[10], lstep2[4]);
1553 lstep3[5] = _mm_add_epi32(lstep2[11], lstep2[5]);
1554 lstep3[6] = _mm_add_epi32(lstep2[8], lstep2[6]);
1555 lstep3[7] = _mm_add_epi32(lstep2[9], lstep2[7]);
1556 lstep3[8] = _mm_sub_epi32(lstep2[6], lstep2[8]);
1557 lstep3[9] = _mm_sub_epi32(lstep2[7], lstep2[9]);
1558 lstep3[10] = _mm_sub_epi32(lstep2[4], lstep2[10]);
1559 lstep3[11] = _mm_sub_epi32(lstep2[5], lstep2[11]);
1560 lstep3[12] = _mm_sub_epi32(lstep2[2], lstep2[12]);
1561 lstep3[13] = _mm_sub_epi32(lstep2[3], lstep2[13]);
1562 lstep3[14] = _mm_sub_epi32(lstep2[0], lstep2[14]);
1563 lstep3[15] = _mm_sub_epi32(lstep2[1], lstep2[15]);
1564 }1543 }
1565 {1544 {
1566 const __m128i s3_10_0 = _mm_unpacklo_epi16(step2[13], step2[10]);1545 const __m128i s3_10_0 = _mm_unpacklo_epi16(step2[13], step2[10]);
@@ -1594,126 +1573,76 @@ void FDCT32x32_2D(const int16_t *input, tran_low_t *output_org, int stride) { · 27 unmodified lines
1594 lstep3[27] = _mm_srai_epi32(s3_13_5, DCT_CONST_BITS);1573 lstep3[27] = _mm_srai_epi32(s3_13_5, DCT_CONST_BITS);
1595 }1574 }
1596 {1575 {
1597 lstep2[40] = _mm_unpacklo_epi16(step2[20], kZero);1576 lstep1[32] = _mm_unpacklo_epi16(step1[16], step2[23]);
1598 lstep2[41] = _mm_unpackhi_epi16(step2[20], kZero);1577 lstep1[33] = _mm_unpackhi_epi16(step1[16], step2[23]);
1599 lstep2[42] = _mm_unpacklo_epi16(step2[21], kZero);1578 lstep1[34] = _mm_unpacklo_epi16(step1[17], step2[22]);
1600 lstep2[43] = _mm_unpackhi_epi16(step2[21], kZero);1579 lstep1[35] = _mm_unpackhi_epi16(step1[17], step2[22]);
1601 lstep2[44] = _mm_unpacklo_epi16(step2[22], kZero);1580 lstep1[36] = _mm_unpacklo_epi16(step1[18], step2[21]);
1602 lstep2[45] = _mm_unpackhi_epi16(step2[22], kZero);1581 lstep1[37] = _mm_unpackhi_epi16(step1[18], step2[21]);
1603 lstep2[46] = _mm_unpacklo_epi16(step2[23], kZero);1582 lstep1[38] = _mm_unpacklo_epi16(step1[19], step2[20]);
1604 lstep2[47] = _mm_unpackhi_epi16(step2[23], kZero);1583 lstep1[39] = _mm_unpackhi_epi16(step1[19], step2[20]);
1605 lstep2[48] = _mm_unpacklo_epi16(step2[24], kZero);1584
1606 lstep2[49] = _mm_unpackhi_epi16(step2[24], kZero);1585 lstep1[56] = _mm_unpacklo_epi16(step1[28], step2[27]);
1607 lstep2[50] = _mm_unpacklo_epi16(step2[25], kZero);1586 lstep1[57] = _mm_unpackhi_epi16(step1[28], step2[27]);
1608 lstep2[51] = _mm_unpackhi_epi16(step2[25], kZero);1587 lstep1[58] = _mm_unpacklo_epi16(step1[29], step2[26]);
1609 lstep2[52] = _mm_unpacklo_epi16(step2[26], kZero);1588 lstep1[59] = _mm_unpackhi_epi16(step1[29], step2[26]);
1610 lstep2[53] = _mm_unpackhi_epi16(step2[26], kZero);1589 lstep1[60] = _mm_unpacklo_epi16(step1[30], step2[25]);
1611 lstep2[54] = _mm_unpacklo_epi16(step2[27], kZero);1590 lstep1[61] = _mm_unpackhi_epi16(step1[30], step2[25]);
1612 lstep2[55] = _mm_unpackhi_epi16(step2[27], kZero);1591 lstep1[62] = _mm_unpacklo_epi16(step1[31], step2[24]);
1613 lstep2[40] = _mm_madd_epi16(lstep2[40], kOne);1592 lstep1[63] = _mm_unpackhi_epi16(step1[31], step2[24]);
1614 lstep2[41] = _mm_madd_epi16(lstep2[41], kOne);1593
1615 lstep2[42] = _mm_madd_epi16(lstep2[42], kOne);1594 lstep3[32] = _mm_madd_epi16(lstep1[32], kOne);
1616 lstep2[43] = _mm_madd_epi16(lstep2[43], kOne);1595 lstep3[33] = _mm_madd_epi16(lstep1[33], kOne);
1617 lstep2[44] = _mm_madd_epi16(lstep2[44], kOne);1596 lstep3[34] = _mm_madd_epi16(lstep1[34], kOne);
1618 lstep2[45] = _mm_madd_epi16(lstep2[45], kOne);1597 lstep3[35] = _mm_madd_epi16(lstep1[35], kOne);
1619 lstep2[46] = _mm_madd_epi16(lstep2[46], kOne);1598 lstep3[36] = _mm_madd_epi16(lstep1[36], kOne);
1620 lstep2[47] = _mm_madd_epi16(lstep2[47], kOne);1599 lstep3[37] = _mm_madd_epi16(lstep1[37], kOne);
1621 lstep2[48] = _mm_madd_epi16(lstep2[48], kOne);1600 lstep3[38] = _mm_madd_epi16(lstep1[38], kOne);
1622 lstep2[49] = _mm_madd_epi16(lstep2[49], kOne);1601 lstep3[39] = _mm_madd_epi16(lstep1[39], kOne);
1623 lstep2[50] = _mm_madd_epi16(lstep2[50], kOne);1602
1624 lstep2[51] = _mm_madd_epi16(lstep2[51], kOne);1603 lstep3[40] = _mm_madd_epi16(lstep1[38], k__pOne_mOne);
1625 lstep2[52] = _mm_madd_epi16(lstep2[52], kOne);1604 lstep3[41] = _mm_madd_epi16(lstep1[39], k__pOne_mOne);
1626 lstep2[53] = _mm_madd_epi16(lstep2[53], kOne);1605 lstep3[42] = _mm_madd_epi16(lstep1[36], k__pOne_mOne);
1627 lstep2[54] = _mm_madd_epi16(lstep2[54], kOne);1606 lstep3[43] = _mm_madd_epi16(lstep1[37], k__pOne_mOne);
1628 lstep2[55] = _mm_madd_epi16(lstep2[55], kOne);1607 lstep3[44] = _mm_madd_epi16(lstep1[34], k__pOne_mOne);
1629 1608 lstep3[45] = _mm_madd_epi16(lstep1[35], k__pOne_mOne);
1630 lstep1[32] = _mm_unpacklo_epi16(step1[16], kZero);1609 lstep3[46] = _mm_madd_epi16(lstep1[32], k__pOne_mOne);
1631 lstep1[33] = _mm_unpackhi_epi16(step1[16], kZero);1610 lstep3[47] = _mm_madd_epi16(lstep1[33], k__pOne_mOne);
1632 lstep1[34] = _mm_unpacklo_epi16(step1[17], kZero);1611
1633 lstep1[35] = _mm_unpackhi_epi16(step1[17], kZero);1612 lstep3[48] = _mm_madd_epi16(lstep1[62], k__pOne_mOne);
1634 lstep1[36] = _mm_unpacklo_epi16(step1[18], kZero);1613 lstep3[49] = _mm_madd_epi16(lstep1[63], k__pOne_mOne);
1635 lstep1[37] = _mm_unpackhi_epi16(step1[18], kZero);1614 lstep3[50] = _mm_madd_epi16(lstep1[60], k__pOne_mOne);
1636 lstep1[38] = _mm_unpacklo_epi16(step1[19], kZero);1615 lstep3[51] = _mm_madd_epi16(lstep1[61], k__pOne_mOne);
1637 lstep1[39] = _mm_unpackhi_epi16(step1[19], kZero);1616 lstep3[52] = _mm_madd_epi16(lstep1[58], k__pOne_mOne);
1638 lstep1[56] = _mm_unpacklo_epi16(step1[28], kZero);1617 lstep3[53] = _mm_madd_epi16(lstep1[59], k__pOne_mOne);
1639 lstep1[57] = _mm_unpackhi_epi16(step1[28], kZero);1618 lstep3[54] = _mm_madd_epi16(lstep1[56], k__pOne_mOne);
1640 lstep1[58] = _mm_unpacklo_epi16(step1[29], kZero);1619 lstep3[55] = _mm_madd_epi16(lstep1[57], k__pOne_mOne);
1641 lstep1[59] = _mm_unpackhi_epi16(step1[29], kZero);1620
1642 lstep1[60] = _mm_unpacklo_epi16(step1[30], kZero);1621 lstep3[56] = _mm_madd_epi16(lstep1[56], kOne);
1643 lstep1[61] = _mm_unpackhi_epi16(step1[30], kZero);1622 lstep3[57] = _mm_madd_epi16(lstep1[57], kOne);
1644 lstep1[62] = _mm_unpacklo_epi16(step1[31], kZero);1623 lstep3[58] = _mm_madd_epi16(lstep1[58], kOne);
1645 lstep1[63] = _mm_unpackhi_epi16(step1[31], kZero);1624 lstep3[59] = _mm_madd_epi16(lstep1[59], kOne);
1646 lstep1[32] = _mm_madd_epi16(lstep1[32], kOne);1625 lstep3[60] = _mm_madd_epi16(lstep1[60], kOne);
1647 lstep1[33] = _mm_madd_epi16(lstep1[33], kOne);1626 lstep3[61] = _mm_madd_epi16(lstep1[61], kOne);
1648 lstep1[34] = _mm_madd_epi16(lstep1[34], kOne);1627 lstep3[62] = _mm_madd_epi16(lstep1[62], kOne);
1649 lstep1[35] = _mm_madd_epi16(lstep1[35], kOne);1628 lstep3[63] = _mm_madd_epi16(lstep1[63], kOne);
1650 lstep1[36] = _mm_madd_epi16(lstep1[36], kOne);
1651 lstep1[37] = _mm_madd_epi16(lstep1[37], kOne);
1652 lstep1[38] = _mm_madd_epi16(lstep1[38], kOne);
1653 lstep1[39] = _mm_madd_epi16(lstep1[39], kOne);
1654 lstep1[56] = _mm_madd_epi16(lstep1[56], kOne);
1655 lstep1[57] = _mm_madd_epi16(lstep1[57], kOne);
1656 lstep1[58] = _mm_madd_epi16(lstep1[58], kOne);
1657 lstep1[59] = _mm_madd_epi16(lstep1[59], kOne);
1658 lstep1[60] = _mm_madd_epi16(lstep1[60], kOne);
1659 lstep1[61] = _mm_madd_epi16(lstep1[61], kOne);
1660 lstep1[62] = _mm_madd_epi16(lstep1[62], kOne);
1661 lstep1[63] = _mm_madd_epi16(lstep1[63], kOne);
1662
1663 lstep3[32] = _mm_add_epi32(lstep2[46], lstep1[32]);
1664 lstep3[33] = _mm_add_epi32(lstep2[47], lstep1[33]);
1665
1666 lstep3[34] = _mm_add_epi32(lstep2[44], lstep1[34]);
1667 lstep3[35] = _mm_add_epi32(lstep2[45], lstep1[35]);
1668 lstep3[36] = _mm_add_epi32(lstep2[42], lstep1[36]);
1669 lstep3[37] = _mm_add_epi32(lstep2[43], lstep1[37]);
1670 lstep3[38] = _mm_add_epi32(lstep2[40], lstep1[38]);
1671 lstep3[39] = _mm_add_epi32(lstep2[41], lstep1[39]);
1672 lstep3[40] = _mm_sub_epi32(lstep1[38], lstep2[40]);
1673 lstep3[41] = _mm_sub_epi32(lstep1[39], lstep2[41]);
1674 lstep3[42] = _mm_sub_epi32(lstep1[36], lstep2[42]);
1675 lstep3[43] = _mm_sub_epi32(lstep1[37], lstep2[43]);
1676 lstep3[44] = _mm_sub_epi32(lstep1[34], lstep2[44]);
1677 lstep3[45] = _mm_sub_epi32(lstep1[35], lstep2[45]);
1678 lstep3[46] = _mm_sub_epi32(lstep1[32], lstep2[46]);
1679 lstep3[47] = _mm_sub_epi32(lstep1[33], lstep2[47]);
1680 lstep3[48] = _mm_sub_epi32(lstep1[62], lstep2[48]);
1681 lstep3[49] = _mm_sub_epi32(lstep1[63], lstep2[49]);
1682 lstep3[50] = _mm_sub_epi32(lstep1[60], lstep2[50]);
1683 lstep3[51] = _mm_sub_epi32(lstep1[61], lstep2[51]);
1684 lstep3[52] = _mm_sub_epi32(lstep1[58], lstep2[52]);
1685 lstep3[53] = _mm_sub_epi32(lstep1[59], lstep2[53]);
1686 lstep3[54] = _mm_sub_epi32(lstep1[56], lstep2[54]);
1687 lstep3[55] = _mm_sub_epi32(lstep1[57], lstep2[55]);
1688 lstep3[56] = _mm_add_epi32(lstep2[54], lstep1[56]);
1689 lstep3[57] = _mm_add_epi32(lstep2[55], lstep1[57]);
1690 lstep3[58] = _mm_add_epi32(lstep2[52], lstep1[58]);
1691 lstep3[59] = _mm_add_epi32(lstep2[53], lstep1[59]);
1692 lstep3[60] = _mm_add_epi32(lstep2[50], lstep1[60]);
1693 lstep3[61] = _mm_add_epi32(lstep2[51], lstep1[61]);
1694 lstep3[62] = _mm_add_epi32(lstep2[48], lstep1[62]);
1695 lstep3[63] = _mm_add_epi32(lstep2[49], lstep1[63]);
1696 }1629 }
1697 1630
1698 // stage 41631 // stage 4
1699 {1632 {
1700 // expanding to 32-bit length priori to addition operations1633 // expanding to 32-bit length prior to addition operations
1701 lstep2[16] = _mm_unpacklo_epi16(step2[8], kZero);1634 sign[0] = _mm_cmpgt_epi16(kZero, step2[8]);
1702 lstep2[17] = _mm_unpackhi_epi16(step2[8], kZero);1635 sign[1] = _mm_cmpgt_epi16(kZero, step2[9]);
1703 lstep2[18] = _mm_unpacklo_epi16(step2[9], kZero);1636 sign[2] = _mm_cmpgt_epi16(kZero, step2[14]);
1704 lstep2[19] = _mm_unpackhi_epi16(step2[9], kZero);1637 sign[3] = _mm_cmpgt_epi16(kZero, step2[15]);
1705 lstep2[28] = _mm_unpacklo_epi16(step2[14], kZero);1638 lstep2[16] = _mm_unpacklo_epi16(step2[8], sign[0]);
1706 lstep2[29] = _mm_unpackhi_epi16(step2[14], kZero);1639 lstep2[17] = _mm_unpackhi_epi16(step2[8], sign[0]);
1707 lstep2[30] = _mm_unpacklo_epi16(step2[15], kZero);1640 lstep2[18] = _mm_unpacklo_epi16(step2[9], sign[1]);
1708 lstep2[31] = _mm_unpackhi_epi16(step2[15], kZero);1641 lstep2[19] = _mm_unpackhi_epi16(step2[9], sign[1]);
1709 lstep2[16] = _mm_madd_epi16(lstep2[16], kOne);1642 lstep2[28] = _mm_unpacklo_epi16(step2[14], sign[2]);
1710 lstep2[17] = _mm_madd_epi16(lstep2[17], kOne);1643 lstep2[29] = _mm_unpackhi_epi16(step2[14], sign[2]);
1711 lstep2[18] = _mm_madd_epi16(lstep2[18], kOne);1644 lstep2[30] = _mm_unpacklo_epi16(step2[15], sign[3]);
1712 lstep2[19] = _mm_madd_epi16(lstep2[19], kOne);1645 lstep2[31] = _mm_unpackhi_epi16(step2[15], sign[3]);
1713 lstep2[28] = _mm_madd_epi16(lstep2[28], kOne);
1714 lstep2[29] = _mm_madd_epi16(lstep2[29], kOne);
1715 lstep2[30] = _mm_madd_epi16(lstep2[30], kOne);
1716 lstep2[31] = _mm_madd_epi16(lstep2[31], kOne);
1717 1646
1718 lstep1[0] = _mm_add_epi32(lstep3[6], lstep3[0]);1647 lstep1[0] = _mm_add_epi32(lstep3[6], lstep3[0]);
1719 lstep1[1] = _mm_add_epi32(lstep3[7], lstep3[1]);1648 lstep1[1] = _mm_add_epi32(lstep3[7], lstep3[1]);

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t