History

Merge "Fold adds in 16->32-bit converts in SSE2/AVX2 fDCT"

Kyle Siefring committed 8 years ago

Commit dccb8b42 parents c1435e3 811b2e42 files · +189 −332Browse files

Unified · Split

2 files
  1. vpx_dsp/x86/fwd_dct32x32_impl_avx2.h +94 −166
  2. vpx_dsp/x86/fwd_dct32x32_impl_sse2.h +95 −166
vpx_dsp/x86/fwd_dct32x32_impl_avx2.h+94−166
@@ -1374,59 +1374,37 @@ void FDCT32x32_2D_AVX2(const int16_t *input, int16_t *output_org, int stride) { · 1373 unmodified lines
13741374 __m256i lstep1[64], lstep2[64], lstep3[64];
13751375 __m256i u[32], v[32], sign[16];
13761376 const __m256i K32One = _mm256_set_epi32(1, 1, 1, 1, 1, 1, 1, 1);
1377+ const __m256i k__pOne_mOne = pair256_set_epi16(1, -1);
13771378 // start using 32-bit operations
13781379 // stage 3
13791380 {
1380− // expanding to 32-bit length priori to addition operations
1381− lstep2[0] = _mm256_unpacklo_epi16(step2[0], kZero);
1382− lstep2[1] = _mm256_unpackhi_epi16(step2[0], kZero);
1383− lstep2[2] = _mm256_unpacklo_epi16(step2[1], kZero);
1384− lstep2[3] = _mm256_unpackhi_epi16(step2[1], kZero);
1385− lstep2[4] = _mm256_unpacklo_epi16(step2[2], kZero);
1386− lstep2[5] = _mm256_unpackhi_epi16(step2[2], kZero);
1387− lstep2[6] = _mm256_unpacklo_epi16(step2[3], kZero);
1388− lstep2[7] = _mm256_unpackhi_epi16(step2[3], kZero);
1389− lstep2[8] = _mm256_unpacklo_epi16(step2[4], kZero);
1390− lstep2[9] = _mm256_unpackhi_epi16(step2[4], kZero);
1391− lstep2[10] = _mm256_unpacklo_epi16(step2[5], kZero);
1392− lstep2[11] = _mm256_unpackhi_epi16(step2[5], kZero);
1393− lstep2[12] = _mm256_unpacklo_epi16(step2[6], kZero);
1394− lstep2[13] = _mm256_unpackhi_epi16(step2[6], kZero);
1395− lstep2[14] = _mm256_unpacklo_epi16(step2[7], kZero);
1396− lstep2[15] = _mm256_unpackhi_epi16(step2[7], kZero);
1397− lstep2[0] = _mm256_madd_epi16(lstep2[0], kOne);
1398− lstep2[1] = _mm256_madd_epi16(lstep2[1], kOne);
1399− lstep2[2] = _mm256_madd_epi16(lstep2[2], kOne);
1400− lstep2[3] = _mm256_madd_epi16(lstep2[3], kOne);
1401− lstep2[4] = _mm256_madd_epi16(lstep2[4], kOne);
1402− lstep2[5] = _mm256_madd_epi16(lstep2[5], kOne);
1403− lstep2[6] = _mm256_madd_epi16(lstep2[6], kOne);
1404− lstep2[7] = _mm256_madd_epi16(lstep2[7], kOne);
1405− lstep2[8] = _mm256_madd_epi16(lstep2[8], kOne);
1406− lstep2[9] = _mm256_madd_epi16(lstep2[9], kOne);
1407− lstep2[10] = _mm256_madd_epi16(lstep2[10], kOne);
1408− lstep2[11] = _mm256_madd_epi16(lstep2[11], kOne);
1409− lstep2[12] = _mm256_madd_epi16(lstep2[12], kOne);
1410− lstep2[13] = _mm256_madd_epi16(lstep2[13], kOne);
1411− lstep2[14] = _mm256_madd_epi16(lstep2[14], kOne);
1412− lstep2[15] = _mm256_madd_epi16(lstep2[15], kOne);
1413−
1414− lstep3[0] = _mm256_add_epi32(lstep2[14], lstep2[0]);
1415− lstep3[1] = _mm256_add_epi32(lstep2[15], lstep2[1]);
1416− lstep3[2] = _mm256_add_epi32(lstep2[12], lstep2[2]);
1417− lstep3[3] = _mm256_add_epi32(lstep2[13], lstep2[3]);
1418− lstep3[4] = _mm256_add_epi32(lstep2[10], lstep2[4]);
1419− lstep3[5] = _mm256_add_epi32(lstep2[11], lstep2[5]);
1420− lstep3[6] = _mm256_add_epi32(lstep2[8], lstep2[6]);
1421− lstep3[7] = _mm256_add_epi32(lstep2[9], lstep2[7]);
1422− lstep3[8] = _mm256_sub_epi32(lstep2[6], lstep2[8]);
1423− lstep3[9] = _mm256_sub_epi32(lstep2[7], lstep2[9]);
1424− lstep3[10] = _mm256_sub_epi32(lstep2[4], lstep2[10]);
1425− lstep3[11] = _mm256_sub_epi32(lstep2[5], lstep2[11]);
1426− lstep3[12] = _mm256_sub_epi32(lstep2[2], lstep2[12]);
1427− lstep3[13] = _mm256_sub_epi32(lstep2[3], lstep2[13]);
1428− lstep3[14] = _mm256_sub_epi32(lstep2[0], lstep2[14]);
1429− lstep3[15] = _mm256_sub_epi32(lstep2[1], lstep2[15]);
1381+ // expanding to 32-bit length while adding and subtracting
1382+ lstep2[0] = _mm256_unpacklo_epi16(step2[0], step2[7]);
1383+ lstep2[1] = _mm256_unpackhi_epi16(step2[0], step2[7]);
1384+ lstep2[2] = _mm256_unpacklo_epi16(step2[1], step2[6]);
1385+ lstep2[3] = _mm256_unpackhi_epi16(step2[1], step2[6]);
1386+ lstep2[4] = _mm256_unpacklo_epi16(step2[2], step2[5]);
1387+ lstep2[5] = _mm256_unpackhi_epi16(step2[2], step2[5]);
1388+ lstep2[6] = _mm256_unpacklo_epi16(step2[3], step2[4]);
1389+ lstep2[7] = _mm256_unpackhi_epi16(step2[3], step2[4]);
1390+
1391+ lstep3[0] = _mm256_madd_epi16(lstep2[0], kOne);
1392+ lstep3[1] = _mm256_madd_epi16(lstep2[1], kOne);
1393+ lstep3[2] = _mm256_madd_epi16(lstep2[2], kOne);
1394+ lstep3[3] = _mm256_madd_epi16(lstep2[3], kOne);
1395+ lstep3[4] = _mm256_madd_epi16(lstep2[4], kOne);
1396+ lstep3[5] = _mm256_madd_epi16(lstep2[5], kOne);
1397+ lstep3[6] = _mm256_madd_epi16(lstep2[6], kOne);
1398+ lstep3[7] = _mm256_madd_epi16(lstep2[7], kOne);
1399+
1400+ lstep3[8] = _mm256_madd_epi16(lstep2[6], k__pOne_mOne);
1401+ lstep3[9] = _mm256_madd_epi16(lstep2[7], k__pOne_mOne);
1402+ lstep3[10] = _mm256_madd_epi16(lstep2[4], k__pOne_mOne);
1403+ lstep3[11] = _mm256_madd_epi16(lstep2[5], k__pOne_mOne);
1404+ lstep3[12] = _mm256_madd_epi16(lstep2[2], k__pOne_mOne);
1405+ lstep3[13] = _mm256_madd_epi16(lstep2[3], k__pOne_mOne);
1406+ lstep3[14] = _mm256_madd_epi16(lstep2[0], k__pOne_mOne);
1407+ lstep3[15] = _mm256_madd_epi16(lstep2[1], k__pOne_mOne);
14301408 }
14311409 {
14321410 const __m256i s3_10_0 = _mm256_unpacklo_epi16(step2[13], step2[10]);
@@ -1468,126 +1446,76 @@ void FDCT32x32_2D_AVX2(const int16_t *input, int16_t *output_org, int stride) { · 35 unmodified lines
14681446 lstep3[27] = _mm256_srai_epi32(s3_13_5, DCT_CONST_BITS);
14691447 }
14701448 {
1471− lstep2[40] = _mm256_unpacklo_epi16(step2[20], kZero);
1472− lstep2[41] = _mm256_unpackhi_epi16(step2[20], kZero);
1473− lstep2[42] = _mm256_unpacklo_epi16(step2[21], kZero);
1474− lstep2[43] = _mm256_unpackhi_epi16(step2[21], kZero);
1475− lstep2[44] = _mm256_unpacklo_epi16(step2[22], kZero);
1476− lstep2[45] = _mm256_unpackhi_epi16(step2[22], kZero);
1477− lstep2[46] = _mm256_unpacklo_epi16(step2[23], kZero);
1478− lstep2[47] = _mm256_unpackhi_epi16(step2[23], kZero);
1479− lstep2[48] = _mm256_unpacklo_epi16(step2[24], kZero);
1480− lstep2[49] = _mm256_unpackhi_epi16(step2[24], kZero);
1481− lstep2[50] = _mm256_unpacklo_epi16(step2[25], kZero);
1482− lstep2[51] = _mm256_unpackhi_epi16(step2[25], kZero);
1483− lstep2[52] = _mm256_unpacklo_epi16(step2[26], kZero);
1484− lstep2[53] = _mm256_unpackhi_epi16(step2[26], kZero);
1485− lstep2[54] = _mm256_unpacklo_epi16(step2[27], kZero);
1486− lstep2[55] = _mm256_unpackhi_epi16(step2[27], kZero);
1487− lstep2[40] = _mm256_madd_epi16(lstep2[40], kOne);
1488− lstep2[41] = _mm256_madd_epi16(lstep2[41], kOne);
1489− lstep2[42] = _mm256_madd_epi16(lstep2[42], kOne);
1490− lstep2[43] = _mm256_madd_epi16(lstep2[43], kOne);
1491− lstep2[44] = _mm256_madd_epi16(lstep2[44], kOne);
1492− lstep2[45] = _mm256_madd_epi16(lstep2[45], kOne);
1493− lstep2[46] = _mm256_madd_epi16(lstep2[46], kOne);
1494− lstep2[47] = _mm256_madd_epi16(lstep2[47], kOne);
1495− lstep2[48] = _mm256_madd_epi16(lstep2[48], kOne);
1496− lstep2[49] = _mm256_madd_epi16(lstep2[49], kOne);
1497− lstep2[50] = _mm256_madd_epi16(lstep2[50], kOne);
1498− lstep2[51] = _mm256_madd_epi16(lstep2[51], kOne);
1499− lstep2[52] = _mm256_madd_epi16(lstep2[52], kOne);
1500− lstep2[53] = _mm256_madd_epi16(lstep2[53], kOne);
1501− lstep2[54] = _mm256_madd_epi16(lstep2[54], kOne);
1502− lstep2[55] = _mm256_madd_epi16(lstep2[55], kOne);
1503−
1504− lstep1[32] = _mm256_unpacklo_epi16(step1[16], kZero);
1505− lstep1[33] = _mm256_unpackhi_epi16(step1[16], kZero);
1506− lstep1[34] = _mm256_unpacklo_epi16(step1[17], kZero);
1507− lstep1[35] = _mm256_unpackhi_epi16(step1[17], kZero);
1508− lstep1[36] = _mm256_unpacklo_epi16(step1[18], kZero);
1509− lstep1[37] = _mm256_unpackhi_epi16(step1[18], kZero);
1510− lstep1[38] = _mm256_unpacklo_epi16(step1[19], kZero);
1511− lstep1[39] = _mm256_unpackhi_epi16(step1[19], kZero);
1512− lstep1[56] = _mm256_unpacklo_epi16(step1[28], kZero);
1513− lstep1[57] = _mm256_unpackhi_epi16(step1[28], kZero);
1514− lstep1[58] = _mm256_unpacklo_epi16(step1[29], kZero);
1515− lstep1[59] = _mm256_unpackhi_epi16(step1[29], kZero);
1516− lstep1[60] = _mm256_unpacklo_epi16(step1[30], kZero);
1517− lstep1[61] = _mm256_unpackhi_epi16(step1[30], kZero);
1518− lstep1[62] = _mm256_unpacklo_epi16(step1[31], kZero);
1519− lstep1[63] = _mm256_unpackhi_epi16(step1[31], kZero);
1520− lstep1[32] = _mm256_madd_epi16(lstep1[32], kOne);
1521− lstep1[33] = _mm256_madd_epi16(lstep1[33], kOne);
1522− lstep1[34] = _mm256_madd_epi16(lstep1[34], kOne);
1523− lstep1[35] = _mm256_madd_epi16(lstep1[35], kOne);
1524− lstep1[36] = _mm256_madd_epi16(lstep1[36], kOne);
1525− lstep1[37] = _mm256_madd_epi16(lstep1[37], kOne);
1526− lstep1[38] = _mm256_madd_epi16(lstep1[38], kOne);
1527− lstep1[39] = _mm256_madd_epi16(lstep1[39], kOne);
1528− lstep1[56] = _mm256_madd_epi16(lstep1[56], kOne);
1529− lstep1[57] = _mm256_madd_epi16(lstep1[57], kOne);
1530− lstep1[58] = _mm256_madd_epi16(lstep1[58], kOne);
1531− lstep1[59] = _mm256_madd_epi16(lstep1[59], kOne);
1532− lstep1[60] = _mm256_madd_epi16(lstep1[60], kOne);
1533− lstep1[61] = _mm256_madd_epi16(lstep1[61], kOne);
1534− lstep1[62] = _mm256_madd_epi16(lstep1[62], kOne);
1535− lstep1[63] = _mm256_madd_epi16(lstep1[63], kOne);
1536−
1537− lstep3[32] = _mm256_add_epi32(lstep2[46], lstep1[32]);
1538− lstep3[33] = _mm256_add_epi32(lstep2[47], lstep1[33]);
1539−
1540− lstep3[34] = _mm256_add_epi32(lstep2[44], lstep1[34]);
1541− lstep3[35] = _mm256_add_epi32(lstep2[45], lstep1[35]);
1542− lstep3[36] = _mm256_add_epi32(lstep2[42], lstep1[36]);
1543− lstep3[37] = _mm256_add_epi32(lstep2[43], lstep1[37]);
1544− lstep3[38] = _mm256_add_epi32(lstep2[40], lstep1[38]);
1545− lstep3[39] = _mm256_add_epi32(lstep2[41], lstep1[39]);
1546− lstep3[40] = _mm256_sub_epi32(lstep1[38], lstep2[40]);
1547− lstep3[41] = _mm256_sub_epi32(lstep1[39], lstep2[41]);
1548− lstep3[42] = _mm256_sub_epi32(lstep1[36], lstep2[42]);
1549− lstep3[43] = _mm256_sub_epi32(lstep1[37], lstep2[43]);
1550− lstep3[44] = _mm256_sub_epi32(lstep1[34], lstep2[44]);
1551− lstep3[45] = _mm256_sub_epi32(lstep1[35], lstep2[45]);
1552− lstep3[46] = _mm256_sub_epi32(lstep1[32], lstep2[46]);
1553− lstep3[47] = _mm256_sub_epi32(lstep1[33], lstep2[47]);
1554− lstep3[48] = _mm256_sub_epi32(lstep1[62], lstep2[48]);
1555− lstep3[49] = _mm256_sub_epi32(lstep1[63], lstep2[49]);
1556− lstep3[50] = _mm256_sub_epi32(lstep1[60], lstep2[50]);
1557− lstep3[51] = _mm256_sub_epi32(lstep1[61], lstep2[51]);
1558− lstep3[52] = _mm256_sub_epi32(lstep1[58], lstep2[52]);
1559− lstep3[53] = _mm256_sub_epi32(lstep1[59], lstep2[53]);
1560− lstep3[54] = _mm256_sub_epi32(lstep1[56], lstep2[54]);
1561− lstep3[55] = _mm256_sub_epi32(lstep1[57], lstep2[55]);
1562− lstep3[56] = _mm256_add_epi32(lstep2[54], lstep1[56]);
1563− lstep3[57] = _mm256_add_epi32(lstep2[55], lstep1[57]);
1564− lstep3[58] = _mm256_add_epi32(lstep2[52], lstep1[58]);
1565− lstep3[59] = _mm256_add_epi32(lstep2[53], lstep1[59]);
1566− lstep3[60] = _mm256_add_epi32(lstep2[50], lstep1[60]);
1567− lstep3[61] = _mm256_add_epi32(lstep2[51], lstep1[61]);
1568− lstep3[62] = _mm256_add_epi32(lstep2[48], lstep1[62]);
1569− lstep3[63] = _mm256_add_epi32(lstep2[49], lstep1[63]);
1449+ lstep1[32] = _mm256_unpacklo_epi16(step1[16], step2[23]);
1450+ lstep1[33] = _mm256_unpackhi_epi16(step1[16], step2[23]);
1451+ lstep1[34] = _mm256_unpacklo_epi16(step1[17], step2[22]);
1452+ lstep1[35] = _mm256_unpackhi_epi16(step1[17], step2[22]);
1453+ lstep1[36] = _mm256_unpacklo_epi16(step1[18], step2[21]);
1454+ lstep1[37] = _mm256_unpackhi_epi16(step1[18], step2[21]);
1455+ lstep1[38] = _mm256_unpacklo_epi16(step1[19], step2[20]);
1456+ lstep1[39] = _mm256_unpackhi_epi16(step1[19], step2[20]);
1457+
1458+ lstep1[56] = _mm256_unpacklo_epi16(step1[28], step2[27]);
1459+ lstep1[57] = _mm256_unpackhi_epi16(step1[28], step2[27]);
1460+ lstep1[58] = _mm256_unpacklo_epi16(step1[29], step2[26]);
1461+ lstep1[59] = _mm256_unpackhi_epi16(step1[29], step2[26]);
1462+ lstep1[60] = _mm256_unpacklo_epi16(step1[30], step2[25]);
1463+ lstep1[61] = _mm256_unpackhi_epi16(step1[30], step2[25]);
1464+ lstep1[62] = _mm256_unpacklo_epi16(step1[31], step2[24]);
1465+ lstep1[63] = _mm256_unpackhi_epi16(step1[31], step2[24]);
1466+
1467+ lstep3[32] = _mm256_madd_epi16(lstep1[32], kOne);
1468+ lstep3[33] = _mm256_madd_epi16(lstep1[33], kOne);
1469+ lstep3[34] = _mm256_madd_epi16(lstep1[34], kOne);
1470+ lstep3[35] = _mm256_madd_epi16(lstep1[35], kOne);
1471+ lstep3[36] = _mm256_madd_epi16(lstep1[36], kOne);
1472+ lstep3[37] = _mm256_madd_epi16(lstep1[37], kOne);
1473+ lstep3[38] = _mm256_madd_epi16(lstep1[38], kOne);
1474+ lstep3[39] = _mm256_madd_epi16(lstep1[39], kOne);
1475+
1476+ lstep3[40] = _mm256_madd_epi16(lstep1[38], k__pOne_mOne);
1477+ lstep3[41] = _mm256_madd_epi16(lstep1[39], k__pOne_mOne);
1478+ lstep3[42] = _mm256_madd_epi16(lstep1[36], k__pOne_mOne);
1479+ lstep3[43] = _mm256_madd_epi16(lstep1[37], k__pOne_mOne);
1480+ lstep3[44] = _mm256_madd_epi16(lstep1[34], k__pOne_mOne);
1481+ lstep3[45] = _mm256_madd_epi16(lstep1[35], k__pOne_mOne);
1482+ lstep3[46] = _mm256_madd_epi16(lstep1[32], k__pOne_mOne);
1483+ lstep3[47] = _mm256_madd_epi16(lstep1[33], k__pOne_mOne);
1484+
1485+ lstep3[48] = _mm256_madd_epi16(lstep1[62], k__pOne_mOne);
1486+ lstep3[49] = _mm256_madd_epi16(lstep1[63], k__pOne_mOne);
1487+ lstep3[50] = _mm256_madd_epi16(lstep1[60], k__pOne_mOne);
1488+ lstep3[51] = _mm256_madd_epi16(lstep1[61], k__pOne_mOne);
1489+ lstep3[52] = _mm256_madd_epi16(lstep1[58], k__pOne_mOne);
1490+ lstep3[53] = _mm256_madd_epi16(lstep1[59], k__pOne_mOne);
1491+ lstep3[54] = _mm256_madd_epi16(lstep1[56], k__pOne_mOne);
1492+ lstep3[55] = _mm256_madd_epi16(lstep1[57], k__pOne_mOne);
1493+
1494+ lstep3[56] = _mm256_madd_epi16(lstep1[56], kOne);
1495+ lstep3[57] = _mm256_madd_epi16(lstep1[57], kOne);
1496+ lstep3[58] = _mm256_madd_epi16(lstep1[58], kOne);
1497+ lstep3[59] = _mm256_madd_epi16(lstep1[59], kOne);
1498+ lstep3[60] = _mm256_madd_epi16(lstep1[60], kOne);
1499+ lstep3[61] = _mm256_madd_epi16(lstep1[61], kOne);
1500+ lstep3[62] = _mm256_madd_epi16(lstep1[62], kOne);
1501+ lstep3[63] = _mm256_madd_epi16(lstep1[63], kOne);
15701502 }
15711503
15721504 // stage 4
15731505 {
1574− // expanding to 32-bit length priori to addition operations
1575− lstep2[16] = _mm256_unpacklo_epi16(step2[8], kZero);
1576− lstep2[17] = _mm256_unpackhi_epi16(step2[8], kZero);
1577− lstep2[18] = _mm256_unpacklo_epi16(step2[9], kZero);
1578− lstep2[19] = _mm256_unpackhi_epi16(step2[9], kZero);
1579− lstep2[28] = _mm256_unpacklo_epi16(step2[14], kZero);
1580− lstep2[29] = _mm256_unpackhi_epi16(step2[14], kZero);
1581− lstep2[30] = _mm256_unpacklo_epi16(step2[15], kZero);
1582− lstep2[31] = _mm256_unpackhi_epi16(step2[15], kZero);
1583− lstep2[16] = _mm256_madd_epi16(lstep2[16], kOne);
1584− lstep2[17] = _mm256_madd_epi16(lstep2[17], kOne);
1585− lstep2[18] = _mm256_madd_epi16(lstep2[18], kOne);
1586− lstep2[19] = _mm256_madd_epi16(lstep2[19], kOne);
1587− lstep2[28] = _mm256_madd_epi16(lstep2[28], kOne);
1588− lstep2[29] = _mm256_madd_epi16(lstep2[29], kOne);
1589− lstep2[30] = _mm256_madd_epi16(lstep2[30], kOne);
1590− lstep2[31] = _mm256_madd_epi16(lstep2[31], kOne);
1506+ // expanding to 32-bit length prior to addition operations
1507+ sign[0] = _mm256_cmpgt_epi16(kZero, step2[8]);
1508+ sign[1] = _mm256_cmpgt_epi16(kZero, step2[9]);
1509+ sign[2] = _mm256_cmpgt_epi16(kZero, step2[14]);
1510+ sign[3] = _mm256_cmpgt_epi16(kZero, step2[15]);
1511+ lstep2[16] = _mm256_unpacklo_epi16(step2[8], sign[0]);
1512+ lstep2[17] = _mm256_unpackhi_epi16(step2[8], sign[0]);
1513+ lstep2[18] = _mm256_unpacklo_epi16(step2[9], sign[1]);
1514+ lstep2[19] = _mm256_unpackhi_epi16(step2[9], sign[1]);
1515+ lstep2[28] = _mm256_unpacklo_epi16(step2[14], sign[2]);
1516+ lstep2[29] = _mm256_unpackhi_epi16(step2[14], sign[2]);
1517+ lstep2[30] = _mm256_unpacklo_epi16(step2[15], sign[3]);
1518+ lstep2[31] = _mm256_unpackhi_epi16(step2[15], sign[3]);
15911519
15921520 lstep1[0] = _mm256_add_epi32(lstep3[6], lstep3[0]);
15931521 lstep1[1] = _mm256_add_epi32(lstep3[7], lstep3[1]);
vpx_dsp/x86/fwd_dct32x32_impl_sse2.h+95−166
@@ -101,6 +101,7 @@ void FDCT32x32_2D(const int16_t *input, tran_low_t *output_org, int stride) { · 100 unmodified lines
101101 const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING);
102102 const __m128i kZero = _mm_set1_epi16(0);
103103 const __m128i kOne = _mm_set1_epi16(1);
104+
104105 // Do the two transform/transpose passes
105106 int pass;
106107 #if DCT_HIGH_BIT_DEPTH
@@ -1508,59 +1509,37 @@ void FDCT32x32_2D(const int16_t *input, tran_low_t *output_org, int stride) { · 1401 unmodified lines
15081509 __m128i lstep1[64], lstep2[64], lstep3[64];
15091510 __m128i u[32], v[32], sign[16];
15101511 const __m128i K32One = _mm_set_epi32(1, 1, 1, 1);
1512+ const __m128i k__pOne_mOne = pair_set_epi16(1, -1);
15111513 // start using 32-bit operations
15121514 // stage 3
15131515 {
1514− // expanding to 32-bit length priori to addition operations
1515− lstep2[0] = _mm_unpacklo_epi16(step2[0], kZero);
1516− lstep2[1] = _mm_unpackhi_epi16(step2[0], kZero);
1517− lstep2[2] = _mm_unpacklo_epi16(step2[1], kZero);
1518− lstep2[3] = _mm_unpackhi_epi16(step2[1], kZero);
1519− lstep2[4] = _mm_unpacklo_epi16(step2[2], kZero);
1520− lstep2[5] = _mm_unpackhi_epi16(step2[2], kZero);
1521− lstep2[6] = _mm_unpacklo_epi16(step2[3], kZero);
1522− lstep2[7] = _mm_unpackhi_epi16(step2[3], kZero);
1523− lstep2[8] = _mm_unpacklo_epi16(step2[4], kZero);
1524− lstep2[9] = _mm_unpackhi_epi16(step2[4], kZero);
1525− lstep2[10] = _mm_unpacklo_epi16(step2[5], kZero);
1526− lstep2[11] = _mm_unpackhi_epi16(step2[5], kZero);
1527− lstep2[12] = _mm_unpacklo_epi16(step2[6], kZero);
1528− lstep2[13] = _mm_unpackhi_epi16(step2[6], kZero);
1529− lstep2[14] = _mm_unpacklo_epi16(step2[7], kZero);
1530− lstep2[15] = _mm_unpackhi_epi16(step2[7], kZero);
1531− lstep2[0] = _mm_madd_epi16(lstep2[0], kOne);
1532− lstep2[1] = _mm_madd_epi16(lstep2[1], kOne);
1533− lstep2[2] = _mm_madd_epi16(lstep2[2], kOne);
1534− lstep2[3] = _mm_madd_epi16(lstep2[3], kOne);
1535− lstep2[4] = _mm_madd_epi16(lstep2[4], kOne);
1536− lstep2[5] = _mm_madd_epi16(lstep2[5], kOne);
1537− lstep2[6] = _mm_madd_epi16(lstep2[6], kOne);
1538− lstep2[7] = _mm_madd_epi16(lstep2[7], kOne);
1539− lstep2[8] = _mm_madd_epi16(lstep2[8], kOne);
1540− lstep2[9] = _mm_madd_epi16(lstep2[9], kOne);
1541− lstep2[10] = _mm_madd_epi16(lstep2[10], kOne);
1542− lstep2[11] = _mm_madd_epi16(lstep2[11], kOne);
1543− lstep2[12] = _mm_madd_epi16(lstep2[12], kOne);
1544− lstep2[13] = _mm_madd_epi16(lstep2[13], kOne);
1545− lstep2[14] = _mm_madd_epi16(lstep2[14], kOne);
1546− lstep2[15] = _mm_madd_epi16(lstep2[15], kOne);
1547−
1548− lstep3[0] = _mm_add_epi32(lstep2[14], lstep2[0]);
1549− lstep3[1] = _mm_add_epi32(lstep2[15], lstep2[1]);
1550− lstep3[2] = _mm_add_epi32(lstep2[12], lstep2[2]);
1551− lstep3[3] = _mm_add_epi32(lstep2[13], lstep2[3]);
1552− lstep3[4] = _mm_add_epi32(lstep2[10], lstep2[4]);
1553− lstep3[5] = _mm_add_epi32(lstep2[11], lstep2[5]);
1554− lstep3[6] = _mm_add_epi32(lstep2[8], lstep2[6]);
1555− lstep3[7] = _mm_add_epi32(lstep2[9], lstep2[7]);
1556− lstep3[8] = _mm_sub_epi32(lstep2[6], lstep2[8]);
1557− lstep3[9] = _mm_sub_epi32(lstep2[7], lstep2[9]);
1558− lstep3[10] = _mm_sub_epi32(lstep2[4], lstep2[10]);
1559− lstep3[11] = _mm_sub_epi32(lstep2[5], lstep2[11]);
1560− lstep3[12] = _mm_sub_epi32(lstep2[2], lstep2[12]);
1561− lstep3[13] = _mm_sub_epi32(lstep2[3], lstep2[13]);
1562− lstep3[14] = _mm_sub_epi32(lstep2[0], lstep2[14]);
1563− lstep3[15] = _mm_sub_epi32(lstep2[1], lstep2[15]);
1516+ // expanding to 32-bit length while adding and subtracting
1517+ lstep2[0] = _mm_unpacklo_epi16(step2[0], step2[7]);
1518+ lstep2[1] = _mm_unpackhi_epi16(step2[0], step2[7]);
1519+ lstep2[2] = _mm_unpacklo_epi16(step2[1], step2[6]);
1520+ lstep2[3] = _mm_unpackhi_epi16(step2[1], step2[6]);
1521+ lstep2[4] = _mm_unpacklo_epi16(step2[2], step2[5]);
1522+ lstep2[5] = _mm_unpackhi_epi16(step2[2], step2[5]);
1523+ lstep2[6] = _mm_unpacklo_epi16(step2[3], step2[4]);
1524+ lstep2[7] = _mm_unpackhi_epi16(step2[3], step2[4]);
1525+
1526+ lstep3[0] = _mm_madd_epi16(lstep2[0], kOne);
1527+ lstep3[1] = _mm_madd_epi16(lstep2[1], kOne);
1528+ lstep3[2] = _mm_madd_epi16(lstep2[2], kOne);
1529+ lstep3[3] = _mm_madd_epi16(lstep2[3], kOne);
1530+ lstep3[4] = _mm_madd_epi16(lstep2[4], kOne);
1531+ lstep3[5] = _mm_madd_epi16(lstep2[5], kOne);
1532+ lstep3[6] = _mm_madd_epi16(lstep2[6], kOne);
1533+ lstep3[7] = _mm_madd_epi16(lstep2[7], kOne);
1534+
1535+ lstep3[8] = _mm_madd_epi16(lstep2[6], k__pOne_mOne);
1536+ lstep3[9] = _mm_madd_epi16(lstep2[7], k__pOne_mOne);
1537+ lstep3[10] = _mm_madd_epi16(lstep2[4], k__pOne_mOne);
1538+ lstep3[11] = _mm_madd_epi16(lstep2[5], k__pOne_mOne);
1539+ lstep3[12] = _mm_madd_epi16(lstep2[2], k__pOne_mOne);
1540+ lstep3[13] = _mm_madd_epi16(lstep2[3], k__pOne_mOne);
1541+ lstep3[14] = _mm_madd_epi16(lstep2[0], k__pOne_mOne);
1542+ lstep3[15] = _mm_madd_epi16(lstep2[1], k__pOne_mOne);
15641543 }
15651544 {
15661545 const __m128i s3_10_0 = _mm_unpacklo_epi16(step2[13], step2[10]);
@@ -1594,126 +1573,76 @@ void FDCT32x32_2D(const int16_t *input, tran_low_t *output_org, int stride) { · 27 unmodified lines
15941573 lstep3[27] = _mm_srai_epi32(s3_13_5, DCT_CONST_BITS);
15951574 }
15961575 {
1597− lstep2[40] = _mm_unpacklo_epi16(step2[20], kZero);
1598− lstep2[41] = _mm_unpackhi_epi16(step2[20], kZero);
1599− lstep2[42] = _mm_unpacklo_epi16(step2[21], kZero);
1600− lstep2[43] = _mm_unpackhi_epi16(step2[21], kZero);
1601− lstep2[44] = _mm_unpacklo_epi16(step2[22], kZero);
1602− lstep2[45] = _mm_unpackhi_epi16(step2[22], kZero);
1603− lstep2[46] = _mm_unpacklo_epi16(step2[23], kZero);
1604− lstep2[47] = _mm_unpackhi_epi16(step2[23], kZero);
1605− lstep2[48] = _mm_unpacklo_epi16(step2[24], kZero);
1606− lstep2[49] = _mm_unpackhi_epi16(step2[24], kZero);
1607− lstep2[50] = _mm_unpacklo_epi16(step2[25], kZero);
1608− lstep2[51] = _mm_unpackhi_epi16(step2[25], kZero);
1609− lstep2[52] = _mm_unpacklo_epi16(step2[26], kZero);
1610− lstep2[53] = _mm_unpackhi_epi16(step2[26], kZero);
1611− lstep2[54] = _mm_unpacklo_epi16(step2[27], kZero);
1612− lstep2[55] = _mm_unpackhi_epi16(step2[27], kZero);
1613− lstep2[40] = _mm_madd_epi16(lstep2[40], kOne);
1614− lstep2[41] = _mm_madd_epi16(lstep2[41], kOne);
1615− lstep2[42] = _mm_madd_epi16(lstep2[42], kOne);
1616− lstep2[43] = _mm_madd_epi16(lstep2[43], kOne);
1617− lstep2[44] = _mm_madd_epi16(lstep2[44], kOne);
1618− lstep2[45] = _mm_madd_epi16(lstep2[45], kOne);
1619− lstep2[46] = _mm_madd_epi16(lstep2[46], kOne);
1620− lstep2[47] = _mm_madd_epi16(lstep2[47], kOne);
1621− lstep2[48] = _mm_madd_epi16(lstep2[48], kOne);
1622− lstep2[49] = _mm_madd_epi16(lstep2[49], kOne);
1623− lstep2[50] = _mm_madd_epi16(lstep2[50], kOne);
1624− lstep2[51] = _mm_madd_epi16(lstep2[51], kOne);
1625− lstep2[52] = _mm_madd_epi16(lstep2[52], kOne);
1626− lstep2[53] = _mm_madd_epi16(lstep2[53], kOne);
1627− lstep2[54] = _mm_madd_epi16(lstep2[54], kOne);
1628− lstep2[55] = _mm_madd_epi16(lstep2[55], kOne);
1629−
1630− lstep1[32] = _mm_unpacklo_epi16(step1[16], kZero);
1631− lstep1[33] = _mm_unpackhi_epi16(step1[16], kZero);
1632− lstep1[34] = _mm_unpacklo_epi16(step1[17], kZero);
1633− lstep1[35] = _mm_unpackhi_epi16(step1[17], kZero);
1634− lstep1[36] = _mm_unpacklo_epi16(step1[18], kZero);
1635− lstep1[37] = _mm_unpackhi_epi16(step1[18], kZero);
1636− lstep1[38] = _mm_unpacklo_epi16(step1[19], kZero);
1637− lstep1[39] = _mm_unpackhi_epi16(step1[19], kZero);
1638− lstep1[56] = _mm_unpacklo_epi16(step1[28], kZero);
1639− lstep1[57] = _mm_unpackhi_epi16(step1[28], kZero);
1640− lstep1[58] = _mm_unpacklo_epi16(step1[29], kZero);
1641− lstep1[59] = _mm_unpackhi_epi16(step1[29], kZero);
1642− lstep1[60] = _mm_unpacklo_epi16(step1[30], kZero);
1643− lstep1[61] = _mm_unpackhi_epi16(step1[30], kZero);
1644− lstep1[62] = _mm_unpacklo_epi16(step1[31], kZero);
1645− lstep1[63] = _mm_unpackhi_epi16(step1[31], kZero);
1646− lstep1[32] = _mm_madd_epi16(lstep1[32], kOne);
1647− lstep1[33] = _mm_madd_epi16(lstep1[33], kOne);
1648− lstep1[34] = _mm_madd_epi16(lstep1[34], kOne);
1649− lstep1[35] = _mm_madd_epi16(lstep1[35], kOne);
1650− lstep1[36] = _mm_madd_epi16(lstep1[36], kOne);
1651− lstep1[37] = _mm_madd_epi16(lstep1[37], kOne);
1652− lstep1[38] = _mm_madd_epi16(lstep1[38], kOne);
1653− lstep1[39] = _mm_madd_epi16(lstep1[39], kOne);
1654− lstep1[56] = _mm_madd_epi16(lstep1[56], kOne);
1655− lstep1[57] = _mm_madd_epi16(lstep1[57], kOne);
1656− lstep1[58] = _mm_madd_epi16(lstep1[58], kOne);
1657− lstep1[59] = _mm_madd_epi16(lstep1[59], kOne);
1658− lstep1[60] = _mm_madd_epi16(lstep1[60], kOne);
1659− lstep1[61] = _mm_madd_epi16(lstep1[61], kOne);
1660− lstep1[62] = _mm_madd_epi16(lstep1[62], kOne);
1661− lstep1[63] = _mm_madd_epi16(lstep1[63], kOne);
1662−
1663− lstep3[32] = _mm_add_epi32(lstep2[46], lstep1[32]);
1664− lstep3[33] = _mm_add_epi32(lstep2[47], lstep1[33]);
1665−
1666− lstep3[34] = _mm_add_epi32(lstep2[44], lstep1[34]);
1667− lstep3[35] = _mm_add_epi32(lstep2[45], lstep1[35]);
1668− lstep3[36] = _mm_add_epi32(lstep2[42], lstep1[36]);
1669− lstep3[37] = _mm_add_epi32(lstep2[43], lstep1[37]);
1670− lstep3[38] = _mm_add_epi32(lstep2[40], lstep1[38]);
1671− lstep3[39] = _mm_add_epi32(lstep2[41], lstep1[39]);
1672− lstep3[40] = _mm_sub_epi32(lstep1[38], lstep2[40]);
1673− lstep3[41] = _mm_sub_epi32(lstep1[39], lstep2[41]);
1674− lstep3[42] = _mm_sub_epi32(lstep1[36], lstep2[42]);
1675− lstep3[43] = _mm_sub_epi32(lstep1[37], lstep2[43]);
1676− lstep3[44] = _mm_sub_epi32(lstep1[34], lstep2[44]);
1677− lstep3[45] = _mm_sub_epi32(lstep1[35], lstep2[45]);
1678− lstep3[46] = _mm_sub_epi32(lstep1[32], lstep2[46]);
1679− lstep3[47] = _mm_sub_epi32(lstep1[33], lstep2[47]);
1680− lstep3[48] = _mm_sub_epi32(lstep1[62], lstep2[48]);
1681− lstep3[49] = _mm_sub_epi32(lstep1[63], lstep2[49]);
1682− lstep3[50] = _mm_sub_epi32(lstep1[60], lstep2[50]);
1683− lstep3[51] = _mm_sub_epi32(lstep1[61], lstep2[51]);
1684− lstep3[52] = _mm_sub_epi32(lstep1[58], lstep2[52]);
1685− lstep3[53] = _mm_sub_epi32(lstep1[59], lstep2[53]);
1686− lstep3[54] = _mm_sub_epi32(lstep1[56], lstep2[54]);
1687− lstep3[55] = _mm_sub_epi32(lstep1[57], lstep2[55]);
1688− lstep3[56] = _mm_add_epi32(lstep2[54], lstep1[56]);
1689− lstep3[57] = _mm_add_epi32(lstep2[55], lstep1[57]);
1690− lstep3[58] = _mm_add_epi32(lstep2[52], lstep1[58]);
1691− lstep3[59] = _mm_add_epi32(lstep2[53], lstep1[59]);
1692− lstep3[60] = _mm_add_epi32(lstep2[50], lstep1[60]);
1693− lstep3[61] = _mm_add_epi32(lstep2[51], lstep1[61]);
1694− lstep3[62] = _mm_add_epi32(lstep2[48], lstep1[62]);
1695− lstep3[63] = _mm_add_epi32(lstep2[49], lstep1[63]);
1576+ lstep1[32] = _mm_unpacklo_epi16(step1[16], step2[23]);
1577+ lstep1[33] = _mm_unpackhi_epi16(step1[16], step2[23]);
1578+ lstep1[34] = _mm_unpacklo_epi16(step1[17], step2[22]);
1579+ lstep1[35] = _mm_unpackhi_epi16(step1[17], step2[22]);
1580+ lstep1[36] = _mm_unpacklo_epi16(step1[18], step2[21]);
1581+ lstep1[37] = _mm_unpackhi_epi16(step1[18], step2[21]);
1582+ lstep1[38] = _mm_unpacklo_epi16(step1[19], step2[20]);
1583+ lstep1[39] = _mm_unpackhi_epi16(step1[19], step2[20]);
1584+
1585+ lstep1[56] = _mm_unpacklo_epi16(step1[28], step2[27]);
1586+ lstep1[57] = _mm_unpackhi_epi16(step1[28], step2[27]);
1587+ lstep1[58] = _mm_unpacklo_epi16(step1[29], step2[26]);
1588+ lstep1[59] = _mm_unpackhi_epi16(step1[29], step2[26]);
1589+ lstep1[60] = _mm_unpacklo_epi16(step1[30], step2[25]);
1590+ lstep1[61] = _mm_unpackhi_epi16(step1[30], step2[25]);
1591+ lstep1[62] = _mm_unpacklo_epi16(step1[31], step2[24]);
1592+ lstep1[63] = _mm_unpackhi_epi16(step1[31], step2[24]);
1593+
1594+ lstep3[32] = _mm_madd_epi16(lstep1[32], kOne);
1595+ lstep3[33] = _mm_madd_epi16(lstep1[33], kOne);
1596+ lstep3[34] = _mm_madd_epi16(lstep1[34], kOne);
1597+ lstep3[35] = _mm_madd_epi16(lstep1[35], kOne);
1598+ lstep3[36] = _mm_madd_epi16(lstep1[36], kOne);
1599+ lstep3[37] = _mm_madd_epi16(lstep1[37], kOne);
1600+ lstep3[38] = _mm_madd_epi16(lstep1[38], kOne);
1601+ lstep3[39] = _mm_madd_epi16(lstep1[39], kOne);
1602+
1603+ lstep3[40] = _mm_madd_epi16(lstep1[38], k__pOne_mOne);
1604+ lstep3[41] = _mm_madd_epi16(lstep1[39], k__pOne_mOne);
1605+ lstep3[42] = _mm_madd_epi16(lstep1[36], k__pOne_mOne);
1606+ lstep3[43] = _mm_madd_epi16(lstep1[37], k__pOne_mOne);
1607+ lstep3[44] = _mm_madd_epi16(lstep1[34], k__pOne_mOne);
1608+ lstep3[45] = _mm_madd_epi16(lstep1[35], k__pOne_mOne);
1609+ lstep3[46] = _mm_madd_epi16(lstep1[32], k__pOne_mOne);
1610+ lstep3[47] = _mm_madd_epi16(lstep1[33], k__pOne_mOne);
1611+
1612+ lstep3[48] = _mm_madd_epi16(lstep1[62], k__pOne_mOne);
1613+ lstep3[49] = _mm_madd_epi16(lstep1[63], k__pOne_mOne);
1614+ lstep3[50] = _mm_madd_epi16(lstep1[60], k__pOne_mOne);
1615+ lstep3[51] = _mm_madd_epi16(lstep1[61], k__pOne_mOne);
1616+ lstep3[52] = _mm_madd_epi16(lstep1[58], k__pOne_mOne);
1617+ lstep3[53] = _mm_madd_epi16(lstep1[59], k__pOne_mOne);
1618+ lstep3[54] = _mm_madd_epi16(lstep1[56], k__pOne_mOne);
1619+ lstep3[55] = _mm_madd_epi16(lstep1[57], k__pOne_mOne);
1620+
1621+ lstep3[56] = _mm_madd_epi16(lstep1[56], kOne);
1622+ lstep3[57] = _mm_madd_epi16(lstep1[57], kOne);
1623+ lstep3[58] = _mm_madd_epi16(lstep1[58], kOne);
1624+ lstep3[59] = _mm_madd_epi16(lstep1[59], kOne);
1625+ lstep3[60] = _mm_madd_epi16(lstep1[60], kOne);
1626+ lstep3[61] = _mm_madd_epi16(lstep1[61], kOne);
1627+ lstep3[62] = _mm_madd_epi16(lstep1[62], kOne);
1628+ lstep3[63] = _mm_madd_epi16(lstep1[63], kOne);
16961629 }
16971630
16981631 // stage 4
16991632 {
1700− // expanding to 32-bit length priori to addition operations
1701− lstep2[16] = _mm_unpacklo_epi16(step2[8], kZero);
1702− lstep2[17] = _mm_unpackhi_epi16(step2[8], kZero);
1703− lstep2[18] = _mm_unpacklo_epi16(step2[9], kZero);
1704− lstep2[19] = _mm_unpackhi_epi16(step2[9], kZero);
1705− lstep2[28] = _mm_unpacklo_epi16(step2[14], kZero);
1706− lstep2[29] = _mm_unpackhi_epi16(step2[14], kZero);
1707− lstep2[30] = _mm_unpacklo_epi16(step2[15], kZero);
1708− lstep2[31] = _mm_unpackhi_epi16(step2[15], kZero);
1709− lstep2[16] = _mm_madd_epi16(lstep2[16], kOne);
1710− lstep2[17] = _mm_madd_epi16(lstep2[17], kOne);
1711− lstep2[18] = _mm_madd_epi16(lstep2[18], kOne);
1712− lstep2[19] = _mm_madd_epi16(lstep2[19], kOne);
1713− lstep2[28] = _mm_madd_epi16(lstep2[28], kOne);
1714− lstep2[29] = _mm_madd_epi16(lstep2[29], kOne);
1715− lstep2[30] = _mm_madd_epi16(lstep2[30], kOne);
1716− lstep2[31] = _mm_madd_epi16(lstep2[31], kOne);
1633+ // expanding to 32-bit length prior to addition operations
1634+ sign[0] = _mm_cmpgt_epi16(kZero, step2[8]);
1635+ sign[1] = _mm_cmpgt_epi16(kZero, step2[9]);
1636+ sign[2] = _mm_cmpgt_epi16(kZero, step2[14]);
1637+ sign[3] = _mm_cmpgt_epi16(kZero, step2[15]);
1638+ lstep2[16] = _mm_unpacklo_epi16(step2[8], sign[0]);
1639+ lstep2[17] = _mm_unpackhi_epi16(step2[8], sign[0]);
1640+ lstep2[18] = _mm_unpacklo_epi16(step2[9], sign[1]);
1641+ lstep2[19] = _mm_unpackhi_epi16(step2[9], sign[1]);
1642+ lstep2[28] = _mm_unpacklo_epi16(step2[14], sign[2]);
1643+ lstep2[29] = _mm_unpackhi_epi16(step2[14], sign[2]);
1644+ lstep2[30] = _mm_unpacklo_epi16(step2[15], sign[3]);
1645+ lstep2[31] = _mm_unpackhi_epi16(step2[15], sign[3]);
17171646
17181647 lstep1[0] = _mm_add_epi32(lstep3[6], lstep3[0]);
17191648 lstep1[1] = _mm_add_epi32(lstep3[7], lstep3[1]);

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t