The idct32x32 function actually pushed d8-d15 onto the stack even though it didn't clobber them; there are plenty of registers that can be used to allow keeping all the idct coefficients in registers without having to reload different subsets of them at different stages in the transform. After this, we still can skip pushing d12-d15. Before: vp9_inv_dct_dct_32x32_sub32_add_neon: 8128.3 After: vp9_inv_dct_dct_32x32_sub32_add_neon: 8053.3
Unified · Split
| @@ -1123,18 +1123,14 @@ endfunc · 1122 unmodified lines | |||
| 1123 | .endm | 1123 | .endm |
| 1124 | 1124 | ||
| 1125 | function idct32_odd | 1125 | function idct32_odd |
| 1126 | ld1 {v0.8h,v1.8h}, [x11] | 1126 | dmbutterfly v16, v31, v8.h[0], v8.h[1], v4, v5, v6, v7 // v16 = t16a, v31 = t31a |
| 1127 | 1127 | dmbutterfly v24, v23, v8.h[2], v8.h[3], v4, v5, v6, v7 // v24 = t17a, v23 = t30a | |
| 1128 | dmbutterfly v16, v31, v0.h[0], v0.h[1], v4, v5, v6, v7 // v16 = t16a, v31 = t31a | 1128 | dmbutterfly v20, v27, v8.h[4], v8.h[5], v4, v5, v6, v7 // v20 = t18a, v27 = t29a |
| 1129 | dmbutterfly v24, v23, v0.h[2], v0.h[3], v4, v5, v6, v7 // v24 = t17a, v23 = t30a | 1129 | dmbutterfly v28, v19, v8.h[6], v8.h[7], v4, v5, v6, v7 // v28 = t19a, v19 = t28a |
| 1130 | dmbutterfly v20, v27, v0.h[4], v0.h[5], v4, v5, v6, v7 // v20 = t18a, v27 = t29a | 1130 | dmbutterfly v18, v29, v9.h[0], v9.h[1], v4, v5, v6, v7 // v18 = t20a, v29 = t27a |
| 1131 | dmbutterfly v28, v19, v0.h[6], v0.h[7], v4, v5, v6, v7 // v28 = t19a, v19 = t28a | 1131 | dmbutterfly v26, v21, v9.h[2], v9.h[3], v4, v5, v6, v7 // v26 = t21a, v21 = t26a |
| 1132 | dmbutterfly v18, v29, v1.h[0], v1.h[1], v4, v5, v6, v7 // v18 = t20a, v29 = t27a | 1132 | dmbutterfly v22, v25, v9.h[4], v9.h[5], v4, v5, v6, v7 // v22 = t22a, v25 = t25a |
| 1133 | dmbutterfly v26, v21, v1.h[2], v1.h[3], v4, v5, v6, v7 // v26 = t21a, v21 = t26a | 1133 | dmbutterfly v30, v17, v9.h[6], v9.h[7], v4, v5, v6, v7 // v30 = t23a, v17 = t24a |
| 1134 | dmbutterfly v22, v25, v1.h[4], v1.h[5], v4, v5, v6, v7 // v22 = t22a, v25 = t25a | ||
| 1135 | dmbutterfly v30, v17, v1.h[6], v1.h[7], v4, v5, v6, v7 // v30 = t23a, v17 = t24a | ||
| 1136 | |||
| 1137 | ld1 {v0.8h}, [x10] | ||
| 1138 | 1134 | ||
| 1139 | butterfly_8h v4, v24, v16, v24 // v4 = t16, v24 = t17 | 1135 | butterfly_8h v4, v24, v16, v24 // v4 = t16, v24 = t17 |
| 1140 | butterfly_8h v5, v20, v28, v20 // v5 = t19, v20 = t18 | 1136 | butterfly_8h v5, v20, v28, v20 // v5 = t19, v20 = t18 |
| @@ -1153,18 +1149,14 @@ function idct32_odd · 12 unmodified lines | |||
| 1153 | endfunc | 1149 | endfunc |
| 1154 | 1150 | ||
| 1155 | function idct32_odd_half | 1151 | function idct32_odd_half |
| 1156 | ld1 {v0.8h,v1.8h}, [x11] | 1152 | dmbutterfly_h1 v16, v31, v8.h[0], v8.h[1], v4, v5, v6, v7 // v16 = t16a, v31 = t31a |
| 1157 | 1153 | dmbutterfly_h2 v24, v23, v8.h[2], v8.h[3], v4, v5, v6, v7 // v24 = t17a, v23 = t30a | |
| 1158 | dmbutterfly_h1 v16, v31, v0.h[0], v0.h[1], v4, v5, v6, v7 // v16 = t16a, v31 = t31a | 1154 | dmbutterfly_h1 v20, v27, v8.h[4], v8.h[5], v4, v5, v6, v7 // v20 = t18a, v27 = t29a |
| 1159 | dmbutterfly_h2 v24, v23, v0.h[2], v0.h[3], v4, v5, v6, v7 // v24 = t17a, v23 = t30a | 1155 | dmbutterfly_h2 v28, v19, v8.h[6], v8.h[7], v4, v5, v6, v7 // v28 = t19a, v19 = t28a |
| 1160 | dmbutterfly_h1 v20, v27, v0.h[4], v0.h[5], v4, v5, v6, v7 // v20 = t18a, v27 = t29a | 1156 | dmbutterfly_h1 v18, v29, v9.h[0], v9.h[1], v4, v5, v6, v7 // v18 = t20a, v29 = t27a |
| 1161 | dmbutterfly_h2 v28, v19, v0.h[6], v0.h[7], v4, v5, v6, v7 // v28 = t19a, v19 = t28a | 1157 | dmbutterfly_h2 v26, v21, v9.h[2], v9.h[3], v4, v5, v6, v7 // v26 = t21a, v21 = t26a |
| 1162 | dmbutterfly_h1 v18, v29, v1.h[0], v1.h[1], v4, v5, v6, v7 // v18 = t20a, v29 = t27a | 1158 | dmbutterfly_h1 v22, v25, v9.h[4], v9.h[5], v4, v5, v6, v7 // v22 = t22a, v25 = t25a |
| 1163 | dmbutterfly_h2 v26, v21, v1.h[2], v1.h[3], v4, v5, v6, v7 // v26 = t21a, v21 = t26a | 1159 | dmbutterfly_h2 v30, v17, v9.h[6], v9.h[7], v4, v5, v6, v7 // v30 = t23a, v17 = t24a |
| 1164 | dmbutterfly_h1 v22, v25, v1.h[4], v1.h[5], v4, v5, v6, v7 // v22 = t22a, v25 = t25a | ||
| 1165 | dmbutterfly_h2 v30, v17, v1.h[6], v1.h[7], v4, v5, v6, v7 // v30 = t23a, v17 = t24a | ||
| 1166 | |||
| 1167 | ld1 {v0.8h}, [x10] | ||
| 1168 | 1160 | ||
| 1169 | butterfly_8h v4, v24, v16, v24 // v4 = t16, v24 = t17 | 1161 | butterfly_8h v4, v24, v16, v24 // v4 = t16, v24 = t17 |
| 1170 | butterfly_8h v5, v20, v28, v20 // v5 = t19, v20 = t18 | 1162 | butterfly_8h v5, v20, v28, v20 // v5 = t19, v20 = t18 |
| @@ -1183,18 +1175,14 @@ function idct32_odd_half · 12 unmodified lines | |||
| 1183 | endfunc | 1175 | endfunc |
| 1184 | 1176 | ||
| 1185 | function idct32_odd_quarter | 1177 | function idct32_odd_quarter |
| 1186 | ld1 {v0.8h,v1.8h}, [x11] | 1178 | dsmull_h v4, v5, v16, v8.h[0] |
| 1187 | 1179 | dsmull_h v28, v29, v19, v8.h[7] | |
| 1188 | dsmull_h v4, v5, v16, v0.h[0] | 1180 | dsmull_h v30, v31, v16, v8.h[1] |
| 1189 | dsmull_h v28, v29, v19, v0.h[7] | 1181 | dsmull_h v22, v23, v17, v9.h[6] |
| 1190 | dsmull_h v30, v31, v16, v0.h[1] | 1182 | dsmull_h v7, v6, v17, v9.h[7] |
| 1191 | dsmull_h v22, v23, v17, v1.h[6] | 1183 | dsmull_h v26, v27, v19, v8.h[6] |
| 1192 | dsmull_h v7, v6, v17, v1.h[7] | 1184 | dsmull_h v20, v21, v18, v9.h[0] |
| 1193 | dsmull_h v26, v27, v19, v0.h[6] | 1185 | dsmull_h v24, v25, v18, v9.h[1] |
| 1194 | dsmull_h v20, v21, v18, v1.h[0] | ||
| 1195 | dsmull_h v24, v25, v18, v1.h[1] | ||
| 1196 | |||
| 1197 | ld1 {v0.8h}, [x10] | ||
| 1198 | 1186 | ||
| 1199 | neg v28.4s, v28.4s | 1187 | neg v28.4s, v28.4s |
| 1200 | neg v29.4s, v29.4s | 1188 | neg v29.4s, v29.4s |
| @@ -1240,12 +1228,8 @@ endfunc · 39 unmodified lines | |||
| 1240 | // x1 = unused | 1228 | // x1 = unused |
| 1241 | // x2 = src | 1229 | // x2 = src |
| 1242 | // x9 = double input stride | 1230 | // x9 = double input stride |
| 1243 | // x10 = idct_coeffs | ||
| 1244 | // x11 = idct_coeffs + 32 | ||
| 1245 | function idct32_1d_8x32_pass1\suffix\()_neon | 1231 | function idct32_1d_8x32_pass1\suffix\()_neon |
| 1246 | mov x14, x30 | 1232 | mov x14, x30 |
| 1247 | ld1 {v0.8h,v1.8h}, [x10] | ||
| 1248 | |||
| 1249 | movi v2.8h, #0 | 1233 | movi v2.8h, #0 |
| 1250 | 1234 | ||
| 1251 | // v16 = IN(0), v17 = IN(2) ... v31 = IN(30) | 1235 | // v16 = IN(0), v17 = IN(2) ... v31 = IN(30) |
| @@ -1278,14 +1262,14 @@ function idct32_1d_8x32_pass1\suffix\()_neon · 26 unmodified lines | |||
| 1278 | .macro store_rev a, b | 1262 | .macro store_rev a, b |
| 1279 | // There's no rev128 instruction, but we reverse each 64 bit | 1263 | // There's no rev128 instruction, but we reverse each 64 bit |
| 1280 | // half, and then flip them using an ext with 8 bytes offset. | 1264 | // half, and then flip them using an ext with 8 bytes offset. |
| 1281 | rev64 v1.8h, \b | 1265 | rev64 v3.8h, \b |
| 1282 | st1 {\a}, [x0], #16 | 1266 | st1 {\a}, [x0], #16 |
| 1283 | rev64 v0.8h, \a | 1267 | rev64 v2.8h, \a |
| 1284 | ext v1.16b, v1.16b, v1.16b, #8 | 1268 | ext v3.16b, v3.16b, v3.16b, #8 |
| 1285 | st1 {\b}, [x0], #16 | 1269 | st1 {\b}, [x0], #16 |
| 1286 | ext v0.16b, v0.16b, v0.16b, #8 | 1270 | ext v2.16b, v2.16b, v2.16b, #8 |
| 1287 | st1 {v1.8h}, [x0], #16 | 1271 | st1 {v3.8h}, [x0], #16 |
| 1288 | st1 {v0.8h}, [x0], #16 | 1272 | st1 {v2.8h}, [x0], #16 |
| 1289 | .endm | 1273 | .endm |
| 1290 | store_rev v16.8h, v24.8h | 1274 | store_rev v16.8h, v24.8h |
| 1291 | store_rev v17.8h, v25.8h | 1275 | store_rev v17.8h, v25.8h |
| @@ -1339,20 +1323,20 @@ function idct32_1d_8x32_pass1\suffix\()_neon · 47 unmodified lines | |||
| 1339 | // subtracted from the output. | 1323 | // subtracted from the output. |
| 1340 | .macro store_rev a, b | 1324 | .macro store_rev a, b |
| 1341 | ld1 {v4.8h}, [x0] | 1325 | ld1 {v4.8h}, [x0] |
| 1342 | rev64 v1.8h, \b | 1326 | rev64 v3.8h, \b |
| 1343 | add v4.8h, v4.8h, \a | 1327 | add v4.8h, v4.8h, \a |
| 1344 | rev64 v0.8h, \a | 1328 | rev64 v2.8h, \a |
| 1345 | st1 {v4.8h}, [x0], #16 | 1329 | st1 {v4.8h}, [x0], #16 |
| 1346 | ext v1.16b, v1.16b, v1.16b, #8 | 1330 | ext v3.16b, v3.16b, v3.16b, #8 |
| 1347 | ld1 {v5.8h}, [x0] | 1331 | ld1 {v5.8h}, [x0] |
| 1348 | ext v0.16b, v0.16b, v0.16b, #8 | 1332 | ext v2.16b, v2.16b, v2.16b, #8 |
| 1349 | add v5.8h, v5.8h, \b | 1333 | add v5.8h, v5.8h, \b |
| 1350 | st1 {v5.8h}, [x0], #16 | 1334 | st1 {v5.8h}, [x0], #16 |
| 1351 | ld1 {v6.8h}, [x0] | 1335 | ld1 {v6.8h}, [x0] |
| 1352 | sub v6.8h, v6.8h, v1.8h | 1336 | sub v6.8h, v6.8h, v3.8h |
| 1353 | st1 {v6.8h}, [x0], #16 | 1337 | st1 {v6.8h}, [x0], #16 |
| 1354 | ld1 {v7.8h}, [x0] | 1338 | ld1 {v7.8h}, [x0] |
| 1355 | sub v7.8h, v7.8h, v0.8h | 1339 | sub v7.8h, v7.8h, v2.8h |
| 1356 | st1 {v7.8h}, [x0], #16 | 1340 | st1 {v7.8h}, [x0], #16 |
| 1357 | .endm | 1341 | .endm |
| 1358 | 1342 | ||
| @@ -1376,12 +1360,8 @@ endfunc · 17 unmodified lines | |||
| 1376 | // x2 = src (temp buffer) | 1360 | // x2 = src (temp buffer) |
| 1377 | // x7 = negative double temp buffer stride | 1361 | // x7 = negative double temp buffer stride |
| 1378 | // x9 = double temp buffer stride | 1362 | // x9 = double temp buffer stride |
| 1379 | // x10 = idct_coeffs | ||
| 1380 | // x11 = idct_coeffs + 32 | ||
| 1381 | function idct32_1d_8x32_pass2\suffix\()_neon | 1363 | function idct32_1d_8x32_pass2\suffix\()_neon |
| 1382 | mov x14, x30 | 1364 | mov x14, x30 |
| 1383 | ld1 {v0.8h,v1.8h}, [x10] | ||
| 1384 | |||
| 1385 | // v16 = IN(0), v17 = IN(2) ... v31 = IN(30) | 1365 | // v16 = IN(0), v17 = IN(2) ... v31 = IN(30) |
| 1386 | .ifb \suffix | 1366 | .ifb \suffix |
| 1387 | .irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 | 1367 | .irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 |
| @@ -1454,15 +1434,15 @@ function idct32_1d_8x32_pass2\suffix\()_neon · 66 unmodified lines | |||
| 1454 | sub v6.8h, v6.8h, \c | 1434 | sub v6.8h, v6.8h, \c |
| 1455 | sub v7.8h, v7.8h, \d | 1435 | sub v7.8h, v7.8h, \d |
| 1456 | .endif | 1436 | .endif |
| 1457 | ld1 {v0.8b}, [x0], x1 | 1437 | ld1 {v10.8b}, [x0], x1 |
| 1458 | ld1 {v1.8b}, [x0], x1 | 1438 | ld1 {v11.8b}, [x0], x1 |
| 1459 | srshr v4.8h, v4.8h, #6 | 1439 | srshr v4.8h, v4.8h, #6 |
| 1460 | ld1 {v2.8b}, [x0], x1 | 1440 | ld1 {v2.8b}, [x0], x1 |
| 1461 | srshr v5.8h, v5.8h, #6 | 1441 | srshr v5.8h, v5.8h, #6 |
| 1462 | uaddw v4.8h, v4.8h, v0.8b | 1442 | uaddw v4.8h, v4.8h, v10.8b |
| 1463 | ld1 {v3.8b}, [x0], x1 | 1443 | ld1 {v3.8b}, [x0], x1 |
| 1464 | srshr v6.8h, v6.8h, #6 | 1444 | srshr v6.8h, v6.8h, #6 |
| 1465 | uaddw v5.8h, v5.8h, v1.8b | 1445 | uaddw v5.8h, v5.8h, v11.8b |
| 1466 | srshr v7.8h, v7.8h, #6 | 1446 | srshr v7.8h, v7.8h, #6 |
| 1467 | sub x0, x0, x1, lsl #2 | 1447 | sub x0, x0, x1, lsl #2 |
| 1468 | uaddw v6.8h, v6.8h, v2.8b | 1448 | uaddw v6.8h, v6.8h, v2.8b |
| @@ -1503,13 +1483,10 @@ function ff_vp9_idct_idct_32x32_add_neon, export=1 · 34 unmodified lines | |||
| 1503 | b.eq idct32x32_dc_add_neon | 1483 | b.eq idct32x32_dc_add_neon |
| 1504 | 1484 | ||
| 1505 | movrel x10, idct_coeffs | 1485 | movrel x10, idct_coeffs |
| 1506 | add x11, x10, #32 | ||
| 1507 | movrel x12, min_eob_idct_idct_32, 2 | 1486 | movrel x12, min_eob_idct_idct_32, 2 |
| 1508 | 1487 | ||
| 1509 | mov x15, x30 | 1488 | mov x15, x30 |
| 1510 | 1489 | ||
| 1511 | stp d14, d15, [sp, #-0x10]! | ||
| 1512 | stp d12, d13, [sp, #-0x10]! | ||
| 1513 | stp d10, d11, [sp, #-0x10]! | 1490 | stp d10, d11, [sp, #-0x10]! |
| 1514 | stp d8, d9, [sp, #-0x10]! | 1491 | stp d8, d9, [sp, #-0x10]! |
| 1515 | 1492 | ||
| @@ -1523,6 +1500,9 @@ function ff_vp9_idct_idct_32x32_add_neon, export=1 · 7 unmodified lines | |||
| 1523 | mov x9, #128 | 1500 | mov x9, #128 |
| 1524 | neg x7, x9 | 1501 | neg x7, x9 |
| 1525 | 1502 | ||
| 1503 | ld1 {v0.8h,v1.8h}, [x10], #32 | ||
| 1504 | ld1 {v8.8h,v9.8h}, [x10] | ||
| 1505 | |||
| 1526 | cmp w3, #34 | 1506 | cmp w3, #34 |
| 1527 | b.le idct32x32_quarter_add_neon | 1507 | b.le idct32x32_quarter_add_neon |
| 1528 | cmp w3, #135 | 1508 | cmp w3, #135 |
| @@ -1565,8 +1545,6 @@ function ff_vp9_idct_idct_32x32_add_neon, export=1 · 36 unmodified lines | |||
| 1565 | 1545 | ||
| 1566 | ldp d8, d9, [sp], 0x10 | 1546 | ldp d8, d9, [sp], 0x10 |
| 1567 | ldp d10, d11, [sp], 0x10 | 1547 | ldp d10, d11, [sp], 0x10 |
| 1568 | ldp d12, d13, [sp], 0x10 | ||
| 1569 | ldp d14, d15, [sp], 0x10 | ||
| 1570 | 1548 | ||
| 1571 | br x15 | 1549 | br x15 |
| 1572 | endfunc | 1550 | endfunc |
| @@ -1592,8 +1570,6 @@ function idct32x32_\size\()_add_neon · 19 unmodified lines | |||
| 1592 | 1570 | ||
| 1593 | ldp d8, d9, [sp], 0x10 | 1571 | ldp d8, d9, [sp], 0x10 |
| 1594 | ldp d10, d11, [sp], 0x10 | 1572 | ldp d10, d11, [sp], 0x10 |
| 1595 | ldp d12, d13, [sp], 0x10 | ||
| 1596 | ldp d14, d15, [sp], 0x10 | ||
| 1597 | 1573 | ||
| 1598 | br x15 | 1574 | br x15 |
| 1599 | endfunc | 1575 | endfunc |