History

aarch64: vp9itxfm: Avoid reloading the idct32 coefficients

The idct32x32 function actually pushed d8-d15 onto the stack even though it didn't clobber them; there are plenty of registers that can be used to allow keeping all the idct coefficients in registers without having to reload different subsets of them at different stages in the transform. After this, we still can skip pushing d12-d15. Before: vp9_inv_dct_dct_32x32_sub32_add_neon: 8128.3 After: vp9_inv_dct_dct_32x32_sub32_add_neon: 8053.3

Martin Storsjö committed 9 years ago

Commit 65aa0021 parent 402546a1 file · +43 −67Browse files

Unified · Split

libavcodec/aarch64/vp9itxfm_neon.S+43−67
@@ -1123,18 +1123,14 @@ endfunc · 1122 unmodified lines
11231123 .endm
11241124
11251125 function idct32_odd
1126− ld1 {v0.8h,v1.8h}, [x11]
1127−
1128− dmbutterfly v16, v31, v0.h[0], v0.h[1], v4, v5, v6, v7 // v16 = t16a, v31 = t31a
1129− dmbutterfly v24, v23, v0.h[2], v0.h[3], v4, v5, v6, v7 // v24 = t17a, v23 = t30a
1130− dmbutterfly v20, v27, v0.h[4], v0.h[5], v4, v5, v6, v7 // v20 = t18a, v27 = t29a
1131− dmbutterfly v28, v19, v0.h[6], v0.h[7], v4, v5, v6, v7 // v28 = t19a, v19 = t28a
1132− dmbutterfly v18, v29, v1.h[0], v1.h[1], v4, v5, v6, v7 // v18 = t20a, v29 = t27a
1133− dmbutterfly v26, v21, v1.h[2], v1.h[3], v4, v5, v6, v7 // v26 = t21a, v21 = t26a
1134− dmbutterfly v22, v25, v1.h[4], v1.h[5], v4, v5, v6, v7 // v22 = t22a, v25 = t25a
1135− dmbutterfly v30, v17, v1.h[6], v1.h[7], v4, v5, v6, v7 // v30 = t23a, v17 = t24a
1136−
1137− ld1 {v0.8h}, [x10]
1126+ dmbutterfly v16, v31, v8.h[0], v8.h[1], v4, v5, v6, v7 // v16 = t16a, v31 = t31a
1127+ dmbutterfly v24, v23, v8.h[2], v8.h[3], v4, v5, v6, v7 // v24 = t17a, v23 = t30a
1128+ dmbutterfly v20, v27, v8.h[4], v8.h[5], v4, v5, v6, v7 // v20 = t18a, v27 = t29a
1129+ dmbutterfly v28, v19, v8.h[6], v8.h[7], v4, v5, v6, v7 // v28 = t19a, v19 = t28a
1130+ dmbutterfly v18, v29, v9.h[0], v9.h[1], v4, v5, v6, v7 // v18 = t20a, v29 = t27a
1131+ dmbutterfly v26, v21, v9.h[2], v9.h[3], v4, v5, v6, v7 // v26 = t21a, v21 = t26a
1132+ dmbutterfly v22, v25, v9.h[4], v9.h[5], v4, v5, v6, v7 // v22 = t22a, v25 = t25a
1133+ dmbutterfly v30, v17, v9.h[6], v9.h[7], v4, v5, v6, v7 // v30 = t23a, v17 = t24a
11381134
11391135 butterfly_8h v4, v24, v16, v24 // v4 = t16, v24 = t17
11401136 butterfly_8h v5, v20, v28, v20 // v5 = t19, v20 = t18
@@ -1153,18 +1149,14 @@ function idct32_odd · 12 unmodified lines
11531149 endfunc
11541150
11551151 function idct32_odd_half
1156− ld1 {v0.8h,v1.8h}, [x11]
1157−
1158− dmbutterfly_h1 v16, v31, v0.h[0], v0.h[1], v4, v5, v6, v7 // v16 = t16a, v31 = t31a
1159− dmbutterfly_h2 v24, v23, v0.h[2], v0.h[3], v4, v5, v6, v7 // v24 = t17a, v23 = t30a
1160− dmbutterfly_h1 v20, v27, v0.h[4], v0.h[5], v4, v5, v6, v7 // v20 = t18a, v27 = t29a
1161− dmbutterfly_h2 v28, v19, v0.h[6], v0.h[7], v4, v5, v6, v7 // v28 = t19a, v19 = t28a
1162− dmbutterfly_h1 v18, v29, v1.h[0], v1.h[1], v4, v5, v6, v7 // v18 = t20a, v29 = t27a
1163− dmbutterfly_h2 v26, v21, v1.h[2], v1.h[3], v4, v5, v6, v7 // v26 = t21a, v21 = t26a
1164− dmbutterfly_h1 v22, v25, v1.h[4], v1.h[5], v4, v5, v6, v7 // v22 = t22a, v25 = t25a
1165− dmbutterfly_h2 v30, v17, v1.h[6], v1.h[7], v4, v5, v6, v7 // v30 = t23a, v17 = t24a
1166−
1167− ld1 {v0.8h}, [x10]
1152+ dmbutterfly_h1 v16, v31, v8.h[0], v8.h[1], v4, v5, v6, v7 // v16 = t16a, v31 = t31a
1153+ dmbutterfly_h2 v24, v23, v8.h[2], v8.h[3], v4, v5, v6, v7 // v24 = t17a, v23 = t30a
1154+ dmbutterfly_h1 v20, v27, v8.h[4], v8.h[5], v4, v5, v6, v7 // v20 = t18a, v27 = t29a
1155+ dmbutterfly_h2 v28, v19, v8.h[6], v8.h[7], v4, v5, v6, v7 // v28 = t19a, v19 = t28a
1156+ dmbutterfly_h1 v18, v29, v9.h[0], v9.h[1], v4, v5, v6, v7 // v18 = t20a, v29 = t27a
1157+ dmbutterfly_h2 v26, v21, v9.h[2], v9.h[3], v4, v5, v6, v7 // v26 = t21a, v21 = t26a
1158+ dmbutterfly_h1 v22, v25, v9.h[4], v9.h[5], v4, v5, v6, v7 // v22 = t22a, v25 = t25a
1159+ dmbutterfly_h2 v30, v17, v9.h[6], v9.h[7], v4, v5, v6, v7 // v30 = t23a, v17 = t24a
11681160
11691161 butterfly_8h v4, v24, v16, v24 // v4 = t16, v24 = t17
11701162 butterfly_8h v5, v20, v28, v20 // v5 = t19, v20 = t18
@@ -1183,18 +1175,14 @@ function idct32_odd_half · 12 unmodified lines
11831175 endfunc
11841176
11851177 function idct32_odd_quarter
1186− ld1 {v0.8h,v1.8h}, [x11]
1187−
1188− dsmull_h v4, v5, v16, v0.h[0]
1189− dsmull_h v28, v29, v19, v0.h[7]
1190− dsmull_h v30, v31, v16, v0.h[1]
1191− dsmull_h v22, v23, v17, v1.h[6]
1192− dsmull_h v7, v6, v17, v1.h[7]
1193− dsmull_h v26, v27, v19, v0.h[6]
1194− dsmull_h v20, v21, v18, v1.h[0]
1195− dsmull_h v24, v25, v18, v1.h[1]
1196−
1197− ld1 {v0.8h}, [x10]
1178+ dsmull_h v4, v5, v16, v8.h[0]
1179+ dsmull_h v28, v29, v19, v8.h[7]
1180+ dsmull_h v30, v31, v16, v8.h[1]
1181+ dsmull_h v22, v23, v17, v9.h[6]
1182+ dsmull_h v7, v6, v17, v9.h[7]
1183+ dsmull_h v26, v27, v19, v8.h[6]
1184+ dsmull_h v20, v21, v18, v9.h[0]
1185+ dsmull_h v24, v25, v18, v9.h[1]
11981186
11991187 neg v28.4s, v28.4s
12001188 neg v29.4s, v29.4s
@@ -1240,12 +1228,8 @@ endfunc · 39 unmodified lines
12401228 // x1 = unused
12411229 // x2 = src
12421230 // x9 = double input stride
1243−// x10 = idct_coeffs
1244−// x11 = idct_coeffs + 32
12451231 function idct32_1d_8x32_pass1\suffix\()_neon
12461232 mov x14, x30
1247− ld1 {v0.8h,v1.8h}, [x10]
1248−
12491233 movi v2.8h, #0
12501234
12511235 // v16 = IN(0), v17 = IN(2) ... v31 = IN(30)
@@ -1278,14 +1262,14 @@ function idct32_1d_8x32_pass1\suffix\()_neon · 26 unmodified lines
12781262 .macro store_rev a, b
12791263 // There's no rev128 instruction, but we reverse each 64 bit
12801264 // half, and then flip them using an ext with 8 bytes offset.
1281− rev64 v1.8h, \b
1265+ rev64 v3.8h, \b
12821266 st1 {\a}, [x0], #16
1283− rev64 v0.8h, \a
1284− ext v1.16b, v1.16b, v1.16b, #8
1267+ rev64 v2.8h, \a
1268+ ext v3.16b, v3.16b, v3.16b, #8
12851269 st1 {\b}, [x0], #16
1286− ext v0.16b, v0.16b, v0.16b, #8
1287− st1 {v1.8h}, [x0], #16
1288− st1 {v0.8h}, [x0], #16
1270+ ext v2.16b, v2.16b, v2.16b, #8
1271+ st1 {v3.8h}, [x0], #16
1272+ st1 {v2.8h}, [x0], #16
12891273 .endm
12901274 store_rev v16.8h, v24.8h
12911275 store_rev v17.8h, v25.8h
@@ -1339,20 +1323,20 @@ function idct32_1d_8x32_pass1\suffix\()_neon · 47 unmodified lines
13391323 // subtracted from the output.
13401324 .macro store_rev a, b
13411325 ld1 {v4.8h}, [x0]
1342− rev64 v1.8h, \b
1326+ rev64 v3.8h, \b
13431327 add v4.8h, v4.8h, \a
1344− rev64 v0.8h, \a
1328+ rev64 v2.8h, \a
13451329 st1 {v4.8h}, [x0], #16
1346− ext v1.16b, v1.16b, v1.16b, #8
1330+ ext v3.16b, v3.16b, v3.16b, #8
13471331 ld1 {v5.8h}, [x0]
1348− ext v0.16b, v0.16b, v0.16b, #8
1332+ ext v2.16b, v2.16b, v2.16b, #8
13491333 add v5.8h, v5.8h, \b
13501334 st1 {v5.8h}, [x0], #16
13511335 ld1 {v6.8h}, [x0]
1352− sub v6.8h, v6.8h, v1.8h
1336+ sub v6.8h, v6.8h, v3.8h
13531337 st1 {v6.8h}, [x0], #16
13541338 ld1 {v7.8h}, [x0]
1355− sub v7.8h, v7.8h, v0.8h
1339+ sub v7.8h, v7.8h, v2.8h
13561340 st1 {v7.8h}, [x0], #16
13571341 .endm
13581342
@@ -1376,12 +1360,8 @@ endfunc · 17 unmodified lines
13761360 // x2 = src (temp buffer)
13771361 // x7 = negative double temp buffer stride
13781362 // x9 = double temp buffer stride
1379−// x10 = idct_coeffs
1380−// x11 = idct_coeffs + 32
13811363 function idct32_1d_8x32_pass2\suffix\()_neon
13821364 mov x14, x30
1383− ld1 {v0.8h,v1.8h}, [x10]
1384−
13851365 // v16 = IN(0), v17 = IN(2) ... v31 = IN(30)
13861366 .ifb \suffix
13871367 .irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
@@ -1454,15 +1434,15 @@ function idct32_1d_8x32_pass2\suffix\()_neon · 66 unmodified lines
14541434 sub v6.8h, v6.8h, \c
14551435 sub v7.8h, v7.8h, \d
14561436 .endif
1457− ld1 {v0.8b}, [x0], x1
1458− ld1 {v1.8b}, [x0], x1
1437+ ld1 {v10.8b}, [x0], x1
1438+ ld1 {v11.8b}, [x0], x1
14591439 srshr v4.8h, v4.8h, #6
14601440 ld1 {v2.8b}, [x0], x1
14611441 srshr v5.8h, v5.8h, #6
1462− uaddw v4.8h, v4.8h, v0.8b
1442+ uaddw v4.8h, v4.8h, v10.8b
14631443 ld1 {v3.8b}, [x0], x1
14641444 srshr v6.8h, v6.8h, #6
1465− uaddw v5.8h, v5.8h, v1.8b
1445+ uaddw v5.8h, v5.8h, v11.8b
14661446 srshr v7.8h, v7.8h, #6
14671447 sub x0, x0, x1, lsl #2
14681448 uaddw v6.8h, v6.8h, v2.8b
@@ -1503,13 +1483,10 @@ function ff_vp9_idct_idct_32x32_add_neon, export=1 · 34 unmodified lines
15031483 b.eq idct32x32_dc_add_neon
15041484
15051485 movrel x10, idct_coeffs
1506− add x11, x10, #32
15071486 movrel x12, min_eob_idct_idct_32, 2
15081487
15091488 mov x15, x30
15101489
1511− stp d14, d15, [sp, #-0x10]!
1512− stp d12, d13, [sp, #-0x10]!
15131490 stp d10, d11, [sp, #-0x10]!
15141491 stp d8, d9, [sp, #-0x10]!
15151492
@@ -1523,6 +1500,9 @@ function ff_vp9_idct_idct_32x32_add_neon, export=1 · 7 unmodified lines
15231500 mov x9, #128
15241501 neg x7, x9
15251502
1503+ ld1 {v0.8h,v1.8h}, [x10], #32
1504+ ld1 {v8.8h,v9.8h}, [x10]
1505+
15261506 cmp w3, #34
15271507 b.le idct32x32_quarter_add_neon
15281508 cmp w3, #135
@@ -1565,8 +1545,6 @@ function ff_vp9_idct_idct_32x32_add_neon, export=1 · 36 unmodified lines
15651545
15661546 ldp d8, d9, [sp], 0x10
15671547 ldp d10, d11, [sp], 0x10
1568− ldp d12, d13, [sp], 0x10
1569− ldp d14, d15, [sp], 0x10
15701548
15711549 br x15
15721550 endfunc
@@ -1592,8 +1570,6 @@ function idct32x32_\size\()_add_neon · 19 unmodified lines
15921570
15931571 ldp d8, d9, [sp], 0x10
15941572 ldp d10, d11, [sp], 0x10
1595− ldp d12, d13, [sp], 0x10
1596− ldp d14, d15, [sp], 0x10
15971573
15981574 br x15
15991575 endfunc

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t