| @@ -41,53 +41,53 @@ · 40 unmodified lines |
| 41 | ;----------------------------------------------------------------------------- */ | 41 | ;----------------------------------------------------------------------------- */ |
| 42 | | 42 | |
| 43 | function ff_hscale8to15_X8_neon, export=1 | 43 | function ff_hscale8to15_X8_neon, export=1 |
| 44 | sbfiz x7, x6, #1, #32 // filterSize*2 (*2 because int16) | 44 | sbfiz x7, x6, #1, #32 // filterSize*2 (*2 because int16) |
| 45 | 1: ldr w8, [x5], #4 // filterPos[idx] | 45 | 1: ldr w8, [x5], #4 // filterPos[idx] |
| 46 | ldr w0, [x5], #4 // filterPos[idx + 1] | 46 | ldr w0, [x5], #4 // filterPos[idx + 1] |
| 47 | ldr w11, [x5], #4 // filterPos[idx + 2] | 47 | ldr w11, [x5], #4 // filterPos[idx + 2] |
| 48 | ldr w9, [x5], #4 // filterPos[idx + 3] | 48 | ldr w9, [x5], #4 // filterPos[idx + 3] |
| 49 | mov x16, x4 // filter0 = filter | 49 | mov x16, x4 // filter0 = filter |
| 50 | add x12, x16, x7 // filter1 = filter0 + filterSize*2 | 50 | add x12, x16, x7 // filter1 = filter0 + filterSize*2 |
| 51 | add x13, x12, x7 // filter2 = filter1 + filterSize*2 | 51 | add x13, x12, x7 // filter2 = filter1 + filterSize*2 |
| 52 | add x4, x13, x7 // filter3 = filter2 + filterSize*2 | 52 | add x4, x13, x7 // filter3 = filter2 + filterSize*2 |
| 53 | movi v0.2d, #0 // val sum part 1 (for dst[0]) | 53 | movi v0.2d, #0 // val sum part 1 (for dst[0]) |
| 54 | movi v1.2d, #0 // val sum part 2 (for dst[1]) | 54 | movi v1.2d, #0 // val sum part 2 (for dst[1]) |
| 55 | movi v2.2d, #0 // val sum part 3 (for dst[2]) | 55 | movi v2.2d, #0 // val sum part 3 (for dst[2]) |
| 56 | movi v3.2d, #0 // val sum part 4 (for dst[3]) | 56 | movi v3.2d, #0 // val sum part 4 (for dst[3]) |
| 57 | add x17, x3, w8, uxtw // srcp + filterPos[0] | 57 | add x17, x3, w8, uxtw // srcp + filterPos[0] |
| 58 | add x8, x3, w0, uxtw // srcp + filterPos[1] | 58 | add x8, x3, w0, uxtw // srcp + filterPos[1] |
| 59 | add x0, x3, w11, uxtw // srcp + filterPos[2] | 59 | add x0, x3, w11, uxtw // srcp + filterPos[2] |
| 60 | add x11, x3, w9, uxtw // srcp + filterPos[3] | 60 | add x11, x3, w9, uxtw // srcp + filterPos[3] |
| 61 | mov w15, w6 // filterSize counter | 61 | mov w15, w6 // filterSize counter |
| 62 | 2: ld1 {v4.8b}, [x17], #8 // srcp[filterPos[0] + {0..7}] | 62 | 2: ld1 {v4.8b}, [x17], #8 // srcp[filterPos[0] + {0..7}] |
| 63 | ld1 {v5.8h}, [x16], #16 // load 8x16-bit filter values, part 1 | 63 | ld1 {v5.8h}, [x16], #16 // load 8x16-bit filter values, part 1 |
| 64 | ld1 {v6.8b}, [x8], #8 // srcp[filterPos[1] + {0..7}] | 64 | ld1 {v6.8b}, [x8], #8 // srcp[filterPos[1] + {0..7}] |
| 65 | ld1 {v7.8h}, [x12], #16 // load 8x16-bit at filter+filterSize | 65 | ld1 {v7.8h}, [x12], #16 // load 8x16-bit at filter+filterSize |
| 66 | uxtl v4.8h, v4.8b // unpack part 1 to 16-bit | 66 | uxtl v4.8h, v4.8b // unpack part 1 to 16-bit |
| 67 | smlal v0.4s, v4.4h, v5.4h // v0 accumulates srcp[filterPos[0] + {0..3}] * filter[{0..3}] | 67 | smlal v0.4s, v4.4h, v5.4h // v0 accumulates srcp[filterPos[0] + {0..3}] * filter[{0..3}] |
| 68 | smlal2 v0.4s, v4.8h, v5.8h // v0 accumulates srcp[filterPos[0] + {4..7}] * filter[{4..7}] | 68 | smlal2 v0.4s, v4.8h, v5.8h // v0 accumulates srcp[filterPos[0] + {4..7}] * filter[{4..7}] |
| 69 | ld1 {v16.8b}, [x0], #8 // srcp[filterPos[2] + {0..7}] | 69 | ld1 {v16.8b}, [x0], #8 // srcp[filterPos[2] + {0..7}] |
| 70 | ld1 {v17.8h}, [x13], #16 // load 8x16-bit at filter+2*filterSize | 70 | ld1 {v17.8h}, [x13], #16 // load 8x16-bit at filter+2*filterSize |
| 71 | uxtl v6.8h, v6.8b // unpack part 2 to 16-bit | 71 | uxtl v6.8h, v6.8b // unpack part 2 to 16-bit |
| 72 | smlal v1.4s, v6.4h, v7.4h // v1 accumulates srcp[filterPos[1] + {0..3}] * filter[{0..3}] | 72 | smlal v1.4s, v6.4h, v7.4h // v1 accumulates srcp[filterPos[1] + {0..3}] * filter[{0..3}] |
| 73 | uxtl v16.8h, v16.8b // unpack part 3 to 16-bit | 73 | uxtl v16.8h, v16.8b // unpack part 3 to 16-bit |
| 74 | smlal v2.4s, v16.4h, v17.4h // v2 accumulates srcp[filterPos[2] + {0..3}] * filter[{0..3}] | 74 | smlal v2.4s, v16.4h, v17.4h // v2 accumulates srcp[filterPos[2] + {0..3}] * filter[{0..3}] |
| 75 | smlal2 v2.4s, v16.8h, v17.8h // v2 accumulates srcp[filterPos[2] + {4..7}] * filter[{4..7}] | 75 | smlal2 v2.4s, v16.8h, v17.8h // v2 accumulates srcp[filterPos[2] + {4..7}] * filter[{4..7}] |
| 76 | ld1 {v18.8b}, [x11], #8 // srcp[filterPos[3] + {0..7}] | 76 | ld1 {v18.8b}, [x11], #8 // srcp[filterPos[3] + {0..7}] |
| 77 | smlal2 v1.4s, v6.8h, v7.8h // v1 accumulates srcp[filterPos[1] + {4..7}] * filter[{4..7}] | 77 | smlal2 v1.4s, v6.8h, v7.8h // v1 accumulates srcp[filterPos[1] + {4..7}] * filter[{4..7}] |
| 78 | ld1 {v19.8h}, [x4], #16 // load 8x16-bit at filter+3*filterSize | 78 | ld1 {v19.8h}, [x4], #16 // load 8x16-bit at filter+3*filterSize |
| 79 | subs w15, w15, #8 // j -= 8: processed 8/filterSize | 79 | subs w15, w15, #8 // j -= 8: processed 8/filterSize |
| 80 | uxtl v18.8h, v18.8b // unpack part 4 to 16-bit | 80 | uxtl v18.8h, v18.8b // unpack part 4 to 16-bit |
| 81 | smlal v3.4s, v18.4h, v19.4h // v3 accumulates srcp[filterPos[3] + {0..3}] * filter[{0..3}] | 81 | smlal v3.4s, v18.4h, v19.4h // v3 accumulates srcp[filterPos[3] + {0..3}] * filter[{0..3}] |
| 82 | smlal2 v3.4s, v18.8h, v19.8h // v3 accumulates srcp[filterPos[3] + {4..7}] * filter[{4..7}] | 82 | smlal2 v3.4s, v18.8h, v19.8h // v3 accumulates srcp[filterPos[3] + {4..7}] * filter[{4..7}] |
| 83 | b.gt 2b // inner loop if filterSize not consumed completely | 83 | b.gt 2b // inner loop if filterSize not consumed completely |
| 84 | addp v0.4s, v0.4s, v1.4s // part01 horizontal pair adding | 84 | addp v0.4s, v0.4s, v1.4s // part01 horizontal pair adding |
| 85 | addp v2.4s, v2.4s, v3.4s // part23 horizontal pair adding | 85 | addp v2.4s, v2.4s, v3.4s // part23 horizontal pair adding |
| 86 | addp v0.4s, v0.4s, v2.4s // part0123 horizontal pair adding | 86 | addp v0.4s, v0.4s, v2.4s // part0123 horizontal pair adding |
| 87 | subs w2, w2, #4 // dstW -= 4 | 87 | subs w2, w2, #4 // dstW -= 4 |
| 88 | sqshrn v0.4h, v0.4s, #7 // shift and clip the 2x16-bit final values | 88 | sqshrn v0.4h, v0.4s, #7 // shift and clip the 2x16-bit final values |
| 89 | st1 {v0.4h}, [x1], #8 // write to destination part0123 | 89 | st1 {v0.4h}, [x1], #8 // write to destination part0123 |
| 90 | b.gt 1b // loop until end of line | 90 | b.gt 1b // loop until end of line |
| 91 | ret | 91 | ret |
| 92 | endfunc | 92 | endfunc |
| 93 | | 93 | |
| @@ -103,98 +103,98 @@ function ff_hscale8to15_X4_neon, export=1 · 9 unmodified lines |
| 103 | // This function for filter sizes that are 4 mod 8. In other words, anything that's 0 mod 4 but not | 103 | // This function for filter sizes that are 4 mod 8. In other words, anything that's 0 mod 4 but not |
| 104 | // 0 mod 8. It also assumes that dstW is 0 mod 4. | 104 | // 0 mod 8. It also assumes that dstW is 0 mod 4. |
| 105 | | 105 | |
| 106 | lsl w7, w6, #1 // w7 = filterSize * 2 | 106 | lsl w7, w6, #1 // w7 = filterSize * 2 |
| 107 | 1: | 107 | 1: |
| 108 | ldp w8, w9, [x5] // filterPos[idx + 0], [idx + 1] | 108 | ldp w8, w9, [x5] // filterPos[idx + 0], [idx + 1] |
| 109 | ldp w10, w11, [x5, #8] // filterPos[idx + 2], [idx + 3] | 109 | ldp w10, w11, [x5, #8] // filterPos[idx + 2], [idx + 3] |
| 110 | | 110 | |
| 111 | movi v16.2d, #0 // initialize accumulator for idx + 0 | 111 | movi v16.2d, #0 // initialize accumulator for idx + 0 |
| 112 | movi v17.2d, #0 // initialize accumulator for idx + 1 | 112 | movi v17.2d, #0 // initialize accumulator for idx + 1 |
| 113 | movi v18.2d, #0 // initialize accumulator for idx + 2 | 113 | movi v18.2d, #0 // initialize accumulator for idx + 2 |
| 114 | movi v19.2d, #0 // initialize accumulator for idx + 3 | 114 | movi v19.2d, #0 // initialize accumulator for idx + 3 |
| 115 | | 115 | |
| 116 | mov x12, x4 // filter pointer for idx + 0 | 116 | mov x12, x4 // filter pointer for idx + 0 |
| 117 | add x13, x4, x7 // filter pointer for idx + 1 | 117 | add x13, x4, x7 // filter pointer for idx + 1 |
| 118 | add x8, x3, w8, uxtw // srcp + filterPos[idx + 0] | 118 | add x8, x3, w8, uxtw // srcp + filterPos[idx + 0] |
| 119 | add x9, x3, w9, uxtw // srcp + filterPos[idx + 1] | 119 | add x9, x3, w9, uxtw // srcp + filterPos[idx + 1] |
| 120 | | 120 | |
| 121 | add x14, x13, x7 // filter pointer for idx + 2 | 121 | add x14, x13, x7 // filter pointer for idx + 2 |
| 122 | add x10, x3, w10, uxtw // srcp + filterPos[idx + 2] | 122 | add x10, x3, w10, uxtw // srcp + filterPos[idx + 2] |
| 123 | add x11, x3, w11, uxtw // srcp + filterPos[idx + 3] | 123 | add x11, x3, w11, uxtw // srcp + filterPos[idx + 3] |
| 124 | | 124 | |
| 125 | mov w0, w6 // copy filterSize to a temp register, w0 | 125 | mov w0, w6 // copy filterSize to a temp register, w0 |
| 126 | add x5, x5, #16 // advance the filterPos pointer | 126 | add x5, x5, #16 // advance the filterPos pointer |
| 127 | add x15, x14, x7 // filter pointer for idx + 3 | 127 | add x15, x14, x7 // filter pointer for idx + 3 |
| 128 | mov x16, xzr // temp register for offsetting filter pointers | 128 | mov x16, xzr // temp register for offsetting filter pointers |
| 129 | | 129 | |
| 130 | 2: | 130 | 2: |
| 131 | // This section loops over 8-wide chunks of filter size | 131 | // This section loops over 8-wide chunks of filter size |
| 132 | ldr d4, [x8], #8 // load 8 bytes from srcp for idx + 0 | 132 | ldr d4, [x8], #8 // load 8 bytes from srcp for idx + 0 |
| 133 | ldr q0, [x12, x16] // load 8 values, 16 bytes from filter for idx + 0 | 133 | ldr q0, [x12, x16] // load 8 values, 16 bytes from filter for idx + 0 |
| 134 | | 134 | |
| 135 | ldr d5, [x9], #8 // load 8 bytes from srcp for idx + 1 | 135 | ldr d5, [x9], #8 // load 8 bytes from srcp for idx + 1 |
| 136 | ldr q1, [x13, x16] // load 8 values, 16 bytes from filter for idx + 1 | 136 | ldr q1, [x13, x16] // load 8 values, 16 bytes from filter for idx + 1 |
| 137 | | 137 | |
| 138 | uxtl v4.8h, v4.8b // unsigned extend long for idx + 0 | 138 | uxtl v4.8h, v4.8b // unsigned extend long for idx + 0 |
| 139 | uxtl v5.8h, v5.8b // unsigned extend long for idx + 1 | 139 | uxtl v5.8h, v5.8b // unsigned extend long for idx + 1 |
| 140 | | 140 | |
| 141 | ldr d6, [x10], #8 // load 8 bytes from srcp for idx + 2 | 141 | ldr d6, [x10], #8 // load 8 bytes from srcp for idx + 2 |
| 142 | ldr q2, [x14, x16] // load 8 values, 16 bytes from filter for idx + 2 | 142 | ldr q2, [x14, x16] // load 8 values, 16 bytes from filter for idx + 2 |
| 143 | | 143 | |
| 144 | smlal v16.4s, v0.4h, v4.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 0 | 144 | smlal v16.4s, v0.4h, v4.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 0 |
| 145 | smlal v17.4s, v1.4h, v5.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 1 | 145 | smlal v17.4s, v1.4h, v5.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 1 |
| 146 | | 146 | |
| 147 | ldr d7, [x11], #8 // load 8 bytes from srcp for idx + 3 | 147 | ldr d7, [x11], #8 // load 8 bytes from srcp for idx + 3 |
| 148 | ldr q3, [x15, x16] // load 8 values, 16 bytes from filter for idx + 3 | 148 | ldr q3, [x15, x16] // load 8 values, 16 bytes from filter for idx + 3 |
| 149 | | 149 | |
| 150 | sub w0, w0, #8 // decrement the remaining filterSize counter | 150 | sub w0, w0, #8 // decrement the remaining filterSize counter |
| 151 | smlal2 v16.4s, v0.8h, v4.8h // val += src[srcPos + j + 4..7] * filter[fs * i + j + 4..7], idx + 0 | 151 | smlal2 v16.4s, v0.8h, v4.8h // val += src[srcPos + j + 4..7] * filter[fs * i + j + 4..7], idx + 0 |
| 152 | smlal2 v17.4s, v1.8h, v5.8h // val += src[srcPos + j + 4..7] * filter[fs * i + j + 4..7], idx + 1 | 152 | smlal2 v17.4s, v1.8h, v5.8h // val += src[srcPos + j + 4..7] * filter[fs * i + j + 4..7], idx + 1 |
| 153 | uxtl v6.8h, v6.8b // unsigned extend long for idx + 2 | 153 | uxtl v6.8h, v6.8b // unsigned extend long for idx + 2 |
| 154 | uxtl v7.8h, v7.8b // unsigned extend long for idx + 3 | 154 | uxtl v7.8h, v7.8b // unsigned extend long for idx + 3 |
| 155 | smlal v18.4s, v2.4h, v6.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 2 | 155 | smlal v18.4s, v2.4h, v6.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 2 |
| 156 | smlal v19.4s, v3.4h, v7.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 3 | 156 | smlal v19.4s, v3.4h, v7.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 3 |
| 157 | | 157 | |
| 158 | cmp w0, #8 // are there at least 8 more elements in filter to consume? | 158 | cmp w0, #8 // are there at least 8 more elements in filter to consume? |
| 159 | add x16, x16, #16 // advance the offsetting register for filter values | 159 | add x16, x16, #16 // advance the offsetting register for filter values |
| 160 | | 160 | |
| 161 | smlal2 v18.4s, v2.8h, v6.8h // val += src[srcPos + j + 4..7] * filter[fs * i + j + 4..7], idx + 2 | 161 | smlal2 v18.4s, v2.8h, v6.8h // val += src[srcPos + j + 4..7] * filter[fs * i + j + 4..7], idx + 2 |
| 162 | smlal2 v19.4s, v3.8h, v7.8h // val += src[srcPos + j + 4..7] * filter[fs * i + j + 4..7], idx + 3 | 162 | smlal2 v19.4s, v3.8h, v7.8h // val += src[srcPos + j + 4..7] * filter[fs * i + j + 4..7], idx + 3 |
| 163 | | 163 | |
| 164 | b.ge 2b // branch back to inner loop | 164 | b.ge 2b // branch back to inner loop |
| 165 | | 165 | |
| 166 | // complete the remaining 4 filter elements | 166 | // complete the remaining 4 filter elements |
| 167 | sub x17, x7, #8 // calculate the offset of the filter pointer for the remaining 4 elements | 167 | sub x17, x7, #8 // calculate the offset of the filter pointer for the remaining 4 elements |
| 168 | | 168 | |
| 169 | ldr s4, [x8] // load 4 bytes from srcp for idx + 0 | 169 | ldr s4, [x8] // load 4 bytes from srcp for idx + 0 |
| 170 | ldr d0, [x12, x17] // load 4 values, 8 bytes from filter for idx + 0 | 170 | ldr d0, [x12, x17] // load 4 values, 8 bytes from filter for idx + 0 |
| 171 | ldr s5, [x9] // load 4 bytes from srcp for idx + 1 | 171 | ldr s5, [x9] // load 4 bytes from srcp for idx + 1 |
| 172 | ldr d1, [x13, x17] // load 4 values, 8 bytes from filter for idx + 1 | 172 | ldr d1, [x13, x17] // load 4 values, 8 bytes from filter for idx + 1 |
| 173 | | 173 | |
| 174 | uxtl v4.8h, v4.8b // unsigned extend long for idx + 0 | 174 | uxtl v4.8h, v4.8b // unsigned extend long for idx + 0 |
| 175 | uxtl v5.8h, v5.8b // unsigned extend long for idx + 1 | 175 | uxtl v5.8h, v5.8b // unsigned extend long for idx + 1 |
| 176 | | 176 | |
| 177 | ldr s6, [x10] // load 4 bytes from srcp for idx + 2 | 177 | ldr s6, [x10] // load 4 bytes from srcp for idx + 2 |
| 178 | ldr d2, [x14, x17] // load 4 values, 8 bytes from filter for idx + 2 | 178 | ldr d2, [x14, x17] // load 4 values, 8 bytes from filter for idx + 2 |
| 179 | smlal v16.4s, v0.4h, v4.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 0 | 179 | smlal v16.4s, v0.4h, v4.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 0 |
| 180 | smlal v17.4s, v1.4h, v5.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 1 | 180 | smlal v17.4s, v1.4h, v5.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 1 |
| 181 | ldr s7, [x11] // load 4 bytes from srcp for idx + 3 | 181 | ldr s7, [x11] // load 4 bytes from srcp for idx + 3 |
| 182 | ldr d3, [x15, x17] // load 4 values, 8 bytes from filter for idx + 3 | 182 | ldr d3, [x15, x17] // load 4 values, 8 bytes from filter for idx + 3 |
| 183 | | 183 | |
| 184 | uxtl v6.8h, v6.8b // unsigned extend long for idx + 2 | 184 | uxtl v6.8h, v6.8b // unsigned extend long for idx + 2 |
| 185 | uxtl v7.8h, v7.8b // unsigned extend long for idx + 3 | 185 | uxtl v7.8h, v7.8b // unsigned extend long for idx + 3 |
| 186 | addp v16.4s, v16.4s, v17.4s // horizontal pair adding for idx 0,1 | 186 | addp v16.4s, v16.4s, v17.4s // horizontal pair adding for idx 0,1 |
| 187 | smlal v18.4s, v2.4h, v6.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 2 | 187 | smlal v18.4s, v2.4h, v6.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 2 |
| 188 | smlal v19.4s, v3.4h, v7.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 3 | 188 | smlal v19.4s, v3.4h, v7.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 3 |
| 189 | | 189 | |
| 190 | addp v18.4s, v18.4s, v19.4s // horizontal pair adding for idx 2,3 | 190 | addp v18.4s, v18.4s, v19.4s // horizontal pair adding for idx 2,3 |
| 191 | addp v16.4s, v16.4s, v18.4s // final horizontal pair adding producing one vector with results for idx = 0..3 | 191 | addp v16.4s, v16.4s, v18.4s // final horizontal pair adding producing one vector with results for idx = 0..3 |
| 192 | | 192 | |
| 193 | subs w2, w2, #4 // dstW -= 4 | 193 | subs w2, w2, #4 // dstW -= 4 |
| 194 | sqshrn v0.4h, v16.4s, #7 // shift and clip the 2x16-bit final values | 194 | sqshrn v0.4h, v16.4s, #7 // shift and clip the 2x16-bit final values |
| 195 | st1 {v0.4h}, [x1], #8 // write to destination idx 0..3 | 195 | st1 {v0.4h}, [x1], #8 // write to destination idx 0..3 |
| 196 | add x4, x4, x7, lsl #2 // filter += (filterSize*2) * 4 | 196 | add x4, x4, x7, lsl #2 // filter += (filterSize*2) * 4 |
| 197 | b.gt 1b // loop until end of line | 197 | b.gt 1b // loop until end of line |
| 198 | ret | 198 | ret |
| 199 | endfunc | 199 | endfunc |
| 200 | | 200 | |
| @@ -219,132 +219,132 @@ function ff_hscale8to15_4_neon, export=1 · 18 unmodified lines |
| 219 | // 3. Complete madd | 219 | // 3. Complete madd |
| 220 | // 4. Complete remaining iterations when dstW % 8 != 0 | 220 | // 4. Complete remaining iterations when dstW % 8 != 0 |
| 221 | | 221 | |
| 222 | sub sp, sp, #32 // allocate 32 bytes on the stack | 222 | sub sp, sp, #32 // allocate 32 bytes on the stack |
| 223 | cmp w2, #16 // if dstW <16, skip to the last block used for wrapping up | 223 | cmp w2, #16 // if dstW <16, skip to the last block used for wrapping up |
| 224 | b.lt 2f | 224 | b.lt 2f |
| 225 | | 225 | |
| 226 | // load 8 values from filterPos to be used as offsets into src | 226 | // load 8 values from filterPos to be used as offsets into src |
| 227 | ldp w8, w9, [x5] // filterPos[idx + 0], [idx + 1] | 227 | ldp w8, w9, [x5] // filterPos[idx + 0], [idx + 1] |
| 228 | ldp w10, w11, [x5, #8] // filterPos[idx + 2], [idx + 3] | 228 | ldp w10, w11, [x5, #8] // filterPos[idx + 2], [idx + 3] |
| 229 | ldp w12, w13, [x5, #16] // filterPos[idx + 4], [idx + 5] | 229 | ldp w12, w13, [x5, #16] // filterPos[idx + 4], [idx + 5] |
| 230 | ldp w14, w15, [x5, #24] // filterPos[idx + 6], [idx + 7] | 230 | ldp w14, w15, [x5, #24] // filterPos[idx + 6], [idx + 7] |
| 231 | add x5, x5, #32 // advance filterPos | 231 | add x5, x5, #32 // advance filterPos |
| 232 | | 232 | |
| 233 | // gather random access data from src into contiguous memory | 233 | // gather random access data from src into contiguous memory |
| 234 | ldr w8, [x3, w8, uxtw] // src[filterPos[idx + 0]][0..3] | 234 | ldr w8, [x3, w8, uxtw] // src[filterPos[idx + 0]][0..3] |
| 235 | ldr w9, [x3, w9, uxtw] // src[filterPos[idx + 1]][0..3] | 235 | ldr w9, [x3, w9, uxtw] // src[filterPos[idx + 1]][0..3] |
| 236 | ldr w10, [x3, w10, uxtw] // src[filterPos[idx + 2]][0..3] | 236 | ldr w10, [x3, w10, uxtw] // src[filterPos[idx + 2]][0..3] |
| 237 | ldr w11, [x3, w11, uxtw] // src[filterPos[idx + 3]][0..3] | 237 | ldr w11, [x3, w11, uxtw] // src[filterPos[idx + 3]][0..3] |
| 238 | ldr w12, [x3, w12, uxtw] // src[filterPos[idx + 4]][0..3] | 238 | ldr w12, [x3, w12, uxtw] // src[filterPos[idx + 4]][0..3] |
| 239 | ldr w13, [x3, w13, uxtw] // src[filterPos[idx + 5]][0..3] | 239 | ldr w13, [x3, w13, uxtw] // src[filterPos[idx + 5]][0..3] |
| 240 | ldr w14, [x3, w14, uxtw] // src[filterPos[idx + 6]][0..3] | 240 | ldr w14, [x3, w14, uxtw] // src[filterPos[idx + 6]][0..3] |
| 241 | ldr w15, [x3, w15, uxtw] // src[filterPos[idx + 7]][0..3] | 241 | ldr w15, [x3, w15, uxtw] // src[filterPos[idx + 7]][0..3] |
| 242 | stp w8, w9, [sp] // *scratch_mem = { src[filterPos[idx + 0]][0..3], src[filterPos[idx + 1]][0..3] } | 242 | stp w8, w9, [sp] // *scratch_mem = { src[filterPos[idx + 0]][0..3], src[filterPos[idx + 1]][0..3] } |
| 243 | stp w10, w11, [sp, #8] // *scratch_mem = { src[filterPos[idx + 2]][0..3], src[filterPos[idx + 3]][0..3] } | 243 | stp w10, w11, [sp, #8] // *scratch_mem = { src[filterPos[idx + 2]][0..3], src[filterPos[idx + 3]][0..3] } |
| 244 | stp w12, w13, [sp, #16] // *scratch_mem = { src[filterPos[idx + 4]][0..3], src[filterPos[idx + 5]][0..3] } | 244 | stp w12, w13, [sp, #16] // *scratch_mem = { src[filterPos[idx + 4]][0..3], src[filterPos[idx + 5]][0..3] } |
| 245 | stp w14, w15, [sp, #24] // *scratch_mem = { src[filterPos[idx + 6]][0..3], src[filterPos[idx + 7]][0..3] } | 245 | stp w14, w15, [sp, #24] // *scratch_mem = { src[filterPos[idx + 6]][0..3], src[filterPos[idx + 7]][0..3] } |
| 246 | | 246 | |
| 247 | 1: | 247 | 1: |
| 248 | ld4 {v16.8b, v17.8b, v18.8b, v19.8b}, [sp] // transpose 8 bytes each from src into 4 registers | 248 | ld4 {v16.8b, v17.8b, v18.8b, v19.8b}, [sp] // transpose 8 bytes each from src into 4 registers |
| 249 | | 249 | |
| 250 | // load 8 values from filterPos to be used as offsets into src | 250 | // load 8 values from filterPos to be used as offsets into src |
| 251 | ldp w8, w9, [x5] // filterPos[idx + 0][0..3], [idx + 1][0..3], next iteration | 251 | ldp w8, w9, [x5] // filterPos[idx + 0][0..3], [idx + 1][0..3], next iteration |
| 252 | ldp w10, w11, [x5, #8] // filterPos[idx + 2][0..3], [idx + 3][0..3], next iteration | 252 | ldp w10, w11, [x5, #8] // filterPos[idx + 2][0..3], [idx + 3][0..3], next iteration |
| 253 | ldp w12, w13, [x5, #16] // filterPos[idx + 4][0..3], [idx + 5][0..3], next iteration | 253 | ldp w12, w13, [x5, #16] // filterPos[idx + 4][0..3], [idx + 5][0..3], next iteration |
| 254 | ldp w14, w15, [x5, #24] // filterPos[idx + 6][0..3], [idx + 7][0..3], next iteration | 254 | ldp w14, w15, [x5, #24] // filterPos[idx + 6][0..3], [idx + 7][0..3], next iteration |
| 255 | | 255 | |
| 256 | movi v0.2d, #0 // Clear madd accumulator for idx 0..3 | 256 | movi v0.2d, #0 // Clear madd accumulator for idx 0..3 |
| 257 | movi v5.2d, #0 // Clear madd accumulator for idx 4..7 | 257 | movi v5.2d, #0 // Clear madd accumulator for idx 4..7 |
| 258 | | 258 | |
| 259 | ld4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x4], #64 // load filter idx + 0..7 | 259 | ld4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x4], #64 // load filter idx + 0..7 |
| 260 | | 260 | |
| 261 | add x5, x5, #32 // advance filterPos | 261 | add x5, x5, #32 // advance filterPos |
| 262 | | 262 | |
| 263 | // interleaved SIMD and prefetching intended to keep ld/st and vector pipelines busy | 263 | // interleaved SIMD and prefetching intended to keep ld/st and vector pipelines busy |
| 264 | uxtl v16.8h, v16.8b // unsigned extend long, covert src data to 16-bit | 264 | uxtl v16.8h, v16.8b // unsigned extend long, covert src data to 16-bit |
| 265 | uxtl v17.8h, v17.8b // unsigned extend long, covert src data to 16-bit | 265 | uxtl v17.8h, v17.8b // unsigned extend long, covert src data to 16-bit |
| 266 | ldr w8, [x3, w8, uxtw] // src[filterPos[idx + 0]], next iteration | 266 | ldr w8, [x3, w8, uxtw] // src[filterPos[idx + 0]], next iteration |
| 267 | ldr w9, [x3, w9, uxtw] // src[filterPos[idx + 1]], next iteration | 267 | ldr w9, [x3, w9, uxtw] // src[filterPos[idx + 1]], next iteration |
| 268 | uxtl v18.8h, v18.8b // unsigned extend long, covert src data to 16-bit | 268 | uxtl v18.8h, v18.8b // unsigned extend long, covert src data to 16-bit |
| 269 | uxtl v19.8h, v19.8b // unsigned extend long, covert src data to 16-bit | 269 | uxtl v19.8h, v19.8b // unsigned extend long, covert src data to 16-bit |
| 270 | ldr w10, [x3, w10, uxtw] // src[filterPos[idx + 2]], next iteration | 270 | ldr w10, [x3, w10, uxtw] // src[filterPos[idx + 2]], next iteration |
| 271 | ldr w11, [x3, w11, uxtw] // src[filterPos[idx + 3]], next iteration | 271 | ldr w11, [x3, w11, uxtw] // src[filterPos[idx + 3]], next iteration |
| 272 | | 272 | |
| 273 | smlal v0.4s, v1.4h, v16.4h // multiply accumulate inner loop j = 0, idx = 0..3 | 273 | smlal v0.4s, v1.4h, v16.4h // multiply accumulate inner loop j = 0, idx = 0..3 |
| 274 | smlal v0.4s, v2.4h, v17.4h // multiply accumulate inner loop j = 1, idx = 0..3 | 274 | smlal v0.4s, v2.4h, v17.4h // multiply accumulate inner loop j = 1, idx = 0..3 |
| 275 | ldr w12, [x3, w12, uxtw] // src[filterPos[idx + 4]], next iteration | 275 | ldr w12, [x3, w12, uxtw] // src[filterPos[idx + 4]], next iteration |
| 276 | ldr w13, [x3, w13, uxtw] // src[filterPos[idx + 5]], next iteration | 276 | ldr w13, [x3, w13, uxtw] // src[filterPos[idx + 5]], next iteration |
| 277 | smlal v0.4s, v3.4h, v18.4h // multiply accumulate inner loop j = 2, idx = 0..3 | 277 | smlal v0.4s, v3.4h, v18.4h // multiply accumulate inner loop j = 2, idx = 0..3 |
| 278 | smlal v0.4s, v4.4h, v19.4h // multiply accumulate inner loop j = 3, idx = 0..3 | 278 | smlal v0.4s, v4.4h, v19.4h // multiply accumulate inner loop j = 3, idx = 0..3 |
| 279 | ldr w14, [x3, w14, uxtw] // src[filterPos[idx + 6]], next iteration | 279 | ldr w14, [x3, w14, uxtw] // src[filterPos[idx + 6]], next iteration |
| 280 | ldr w15, [x3, w15, uxtw] // src[filterPos[idx + 7]], next iteration | 280 | ldr w15, [x3, w15, uxtw] // src[filterPos[idx + 7]], next iteration |
| 281 | | 281 | |
| 282 | smlal2 v5.4s, v1.8h, v16.8h // multiply accumulate inner loop j = 0, idx = 4..7 | 282 | smlal2 v5.4s, v1.8h, v16.8h // multiply accumulate inner loop j = 0, idx = 4..7 |
| 283 | smlal2 v5.4s, v2.8h, v17.8h // multiply accumulate inner loop j = 1, idx = 4..7 | 283 | smlal2 v5.4s, v2.8h, v17.8h // multiply accumulate inner loop j = 1, idx = 4..7 |
| 284 | stp w8, w9, [sp] // *scratch_mem = { src[filterPos[idx + 0]][0..3], src[filterPos[idx + 1]][0..3] } | 284 | stp w8, w9, [sp] // *scratch_mem = { src[filterPos[idx + 0]][0..3], src[filterPos[idx + 1]][0..3] } |
| 285 | stp w10, w11, [sp, #8] // *scratch_mem = { src[filterPos[idx + 2]][0..3], src[filterPos[idx + 3]][0..3] } | 285 | stp w10, w11, [sp, #8] // *scratch_mem = { src[filterPos[idx + 2]][0..3], src[filterPos[idx + 3]][0..3] } |
| 286 | smlal2 v5.4s, v3.8h, v18.8h // multiply accumulate inner loop j = 2, idx = 4..7 | 286 | smlal2 v5.4s, v3.8h, v18.8h // multiply accumulate inner loop j = 2, idx = 4..7 |
| 287 | smlal2 v5.4s, v4.8h, v19.8h // multiply accumulate inner loop j = 3, idx = 4..7 | 287 | smlal2 v5.4s, v4.8h, v19.8h // multiply accumulate inner loop j = 3, idx = 4..7 |
| 288 | stp w12, w13, [sp, #16] // *scratch_mem = { src[filterPos[idx + 4]][0..3], src[filterPos[idx + 5]][0..3] } | 288 | stp w12, w13, [sp, #16] // *scratch_mem = { src[filterPos[idx + 4]][0..3], src[filterPos[idx + 5]][0..3] } |
| 289 | stp w14, w15, [sp, #24] // *scratch_mem = { src[filterPos[idx + 6]][0..3], src[filterPos[idx + 7]][0..3] } | 289 | stp w14, w15, [sp, #24] // *scratch_mem = { src[filterPos[idx + 6]][0..3], src[filterPos[idx + 7]][0..3] } |
| 290 | | 290 | |
| 291 | sub w2, w2, #8 // dstW -= 8 | 291 | sub w2, w2, #8 // dstW -= 8 |
| 292 | sqshrn v0.4h, v0.4s, #7 // shift and clip the 2x16-bit final values | 292 | sqshrn v0.4h, v0.4s, #7 // shift and clip the 2x16-bit final values |
| 293 | sqshrn v1.4h, v5.4s, #7 // shift and clip the 2x16-bit final values | 293 | sqshrn v1.4h, v5.4s, #7 // shift and clip the 2x16-bit final values |
| 294 | st1 {v0.4h, v1.4h}, [x1], #16 // write to dst[idx + 0..7] | 294 | st1 {v0.4h, v1.4h}, [x1], #16 // write to dst[idx + 0..7] |
| 295 | cmp w2, #16 // continue on main loop if there are at least 16 iterations left | 295 | cmp w2, #16 // continue on main loop if there are at least 16 iterations left |
| 296 | b.ge 1b | 296 | b.ge 1b |
| 297 | | 297 | |
| 298 | // last full iteration | 298 | // last full iteration |
| 299 | ld4 {v16.8b, v17.8b, v18.8b, v19.8b}, [sp] | 299 | ld4 {v16.8b, v17.8b, v18.8b, v19.8b}, [sp] |
| 300 | ld4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x4], #64 // load filter idx + 0..7 | 300 | ld4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x4], #64 // load filter idx + 0..7 |
| 301 | | 301 | |
| 302 | movi v0.2d, #0 // Clear madd accumulator for idx 0..3 | 302 | movi v0.2d, #0 // Clear madd accumulator for idx 0..3 |
| 303 | movi v5.2d, #0 // Clear madd accumulator for idx 4..7 | 303 | movi v5.2d, #0 // Clear madd accumulator for idx 4..7 |
| 304 | | 304 | |
| 305 | uxtl v16.8h, v16.8b // unsigned extend long, covert src data to 16-bit | 305 | uxtl v16.8h, v16.8b // unsigned extend long, covert src data to 16-bit |
| 306 | uxtl v17.8h, v17.8b // unsigned extend long, covert src data to 16-bit | 306 | uxtl v17.8h, v17.8b // unsigned extend long, covert src data to 16-bit |
| 307 | uxtl v18.8h, v18.8b // unsigned extend long, covert src data to 16-bit | 307 | uxtl v18.8h, v18.8b // unsigned extend long, covert src data to 16-bit |
| 308 | uxtl v19.8h, v19.8b // unsigned extend long, covert src data to 16-bit | 308 | uxtl v19.8h, v19.8b // unsigned extend long, covert src data to 16-bit |
| 309 | | 309 | |
| 310 | smlal v0.4s, v1.4h, v16.4h // multiply accumulate inner loop j = 0, idx = 0..3 | 310 | smlal v0.4s, v1.4h, v16.4h // multiply accumulate inner loop j = 0, idx = 0..3 |
| 311 | smlal v0.4s, v2.4h, v17.4h // multiply accumulate inner loop j = 1, idx = 0..3 | 311 | smlal v0.4s, v2.4h, v17.4h // multiply accumulate inner loop j = 1, idx = 0..3 |
| 312 | smlal v0.4s, v3.4h, v18.4h // multiply accumulate inner loop j = 2, idx = 0..3 | 312 | smlal v0.4s, v3.4h, v18.4h // multiply accumulate inner loop j = 2, idx = 0..3 |
| 313 | smlal v0.4s, v4.4h, v19.4h // multiply accumulate inner loop j = 3, idx = 0..3 | 313 | smlal v0.4s, v4.4h, v19.4h // multiply accumulate inner loop j = 3, idx = 0..3 |
| 314 | | 314 | |
| 315 | smlal2 v5.4s, v1.8h, v16.8h // multiply accumulate inner loop j = 0, idx = 4..7 | 315 | smlal2 v5.4s, v1.8h, v16.8h // multiply accumulate inner loop j = 0, idx = 4..7 |
| 316 | smlal2 v5.4s, v2.8h, v17.8h // multiply accumulate inner loop j = 1, idx = 4..7 | 316 | smlal2 v5.4s, v2.8h, v17.8h // multiply accumulate inner loop j = 1, idx = 4..7 |
| 317 | smlal2 v5.4s, v3.8h, v18.8h // multiply accumulate inner loop j = 2, idx = 4..7 | 317 | smlal2 v5.4s, v3.8h, v18.8h // multiply accumulate inner loop j = 2, idx = 4..7 |
| 318 | smlal2 v5.4s, v4.8h, v19.8h // multiply accumulate inner loop j = 3, idx = 4..7 | 318 | smlal2 v5.4s, v4.8h, v19.8h // multiply accumulate inner loop j = 3, idx = 4..7 |
| 319 | | 319 | |
| 320 | subs w2, w2, #8 // dstW -= 8 | 320 | subs w2, w2, #8 // dstW -= 8 |
| 321 | sqshrn v0.4h, v0.4s, #7 // shift and clip the 2x16-bit final values | 321 | sqshrn v0.4h, v0.4s, #7 // shift and clip the 2x16-bit final values |
| 322 | sqshrn v1.4h, v5.4s, #7 // shift and clip the 2x16-bit final values | 322 | sqshrn v1.4h, v5.4s, #7 // shift and clip the 2x16-bit final values |
| 323 | st1 {v0.4h, v1.4h}, [x1], #16 // write to dst[idx + 0..7] | 323 | st1 {v0.4h, v1.4h}, [x1], #16 // write to dst[idx + 0..7] |
| 324 | | 324 | |
| 325 | cbnz w2, 2f // if >0 iterations remain, jump to the wrap up section | 325 | cbnz w2, 2f // if >0 iterations remain, jump to the wrap up section |
| 326 | | 326 | |
| 327 | add sp, sp, #32 // clean up stack | 327 | add sp, sp, #32 // clean up stack |
| 328 | ret | 328 | ret |
| 329 | | 329 | |
| 330 | // finish up when dstW % 8 != 0 or dstW < 16 | 330 | // finish up when dstW % 8 != 0 or dstW < 16 |
| 331 | 2: | 331 | 2: |
| 332 | // load src | 332 | // load src |
| 333 | ldr w8, [x5], #4 // filterPos[i] | 333 | ldr w8, [x5], #4 // filterPos[i] |
| 334 | add x9, x3, w8, uxtw // calculate the address for src load | 334 | add x9, x3, w8, uxtw // calculate the address for src load |
| 335 | ld1 {v5.s}[0], [x9] // src[filterPos[i] + 0..3] | 335 | ld1 {v5.s}[0], [x9] // src[filterPos[i] + 0..3] |
| 336 | // load filter | 336 | // load filter |
| 337 | ld1 {v6.4h}, [x4], #8 // filter[filterSize * i + 0..3] | 337 | ld1 {v6.4h}, [x4], #8 // filter[filterSize * i + 0..3] |
| 338 | | 338 | |
| 339 | uxtl v5.8h, v5.8b // unsigned exten long, convert src data to 16-bit | 339 | uxtl v5.8h, v5.8b // unsigned exten long, convert src data to 16-bit |
| 340 | smull v0.4s, v5.4h, v6.4h // 4 iterations of src[...] * filter[...] | 340 | smull v0.4s, v5.4h, v6.4h // 4 iterations of src[...] * filter[...] |
| 341 | addv s0, v0.4s // add up products of src and filter values | 341 | addv s0, v0.4s // add up products of src and filter values |
| 342 | sqshrn h0, s0, #7 // shift and clip the 2x16-bit final value | 342 | sqshrn h0, s0, #7 // shift and clip the 2x16-bit final value |
| 343 | st1 {v0.h}[0], [x1], #2 // dst[i] = ... | 343 | st1 {v0.h}[0], [x1], #2 // dst[i] = ... |
| 344 | sub w2, w2, #1 // dstW-- | 344 | sub w2, w2, #1 // dstW-- |
| 345 | cbnz w2, 2b | 345 | cbnz w2, 2b |
| 346 | | 346 | |
| 347 | add sp, sp, #32 // clean up stack | 347 | add sp, sp, #32 // clean up stack |
| 348 | ret | 348 | ret |
| 349 | endfunc | 349 | endfunc |
| 350 | | 350 | |
| @@ -357,187 +357,187 @@ function ff_hscale8to19_4_neon, export=1 · 6 unmodified lines |
| 357 | // x5 const int32_t *filterPos | 357 | // x5 const int32_t *filterPos |
| 358 | // w6 int filterSize | 358 | // w6 int filterSize |
| 359 | | 359 | |
| 360 | movi v18.4s, #1 | 360 | movi v18.4s, #1 |
| 361 | movi v17.4s, #1 | 361 | movi v17.4s, #1 |
| 362 | shl v18.4s, v18.4s, #19 | 362 | shl v18.4s, v18.4s, #19 |
| 363 | sub v18.4s, v18.4s, v17.4s // max allowed value | 363 | sub v18.4s, v18.4s, v17.4s // max allowed value |
| 364 | | 364 | |
| 365 | cmp w2, #16 | 365 | cmp w2, #16 |
| 366 | b.lt 2f // move to last block | 366 | b.lt 2f // move to last block |
| 367 | | 367 | |
| 368 | ldp w8, w9, [x5] // filterPos[0], filterPos[1] | 368 | ldp w8, w9, [x5] // filterPos[0], filterPos[1] |
| 369 | ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3] | 369 | ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3] |
| 370 | ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5] | 370 | ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5] |
| 371 | ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7] | 371 | ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7] |
| 372 | add x5, x5, #32 | 372 | add x5, x5, #32 |
| 373 | | 373 | |
| 374 | // load data from | 374 | // load data from |
| 375 | ldr w8, [x3, w8, uxtw] | 375 | ldr w8, [x3, w8, uxtw] |
| 376 | ldr w9, [x3, w9, uxtw] | 376 | ldr w9, [x3, w9, uxtw] |
| 377 | ldr w10, [x3, w10, uxtw] | 377 | ldr w10, [x3, w10, uxtw] |
| 378 | ldr w11, [x3, w11, uxtw] | 378 | ldr w11, [x3, w11, uxtw] |
| 379 | ldr w12, [x3, w12, uxtw] | 379 | ldr w12, [x3, w12, uxtw] |
| 380 | ldr w13, [x3, w13, uxtw] | 380 | ldr w13, [x3, w13, uxtw] |
| 381 | ldr w14, [x3, w14, uxtw] | 381 | ldr w14, [x3, w14, uxtw] |
| 382 | ldr w15, [x3, w15, uxtw] | 382 | ldr w15, [x3, w15, uxtw] |
| 383 | | 383 | |
| 384 | sub sp, sp, #32 | 384 | sub sp, sp, #32 |
| 385 | | 385 | |
| 386 | stp w8, w9, [sp] | 386 | stp w8, w9, [sp] |
| 387 | stp w10, w11, [sp, #8] | 387 | stp w10, w11, [sp, #8] |
| 388 | stp w12, w13, [sp, #16] | 388 | stp w12, w13, [sp, #16] |
| 389 | stp w14, w15, [sp, #24] | 389 | stp w14, w15, [sp, #24] |
| 390 | | 390 | |
| 391 | 1: | 391 | 1: |
| 392 | ld4 {v0.8b, v1.8b, v2.8b, v3.8b}, [sp] | 392 | ld4 {v0.8b, v1.8b, v2.8b, v3.8b}, [sp] |
| 393 | ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64 // filter[0..7] | 393 | ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64 // filter[0..7] |
| 394 | // load filterPositions into registers for next iteration | 394 | // load filterPositions into registers for next iteration |
| 395 | | 395 | |
| 396 | ldp w8, w9, [x5] // filterPos[0], filterPos[1] | 396 | ldp w8, w9, [x5] // filterPos[0], filterPos[1] |
| 397 | ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3] | 397 | ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3] |
| 398 | ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5] | 398 | ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5] |
| 399 | ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7] | 399 | ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7] |
| 400 | add x5, x5, #32 | 400 | add x5, x5, #32 |
| 401 | uxtl v0.8h, v0.8b | 401 | uxtl v0.8h, v0.8b |
| 402 | ldr w8, [x3, w8, uxtw] | 402 | ldr w8, [x3, w8, uxtw] |
| 403 | smull v5.4s, v0.4h, v28.4h // multiply first column of src | 403 | smull v5.4s, v0.4h, v28.4h // multiply first column of src |
| 404 | ldr w9, [x3, w9, uxtw] | 404 | ldr w9, [x3, w9, uxtw] |
| 405 | smull2 v6.4s, v0.8h, v28.8h | 405 | smull2 v6.4s, v0.8h, v28.8h |
| 406 | stp w8, w9, [sp] | 406 | stp w8, w9, [sp] |
| 407 | | 407 | |
| 408 | uxtl v1.8h, v1.8b | 408 | uxtl v1.8h, v1.8b |
| 409 | ldr w10, [x3, w10, uxtw] | 409 | ldr w10, [x3, w10, uxtw] |
| 410 | smlal v5.4s, v1.4h, v29.4h // multiply second column of src | 410 | smlal v5.4s, v1.4h, v29.4h // multiply second column of src |
| 411 | ldr w11, [x3, w11, uxtw] | 411 | ldr w11, [x3, w11, uxtw] |
| 412 | smlal2 v6.4s, v1.8h, v29.8h | 412 | smlal2 v6.4s, v1.8h, v29.8h |
| 413 | stp w10, w11, [sp, #8] | 413 | stp w10, w11, [sp, #8] |
| 414 | | 414 | |
| 415 | uxtl v2.8h, v2.8b | 415 | uxtl v2.8h, v2.8b |
| 416 | ldr w12, [x3, w12, uxtw] | 416 | ldr w12, [x3, w12, uxtw] |
| 417 | smlal v5.4s, v2.4h, v30.4h // multiply third column of src | 417 | smlal v5.4s, v2.4h, v30.4h // multiply third column of src |
| 418 | ldr w13, [x3, w13, uxtw] | 418 | ldr w13, [x3, w13, uxtw] |
| 419 | smlal2 v6.4s, v2.8h, v30.8h | 419 | smlal2 v6.4s, v2.8h, v30.8h |
| 420 | stp w12, w13, [sp, #16] | 420 | stp w12, w13, [sp, #16] |
| 421 | | 421 | |
| 422 | uxtl v3.8h, v3.8b | 422 | uxtl v3.8h, v3.8b |
| 423 | ldr w14, [x3, w14, uxtw] | 423 | ldr w14, [x3, w14, uxtw] |
| 424 | smlal v5.4s, v3.4h, v31.4h // multiply fourth column of src | 424 | smlal v5.4s, v3.4h, v31.4h // multiply fourth column of src |
| 425 | ldr w15, [x3, w15, uxtw] | 425 | ldr w15, [x3, w15, uxtw] |
| 426 | smlal2 v6.4s, v3.8h, v31.8h | 426 | smlal2 v6.4s, v3.8h, v31.8h |
| 427 | stp w14, w15, [sp, #24] | 427 | stp w14, w15, [sp, #24] |
| 428 | | 428 | |
| 429 | sub w2, w2, #8 | 429 | sub w2, w2, #8 |
| 430 | sshr v5.4s, v5.4s, #3 | 430 | sshr v5.4s, v5.4s, #3 |
| 431 | sshr v6.4s, v6.4s, #3 | 431 | sshr v6.4s, v6.4s, #3 |
| 432 | smin v5.4s, v5.4s, v18.4s | 432 | smin v5.4s, v5.4s, v18.4s |
| 433 | smin v6.4s, v6.4s, v18.4s | 433 | smin v6.4s, v6.4s, v18.4s |
| 434 | | 434 | |
| 435 | st1 {v5.4s, v6.4s}, [x1], #32 | 435 | st1 {v5.4s, v6.4s}, [x1], #32 |
| 436 | cmp w2, #16 | 436 | cmp w2, #16 |
| 437 | b.ge 1b | 437 | b.ge 1b |
| 438 | | 438 | |
| 439 | // here we make last iteration, without updating the registers | 439 | // here we make last iteration, without updating the registers |
| 440 | ld4 {v0.8b, v1.8b, v2.8b, v3.8b}, [sp] | 440 | ld4 {v0.8b, v1.8b, v2.8b, v3.8b}, [sp] |
| 441 | ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64 // filter[0..7] | 441 | ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64 // filter[0..7] |
| 442 | | 442 | |
| 443 | uxtl v0.8h, v0.8b | 443 | uxtl v0.8h, v0.8b |
| 444 | uxtl v1.8h, v1.8b | 444 | uxtl v1.8h, v1.8b |
| 445 | smull v5.4s, v0.4h, v28.4h | 445 | smull v5.4s, v0.4h, v28.4h |
| 446 | smull2 v6.4s, v0.8h, v28.8h | 446 | smull2 v6.4s, v0.8h, v28.8h |
| 447 | uxtl v2.8h, v2.8b | 447 | uxtl v2.8h, v2.8b |
| 448 | smlal v5.4s, v1.4h, v29.4h | 448 | smlal v5.4s, v1.4h, v29.4h |
| 449 | smlal2 v6.4s, v1.8h, v29.8h | 449 | smlal2 v6.4s, v1.8h, v29.8h |
| 450 | uxtl v3.8h, v3.8b | 450 | uxtl v3.8h, v3.8b |
| 451 | smlal v5.4s, v2.4h, v30.4h | 451 | smlal v5.4s, v2.4h, v30.4h |
| 452 | smlal2 v6.4s, v2.8h, v30.8h | 452 | smlal2 v6.4s, v2.8h, v30.8h |
| 453 | smlal v5.4s, v3.4h, v31.4h | 453 | smlal v5.4s, v3.4h, v31.4h |
| 454 | smlal2 v6.4s, v3.8h, v31.8h | 454 | smlal2 v6.4s, v3.8h, v31.8h |
| 455 | | 455 | |
| 456 | sshr v5.4s, v5.4s, #3 | 456 | sshr v5.4s, v5.4s, #3 |
| 457 | sshr v6.4s, v6.4s, #3 | 457 | sshr v6.4s, v6.4s, #3 |
| 458 | | 458 | |
| 459 | smin v5.4s, v5.4s, v18.4s | 459 | smin v5.4s, v5.4s, v18.4s |
| 460 | smin v6.4s, v6.4s, v18.4s | 460 | smin v6.4s, v6.4s, v18.4s |
| 461 | | 461 | |
| 462 | sub w2, w2, #8 | 462 | sub w2, w2, #8 |
| 463 | st1 {v5.4s, v6.4s}, [x1], #32 | 463 | st1 {v5.4s, v6.4s}, [x1], #32 |
| 464 | add sp, sp, #32 // restore stack | 464 | add sp, sp, #32 // restore stack |
| 465 | cbnz w2, 2f | 465 | cbnz w2, 2f |
| 466 | | 466 | |
| 467 | ret | 467 | ret |
| 468 | | 468 | |
| 469 | 2: | 469 | 2: |
| 470 | ldr w8, [x5], #4 // load filterPos | 470 | ldr w8, [x5], #4 // load filterPos |
| 471 | add x9, x3, w8, uxtw // src + filterPos | 471 | add x9, x3, w8, uxtw // src + filterPos |
| 472 | ld1 {v0.s}[0], [x9] // load 4 * uint8_t* into one single | 472 | ld1 {v0.s}[0], [x9] // load 4 * uint8_t* into one single |
| 473 | ld1 {v31.4h}, [x4], #8 | 473 | ld1 {v31.4h}, [x4], #8 |
| 474 | uxtl v0.8h, v0.8b | 474 | uxtl v0.8h, v0.8b |
| 475 | smull v5.4s, v0.4h, v31.4h | 475 | smull v5.4s, v0.4h, v31.4h |
| 476 | saddlv d0, v5.4s | 476 | saddlv d0, v5.4s |
| 477 | sqshrn s0, d0, #3 | 477 | sqshrn s0, d0, #3 |
| 478 | smin v0.4s, v0.4s, v18.4s | 478 | smin v0.4s, v0.4s, v18.4s |
| 479 | st1 {v0.s}[0], [x1], #4 | 479 | st1 {v0.s}[0], [x1], #4 |
| 480 | sub w2, w2, #1 | 480 | sub w2, w2, #1 |
| 481 | cbnz w2, 2b // if iterations remain jump to beginning | 481 | cbnz w2, 2b // if iterations remain jump to beginning |
| 482 | | 482 | |
| 483 | ret | 483 | ret |
| 484 | endfunc | 484 | endfunc |
| 485 | | 485 | |
| 486 | function ff_hscale8to19_X8_neon, export=1 | 486 | function ff_hscale8to19_X8_neon, export=1 |
| 487 | movi v20.4s, #1 | 487 | movi v20.4s, #1 |
| 488 | movi v17.4s, #1 | 488 | movi v17.4s, #1 |
| 489 | shl v20.4s, v20.4s, #19 | 489 | shl v20.4s, v20.4s, #19 |
| 490 | sub v20.4s, v20.4s, v17.4s | 490 | sub v20.4s, v20.4s, v17.4s |
| 491 | | 491 | |
| 492 | sbfiz x7, x6, #1, #32 // filterSize*2 (*2 because int16) | 492 | sbfiz x7, x6, #1, #32 // filterSize*2 (*2 because int16) |
| 493 | 1: | 493 | 1: |
| 494 | mov x16, x4 // filter0 = filter | 494 | mov x16, x4 // filter0 = filter |
| 495 | ldr w8, [x5], #4 // filterPos[idx] | 495 | ldr w8, [x5], #4 // filterPos[idx] |
| 496 | add x12, x16, x7 // filter1 = filter0 + filterSize*2 | 496 | add x12, x16, x7 // filter1 = filter0 + filterSize*2 |
| 497 | ldr w0, [x5], #4 // filterPos[idx + 1] | 497 | ldr w0, [x5], #4 // filterPos[idx + 1] |
| 498 | add x13, x12, x7 // filter2 = filter1 + filterSize*2 | 498 | add x13, x12, x7 // filter2 = filter1 + filterSize*2 |
| 499 | ldr w11, [x5], #4 // filterPos[idx + 2] | 499 | ldr w11, [x5], #4 // filterPos[idx + 2] |
| 500 | add x4, x13, x7 // filter3 = filter2 + filterSize*2 | 500 | add x4, x13, x7 // filter3 = filter2 + filterSize*2 |
| 501 | ldr w9, [x5], #4 // filterPos[idx + 3] | 501 | ldr w9, [x5], #4 // filterPos[idx + 3] |
| 502 | movi v0.2d, #0 // val sum part 1 (for dst[0]) | 502 | movi v0.2d, #0 // val sum part 1 (for dst[0]) |
| 503 | movi v1.2d, #0 // val sum part 2 (for dst[1]) | 503 | movi v1.2d, #0 // val sum part 2 (for dst[1]) |
| 504 | movi v2.2d, #0 // val sum part 3 (for dst[2]) | 504 | movi v2.2d, #0 // val sum part 3 (for dst[2]) |
| 505 | movi v3.2d, #0 // val sum part 4 (for dst[3]) | 505 | movi v3.2d, #0 // val sum part 4 (for dst[3]) |
| 506 | add x17, x3, w8, uxtw // srcp + filterPos[0] | 506 | add x17, x3, w8, uxtw // srcp + filterPos[0] |
| 507 | add x8, x3, w0, uxtw // srcp + filterPos[1] | 507 | add x8, x3, w0, uxtw // srcp + filterPos[1] |
| 508 | add x0, x3, w11, uxtw // srcp + filterPos[2] | 508 | add x0, x3, w11, uxtw // srcp + filterPos[2] |
| 509 | add x11, x3, w9, uxtw // srcp + filterPos[3] | 509 | add x11, x3, w9, uxtw // srcp + filterPos[3] |
| 510 | mov w15, w6 // filterSize counter | 510 | mov w15, w6 // filterSize counter |
| 511 | 2: ld1 {v4.8b}, [x17], #8 // srcp[filterPos[0] + {0..7}] | 511 | 2: ld1 {v4.8b}, [x17], #8 // srcp[filterPos[0] + {0..7}] |
| 512 | ld1 {v5.8h}, [x16], #16 // load 8x16-bit filter values, part 1 | 512 | ld1 {v5.8h}, [x16], #16 // load 8x16-bit filter values, part 1 |
| 513 | uxtl v4.8h, v4.8b // unpack part 1 to 16-bit | 513 | uxtl v4.8h, v4.8b // unpack part 1 to 16-bit |
| 514 | smlal v0.4s, v4.4h, v5.4h // v0 accumulates srcp[filterPos[0] + {0..3}] * filter[{0..3}] | 514 | smlal v0.4s, v4.4h, v5.4h // v0 accumulates srcp[filterPos[0] + {0..3}] * filter[{0..3}] |
| 515 | ld1 {v6.8b}, [x8], #8 // srcp[filterPos[1] + {0..7}] | 515 | ld1 {v6.8b}, [x8], #8 // srcp[filterPos[1] + {0..7}] |
| 516 | smlal2 v0.4s, v4.8h, v5.8h // v0 accumulates srcp[filterPos[0] + {4..7}] * filter[{4..7}] | 516 | smlal2 v0.4s, v4.8h, v5.8h // v0 accumulates srcp[filterPos[0] + {4..7}] * filter[{4..7}] |
| 517 | ld1 {v7.8h}, [x12], #16 // load 8x16-bit at filter+filterSize | 517 | ld1 {v7.8h}, [x12], #16 // load 8x16-bit at filter+filterSize |
| 518 | ld1 {v16.8b}, [x0], #8 // srcp[filterPos[2] + {0..7}] | 518 | ld1 {v16.8b}, [x0], #8 // srcp[filterPos[2] + {0..7}] |
| 519 | uxtl v6.8h, v6.8b // unpack part 2 to 16-bit | 519 | uxtl v6.8h, v6.8b // unpack part 2 to 16-bit |
| 520 | ld1 {v17.8h}, [x13], #16 // load 8x16-bit at filter+2*filterSize | 520 | ld1 {v17.8h}, [x13], #16 // load 8x16-bit at filter+2*filterSize |
| 521 | uxtl v16.8h, v16.8b // unpack part 3 to 16-bit | 521 | uxtl v16.8h, v16.8b // unpack part 3 to 16-bit |
| 522 | smlal v1.4s, v6.4h, v7.4h // v1 accumulates srcp[filterPos[1] + {0..3}] * filter[{0..3}] | 522 | smlal v1.4s, v6.4h, v7.4h // v1 accumulates srcp[filterPos[1] + {0..3}] * filter[{0..3}] |
| 523 | ld1 {v18.8b}, [x11], #8 // srcp[filterPos[3] + {0..7}] | 523 | ld1 {v18.8b}, [x11], #8 // srcp[filterPos[3] + {0..7}] |
| 524 | smlal v2.4s, v16.4h, v17.4h // v2 accumulates srcp[filterPos[2] + {0..3}] * filter[{0..3}] | 524 | smlal v2.4s, v16.4h, v17.4h // v2 accumulates srcp[filterPos[2] + {0..3}] * filter[{0..3}] |
| 525 | ld1 {v19.8h}, [x4], #16 // load 8x16-bit at filter+3*filterSize | 525 | ld1 {v19.8h}, [x4], #16 // load 8x16-bit at filter+3*filterSize |
| 526 | smlal2 v2.4s, v16.8h, v17.8h // v2 accumulates srcp[filterPos[2] + {4..7}] * filter[{4..7}] | 526 | smlal2 v2.4s, v16.8h, v17.8h // v2 accumulates srcp[filterPos[2] + {4..7}] * filter[{4..7}] |
| 527 | uxtl v18.8h, v18.8b // unpack part 4 to 16-bit | 527 | uxtl v18.8h, v18.8b // unpack part 4 to 16-bit |
| 528 | smlal2 v1.4s, v6.8h, v7.8h // v1 accumulates srcp[filterPos[1] + {4..7}] * filter[{4..7}] | 528 | smlal2 v1.4s, v6.8h, v7.8h // v1 accumulates srcp[filterPos[1] + {4..7}] * filter[{4..7}] |
| 529 | smlal v3.4s, v18.4h, v19.4h // v3 accumulates srcp[filterPos[3] + {0..3}] * filter[{0..3}] | 529 | smlal v3.4s, v18.4h, v19.4h // v3 accumulates srcp[filterPos[3] + {0..3}] * filter[{0..3}] |
| 530 | subs w15, w15, #8 // j -= 8: processed 8/filterSize | 530 | subs w15, w15, #8 // j -= 8: processed 8/filterSize |
| 531 | smlal2 v3.4s, v18.8h, v19.8h // v3 accumulates srcp[filterPos[3] + {4..7}] * filter[{4..7}] | 531 | smlal2 v3.4s, v18.8h, v19.8h // v3 accumulates srcp[filterPos[3] + {4..7}] * filter[{4..7}] |
| 532 | b.gt 2b // inner loop if filterSize not consumed completely | 532 | b.gt 2b // inner loop if filterSize not consumed completely |
| 533 | addp v0.4s, v0.4s, v1.4s // part01 horizontal pair adding | 533 | addp v0.4s, v0.4s, v1.4s // part01 horizontal pair adding |
| 534 | addp v2.4s, v2.4s, v3.4s // part23 horizontal pair adding | 534 | addp v2.4s, v2.4s, v3.4s // part23 horizontal pair adding |
| 535 | addp v0.4s, v0.4s, v2.4s // part0123 horizontal pair adding | 535 | addp v0.4s, v0.4s, v2.4s // part0123 horizontal pair adding |
| 536 | subs w2, w2, #4 // dstW -= 4 | 536 | subs w2, w2, #4 // dstW -= 4 |
| 537 | sshr v0.4s, v0.4s, #3 // shift and clip the 2x16-bit final values | 537 | sshr v0.4s, v0.4s, #3 // shift and clip the 2x16-bit final values |
| 538 | smin v0.4s, v0.4s, v20.4s | 538 | smin v0.4s, v0.4s, v20.4s |
| 539 | st1 {v0.4s}, [x1], #16 // write to destination part0123 | 539 | st1 {v0.4s}, [x1], #16 // write to destination part0123 |
| 540 | b.gt 1b // loop until end of line | 540 | b.gt 1b // loop until end of line |
| 541 | ret | 541 | ret |
| 542 | endfunc | 542 | endfunc |
| 543 | | 543 | |
| @@ -550,91 +550,91 @@ function ff_hscale8to19_X4_neon, export=1 · 6 unmodified lines |
| 550 | // x5 const int32_t *filterPos | 550 | // x5 const int32_t *filterPos |
| 551 | // w6 int filterSize | 551 | // w6 int filterSize |
| 552 | | 552 | |
| 553 | movi v20.4s, #1 | 553 | movi v20.4s, #1 |
| 554 | movi v17.4s, #1 | 554 | movi v17.4s, #1 |
| 555 | shl v20.4s, v20.4s, #19 | 555 | shl v20.4s, v20.4s, #19 |
| 556 | sub v20.4s, v20.4s, v17.4s | 556 | sub v20.4s, v20.4s, v17.4s |
| 557 | | 557 | |
| 558 | lsl w7, w6, #1 | 558 | lsl w7, w6, #1 |
| 559 | 1: | 559 | 1: |
| 560 | ldp w8, w9, [x5] | 560 | ldp w8, w9, [x5] |
| 561 | ldp w10, w11, [x5, #8] | 561 | ldp w10, w11, [x5, #8] |
| 562 | | 562 | |
| 563 | movi v16.2d, #0 // initialize accumulator for idx + 0 | 563 | movi v16.2d, #0 // initialize accumulator for idx + 0 |
| 564 | movi v17.2d, #0 // initialize accumulator for idx + 1 | 564 | movi v17.2d, #0 // initialize accumulator for idx + 1 |
| 565 | movi v18.2d, #0 // initialize accumulator for idx + 2 | 565 | movi v18.2d, #0 // initialize accumulator for idx + 2 |
| 566 | movi v19.2d, #0 // initialize accumulator for idx + 3 | 566 | movi v19.2d, #0 // initialize accumulator for idx + 3 |
| 567 | | 567 | |
| 568 | mov x12, x4 // filter + 0 | 568 | mov x12, x4 // filter + 0 |
| 569 | add x13, x4, x7 // filter + 1 | 569 | add x13, x4, x7 // filter + 1 |
| 570 | add x8, x3, w8, uxtw // srcp + filterPos 0 | 570 | add x8, x3, w8, uxtw // srcp + filterPos 0 |
| 571 | add x14, x13, x7 // filter + 2 | 571 | add x14, x13, x7 // filter + 2 |
| 572 | add x9, x3, w9, uxtw // srcp + filterPos 1 | 572 | add x9, x3, w9, uxtw // srcp + filterPos 1 |
| 573 | add x15, x14, x7 // filter + 3 | 573 | add x15, x14, x7 // filter + 3 |
| 574 | add x10, x3, w10, uxtw // srcp + filterPos 2 | 574 | add x10, x3, w10, uxtw // srcp + filterPos 2 |
| 575 | mov w0, w6 // save the filterSize to temporary variable | 575 | mov w0, w6 // save the filterSize to temporary variable |
| 576 | add x11, x3, w11, uxtw // srcp + filterPos 3 | 576 | add x11, x3, w11, uxtw // srcp + filterPos 3 |
| 577 | add x5, x5, #16 // advance filter position | 577 | add x5, x5, #16 // advance filter position |
| 578 | mov x16, xzr // clear the register x16 used for offsetting the filter values | 578 | mov x16, xzr // clear the register x16 used for offsetting the filter values |
| 579 | | 579 | |
| 580 | 2: | 580 | 2: |
| 581 | ldr d4, [x8], #8 // load src values for idx 0 | 581 | ldr d4, [x8], #8 // load src values for idx 0 |
| 582 | ldr q31, [x12, x16] // load filter values for idx 0 | 582 | ldr q31, [x12, x16] // load filter values for idx 0 |
| 583 | uxtl v4.8h, v4.8b // extend type to match the filter' size | 583 | uxtl v4.8h, v4.8b // extend type to match the filter' size |
| 584 | ldr d5, [x9], #8 // load src values for idx 1 | 584 | ldr d5, [x9], #8 // load src values for idx 1 |
| 585 | smlal v16.4s, v4.4h, v31.4h // multiplication of lower half for idx 0 | 585 | smlal v16.4s, v4.4h, v31.4h // multiplication of lower half for idx 0 |
| 586 | uxtl v5.8h, v5.8b // extend type to match the filter' size | 586 | uxtl v5.8h, v5.8b // extend type to match the filter' size |
| 587 | ldr q30, [x13, x16] // load filter values for idx 1 | 587 | ldr q30, [x13, x16] // load filter values for idx 1 |
| 588 | smlal2 v16.4s, v4.8h, v31.8h // multiplication of upper half for idx 0 | 588 | smlal2 v16.4s, v4.8h, v31.8h // multiplication of upper half for idx 0 |
| 589 | ldr d6, [x10], #8 // load src values for idx 2 | 589 | ldr d6, [x10], #8 // load src values for idx 2 |
| 590 | ldr q29, [x14, x16] // load filter values for idx 2 | 590 | ldr q29, [x14, x16] // load filter values for idx 2 |
| 591 | smlal v17.4s, v5.4h, v30.4h // multiplication of lower half for idx 1 | 591 | smlal v17.4s, v5.4h, v30.4h // multiplication of lower half for idx 1 |
| 592 | ldr d7, [x11], #8 // load src values for idx 3 | 592 | ldr d7, [x11], #8 // load src values for idx 3 |
| 593 | smlal2 v17.4s, v5.8h, v30.8h // multiplication of upper half for idx 1 | 593 | smlal2 v17.4s, v5.8h, v30.8h // multiplication of upper half for idx 1 |
| 594 | uxtl v6.8h, v6.8b // extend tpye to matchi the filter's size | 594 | uxtl v6.8h, v6.8b // extend tpye to matchi the filter's size |
| 595 | ldr q28, [x15, x16] // load filter values for idx 3 | 595 | ldr q28, [x15, x16] // load filter values for idx 3 |
| 596 | smlal v18.4s, v6.4h, v29.4h // multiplication of lower half for idx 2 | 596 | smlal v18.4s, v6.4h, v29.4h // multiplication of lower half for idx 2 |
| 597 | uxtl v7.8h, v7.8b | 597 | uxtl v7.8h, v7.8b |
| 598 | smlal2 v18.4s, v6.8h, v29.8h // multiplication of upper half for idx 2 | 598 | smlal2 v18.4s, v6.8h, v29.8h // multiplication of upper half for idx 2 |
| 599 | sub w0, w0, #8 | 599 | sub w0, w0, #8 |
| 600 | smlal v19.4s, v7.4h, v28.4h // multiplication of lower half for idx 3 | 600 | smlal v19.4s, v7.4h, v28.4h // multiplication of lower half for idx 3 |
| 601 | cmp w0, #8 | 601 | cmp w0, #8 |
| 602 | smlal2 v19.4s, v7.8h, v28.8h // multiplication of upper half for idx 3 | 602 | smlal2 v19.4s, v7.8h, v28.8h // multiplication of upper half for idx 3 |
| 603 | add x16, x16, #16 // advance filter values indexing | 603 | add x16, x16, #16 // advance filter values indexing |
| 604 | | 604 | |
| 605 | b.ge 2b | 605 | b.ge 2b |
| 606 | | 606 | |
| 607 | | 607 | |
| 608 | // 4 iterations left | 608 | // 4 iterations left |
| 609 | | 609 | |
| 610 | sub x17, x7, #8 // step back to wrap up the filter pos for last 4 elements | 610 | sub x17, x7, #8 // step back to wrap up the filter pos for last 4 elements |
| 611 | | 611 | |
| 612 | ldr s4, [x8] // load src values for idx 0 | 612 | ldr s4, [x8] // load src values for idx 0 |
| 613 | ldr d31, [x12, x17] // load filter values for idx 0 | 613 | ldr d31, [x12, x17] // load filter values for idx 0 |
| 614 | uxtl v4.8h, v4.8b // extend type to match the filter' size | 614 | uxtl v4.8h, v4.8b // extend type to match the filter' size |
| 615 | ldr s5, [x9] // load src values for idx 1 | 615 | ldr s5, [x9] // load src values for idx 1 |
| 616 | smlal v16.4s, v4.4h, v31.4h | 616 | smlal v16.4s, v4.4h, v31.4h |
| 617 | ldr d30, [x13, x17] // load filter values for idx 1 | 617 | ldr d30, [x13, x17] // load filter values for idx 1 |
| 618 | uxtl v5.8h, v5.8b // extend type to match the filter' size | 618 | uxtl v5.8h, v5.8b // extend type to match the filter' size |
| 619 | ldr s6, [x10] // load src values for idx 2 | 619 | ldr s6, [x10] // load src values for idx 2 |
| 620 | smlal v17.4s, v5.4h, v30.4h | 620 | smlal v17.4s, v5.4h, v30.4h |
| 621 | uxtl v6.8h, v6.8b // extend type to match the filter's size | 621 | uxtl v6.8h, v6.8b // extend type to match the filter's size |
| 622 | ldr d29, [x14, x17] // load filter values for idx 2 | 622 | ldr d29, [x14, x17] // load filter values for idx 2 |
| 623 | ldr s7, [x11] // load src values for idx 3 | 623 | ldr s7, [x11] // load src values for idx 3 |
| 624 | addp v16.4s, v16.4s, v17.4s | 624 | addp v16.4s, v16.4s, v17.4s |
| 625 | uxtl v7.8h, v7.8b | 625 | uxtl v7.8h, v7.8b |
| 626 | ldr d28, [x15, x17] // load filter values for idx 3 | 626 | ldr d28, [x15, x17] // load filter values for idx 3 |
| 627 | smlal v18.4s, v6.4h, v29.4h | 627 | smlal v18.4s, v6.4h, v29.4h |
| 628 | smlal v19.4s, v7.4h, v28.4h | 628 | smlal v19.4s, v7.4h, v28.4h |
| 629 | subs w2, w2, #4 | 629 | subs w2, w2, #4 |
| 630 | addp v18.4s, v18.4s, v19.4s | 630 | addp v18.4s, v18.4s, v19.4s |
| 631 | addp v16.4s, v16.4s, v18.4s | 631 | addp v16.4s, v16.4s, v18.4s |
| 632 | sshr v16.4s, v16.4s, #3 | 632 | sshr v16.4s, v16.4s, #3 |
| 633 | smin v16.4s, v16.4s, v20.4s | 633 | smin v16.4s, v16.4s, v20.4s |
| 634 | | 634 | |
| 635 | st1 {v16.4s}, [x1], #16 | 635 | st1 {v16.4s}, [x1], #16 |
| 636 | add x4, x4, x7, lsl #2 | 636 | add x4, x4, x7, lsl #2 |
| 637 | b.gt 1b | 637 | b.gt 1b |
| 638 | ret | 638 | ret |
| 639 | endfunc | 639 | endfunc |
| 640 | | 640 | |
| @@ -647,191 +647,191 @@ function ff_hscale16to15_4_neon_asm, export=1 · 6 unmodified lines |
| 647 | // x5 const int32_t *filterPos | 647 | // x5 const int32_t *filterPos |
| 648 | // w6 int filterSize | 648 | // w6 int filterSize |
| 649 | | 649 | |
| 650 | movi v18.4s, #1 | 650 | movi v18.4s, #1 |
| 651 | movi v17.4s, #1 | 651 | movi v17.4s, #1 |
| 652 | shl v18.4s, v18.4s, #15 | 652 | shl v18.4s, v18.4s, #15 |
| 653 | sub v18.4s, v18.4s, v17.4s // max allowed value | 653 | sub v18.4s, v18.4s, v17.4s // max allowed value |
| 654 | dup v17.4s, w0 // read shift | 654 | dup v17.4s, w0 // read shift |
| 655 | neg v17.4s, v17.4s // negate it, so it can be used in sshl (effectively shift right) | 655 | neg v17.4s, v17.4s // negate it, so it can be used in sshl (effectively shift right) |
| 656 | | 656 | |
| 657 | cmp w2, #16 | 657 | cmp w2, #16 |
| 658 | b.lt 2f // move to last block | 658 | b.lt 2f // move to last block |
| 659 | | 659 | |
| 660 | ldp w8, w9, [x5] // filterPos[0], filterPos[1] | 660 | ldp w8, w9, [x5] // filterPos[0], filterPos[1] |
| 661 | ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3] | 661 | ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3] |
| 662 | ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5] | 662 | ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5] |
| 663 | ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7] | 663 | ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7] |
| 664 | add x5, x5, #32 | 664 | add x5, x5, #32 |
| 665 | | 665 | |
| 666 | // shift all filterPos left by one, as uint16_t will be read | 666 | // shift all filterPos left by one, as uint16_t will be read |
| 667 | lsl x8, x8, #1 | 667 | lsl x8, x8, #1 |
| 668 | lsl x9, x9, #1 | 668 | lsl x9, x9, #1 |
| 669 | lsl x10, x10, #1 | 669 | lsl x10, x10, #1 |
| 670 | lsl x11, x11, #1 | 670 | lsl x11, x11, #1 |
| 671 | lsl x12, x12, #1 | 671 | lsl x12, x12, #1 |
| 672 | lsl x13, x13, #1 | 672 | lsl x13, x13, #1 |
| 673 | lsl x14, x14, #1 | 673 | lsl x14, x14, #1 |
| 674 | lsl x15, x15, #1 | 674 | lsl x15, x15, #1 |
| 675 | | 675 | |
| 676 | // load src with given offset | 676 | // load src with given offset |
| 677 | ldr x8, [x3, w8, uxtw] | 677 | ldr x8, [x3, w8, uxtw] |
| 678 | ldr x9, [x3, w9, uxtw] | 678 | ldr x9, [x3, w9, uxtw] |
| 679 | ldr x10, [x3, w10, uxtw] | 679 | ldr x10, [x3, w10, uxtw] |
| 680 | ldr x11, [x3, w11, uxtw] | 680 | ldr x11, [x3, w11, uxtw] |
| 681 | ldr x12, [x3, w12, uxtw] | 681 | ldr x12, [x3, w12, uxtw] |
| 682 | ldr x13, [x3, w13, uxtw] | 682 | ldr x13, [x3, w13, uxtw] |
| 683 | ldr x14, [x3, w14, uxtw] | 683 | ldr x14, [x3, w14, uxtw] |
| 684 | ldr x15, [x3, w15, uxtw] | 684 | ldr x15, [x3, w15, uxtw] |
| 685 | | 685 | |
| 686 | sub sp, sp, #64 | 686 | sub sp, sp, #64 |
| 687 | // push src on stack so it can be loaded into vectors later | 687 | // push src on stack so it can be loaded into vectors later |
| 688 | stp x8, x9, [sp] | 688 | stp x8, x9, [sp] |
| 689 | stp x10, x11, [sp, #16] | 689 | stp x10, x11, [sp, #16] |
| 690 | stp x12, x13, [sp, #32] | 690 | stp x12, x13, [sp, #32] |
| 691 | stp x14, x15, [sp, #48] | 691 | stp x14, x15, [sp, #48] |
| 692 | | 692 | |
| 693 | 1: | 693 | 1: |
| 694 | ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [sp] | 694 | ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [sp] |
| 695 | ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64 // filter[0..7] | 695 | ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64 // filter[0..7] |
| 696 | | 696 | |
| 697 | // Each of blocks does the following: | 697 | // Each of blocks does the following: |
| 698 | // Extend src and filter to 32 bits with uxtl and sxtl | 698 | // Extend src and filter to 32 bits with uxtl and sxtl |
| 699 | // multiply or multiply and accumulate results | 699 | // multiply or multiply and accumulate results |
| 700 | // Extending to 32 bits is necessary, as unit16_t values can't | 700 | // Extending to 32 bits is necessary, as unit16_t values can't |
| 701 | // be represented as int16_t without type promotion. | 701 | // be represented as int16_t without type promotion. |
| 702 | uxtl v26.4s, v0.4h | 702 | uxtl v26.4s, v0.4h |
| 703 | sxtl v27.4s, v28.4h | 703 | sxtl v27.4s, v28.4h |
| 704 | uxtl2 v0.4s, v0.8h | 704 | uxtl2 v0.4s, v0.8h |
| 705 | mul v5.4s, v26.4s, v27.4s | 705 | mul v5.4s, v26.4s, v27.4s |
| 706 | sxtl2 v28.4s, v28.8h | 706 | sxtl2 v28.4s, v28.8h |
| 707 | uxtl v26.4s, v1.4h | 707 | uxtl v26.4s, v1.4h |
| 708 | mul v6.4s, v0.4s, v28.4s | 708 | mul v6.4s, v0.4s, v28.4s |
| 709 | | 709 | |
| 710 | sxtl v27.4s, v29.4h | 710 | sxtl v27.4s, v29.4h |
| 711 | uxtl2 v0.4s, v1.8h | 711 | uxtl2 v0.4s, v1.8h |
| 712 | mla v5.4s, v27.4s, v26.4s | 712 | mla v5.4s, v27.4s, v26.4s |
| 713 | sxtl2 v28.4s, v29.8h | 713 | sxtl2 v28.4s, v29.8h |
| 714 | uxtl v26.4s, v2.4h | 714 | uxtl v26.4s, v2.4h |
| 715 | mla v6.4s, v28.4s, v0.4s | 715 | mla v6.4s, v28.4s, v0.4s |
| 716 | | 716 | |
| 717 | sxtl v27.4s, v30.4h | 717 | sxtl v27.4s, v30.4h |
| 718 | uxtl2 v0.4s, v2.8h | 718 | uxtl2 v0.4s, v2.8h |
| 719 | mla v5.4s, v27.4s, v26.4s | 719 | mla v5.4s, v27.4s, v26.4s |
| 720 | sxtl2 v28.4s, v30.8h | 720 | sxtl2 v28.4s, v30.8h |
| 721 | uxtl v26.4s, v3.4h | 721 | uxtl v26.4s, v3.4h |
| 722 | mla v6.4s, v28.4s, v0.4s | 722 | mla v6.4s, v28.4s, v0.4s |
| 723 | | 723 | |
| 724 | sxtl v27.4s, v31.4h | 724 | sxtl v27.4s, v31.4h |
| 725 | uxtl2 v0.4s, v3.8h | 725 | uxtl2 v0.4s, v3.8h |
| 726 | mla v5.4s, v27.4s, v26.4s | 726 | mla v5.4s, v27.4s, v26.4s |
| 727 | sxtl2 v28.4s, v31.8h | 727 | sxtl2 v28.4s, v31.8h |
| 728 | sub w2, w2, #8 | 728 | sub w2, w2, #8 |
| 729 | mla v6.4s, v28.4s, v0.4s | 729 | mla v6.4s, v28.4s, v0.4s |
| 730 | | 730 | |
| 731 | sshl v5.4s, v5.4s, v17.4s | 731 | sshl v5.4s, v5.4s, v17.4s |
| 732 | sshl v6.4s, v6.4s, v17.4s | 732 | sshl v6.4s, v6.4s, v17.4s |
| 733 | smin v5.4s, v5.4s, v18.4s | 733 | smin v5.4s, v5.4s, v18.4s |
| 734 | smin v6.4s, v6.4s, v18.4s | 734 | smin v6.4s, v6.4s, v18.4s |
| 735 | xtn v5.4h, v5.4s | 735 | xtn v5.4h, v5.4s |
| 736 | xtn2 v5.8h, v6.4s | 736 | xtn2 v5.8h, v6.4s |
| 737 | | 737 | |
| 738 | st1 {v5.8h}, [x1], #16 | 738 | st1 {v5.8h}, [x1], #16 |
| 739 | cmp w2, #16 | 739 | cmp w2, #16 |
| 740 | | 740 | |
| 741 | // load filterPositions into registers for next iteration | 741 | // load filterPositions into registers for next iteration |
| 742 | ldp w8, w9, [x5] // filterPos[0], filterPos[1] | 742 | ldp w8, w9, [x5] // filterPos[0], filterPos[1] |
| 743 | ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3] | 743 | ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3] |
| 744 | ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5] | 744 | ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5] |
| 745 | ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7] | 745 | ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7] |
| 746 | add x5, x5, #32 | 746 | add x5, x5, #32 |
| 747 | | 747 | |
| 748 | lsl x8, x8, #1 | 748 | lsl x8, x8, #1 |
| 749 | lsl x9, x9, #1 | 749 | lsl x9, x9, #1 |
| 750 | lsl x10, x10, #1 | 750 | lsl x10, x10, #1 |
| 751 | lsl x11, x11, #1 | 751 | lsl x11, x11, #1 |
| 752 | lsl x12, x12, #1 | 752 | lsl x12, x12, #1 |
| 753 | lsl x13, x13, #1 | 753 | lsl x13, x13, #1 |
| 754 | lsl x14, x14, #1 | 754 | lsl x14, x14, #1 |
| 755 | lsl x15, x15, #1 | 755 | lsl x15, x15, #1 |
| 756 | | 756 | |
| 757 | ldr x8, [x3, w8, uxtw] | 757 | ldr x8, [x3, w8, uxtw] |
| 758 | ldr x9, [x3, w9, uxtw] | 758 | ldr x9, [x3, w9, uxtw] |
| 759 | ldr x10, [x3, w10, uxtw] | 759 | ldr x10, [x3, w10, uxtw] |
| 760 | ldr x11, [x3, w11, uxtw] | 760 | ldr x11, [x3, w11, uxtw] |
| 761 | ldr x12, [x3, w12, uxtw] | 761 | ldr x12, [x3, w12, uxtw] |
| 762 | ldr x13, [x3, w13, uxtw] | 762 | ldr x13, [x3, w13, uxtw] |
| 763 | ldr x14, [x3, w14, uxtw] | 763 | ldr x14, [x3, w14, uxtw] |
| 764 | ldr x15, [x3, w15, uxtw] | 764 | ldr x15, [x3, w15, uxtw] |
| 765 | | 765 | |
| 766 | stp x8, x9, [sp] | 766 | stp x8, x9, [sp] |
| 767 | stp x10, x11, [sp, #16] | 767 | stp x10, x11, [sp, #16] |
| 768 | stp x12, x13, [sp, #32] | 768 | stp x12, x13, [sp, #32] |
| 769 | stp x14, x15, [sp, #48] | 769 | stp x14, x15, [sp, #48] |
| 770 | | 770 | |
| 771 | b.ge 1b | 771 | b.ge 1b |
| 772 | | 772 | |
| 773 | // here we make last iteration, without updating the registers | 773 | // here we make last iteration, without updating the registers |
| 774 | ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [sp] | 774 | ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [sp] |
| 775 | ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64 | 775 | ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64 |
| 776 | | 776 | |
| 777 | uxtl v26.4s, v0.4h | 777 | uxtl v26.4s, v0.4h |
| 778 | sxtl v27.4s, v28.4h | 778 | sxtl v27.4s, v28.4h |
| 779 | uxtl2 v0.4s, v0.8h | 779 | uxtl2 v0.4s, v0.8h |
| 780 | mul v5.4s, v26.4s, v27.4s | 780 | mul v5.4s, v26.4s, v27.4s |
| 781 | sxtl2 v28.4s, v28.8h | 781 | sxtl2 v28.4s, v28.8h |
| 782 | uxtl v26.4s, v1.4h | 782 | uxtl v26.4s, v1.4h |
| 783 | mul v6.4s, v0.4s, v28.4s | 783 | mul v6.4s, v0.4s, v28.4s |
| 784 | | 784 | |
| 785 | sxtl v27.4s, v29.4h | 785 | sxtl v27.4s, v29.4h |
| 786 | uxtl2 v0.4s, v1.8h | 786 | uxtl2 v0.4s, v1.8h |
| 787 | mla v5.4s, v26.4s, v27.4s | 787 | mla v5.4s, v26.4s, v27.4s |
| 788 | sxtl2 v28.4s, v29.8h | 788 | sxtl2 v28.4s, v29.8h |
| 789 | uxtl v26.4s, v2.4h | 789 | uxtl v26.4s, v2.4h |
| 790 | mla v6.4s, v0.4s, v28.4s | 790 | mla v6.4s, v0.4s, v28.4s |
| 791 | | 791 | |
| 792 | sxtl v27.4s, v30.4h | 792 | sxtl v27.4s, v30.4h |
| 793 | uxtl2 v0.4s, v2.8h | 793 | uxtl2 v0.4s, v2.8h |
| 794 | mla v5.4s, v26.4s, v27.4s | 794 | mla v5.4s, v26.4s, v27.4s |
| 795 | sxtl2 v28.4s, v30.8h | 795 | sxtl2 v28.4s, v30.8h |
| 796 | uxtl v26.4s, v3.4h | 796 | uxtl v26.4s, v3.4h |
| 797 | mla v6.4s, v0.4s, v28.4s | 797 | mla v6.4s, v0.4s, v28.4s |
| 798 | | 798 | |
| 799 | sxtl v27.4s, v31.4h | 799 | sxtl v27.4s, v31.4h |
| 800 | uxtl2 v0.4s, v3.8h | 800 | uxtl2 v0.4s, v3.8h |
| 801 | mla v5.4s, v26.4s, v27.4s | 801 | mla v5.4s, v26.4s, v27.4s |
| 802 | sxtl2 v28.4s, v31.8h | 802 | sxtl2 v28.4s, v31.8h |
| 803 | subs w2, w2, #8 | 803 | subs w2, w2, #8 |
| 804 | mla v6.4s, v0.4s, v28.4s | 804 | mla v6.4s, v0.4s, v28.4s |
| 805 | | 805 | |
| 806 | sshl v5.4s, v5.4s, v17.4s | 806 | sshl v5.4s, v5.4s, v17.4s |
| 807 | sshl v6.4s, v6.4s, v17.4s | 807 | sshl v6.4s, v6.4s, v17.4s |
| 808 | smin v5.4s, v5.4s, v18.4s | 808 | smin v5.4s, v5.4s, v18.4s |
| 809 | smin v6.4s, v6.4s, v18.4s | 809 | smin v6.4s, v6.4s, v18.4s |
| 810 | xtn v5.4h, v5.4s | 810 | xtn v5.4h, v5.4s |
| 811 | xtn2 v5.8h, v6.4s | 811 | xtn2 v5.8h, v6.4s |
| 812 | | 812 | |
| 813 | st1 {v5.8h}, [x1], #16 | 813 | st1 {v5.8h}, [x1], #16 |
| 814 | add sp, sp, #64 // restore stack | 814 | add sp, sp, #64 // restore stack |
| 815 | cbnz w2, 2f | 815 | cbnz w2, 2f |
| 816 | | 816 | |
| 817 | ret | 817 | ret |
| 818 | | 818 | |
| 819 | 2: | 819 | 2: |
| 820 | ldr w8, [x5], #4 // load filterPos | 820 | ldr w8, [x5], #4 // load filterPos |
| 821 | lsl w8, w8, #1 | 821 | lsl w8, w8, #1 |
| 822 | add x9, x3, w8, uxtw // src + filterPos | 822 | add x9, x3, w8, uxtw // src + filterPos |
| 823 | ld1 {v0.4h}, [x9] // load 4 * uint16_t | 823 | ld1 {v0.4h}, [x9] // load 4 * uint16_t |
| 824 | ld1 {v31.4h}, [x4], #8 | 824 | ld1 {v31.4h}, [x4], #8 |
| 825 | | 825 | |
| 826 | uxtl v0.4s, v0.4h | 826 | uxtl v0.4s, v0.4h |
| 827 | sxtl v31.4s, v31.4h | 827 | sxtl v31.4s, v31.4h |
| 828 | mul v5.4s, v0.4s, v31.4s | 828 | mul v5.4s, v0.4s, v31.4s |
| 829 | addv s0, v5.4s | 829 | addv s0, v5.4s |
| 830 | sshl v0.4s, v0.4s, v17.4s | 830 | sshl v0.4s, v0.4s, v17.4s |
| 831 | smin v0.4s, v0.4s, v18.4s | 831 | smin v0.4s, v0.4s, v18.4s |
| 832 | st1 {v0.h}[0], [x1], #2 | 832 | st1 {v0.h}[0], [x1], #2 |
| 833 | sub w2, w2, #1 | 833 | sub w2, w2, #1 |
| 834 | cbnz w2, 2b // if iterations remain jump to beginning | 834 | cbnz w2, 2b // if iterations remain jump to beginning |
| 835 | | 835 | |
| 836 | ret | 836 | ret |
| 837 | endfunc | 837 | endfunc |
| @@ -845,79 +845,79 @@ function ff_hscale16to15_X8_neon_asm, export=1 · 7 unmodified lines |
| 845 | // x5 const int32_t *filterPos | 845 | // x5 const int32_t *filterPos |
| 846 | // w6 int filterSize | 846 | // w6 int filterSize |
| 847 | | 847 | |
| 848 | movi v20.4s, #1 | 848 | movi v20.4s, #1 |
| 849 | movi v21.4s, #1 | 849 | movi v21.4s, #1 |
| 850 | shl v20.4s, v20.4s, #15 | 850 | shl v20.4s, v20.4s, #15 |
| 851 | sub v20.4s, v20.4s, v21.4s | 851 | sub v20.4s, v20.4s, v21.4s |
| 852 | dup v21.4s, w0 | 852 | dup v21.4s, w0 |
| 853 | neg v21.4s, v21.4s | 853 | neg v21.4s, v21.4s |
| 854 | | 854 | |
| 855 | sbfiz x7, x6, #1, #32 // filterSize*2 (*2 because int16) | 855 | sbfiz x7, x6, #1, #32 // filterSize*2 (*2 because int16) |
| 856 | 1: ldr w8, [x5], #4 // filterPos[idx] | 856 | 1: ldr w8, [x5], #4 // filterPos[idx] |
| 857 | lsl w8, w8, #1 | 857 | lsl w8, w8, #1 |
| 858 | ldr w10, [x5], #4 // filterPos[idx + 1] | 858 | ldr w10, [x5], #4 // filterPos[idx + 1] |
| 859 | lsl w10, w10, #1 | 859 | lsl w10, w10, #1 |
| 860 | ldr w11, [x5], #4 // filterPos[idx + 2] | 860 | ldr w11, [x5], #4 // filterPos[idx + 2] |
| 861 | lsl w11, w11, #1 | 861 | lsl w11, w11, #1 |
| 862 | ldr w9, [x5], #4 // filterPos[idx + 3] | 862 | ldr w9, [x5], #4 // filterPos[idx + 3] |
| 863 | lsl w9, w9, #1 | 863 | lsl w9, w9, #1 |
| 864 | mov x16, x4 // filter0 = filter | 864 | mov x16, x4 // filter0 = filter |
| 865 | add x12, x16, x7 // filter1 = filter0 + filterSize*2 | 865 | add x12, x16, x7 // filter1 = filter0 + filterSize*2 |
| 866 | add x13, x12, x7 // filter2 = filter1 + filterSize*2 | 866 | add x13, x12, x7 // filter2 = filter1 + filterSize*2 |
| 867 | add x4, x13, x7 // filter3 = filter2 + filterSize*2 | 867 | add x4, x13, x7 // filter3 = filter2 + filterSize*2 |
| 868 | movi v0.2d, #0 // val sum part 1 (for dst[0]) | 868 | movi v0.2d, #0 // val sum part 1 (for dst[0]) |
| 869 | movi v1.2d, #0 // val sum part 2 (for dst[1]) | 869 | movi v1.2d, #0 // val sum part 2 (for dst[1]) |
| 870 | movi v2.2d, #0 // val sum part 3 (for dst[2]) | 870 | movi v2.2d, #0 // val sum part 3 (for dst[2]) |
| 871 | movi v3.2d, #0 // val sum part 4 (for dst[3]) | 871 | movi v3.2d, #0 // val sum part 4 (for dst[3]) |
| 872 | add x17, x3, w8, uxtw // srcp + filterPos[0] | 872 | add x17, x3, w8, uxtw // srcp + filterPos[0] |
| 873 | add x8, x3, w10, uxtw // srcp + filterPos[1] | 873 | add x8, x3, w10, uxtw // srcp + filterPos[1] |
| 874 | add x10, x3, w11, uxtw // srcp + filterPos[2] | 874 | add x10, x3, w11, uxtw // srcp + filterPos[2] |
| 875 | add x11, x3, w9, uxtw // srcp + filterPos[3] | 875 | add x11, x3, w9, uxtw // srcp + filterPos[3] |
| 876 | mov w15, w6 // filterSize counter | 876 | mov w15, w6 // filterSize counter |
| 877 | 2: ld1 {v4.8h}, [x17], #16 // srcp[filterPos[0] + {0..7}] | 877 | 2: ld1 {v4.8h}, [x17], #16 // srcp[filterPos[0] + {0..7}] |
| 878 | ld1 {v5.8h}, [x16], #16 // load 8x16-bit filter values, part 1 | 878 | ld1 {v5.8h}, [x16], #16 // load 8x16-bit filter values, part 1 |
| 879 | ld1 {v6.8h}, [x8], #16 // srcp[filterPos[1] + {0..7}] | 879 | ld1 {v6.8h}, [x8], #16 // srcp[filterPos[1] + {0..7}] |
| 880 | ld1 {v7.8h}, [x12], #16 // load 8x16-bit at filter+filterSize | 880 | ld1 {v7.8h}, [x12], #16 // load 8x16-bit at filter+filterSize |
| 881 | uxtl v24.4s, v4.4h // extend srcp lower half to 32 bits to preserve sign | 881 | uxtl v24.4s, v4.4h // extend srcp lower half to 32 bits to preserve sign |
| 882 | sxtl v25.4s, v5.4h // extend filter lower half to 32 bits to match srcp size | 882 | sxtl v25.4s, v5.4h // extend filter lower half to 32 bits to match srcp size |
| 883 | uxtl2 v4.4s, v4.8h // extend srcp upper half to 32 bits | 883 | uxtl2 v4.4s, v4.8h // extend srcp upper half to 32 bits |
| 884 | mla v0.4s, v24.4s, v25.4s // multiply accumulate lower half of v4 * v5 | 884 | mla v0.4s, v24.4s, v25.4s // multiply accumulate lower half of v4 * v5 |
| 885 | sxtl2 v5.4s, v5.8h // extend filter upper half to 32 bits | 885 | sxtl2 v5.4s, v5.8h // extend filter upper half to 32 bits |
| 886 | uxtl v26.4s, v6.4h // extend srcp lower half to 32 bits | 886 | uxtl v26.4s, v6.4h // extend srcp lower half to 32 bits |
| 887 | mla v0.4s, v4.4s, v5.4s // multiply accumulate upper half of v4 * v5 | 887 | mla v0.4s, v4.4s, v5.4s // multiply accumulate upper half of v4 * v5 |
| 888 | sxtl v27.4s, v7.4h // exted filter lower half | 888 | sxtl v27.4s, v7.4h // exted filter lower half |
| 889 | uxtl2 v6.4s, v6.8h // extend srcp upper half | 889 | uxtl2 v6.4s, v6.8h // extend srcp upper half |
| 890 | sxtl2 v7.4s, v7.8h // extend filter upper half | 890 | sxtl2 v7.4s, v7.8h // extend filter upper half |
| 891 | ld1 {v16.8h}, [x10], #16 // srcp[filterPos[2] + {0..7}] | 891 | ld1 {v16.8h}, [x10], #16 // srcp[filterPos[2] + {0..7}] |
| 892 | mla v1.4s, v26.4s, v27.4s // v1 accumulates srcp[filterPos[1] + {0..3}] * filter[{0..3}] | 892 | mla v1.4s, v26.4s, v27.4s // v1 accumulates srcp[filterPos[1] + {0..3}] * filter[{0..3}] |
| 893 | ld1 {v17.8h}, [x13], #16 // load 8x16-bit at filter+2*filterSize | 893 | ld1 {v17.8h}, [x13], #16 // load 8x16-bit at filter+2*filterSize |
| 894 | uxtl v22.4s, v16.4h // extend srcp lower half | 894 | uxtl v22.4s, v16.4h // extend srcp lower half |
| 895 | sxtl v23.4s, v17.4h // extend filter lower half | 895 | sxtl v23.4s, v17.4h // extend filter lower half |
| 896 | uxtl2 v16.4s, v16.8h // extend srcp upper half | 896 | uxtl2 v16.4s, v16.8h // extend srcp upper half |
| 897 | sxtl2 v17.4s, v17.8h // extend filter upper half | 897 | sxtl2 v17.4s, v17.8h // extend filter upper half |
| 898 | mla v2.4s, v22.4s, v23.4s // v2 accumulates srcp[filterPos[2] + {0..3}] * filter[{0..3}] | 898 | mla v2.4s, v22.4s, v23.4s // v2 accumulates srcp[filterPos[2] + {0..3}] * filter[{0..3}] |
| 899 | mla v2.4s, v16.4s, v17.4s // v2 accumulates srcp[filterPos[2] + {4..7}] * filter[{4..7}] | 899 | mla v2.4s, v16.4s, v17.4s // v2 accumulates srcp[filterPos[2] + {4..7}] * filter[{4..7}] |
| 900 | ld1 {v18.8h}, [x11], #16 // srcp[filterPos[3] + {0..7}] | 900 | ld1 {v18.8h}, [x11], #16 // srcp[filterPos[3] + {0..7}] |
| 901 | mla v1.4s, v6.4s, v7.4s // v1 accumulates srcp[filterPos[1] + {4..7}] * filter[{4..7}] | 901 | mla v1.4s, v6.4s, v7.4s // v1 accumulates srcp[filterPos[1] + {4..7}] * filter[{4..7}] |
| 902 | ld1 {v19.8h}, [x4], #16 // load 8x16-bit at filter+3*filterSize | 902 | ld1 {v19.8h}, [x4], #16 // load 8x16-bit at filter+3*filterSize |
| 903 | subs w15, w15, #8 // j -= 8: processed 8/filterSize | 903 | subs w15, w15, #8 // j -= 8: processed 8/filterSize |
| 904 | uxtl v28.4s, v18.4h // extend srcp lower half | 904 | uxtl v28.4s, v18.4h // extend srcp lower half |
| 905 | sxtl v29.4s, v19.4h // extend filter lower half | 905 | sxtl v29.4s, v19.4h // extend filter lower half |
| 906 | uxtl2 v18.4s, v18.8h // extend srcp upper half | 906 | uxtl2 v18.4s, v18.8h // extend srcp upper half |
| 907 | sxtl2 v19.4s, v19.8h // extend filter upper half | 907 | sxtl2 v19.4s, v19.8h // extend filter upper half |
| 908 | mla v3.4s, v28.4s, v29.4s // v3 accumulates srcp[filterPos[3] + {0..3}] * filter[{0..3}] | 908 | mla v3.4s, v28.4s, v29.4s // v3 accumulates srcp[filterPos[3] + {0..3}] * filter[{0..3}] |
| 909 | mla v3.4s, v18.4s, v19.4s // v3 accumulates srcp[filterPos[3] + {4..7}] * filter[{4..7}] | 909 | mla v3.4s, v18.4s, v19.4s // v3 accumulates srcp[filterPos[3] + {4..7}] * filter[{4..7}] |
| 910 | b.gt 2b // inner loop if filterSize not consumed completely | 910 | b.gt 2b // inner loop if filterSize not consumed completely |
| 911 | addp v0.4s, v0.4s, v1.4s // part01 horizontal pair adding | 911 | addp v0.4s, v0.4s, v1.4s // part01 horizontal pair adding |
| 912 | addp v2.4s, v2.4s, v3.4s // part23 horizontal pair adding | 912 | addp v2.4s, v2.4s, v3.4s // part23 horizontal pair adding |
| 913 | addp v0.4s, v0.4s, v2.4s // part0123 horizontal pair adding | 913 | addp v0.4s, v0.4s, v2.4s // part0123 horizontal pair adding |
| 914 | subs w2, w2, #4 // dstW -= 4 | 914 | subs w2, w2, #4 // dstW -= 4 |
| 915 | sshl v0.4s, v0.4s, v21.4s // shift right (effectively rigth, as shift is negative); overflow expected | 915 | sshl v0.4s, v0.4s, v21.4s // shift right (effectively rigth, as shift is negative); overflow expected |
| 916 | smin v0.4s, v0.4s, v20.4s // apply min (do not use sqshl) | 916 | smin v0.4s, v0.4s, v20.4s // apply min (do not use sqshl) |
| 917 | xtn v0.4h, v0.4s // narrow down to 16 bits | 917 | xtn v0.4h, v0.4s // narrow down to 16 bits |
| 918 | | 918 | |
| 919 | st1 {v0.4h}, [x1], #8 // write to destination part0123 | 919 | st1 {v0.4h}, [x1], #8 // write to destination part0123 |
| 920 | b.gt 1b // loop until end of line | 920 | b.gt 1b // loop until end of line |
| 921 | ret | 921 | ret |
| 922 | endfunc | 922 | endfunc |
| 923 | | 923 | |
| @@ -930,118 +930,118 @@ function ff_hscale16to15_X4_neon_asm, export=1 · 6 unmodified lines |
| 930 | // x5 const int32_t *filterPos | 930 | // x5 const int32_t *filterPos |
| 931 | // w6 int filterSize | 931 | // w6 int filterSize |
| 932 | | 932 | |
| 933 | stp d8, d9, [sp, #-0x20]! | 933 | stp d8, d9, [sp, #-0x20]! |
| 934 | stp d10, d11, [sp, #0x10] | 934 | stp d10, d11, [sp, #0x10] |
| 935 | | 935 | |
| 936 | movi v18.4s, #1 | 936 | movi v18.4s, #1 |
| 937 | movi v17.4s, #1 | 937 | movi v17.4s, #1 |
| 938 | shl v18.4s, v18.4s, #15 | 938 | shl v18.4s, v18.4s, #15 |
| 939 | sub v21.4s, v18.4s, v17.4s // max allowed value | 939 | sub v21.4s, v18.4s, v17.4s // max allowed value |
| 940 | dup v17.4s, w0 // read shift | 940 | dup v17.4s, w0 // read shift |
| 941 | neg v20.4s, v17.4s // negate it, so it can be used in sshl (effectively shift right) | 941 | neg v20.4s, v17.4s // negate it, so it can be used in sshl (effectively shift right) |
| 942 | | 942 | |
| 943 | lsl w7, w6, #1 | 943 | lsl w7, w6, #1 |
| 944 | 1: | 944 | 1: |
| 945 | ldp w8, w9, [x5] | 945 | ldp w8, w9, [x5] |
| 946 | ldp w10, w11, [x5, #8] | 946 | ldp w10, w11, [x5, #8] |
| 947 | | 947 | |
| 948 | movi v16.2d, #0 // initialize accumulator for idx + 0 | 948 | movi v16.2d, #0 // initialize accumulator for idx + 0 |
| 949 | movi v17.2d, #0 // initialize accumulator for idx + 1 | 949 | movi v17.2d, #0 // initialize accumulator for idx + 1 |
| 950 | movi v18.2d, #0 // initialize accumulator for idx + 2 | 950 | movi v18.2d, #0 // initialize accumulator for idx + 2 |
| 951 | movi v19.2d, #0 // initialize accumulator for idx + 3 | 951 | movi v19.2d, #0 // initialize accumulator for idx + 3 |
| 952 | | 952 | |
| 953 | mov x12, x4 // filter + 0 | 953 | mov x12, x4 // filter + 0 |
| 954 | add x13, x4, x7 // filter + 1 | 954 | add x13, x4, x7 // filter + 1 |
| 955 | add x8, x3, x8, lsl #1 // srcp + filterPos 0 | 955 | add x8, x3, x8, lsl #1 // srcp + filterPos 0 |
| 956 | add x14, x13, x7 // filter + 2 | 956 | add x14, x13, x7 // filter + 2 |
| 957 | add x9, x3, x9, lsl #1 // srcp + filterPos 1 | 957 | add x9, x3, x9, lsl #1 // srcp + filterPos 1 |
| 958 | add x15, x14, x7 // filter + 3 | 958 | add x15, x14, x7 // filter + 3 |
| 959 | add x10, x3, x10, lsl #1 // srcp + filterPos 2 | 959 | add x10, x3, x10, lsl #1 // srcp + filterPos 2 |
| 960 | mov w0, w6 // save the filterSize to temporary variable | 960 | mov w0, w6 // save the filterSize to temporary variable |
| 961 | add x11, x3, x11, lsl #1 // srcp + filterPos 3 | 961 | add x11, x3, x11, lsl #1 // srcp + filterPos 3 |
| 962 | add x5, x5, #16 // advance filter position | 962 | add x5, x5, #16 // advance filter position |
| 963 | mov x16, xzr // clear the register x16 used for offsetting the filter values | 963 | mov x16, xzr // clear the register x16 used for offsetting the filter values |
| 964 | | 964 | |
| 965 | 2: | 965 | 2: |
| 966 | ldr q4, [x8], #16 // load src values for idx 0 | 966 | ldr q4, [x8], #16 // load src values for idx 0 |
| 967 | ldr q5, [x9], #16 // load src values for idx 1 | 967 | ldr q5, [x9], #16 // load src values for idx 1 |
| 968 | uxtl v26.4s, v4.4h | 968 | uxtl v26.4s, v4.4h |
| 969 | uxtl2 v4.4s, v4.8h | 969 | uxtl2 v4.4s, v4.8h |
| 970 | ldr q31, [x12, x16] // load filter values for idx 0 | 970 | ldr q31, [x12, x16] // load filter values for idx 0 |
| 971 | ldr q6, [x10], #16 // load src values for idx 2 | 971 | ldr q6, [x10], #16 // load src values for idx 2 |
| 972 | sxtl v22.4s, v31.4h | 972 | sxtl v22.4s, v31.4h |
| 973 | sxtl2 v31.4s, v31.8h | 973 | sxtl2 v31.4s, v31.8h |
| 974 | mla v16.4s, v26.4s, v22.4s // multiplication of lower half for idx 0 | 974 | mla v16.4s, v26.4s, v22.4s // multiplication of lower half for idx 0 |
| 975 | uxtl v25.4s, v5.4h | 975 | uxtl v25.4s, v5.4h |
| 976 | uxtl2 v5.4s, v5.8h | 976 | uxtl2 v5.4s, v5.8h |
| 977 | ldr q30, [x13, x16] // load filter values for idx 1 | 977 | ldr q30, [x13, x16] // load filter values for idx 1 |
| 978 | ldr q7, [x11], #16 // load src values for idx 3 | 978 | ldr q7, [x11], #16 // load src values for idx 3 |
| 979 | mla v16.4s, v4.4s, v31.4s // multiplication of upper half for idx 0 | 979 | mla v16.4s, v4.4s, v31.4s // multiplication of upper half for idx 0 |
| 980 | uxtl v24.4s, v6.4h | 980 | uxtl v24.4s, v6.4h |
| 981 | sxtl v8.4s, v30.4h | 981 | sxtl v8.4s, v30.4h |
| 982 | sxtl2 v30.4s, v30.8h | 982 | sxtl2 v30.4s, v30.8h |
| 983 | mla v17.4s, v25.4s, v8.4s // multiplication of lower half for idx 1 | 983 | mla v17.4s, v25.4s, v8.4s // multiplication of lower half for idx 1 |
| 984 | ldr q29, [x14, x16] // load filter values for idx 2 | 984 | ldr q29, [x14, x16] // load filter values for idx 2 |
| 985 | uxtl2 v6.4s, v6.8h | 985 | uxtl2 v6.4s, v6.8h |
| 986 | sxtl v9.4s, v29.4h | 986 | sxtl v9.4s, v29.4h |
| 987 | sxtl2 v29.4s, v29.8h | 987 | sxtl2 v29.4s, v29.8h |
| 988 | mla v17.4s, v5.4s, v30.4s // multiplication of upper half for idx 1 | 988 | mla v17.4s, v5.4s, v30.4s // multiplication of upper half for idx 1 |
| 989 | mla v18.4s, v24.4s, v9.4s // multiplication of lower half for idx 2 | 989 | mla v18.4s, v24.4s, v9.4s // multiplication of lower half for idx 2 |
| 990 | ldr q28, [x15, x16] // load filter values for idx 3 | 990 | ldr q28, [x15, x16] // load filter values for idx 3 |
| 991 | uxtl v23.4s, v7.4h | 991 | uxtl v23.4s, v7.4h |
| 992 | sxtl v10.4s, v28.4h | 992 | sxtl v10.4s, v28.4h |
| 993 | mla v18.4s, v6.4s, v29.4s // multiplication of upper half for idx 2 | 993 | mla v18.4s, v6.4s, v29.4s // multiplication of upper half for idx 2 |
| 994 | uxtl2 v7.4s, v7.8h | 994 | uxtl2 v7.4s, v7.8h |
| 995 | sxtl2 v28.4s, v28.8h | 995 | sxtl2 v28.4s, v28.8h |
| 996 | mla v19.4s, v23.4s, v10.4s // multiplication of lower half for idx 3 | 996 | mla v19.4s, v23.4s, v10.4s // multiplication of lower half for idx 3 |
| 997 | sub w0, w0, #8 | 997 | sub w0, w0, #8 |
| 998 | cmp w0, #8 | 998 | cmp w0, #8 |
| 999 | mla v19.4s, v7.4s, v28.4s // multiplication of upper half for idx 3 | 999 | mla v19.4s, v7.4s, v28.4s // multiplication of upper half for idx 3 |
| 1000 | | 1000 | |
| 1001 | add x16, x16, #16 // advance filter values indexing | 1001 | add x16, x16, #16 // advance filter values indexing |
| 1002 | | 1002 | |
| 1003 | b.ge 2b | 1003 | b.ge 2b |
| 1004 | | 1004 | |
| 1005 | // 4 iterations left | 1005 | // 4 iterations left |
| 1006 | | 1006 | |
| 1007 | sub x17, x7, #8 // step back to wrap up the filter pos for last 4 elements | 1007 | sub x17, x7, #8 // step back to wrap up the filter pos for last 4 elements |
| 1008 | | 1008 | |
| 1009 | ldr d4, [x8] // load src values for idx 0 | 1009 | ldr d4, [x8] // load src values for idx 0 |
| 1010 | ldr d31, [x12, x17] // load filter values for idx 0 | 1010 | ldr d31, [x12, x17] // load filter values for idx 0 |
| 1011 | uxtl v4.4s, v4.4h | 1011 | uxtl v4.4s, v4.4h |
| 1012 | sxtl v31.4s, v31.4h | 1012 | sxtl v31.4s, v31.4h |
| 1013 | ldr d5, [x9] // load src values for idx 1 | 1013 | ldr d5, [x9] // load src values for idx 1 |
| 1014 | mla v16.4s, v4.4s, v31.4s // multiplication of upper half for idx 0 | 1014 | mla v16.4s, v4.4s, v31.4s // multiplication of upper half for idx 0 |
| 1015 | ldr d30, [x13, x17] // load filter values for idx 1 | 1015 | ldr d30, [x13, x17] // load filter values for idx 1 |
| 1016 | uxtl v5.4s, v5.4h | 1016 | uxtl v5.4s, v5.4h |
| 1017 | sxtl v30.4s, v30.4h | 1017 | sxtl v30.4s, v30.4h |
| 1018 | ldr d6, [x10] // load src values for idx 2 | 1018 | ldr d6, [x10] // load src values for idx 2 |
| 1019 | mla v17.4s, v5.4s, v30.4s // multiplication of upper half for idx 1 | 1019 | mla v17.4s, v5.4s, v30.4s // multiplication of upper half for idx 1 |
| 1020 | ldr d29, [x14, x17] // load filter values for idx 2 | 1020 | ldr d29, [x14, x17] // load filter values for idx 2 |
| 1021 | uxtl v6.4s, v6.4h | 1021 | uxtl v6.4s, v6.4h |
| 1022 | sxtl v29.4s, v29.4h | 1022 | sxtl v29.4s, v29.4h |
| 1023 | ldr d7, [x11] // load src values for idx 3 | 1023 | ldr d7, [x11] // load src values for idx 3 |
| 1024 | ldr d28, [x15, x17] // load filter values for idx 3 | 1024 | ldr d28, [x15, x17] // load filter values for idx 3 |
| 1025 | mla v18.4s, v6.4s, v29.4s // multiplication of upper half for idx 2 | 1025 | mla v18.4s, v6.4s, v29.4s // multiplication of upper half for idx 2 |
| 1026 | uxtl v7.4s, v7.4h | 1026 | uxtl v7.4s, v7.4h |
| 1027 | sxtl v28.4s, v28.4h | 1027 | sxtl v28.4s, v28.4h |
| 1028 | addp v16.4s, v16.4s, v17.4s | 1028 | addp v16.4s, v16.4s, v17.4s |
| 1029 | mla v19.4s, v7.4s, v28.4s // multiplication of upper half for idx 3 | 1029 | mla v19.4s, v7.4s, v28.4s // multiplication of upper half for idx 3 |
| 1030 | subs w2, w2, #4 | 1030 | subs w2, w2, #4 |
| 1031 | addp v18.4s, v18.4s, v19.4s | 1031 | addp v18.4s, v18.4s, v19.4s |
| 1032 | addp v16.4s, v16.4s, v18.4s | 1032 | addp v16.4s, v16.4s, v18.4s |
| 1033 | sshl v16.4s, v16.4s, v20.4s | 1033 | sshl v16.4s, v16.4s, v20.4s |
| 1034 | smin v16.4s, v16.4s, v21.4s | 1034 | smin v16.4s, v16.4s, v21.4s |
| 1035 | xtn v16.4h, v16.4s | 1035 | xtn v16.4h, v16.4s |
| 1036 | | 1036 | |
| 1037 | st1 {v16.4h}, [x1], #8 | 1037 | st1 {v16.4h}, [x1], #8 |
| 1038 | add x4, x4, x7, lsl #2 | 1038 | add x4, x4, x7, lsl #2 |
| 1039 | b.gt 1b | 1039 | b.gt 1b |
| 1040 | | 1040 | |
| 1041 | ldp d8, d9, [sp] | 1041 | ldp d8, d9, [sp] |
| 1042 | ldp d10, d11, [sp, #0x10] | 1042 | ldp d10, d11, [sp, #0x10] |
| 1043 | | 1043 | |
| 1044 | add sp, sp, #0x20 | 1044 | add sp, sp, #0x20 |
| 1045 | | 1045 | |
| 1046 | ret | 1046 | ret |
| 1047 | endfunc | 1047 | endfunc |
| @@ -1055,188 +1055,188 @@ function ff_hscale16to19_4_neon_asm, export=1 · 7 unmodified lines |
| 1055 | // x5 const int32_t *filterPos | 1055 | // x5 const int32_t *filterPos |
| 1056 | // w6 int filterSize | 1056 | // w6 int filterSize |
| 1057 | | 1057 | |
| 1058 | movi v18.4s, #1 | 1058 | movi v18.4s, #1 |
| 1059 | movi v17.4s, #1 | 1059 | movi v17.4s, #1 |
| 1060 | shl v18.4s, v18.4s, #19 | 1060 | shl v18.4s, v18.4s, #19 |
| 1061 | sub v18.4s, v18.4s, v17.4s // max allowed value | 1061 | sub v18.4s, v18.4s, v17.4s // max allowed value |
| 1062 | dup v17.4s, w0 // read shift | 1062 | dup v17.4s, w0 // read shift |
| 1063 | neg v17.4s, v17.4s // negate it, so it can be used in sshl (effectively shift right) | 1063 | neg v17.4s, v17.4s // negate it, so it can be used in sshl (effectively shift right) |
| 1064 | | 1064 | |
| 1065 | cmp w2, #16 | 1065 | cmp w2, #16 |
| 1066 | b.lt 2f // move to last block | 1066 | b.lt 2f // move to last block |
| 1067 | | 1067 | |
| 1068 | ldp w8, w9, [x5] // filterPos[0], filterPos[1] | 1068 | ldp w8, w9, [x5] // filterPos[0], filterPos[1] |
| 1069 | ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3] | 1069 | ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3] |
| 1070 | ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5] | 1070 | ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5] |
| 1071 | ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7] | 1071 | ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7] |
| 1072 | add x5, x5, #32 | 1072 | add x5, x5, #32 |
| 1073 | | 1073 | |
| 1074 | // shift all filterPos left by one, as uint16_t will be read | 1074 | // shift all filterPos left by one, as uint16_t will be read |
| 1075 | lsl x8, x8, #1 | 1075 | lsl x8, x8, #1 |
| 1076 | lsl x9, x9, #1 | 1076 | lsl x9, x9, #1 |
| 1077 | lsl x10, x10, #1 | 1077 | lsl x10, x10, #1 |
| 1078 | lsl x11, x11, #1 | 1078 | lsl x11, x11, #1 |
| 1079 | lsl x12, x12, #1 | 1079 | lsl x12, x12, #1 |
| 1080 | lsl x13, x13, #1 | 1080 | lsl x13, x13, #1 |
| 1081 | lsl x14, x14, #1 | 1081 | lsl x14, x14, #1 |
| 1082 | lsl x15, x15, #1 | 1082 | lsl x15, x15, #1 |
| 1083 | | 1083 | |
| 1084 | // load src with given offset | 1084 | // load src with given offset |
| 1085 | ldr x8, [x3, w8, uxtw] | 1085 | ldr x8, [x3, w8, uxtw] |
| 1086 | ldr x9, [x3, w9, uxtw] | 1086 | ldr x9, [x3, w9, uxtw] |
| 1087 | ldr x10, [x3, w10, uxtw] | 1087 | ldr x10, [x3, w10, uxtw] |
| 1088 | ldr x11, [x3, w11, uxtw] | 1088 | ldr x11, [x3, w11, uxtw] |
| 1089 | ldr x12, [x3, w12, uxtw] | 1089 | ldr x12, [x3, w12, uxtw] |
| 1090 | ldr x13, [x3, w13, uxtw] | 1090 | ldr x13, [x3, w13, uxtw] |
| 1091 | ldr x14, [x3, w14, uxtw] | 1091 | ldr x14, [x3, w14, uxtw] |
| 1092 | ldr x15, [x3, w15, uxtw] | 1092 | ldr x15, [x3, w15, uxtw] |
| 1093 | | 1093 | |
| 1094 | sub sp, sp, #64 | 1094 | sub sp, sp, #64 |
| 1095 | // push src on stack so it can be loaded into vectors later | 1095 | // push src on stack so it can be loaded into vectors later |
| 1096 | stp x8, x9, [sp] | 1096 | stp x8, x9, [sp] |
| 1097 | stp x10, x11, [sp, #16] | 1097 | stp x10, x11, [sp, #16] |
| 1098 | stp x12, x13, [sp, #32] | 1098 | stp x12, x13, [sp, #32] |
| 1099 | stp x14, x15, [sp, #48] | 1099 | stp x14, x15, [sp, #48] |
| 1100 | | 1100 | |
| 1101 | 1: | 1101 | 1: |
| 1102 | ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [sp] | 1102 | ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [sp] |
| 1103 | ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64 // filter[0..7] | 1103 | ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64 // filter[0..7] |
| 1104 | | 1104 | |
| 1105 | // Each of blocks does the following: | 1105 | // Each of blocks does the following: |
| 1106 | // Extend src and filter to 32 bits with uxtl and sxtl | 1106 | // Extend src and filter to 32 bits with uxtl and sxtl |
| 1107 | // multiply or multiply and accumulate results | 1107 | // multiply or multiply and accumulate results |
| 1108 | // Extending to 32 bits is necessary, as unit16_t values can't | 1108 | // Extending to 32 bits is necessary, as unit16_t values can't |
| 1109 | // be represented as int16_t without type promotion. | 1109 | // be represented as int16_t without type promotion. |
| 1110 | uxtl v26.4s, v0.4h | 1110 | uxtl v26.4s, v0.4h |
| 1111 | sxtl v27.4s, v28.4h | 1111 | sxtl v27.4s, v28.4h |
| 1112 | uxtl2 v0.4s, v0.8h | 1112 | uxtl2 v0.4s, v0.8h |
| 1113 | mul v5.4s, v26.4s, v27.4s | 1113 | mul v5.4s, v26.4s, v27.4s |
| 1114 | sxtl2 v28.4s, v28.8h | 1114 | sxtl2 v28.4s, v28.8h |
| 1115 | uxtl v26.4s, v1.4h | 1115 | uxtl v26.4s, v1.4h |
| 1116 | mul v6.4s, v0.4s, v28.4s | 1116 | mul v6.4s, v0.4s, v28.4s |
| 1117 | | 1117 | |
| 1118 | sxtl v27.4s, v29.4h | 1118 | sxtl v27.4s, v29.4h |
| 1119 | uxtl2 v0.4s, v1.8h | 1119 | uxtl2 v0.4s, v1.8h |
| 1120 | mla v5.4s, v27.4s, v26.4s | 1120 | mla v5.4s, v27.4s, v26.4s |
| 1121 | sxtl2 v28.4s, v29.8h | 1121 | sxtl2 v28.4s, v29.8h |
| 1122 | uxtl v26.4s, v2.4h | 1122 | uxtl v26.4s, v2.4h |
| 1123 | mla v6.4s, v28.4s, v0.4s | 1123 | mla v6.4s, v28.4s, v0.4s |
| 1124 | | 1124 | |
| 1125 | sxtl v27.4s, v30.4h | 1125 | sxtl v27.4s, v30.4h |
| 1126 | uxtl2 v0.4s, v2.8h | 1126 | uxtl2 v0.4s, v2.8h |
| 1127 | mla v5.4s, v27.4s, v26.4s | 1127 | mla v5.4s, v27.4s, v26.4s |
| 1128 | sxtl2 v28.4s, v30.8h | 1128 | sxtl2 v28.4s, v30.8h |
| 1129 | uxtl v26.4s, v3.4h | 1129 | uxtl v26.4s, v3.4h |
| 1130 | mla v6.4s, v28.4s, v0.4s | 1130 | mla v6.4s, v28.4s, v0.4s |
| 1131 | | 1131 | |
| 1132 | sxtl v27.4s, v31.4h | 1132 | sxtl v27.4s, v31.4h |
| 1133 | uxtl2 v0.4s, v3.8h | 1133 | uxtl2 v0.4s, v3.8h |
| 1134 | mla v5.4s, v27.4s, v26.4s | 1134 | mla v5.4s, v27.4s, v26.4s |
| 1135 | sxtl2 v28.4s, v31.8h | 1135 | sxtl2 v28.4s, v31.8h |
| 1136 | sub w2, w2, #8 | 1136 | sub w2, w2, #8 |
| 1137 | mla v6.4s, v28.4s, v0.4s | 1137 | mla v6.4s, v28.4s, v0.4s |
| 1138 | | 1138 | |
| 1139 | sshl v5.4s, v5.4s, v17.4s | 1139 | sshl v5.4s, v5.4s, v17.4s |
| 1140 | sshl v6.4s, v6.4s, v17.4s | 1140 | sshl v6.4s, v6.4s, v17.4s |
| 1141 | smin v5.4s, v5.4s, v18.4s | 1141 | smin v5.4s, v5.4s, v18.4s |
| 1142 | smin v6.4s, v6.4s, v18.4s | 1142 | smin v6.4s, v6.4s, v18.4s |
| 1143 | | 1143 | |
| 1144 | st1 {v5.4s, v6.4s}, [x1], #32 | 1144 | st1 {v5.4s, v6.4s}, [x1], #32 |
| 1145 | cmp w2, #16 | 1145 | cmp w2, #16 |
| 1146 | | 1146 | |
| 1147 | // load filterPositions into registers for next iteration | 1147 | // load filterPositions into registers for next iteration |
| 1148 | ldp w8, w9, [x5] // filterPos[0], filterPos[1] | 1148 | ldp w8, w9, [x5] // filterPos[0], filterPos[1] |
| 1149 | ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3] | 1149 | ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3] |
| 1150 | ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5] | 1150 | ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5] |
| 1151 | ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7] | 1151 | ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7] |
| 1152 | add x5, x5, #32 | 1152 | add x5, x5, #32 |
| 1153 | | 1153 | |
| 1154 | lsl x8, x8, #1 | 1154 | lsl x8, x8, #1 |
| 1155 | lsl x9, x9, #1 | 1155 | lsl x9, x9, #1 |
| 1156 | lsl x10, x10, #1 | 1156 | lsl x10, x10, #1 |
| 1157 | lsl x11, x11, #1 | 1157 | lsl x11, x11, #1 |
| 1158 | lsl x12, x12, #1 | 1158 | lsl x12, x12, #1 |
| 1159 | lsl x13, x13, #1 | 1159 | lsl x13, x13, #1 |
| 1160 | lsl x14, x14, #1 | 1160 | lsl x14, x14, #1 |
| 1161 | lsl x15, x15, #1 | 1161 | lsl x15, x15, #1 |
| 1162 | | 1162 | |
| 1163 | ldr x8, [x3, w8, uxtw] | 1163 | ldr x8, [x3, w8, uxtw] |
| 1164 | ldr x9, [x3, w9, uxtw] | 1164 | ldr x9, [x3, w9, uxtw] |
| 1165 | ldr x10, [x3, w10, uxtw] | 1165 | ldr x10, [x3, w10, uxtw] |
| 1166 | ldr x11, [x3, w11, uxtw] | 1166 | ldr x11, [x3, w11, uxtw] |
| 1167 | ldr x12, [x3, w12, uxtw] | 1167 | ldr x12, [x3, w12, uxtw] |
| 1168 | ldr x13, [x3, w13, uxtw] | 1168 | ldr x13, [x3, w13, uxtw] |
| 1169 | ldr x14, [x3, w14, uxtw] | 1169 | ldr x14, [x3, w14, uxtw] |
| 1170 | ldr x15, [x3, w15, uxtw] | 1170 | ldr x15, [x3, w15, uxtw] |
| 1171 | | 1171 | |
| 1172 | stp x8, x9, [sp] | 1172 | stp x8, x9, [sp] |
| 1173 | stp x10, x11, [sp, #16] | 1173 | stp x10, x11, [sp, #16] |
| 1174 | stp x12, x13, [sp, #32] | 1174 | stp x12, x13, [sp, #32] |
| 1175 | stp x14, x15, [sp, #48] | 1175 | stp x14, x15, [sp, #48] |
| 1176 | | 1176 | |
| 1177 | b.ge 1b | 1177 | b.ge 1b |
| 1178 | | 1178 | |
| 1179 | // here we make last iteration, without updating the registers | 1179 | // here we make last iteration, without updating the registers |
| 1180 | ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [sp] | 1180 | ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [sp] |
| 1181 | ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64 | 1181 | ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64 |
| 1182 | | 1182 | |
| 1183 | uxtl v26.4s, v0.4h | 1183 | uxtl v26.4s, v0.4h |
| 1184 | sxtl v27.4s, v28.4h | 1184 | sxtl v27.4s, v28.4h |
| 1185 | uxtl2 v0.4s, v0.8h | 1185 | uxtl2 v0.4s, v0.8h |
| 1186 | mul v5.4s, v26.4s, v27.4s | 1186 | mul v5.4s, v26.4s, v27.4s |
| 1187 | sxtl2 v28.4s, v28.8h | 1187 | sxtl2 v28.4s, v28.8h |
| 1188 | uxtl v26.4s, v1.4h | 1188 | uxtl v26.4s, v1.4h |
| 1189 | mul v6.4s, v0.4s, v28.4s | 1189 | mul v6.4s, v0.4s, v28.4s |
| 1190 | | 1190 | |
| 1191 | sxtl v27.4s, v29.4h | 1191 | sxtl v27.4s, v29.4h |
| 1192 | uxtl2 v0.4s, v1.8h | 1192 | uxtl2 v0.4s, v1.8h |
| 1193 | mla v5.4s, v26.4s, v27.4s | 1193 | mla v5.4s, v26.4s, v27.4s |
| 1194 | sxtl2 v28.4s, v29.8h | 1194 | sxtl2 v28.4s, v29.8h |
| 1195 | uxtl v26.4s, v2.4h | 1195 | uxtl v26.4s, v2.4h |
| 1196 | mla v6.4s, v0.4s, v28.4s | 1196 | mla v6.4s, v0.4s, v28.4s |
| 1197 | | 1197 | |
| 1198 | sxtl v27.4s, v30.4h | 1198 | sxtl v27.4s, v30.4h |
| 1199 | uxtl2 v0.4s, v2.8h | 1199 | uxtl2 v0.4s, v2.8h |
| 1200 | mla v5.4s, v26.4s, v27.4s | 1200 | mla v5.4s, v26.4s, v27.4s |
| 1201 | sxtl2 v28.4s, v30.8h | 1201 | sxtl2 v28.4s, v30.8h |
| 1202 | uxtl v26.4s, v3.4h | 1202 | uxtl v26.4s, v3.4h |
| 1203 | mla v6.4s, v0.4s, v28.4s | 1203 | mla v6.4s, v0.4s, v28.4s |
| 1204 | | 1204 | |
| 1205 | sxtl v27.4s, v31.4h | 1205 | sxtl v27.4s, v31.4h |
| 1206 | uxtl2 v0.4s, v3.8h | 1206 | uxtl2 v0.4s, v3.8h |
| 1207 | mla v5.4s, v26.4s, v27.4s | 1207 | mla v5.4s, v26.4s, v27.4s |
| 1208 | sxtl2 v28.4s, v31.8h | 1208 | sxtl2 v28.4s, v31.8h |
| 1209 | subs w2, w2, #8 | 1209 | subs w2, w2, #8 |
| 1210 | mla v6.4s, v0.4s, v28.4s | 1210 | mla v6.4s, v0.4s, v28.4s |
| 1211 | | 1211 | |
| 1212 | sshl v5.4s, v5.4s, v17.4s | 1212 | sshl v5.4s, v5.4s, v17.4s |
| 1213 | sshl v6.4s, v6.4s, v17.4s | 1213 | sshl v6.4s, v6.4s, v17.4s |
| 1214 | | 1214 | |
| 1215 | smin v5.4s, v5.4s, v18.4s | 1215 | smin v5.4s, v5.4s, v18.4s |
| 1216 | smin v6.4s, v6.4s, v18.4s | 1216 | smin v6.4s, v6.4s, v18.4s |
| 1217 | | 1217 | |
| 1218 | st1 {v5.4s, v6.4s}, [x1], #32 | 1218 | st1 {v5.4s, v6.4s}, [x1], #32 |
| 1219 | add sp, sp, #64 // restore stack | 1219 | add sp, sp, #64 // restore stack |
| 1220 | cbnz w2, 2f | 1220 | cbnz w2, 2f |
| 1221 | | 1221 | |
| 1222 | ret | 1222 | ret |
| 1223 | | 1223 | |
| 1224 | 2: | 1224 | 2: |
| 1225 | ldr w8, [x5], #4 // load filterPos | 1225 | ldr w8, [x5], #4 // load filterPos |
| 1226 | lsl w8, w8, #1 | 1226 | lsl w8, w8, #1 |
| 1227 | add x9, x3, w8, uxtw // src + filterPos | 1227 | add x9, x3, w8, uxtw // src + filterPos |
| 1228 | ld1 {v0.4h}, [x9] // load 4 * uint16_t | 1228 | ld1 {v0.4h}, [x9] // load 4 * uint16_t |
| 1229 | ld1 {v31.4h}, [x4], #8 | 1229 | ld1 {v31.4h}, [x4], #8 |
| 1230 | | 1230 | |
| 1231 | uxtl v0.4s, v0.4h | 1231 | uxtl v0.4s, v0.4h |
| 1232 | sxtl v31.4s, v31.4h | 1232 | sxtl v31.4s, v31.4h |
| 1233 | subs w2, w2, #1 | 1233 | subs w2, w2, #1 |
| 1234 | mul v5.4s, v0.4s, v31.4s | 1234 | mul v5.4s, v0.4s, v31.4s |
| 1235 | addv s0, v5.4s | 1235 | addv s0, v5.4s |
| 1236 | sshl v0.4s, v0.4s, v17.4s | 1236 | sshl v0.4s, v0.4s, v17.4s |
| 1237 | smin v0.4s, v0.4s, v18.4s | 1237 | smin v0.4s, v0.4s, v18.4s |
| 1238 | st1 {v0.s}[0], [x1], #4 | 1238 | st1 {v0.s}[0], [x1], #4 |
| 1239 | cbnz w2, 2b // if iterations remain jump to beginning | 1239 | cbnz w2, 2b // if iterations remain jump to beginning |
| 1240 | | 1240 | |
| 1241 | ret | 1241 | ret |
| 1242 | endfunc | 1242 | endfunc |
| @@ -1250,77 +1250,77 @@ function ff_hscale16to19_X8_neon_asm, export=1 · 7 unmodified lines |
| 1250 | // x5 const int32_t *filterPos | 1250 | // x5 const int32_t *filterPos |
| 1251 | // w6 int filterSize | 1251 | // w6 int filterSize |
| 1252 | | 1252 | |
| 1253 | movi v20.4s, #1 | 1253 | movi v20.4s, #1 |
| 1254 | movi v21.4s, #1 | 1254 | movi v21.4s, #1 |
| 1255 | shl v20.4s, v20.4s, #19 | 1255 | shl v20.4s, v20.4s, #19 |
| 1256 | sub v20.4s, v20.4s, v21.4s | 1256 | sub v20.4s, v20.4s, v21.4s |
| 1257 | dup v21.4s, w0 | 1257 | dup v21.4s, w0 |
| 1258 | neg v21.4s, v21.4s | 1258 | neg v21.4s, v21.4s |
| 1259 | | 1259 | |
| 1260 | sbfiz x7, x6, #1, #32 // filterSize*2 (*2 because int16) | 1260 | sbfiz x7, x6, #1, #32 // filterSize*2 (*2 because int16) |
| 1261 | 1: ldr w8, [x5], #4 // filterPos[idx] | 1261 | 1: ldr w8, [x5], #4 // filterPos[idx] |
| 1262 | ldr w10, [x5], #4 // filterPos[idx + 1] | 1262 | ldr w10, [x5], #4 // filterPos[idx + 1] |
| 1263 | lsl w8, w8, #1 | 1263 | lsl w8, w8, #1 |
| 1264 | ldr w11, [x5], #4 // filterPos[idx + 2] | 1264 | ldr w11, [x5], #4 // filterPos[idx + 2] |
| 1265 | ldr w9, [x5], #4 // filterPos[idx + 3] | 1265 | ldr w9, [x5], #4 // filterPos[idx + 3] |
| 1266 | mov x16, x4 // filter0 = filter | 1266 | mov x16, x4 // filter0 = filter |
| 1267 | lsl w11, w11, #1 | 1267 | lsl w11, w11, #1 |
| 1268 | add x12, x16, x7 // filter1 = filter0 + filterSize*2 | 1268 | add x12, x16, x7 // filter1 = filter0 + filterSize*2 |
| 1269 | lsl w9, w9, #1 | 1269 | lsl w9, w9, #1 |
| 1270 | add x13, x12, x7 // filter2 = filter1 + filterSize*2 | 1270 | add x13, x12, x7 // filter2 = filter1 + filterSize*2 |
| 1271 | lsl w10, w10, #1 | 1271 | lsl w10, w10, #1 |
| 1272 | add x4, x13, x7 // filter3 = filter2 + filterSize*2 | 1272 | add x4, x13, x7 // filter3 = filter2 + filterSize*2 |
| 1273 | movi v0.2d, #0 // val sum part 1 (for dst[0]) | 1273 | movi v0.2d, #0 // val sum part 1 (for dst[0]) |
| 1274 | movi v1.2d, #0 // val sum part 2 (for dst[1]) | 1274 | movi v1.2d, #0 // val sum part 2 (for dst[1]) |
| 1275 | movi v2.2d, #0 // val sum part 3 (for dst[2]) | 1275 | movi v2.2d, #0 // val sum part 3 (for dst[2]) |
| 1276 | movi v3.2d, #0 // val sum part 4 (for dst[3]) | 1276 | movi v3.2d, #0 // val sum part 4 (for dst[3]) |
| 1277 | add x17, x3, w8, uxtw // srcp + filterPos[0] | 1277 | add x17, x3, w8, uxtw // srcp + filterPos[0] |
| 1278 | add x8, x3, w10, uxtw // srcp + filterPos[1] | 1278 | add x8, x3, w10, uxtw // srcp + filterPos[1] |
| 1279 | add x10, x3, w11, uxtw // srcp + filterPos[2] | 1279 | add x10, x3, w11, uxtw // srcp + filterPos[2] |
| 1280 | add x11, x3, w9, uxtw // srcp + filterPos[3] | 1280 | add x11, x3, w9, uxtw // srcp + filterPos[3] |
| 1281 | mov w15, w6 // filterSize counter | 1281 | mov w15, w6 // filterSize counter |
| 1282 | 2: ld1 {v4.8h}, [x17], #16 // srcp[filterPos[0] + {0..7}] | 1282 | 2: ld1 {v4.8h}, [x17], #16 // srcp[filterPos[0] + {0..7}] |
| 1283 | ld1 {v5.8h}, [x16], #16 // load 8x16-bit filter values, part 1 | 1283 | ld1 {v5.8h}, [x16], #16 // load 8x16-bit filter values, part 1 |
| 1284 | ld1 {v6.8h}, [x8], #16 // srcp[filterPos[1] + {0..7}] | 1284 | ld1 {v6.8h}, [x8], #16 // srcp[filterPos[1] + {0..7}] |
| 1285 | ld1 {v7.8h}, [x12], #16 // load 8x16-bit at filter+filterSize | 1285 | ld1 {v7.8h}, [x12], #16 // load 8x16-bit at filter+filterSize |
| 1286 | uxtl v24.4s, v4.4h // extend srcp lower half to 32 bits to preserve sign | 1286 | uxtl v24.4s, v4.4h // extend srcp lower half to 32 bits to preserve sign |
| 1287 | sxtl v25.4s, v5.4h // extend filter lower half to 32 bits to match srcp size | 1287 | sxtl v25.4s, v5.4h // extend filter lower half to 32 bits to match srcp size |
| 1288 | uxtl2 v4.4s, v4.8h // extend srcp upper half to 32 bits | 1288 | uxtl2 v4.4s, v4.8h // extend srcp upper half to 32 bits |
| 1289 | mla v0.4s, v24.4s, v25.4s // multiply accumulate lower half of v4 * v5 | 1289 | mla v0.4s, v24.4s, v25.4s // multiply accumulate lower half of v4 * v5 |
| 1290 | sxtl2 v5.4s, v5.8h // extend filter upper half to 32 bits | 1290 | sxtl2 v5.4s, v5.8h // extend filter upper half to 32 bits |
| 1291 | uxtl v26.4s, v6.4h // extend srcp lower half to 32 bits | 1291 | uxtl v26.4s, v6.4h // extend srcp lower half to 32 bits |
| 1292 | mla v0.4s, v4.4s, v5.4s // multiply accumulate upper half of v4 * v5 | 1292 | mla v0.4s, v4.4s, v5.4s // multiply accumulate upper half of v4 * v5 |
| 1293 | sxtl v27.4s, v7.4h // exted filter lower half | 1293 | sxtl v27.4s, v7.4h // exted filter lower half |
| 1294 | uxtl2 v6.4s, v6.8h // extend srcp upper half | 1294 | uxtl2 v6.4s, v6.8h // extend srcp upper half |
| 1295 | sxtl2 v7.4s, v7.8h // extend filter upper half | 1295 | sxtl2 v7.4s, v7.8h // extend filter upper half |
| 1296 | ld1 {v16.8h}, [x10], #16 // srcp[filterPos[2] + {0..7}] | 1296 | ld1 {v16.8h}, [x10], #16 // srcp[filterPos[2] + {0..7}] |
| 1297 | mla v1.4s, v26.4s, v27.4s // v1 accumulates srcp[filterPos[1] + {0..3}] * filter[{0..3}] | 1297 | mla v1.4s, v26.4s, v27.4s // v1 accumulates srcp[filterPos[1] + {0..3}] * filter[{0..3}] |
| 1298 | ld1 {v17.8h}, [x13], #16 // load 8x16-bit at filter+2*filterSize | 1298 | ld1 {v17.8h}, [x13], #16 // load 8x16-bit at filter+2*filterSize |
| 1299 | uxtl v22.4s, v16.4h // extend srcp lower half | 1299 | uxtl v22.4s, v16.4h // extend srcp lower half |
| 1300 | sxtl v23.4s, v17.4h // extend filter lower half | 1300 | sxtl v23.4s, v17.4h // extend filter lower half |
| 1301 | uxtl2 v16.4s, v16.8h // extend srcp upper half | 1301 | uxtl2 v16.4s, v16.8h // extend srcp upper half |
| 1302 | sxtl2 v17.4s, v17.8h // extend filter upper half | 1302 | sxtl2 v17.4s, v17.8h // extend filter upper half |
| 1303 | mla v2.4s, v22.4s, v23.4s // v2 accumulates srcp[filterPos[2] + {0..3}] * filter[{0..3}] | 1303 | mla v2.4s, v22.4s, v23.4s // v2 accumulates srcp[filterPos[2] + {0..3}] * filter[{0..3}] |
| 1304 | mla v2.4s, v16.4s, v17.4s // v2 accumulates srcp[filterPos[2] + {4..7}] * filter[{4..7}] | 1304 | mla v2.4s, v16.4s, v17.4s // v2 accumulates srcp[filterPos[2] + {4..7}] * filter[{4..7}] |
| 1305 | ld1 {v18.8h}, [x11], #16 // srcp[filterPos[3] + {0..7}] | 1305 | ld1 {v18.8h}, [x11], #16 // srcp[filterPos[3] + {0..7}] |
| 1306 | mla v1.4s, v6.4s, v7.4s // v1 accumulates srcp[filterPos[1] + {4..7}] * filter[{4..7}] | 1306 | mla v1.4s, v6.4s, v7.4s // v1 accumulates srcp[filterPos[1] + {4..7}] * filter[{4..7}] |
| 1307 | ld1 {v19.8h}, [x4], #16 // load 8x16-bit at filter+3*filterSize | 1307 | ld1 {v19.8h}, [x4], #16 // load 8x16-bit at filter+3*filterSize |
| 1308 | subs w15, w15, #8 // j -= 8: processed 8/filterSize | 1308 | subs w15, w15, #8 // j -= 8: processed 8/filterSize |
| 1309 | uxtl v28.4s, v18.4h // extend srcp lower half | 1309 | uxtl v28.4s, v18.4h // extend srcp lower half |
| 1310 | sxtl v29.4s, v19.4h // extend filter lower half | 1310 | sxtl v29.4s, v19.4h // extend filter lower half |
| 1311 | uxtl2 v18.4s, v18.8h // extend srcp upper half | 1311 | uxtl2 v18.4s, v18.8h // extend srcp upper half |
| 1312 | sxtl2 v19.4s, v19.8h // extend filter upper half | 1312 | sxtl2 v19.4s, v19.8h // extend filter upper half |
| 1313 | mla v3.4s, v28.4s, v29.4s // v3 accumulates srcp[filterPos[3] + {0..3}] * filter[{0..3}] | 1313 | mla v3.4s, v28.4s, v29.4s // v3 accumulates srcp[filterPos[3] + {0..3}] * filter[{0..3}] |
| 1314 | mla v3.4s, v18.4s, v19.4s // v3 accumulates srcp[filterPos[3] + {4..7}] * filter[{4..7}] | 1314 | mla v3.4s, v18.4s, v19.4s // v3 accumulates srcp[filterPos[3] + {4..7}] * filter[{4..7}] |
| 1315 | b.gt 2b // inner loop if filterSize not consumed completely | 1315 | b.gt 2b // inner loop if filterSize not consumed completely |
| 1316 | addp v0.4s, v0.4s, v1.4s // part01 horizontal pair adding | 1316 | addp v0.4s, v0.4s, v1.4s // part01 horizontal pair adding |
| 1317 | addp v2.4s, v2.4s, v3.4s // part23 horizontal pair adding | 1317 | addp v2.4s, v2.4s, v3.4s // part23 horizontal pair adding |
| 1318 | addp v0.4s, v0.4s, v2.4s // part0123 horizontal pair adding | 1318 | addp v0.4s, v0.4s, v2.4s // part0123 horizontal pair adding |
| 1319 | subs w2, w2, #4 // dstW -= 4 | 1319 | subs w2, w2, #4 // dstW -= 4 |
| 1320 | sshl v0.4s, v0.4s, v21.4s // shift right (effectively rigth, as shift is negative); overflow expected | 1320 | sshl v0.4s, v0.4s, v21.4s // shift right (effectively rigth, as shift is negative); overflow expected |
| 1321 | smin v0.4s, v0.4s, v20.4s // apply min (do not use sqshl) | 1321 | smin v0.4s, v0.4s, v20.4s // apply min (do not use sqshl) |
| 1322 | st1 {v0.4s}, [x1], #16 // write to destination part0123 | 1322 | st1 {v0.4s}, [x1], #16 // write to destination part0123 |
| 1323 | b.gt 1b // loop until end of line | 1323 | b.gt 1b // loop until end of line |
| 1324 | ret | 1324 | ret |
| 1325 | endfunc | 1325 | endfunc |
| 1326 | | 1326 | |
| @@ -1333,117 +1333,117 @@ function ff_hscale16to19_X4_neon_asm, export=1 · 6 unmodified lines |
| 1333 | // x5 const int32_t *filterPos | 1333 | // x5 const int32_t *filterPos |
| 1334 | // w6 int filterSize | 1334 | // w6 int filterSize |
| 1335 | | 1335 | |
| 1336 | stp d8, d9, [sp, #-0x20]! | 1336 | stp d8, d9, [sp, #-0x20]! |
| 1337 | stp d10, d11, [sp, #0x10] | 1337 | stp d10, d11, [sp, #0x10] |
| 1338 | | 1338 | |
| 1339 | movi v18.4s, #1 | 1339 | movi v18.4s, #1 |
| 1340 | movi v17.4s, #1 | 1340 | movi v17.4s, #1 |
| 1341 | shl v18.4s, v18.4s, #19 | 1341 | shl v18.4s, v18.4s, #19 |
| 1342 | sub v21.4s, v18.4s, v17.4s // max allowed value | 1342 | sub v21.4s, v18.4s, v17.4s // max allowed value |
| 1343 | dup v17.4s, w0 // read shift | 1343 | dup v17.4s, w0 // read shift |
| 1344 | neg v20.4s, v17.4s // negate it, so it can be used in sshl (effectively shift right) | 1344 | neg v20.4s, v17.4s // negate it, so it can be used in sshl (effectively shift right) |
| 1345 | | 1345 | |
| 1346 | lsl w7, w6, #1 | 1346 | lsl w7, w6, #1 |
| 1347 | 1: | 1347 | 1: |
| 1348 | ldp w8, w9, [x5] | 1348 | ldp w8, w9, [x5] |
| 1349 | ldp w10, w11, [x5, #8] | 1349 | ldp w10, w11, [x5, #8] |
| 1350 | | 1350 | |
| 1351 | movi v16.2d, #0 // initialize accumulator for idx + 0 | 1351 | movi v16.2d, #0 // initialize accumulator for idx + 0 |
| 1352 | movi v17.2d, #0 // initialize accumulator for idx + 1 | 1352 | movi v17.2d, #0 // initialize accumulator for idx + 1 |
| 1353 | movi v18.2d, #0 // initialize accumulator for idx + 2 | 1353 | movi v18.2d, #0 // initialize accumulator for idx + 2 |
| 1354 | movi v19.2d, #0 // initialize accumulator for idx + 3 | 1354 | movi v19.2d, #0 // initialize accumulator for idx + 3 |
| 1355 | | 1355 | |
| 1356 | mov x12, x4 // filter + 0 | 1356 | mov x12, x4 // filter + 0 |
| 1357 | add x13, x4, x7 // filter + 1 | 1357 | add x13, x4, x7 // filter + 1 |
| 1358 | add x8, x3, x8, lsl #1 // srcp + filterPos 0 | 1358 | add x8, x3, x8, lsl #1 // srcp + filterPos 0 |
| 1359 | add x14, x13, x7 // filter + 2 | 1359 | add x14, x13, x7 // filter + 2 |
| 1360 | add x9, x3, x9, lsl #1 // srcp + filterPos 1 | 1360 | add x9, x3, x9, lsl #1 // srcp + filterPos 1 |
| 1361 | add x15, x14, x7 // filter + 3 | 1361 | add x15, x14, x7 // filter + 3 |
| 1362 | add x10, x3, x10, lsl #1 // srcp + filterPos 2 | 1362 | add x10, x3, x10, lsl #1 // srcp + filterPos 2 |
| 1363 | mov w0, w6 // save the filterSize to temporary variable | 1363 | mov w0, w6 // save the filterSize to temporary variable |
| 1364 | add x11, x3, x11, lsl #1 // srcp + filterPos 3 | 1364 | add x11, x3, x11, lsl #1 // srcp + filterPos 3 |
| 1365 | add x5, x5, #16 // advance filter position | 1365 | add x5, x5, #16 // advance filter position |
| 1366 | mov x16, xzr // clear the register x16 used for offsetting the filter values | 1366 | mov x16, xzr // clear the register x16 used for offsetting the filter values |
| 1367 | | 1367 | |
| 1368 | 2: | 1368 | 2: |
| 1369 | ldr q4, [x8], #16 // load src values for idx 0 | 1369 | ldr q4, [x8], #16 // load src values for idx 0 |
| 1370 | ldr q5, [x9], #16 // load src values for idx 1 | 1370 | ldr q5, [x9], #16 // load src values for idx 1 |
| 1371 | uxtl v26.4s, v4.4h | 1371 | uxtl v26.4s, v4.4h |
| 1372 | uxtl2 v4.4s, v4.8h | 1372 | uxtl2 v4.4s, v4.8h |
| 1373 | ldr q31, [x12, x16] // load filter values for idx 0 | 1373 | ldr q31, [x12, x16] // load filter values for idx 0 |
| 1374 | ldr q6, [x10], #16 // load src values for idx 2 | 1374 | ldr q6, [x10], #16 // load src values for idx 2 |
| 1375 | sxtl v22.4s, v31.4h | 1375 | sxtl v22.4s, v31.4h |
| 1376 | sxtl2 v31.4s, v31.8h | 1376 | sxtl2 v31.4s, v31.8h |
| 1377 | mla v16.4s, v26.4s, v22.4s // multiplication of lower half for idx 0 | 1377 | mla v16.4s, v26.4s, v22.4s // multiplication of lower half for idx 0 |
| 1378 | uxtl v25.4s, v5.4h | 1378 | uxtl v25.4s, v5.4h |
| 1379 | uxtl2 v5.4s, v5.8h | 1379 | uxtl2 v5.4s, v5.8h |
| 1380 | ldr q30, [x13, x16] // load filter values for idx 1 | 1380 | ldr q30, [x13, x16] // load filter values for idx 1 |
| 1381 | ldr q7, [x11], #16 // load src values for idx 3 | 1381 | ldr q7, [x11], #16 // load src values for idx 3 |
| 1382 | mla v16.4s, v4.4s, v31.4s // multiplication of upper half for idx 0 | 1382 | mla v16.4s, v4.4s, v31.4s // multiplication of upper half for idx 0 |
| 1383 | uxtl v24.4s, v6.4h | 1383 | uxtl v24.4s, v6.4h |
| 1384 | sxtl v8.4s, v30.4h | 1384 | sxtl v8.4s, v30.4h |
| 1385 | sxtl2 v30.4s, v30.8h | 1385 | sxtl2 v30.4s, v30.8h |
| 1386 | mla v17.4s, v25.4s, v8.4s // multiplication of lower half for idx 1 | 1386 | mla v17.4s, v25.4s, v8.4s // multiplication of lower half for idx 1 |
| 1387 | ldr q29, [x14, x16] // load filter values for idx 2 | 1387 | ldr q29, [x14, x16] // load filter values for idx 2 |
| 1388 | uxtl2 v6.4s, v6.8h | 1388 | uxtl2 v6.4s, v6.8h |
| 1389 | sxtl v9.4s, v29.4h | 1389 | sxtl v9.4s, v29.4h |
| 1390 | sxtl2 v29.4s, v29.8h | 1390 | sxtl2 v29.4s, v29.8h |
| 1391 | mla v17.4s, v5.4s, v30.4s // multiplication of upper half for idx 1 | 1391 | mla v17.4s, v5.4s, v30.4s // multiplication of upper half for idx 1 |
| 1392 | ldr q28, [x15, x16] // load filter values for idx 3 | 1392 | ldr q28, [x15, x16] // load filter values for idx 3 |
| 1393 | mla v18.4s, v24.4s, v9.4s // multiplication of lower half for idx 2 | 1393 | mla v18.4s, v24.4s, v9.4s // multiplication of lower half for idx 2 |
| 1394 | uxtl v23.4s, v7.4h | 1394 | uxtl v23.4s, v7.4h |
| 1395 | sxtl v10.4s, v28.4h | 1395 | sxtl v10.4s, v28.4h |
| 1396 | mla v18.4s, v6.4s, v29.4s // multiplication of upper half for idx 2 | 1396 | mla v18.4s, v6.4s, v29.4s // multiplication of upper half for idx 2 |
| 1397 | uxtl2 v7.4s, v7.8h | 1397 | uxtl2 v7.4s, v7.8h |
| 1398 | sxtl2 v28.4s, v28.8h | 1398 | sxtl2 v28.4s, v28.8h |
| 1399 | mla v19.4s, v23.4s, v10.4s // multiplication of lower half for idx 3 | 1399 | mla v19.4s, v23.4s, v10.4s // multiplication of lower half for idx 3 |
| 1400 | sub w0, w0, #8 | 1400 | sub w0, w0, #8 |
| 1401 | cmp w0, #8 | 1401 | cmp w0, #8 |
| 1402 | mla v19.4s, v7.4s, v28.4s // multiplication of upper half for idx 3 | 1402 | mla v19.4s, v7.4s, v28.4s // multiplication of upper half for idx 3 |
| 1403 | | 1403 | |
| 1404 | add x16, x16, #16 // advance filter values indexing | 1404 | add x16, x16, #16 // advance filter values indexing |
| 1405 | | 1405 | |
| 1406 | b.ge 2b | 1406 | b.ge 2b |
| 1407 | | 1407 | |
| 1408 | // 4 iterations left | 1408 | // 4 iterations left |
| 1409 | | 1409 | |
| 1410 | sub x17, x7, #8 // step back to wrap up the filter pos for last 4 elements | 1410 | sub x17, x7, #8 // step back to wrap up the filter pos for last 4 elements |
| 1411 | | 1411 | |
| 1412 | ldr d4, [x8] // load src values for idx 0 | 1412 | ldr d4, [x8] // load src values for idx 0 |
| 1413 | ldr d31, [x12, x17] // load filter values for idx 0 | 1413 | ldr d31, [x12, x17] // load filter values for idx 0 |
| 1414 | uxtl v4.4s, v4.4h | 1414 | uxtl v4.4s, v4.4h |
| 1415 | sxtl v31.4s, v31.4h | 1415 | sxtl v31.4s, v31.4h |
| 1416 | ldr d5, [x9] // load src values for idx 1 | 1416 | ldr d5, [x9] // load src values for idx 1 |
| 1417 | mla v16.4s, v4.4s, v31.4s // multiplication of upper half for idx 0 | 1417 | mla v16.4s, v4.4s, v31.4s // multiplication of upper half for idx 0 |
| 1418 | ldr d30, [x13, x17] // load filter values for idx 1 | 1418 | ldr d30, [x13, x17] // load filter values for idx 1 |
| 1419 | uxtl v5.4s, v5.4h | 1419 | uxtl v5.4s, v5.4h |
| 1420 | sxtl v30.4s, v30.4h | 1420 | sxtl v30.4s, v30.4h |
| 1421 | ldr d6, [x10] // load src values for idx 2 | 1421 | ldr d6, [x10] // load src values for idx 2 |
| 1422 | mla v17.4s, v5.4s, v30.4s // multiplication of upper half for idx 1 | 1422 | mla v17.4s, v5.4s, v30.4s // multiplication of upper half for idx 1 |
| 1423 | ldr d29, [x14, x17] // load filter values for idx 2 | 1423 | ldr d29, [x14, x17] // load filter values for idx 2 |
| 1424 | uxtl v6.4s, v6.4h | 1424 | uxtl v6.4s, v6.4h |
| 1425 | sxtl v29.4s, v29.4h | 1425 | sxtl v29.4s, v29.4h |
| 1426 | ldr d7, [x11] // load src values for idx 3 | 1426 | ldr d7, [x11] // load src values for idx 3 |
| 1427 | ldr d28, [x15, x17] // load filter values for idx 3 | 1427 | ldr d28, [x15, x17] // load filter values for idx 3 |
| 1428 | mla v18.4s, v6.4s, v29.4s // multiplication of upper half for idx 2 | 1428 | mla v18.4s, v6.4s, v29.4s // multiplication of upper half for idx 2 |
| 1429 | uxtl v7.4s, v7.4h | 1429 | uxtl v7.4s, v7.4h |
| 1430 | sxtl v28.4s, v28.4h | 1430 | sxtl v28.4s, v28.4h |
| 1431 | addp v16.4s, v16.4s, v17.4s | 1431 | addp v16.4s, v16.4s, v17.4s |
| 1432 | mla v19.4s, v7.4s, v28.4s // multiplication of upper half for idx 3 | 1432 | mla v19.4s, v7.4s, v28.4s // multiplication of upper half for idx 3 |
| 1433 | subs w2, w2, #4 | 1433 | subs w2, w2, #4 |
| 1434 | addp v18.4s, v18.4s, v19.4s | 1434 | addp v18.4s, v18.4s, v19.4s |
| 1435 | addp v16.4s, v16.4s, v18.4s | 1435 | addp v16.4s, v16.4s, v18.4s |
| 1436 | sshl v16.4s, v16.4s, v20.4s | 1436 | sshl v16.4s, v16.4s, v20.4s |
| 1437 | smin v16.4s, v16.4s, v21.4s | 1437 | smin v16.4s, v16.4s, v21.4s |
| 1438 | | 1438 | |
| 1439 | st1 {v16.4s}, [x1], #16 | 1439 | st1 {v16.4s}, [x1], #16 |
| 1440 | add x4, x4, x7, lsl #2 | 1440 | add x4, x4, x7, lsl #2 |
| 1441 | b.gt 1b | 1441 | b.gt 1b |
| 1442 | | 1442 | |
| 1443 | ldp d8, d9, [sp] | 1443 | ldp d8, d9, [sp] |
| 1444 | ldp d10, d11, [sp, #0x10] | 1444 | ldp d10, d11, [sp, #0x10] |
| 1445 | | 1445 | |
| 1446 | add sp, sp, #0x20 | 1446 | add sp, sp, #0x20 |
| 1447 | | 1447 | |
| 1448 | ret | 1448 | ret |
| 1449 | endfunc | 1449 | endfunc |