| @@ -1,497 +0,0 @@ |
| 1 | | −; |
| 2 | | −; Copyright (c) 2015 The WebM project authors. All Rights Reserved. |
| 3 | | −; |
| 4 | | −; Use of this source code is governed by a BSD-style license |
| 5 | | −; that can be found in the LICENSE file in the root of the source |
| 6 | | −; tree. An additional intellectual property rights grant can be found |
| 7 | | −; in the file PATENTS. All contributing project authors may |
| 8 | | −; be found in the AUTHORS file in the root of the source tree. |
| 9 | | −; |
| 10 | | − |
| 11 | | −%include "third_party/x86inc/x86inc.asm" |
| 12 | | − |
| 13 | | −SECTION .text |
| 14 | | − |
| 15 | | −%macro QUANTIZE_FN 2 |
| 16 | | −cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, skip, zbin, round, quant, \ |
| 17 | | − shift, qcoeff, dqcoeff, dequant, \ |
| 18 | | − eob, scan, iscan |
| 19 | | − |
| 20 | | − vzeroupper |
| 21 | | − |
| 22 | | −%ifnidn %1, b_32x32 |
| 23 | | − |
| 24 | | − ; Special case for ncoeff == 16, as it is frequent and we can save on |
| 25 | | − ; not setting up a loop. |
| 26 | | − cmp ncoeffmp, 16 |
| 27 | | − jne .generic |
| 28 | | − |
| 29 | | − ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; |
| 30 | | − ;; Special case of ncoeff == 16 |
| 31 | | − ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; |
| 32 | | − |
| 33 | | −.single: |
| 34 | | − |
| 35 | | − movifnidn coeffq, coeffmp |
| 36 | | − movifnidn zbinq, zbinmp |
| 37 | | − mova m0, [zbinq] ; m0 = zbin |
| 38 | | − |
| 39 | | − ; Get DC and first 15 AC coeffs - in this special case, that is all. |
| 40 | | −%if CONFIG_VP9_HIGHBITDEPTH |
| 41 | | − ; coeff stored as 32bit numbers but we process them as 16 bit numbers |
| 42 | | − mova m9, [coeffq] |
| 43 | | − packssdw m9, [coeffq+16] ; m9 = c[i] |
| 44 | | − mova m10, [coeffq+32] |
| 45 | | − packssdw m10, [coeffq+48] ; m10 = c[i] |
| 46 | | −%else |
| 47 | | − mova m9, [coeffq] ; m9 = c[i] |
| 48 | | − mova m10, [coeffq+16] ; m10 = c[i] |
| 49 | | −%endif |
| 50 | | − |
| 51 | | − mov r0, eobmp ; Output pointer |
| 52 | | − mov r1, qcoeffmp ; Output pointer |
| 53 | | − mov r2, dqcoeffmp ; Output pointer |
| 54 | | − |
| 55 | | − pxor m5, m5 ; m5 = dedicated zero |
| 56 | | − |
| 57 | | − pcmpeqw m4, m4 ; All word lanes -1 |
| 58 | | − paddw m0, m4 ; m0 = zbin - 1 |
| 59 | | − |
| 60 | | − pabsw m6, m9 ; m6 = abs(m9) |
| 61 | | − pabsw m11, m10 ; m11 = abs(m10) |
| 62 | | − pcmpgtw m7, m6, m0 ; m7 = c[i] >= zbin |
| 63 | | − punpckhqdq m0, m0 |
| 64 | | − pcmpgtw m12, m11, m0 ; m12 = c[i] >= zbin |
| 65 | | − |
| 66 | | − ; Check if all coeffs are less than zbin. If yes, we just write zeros |
| 67 | | − ; to the outputs and we are done. |
| 68 | | − por m14, m7, m12 |
| 69 | | − ptest m14, m14 |
| 70 | | − jnz .single_nonzero |
| 71 | | − |
| 72 | | −%if CONFIG_VP9_HIGHBITDEPTH |
| 73 | | − mova [r1 ], ymm5 |
| 74 | | − mova [r1+32], ymm5 |
| 75 | | − mova [r2 ], ymm5 |
| 76 | | − mova [r2+32], ymm5 |
| 77 | | −%else |
| 78 | | − mova [r1], ymm5 |
| 79 | | − mova [r2], ymm5 |
| 80 | | −%endif |
| 81 | | − mov [r0], word 0 |
| 82 | | − |
| 83 | | − vzeroupper |
| 84 | | − RET |
| 85 | | − |
| 86 | | −.single_nonzero: |
| 87 | | − |
| 88 | | − ; Actual quantization of size 16 block - setup pointers, rounders, etc. |
| 89 | | − movifnidn r4, roundmp |
| 90 | | − movifnidn r5, quantmp |
| 91 | | − mov r3, dequantmp |
| 92 | | − mov r6, shiftmp |
| 93 | | − mova m1, [r4] ; m1 = round |
| 94 | | − mova m2, [r5] ; m2 = quant |
| 95 | | − mova m3, [r3] ; m3 = dequant |
| 96 | | − mova m4, [r6] ; m4 = shift |
| 97 | | − |
| 98 | | − mov r3, iscanmp |
| 99 | | − |
| 100 | | − DEFINE_ARGS eob, qcoeff, dqcoeff, iscan |
| 101 | | − |
| 102 | | − ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; |
| 103 | | − |
| 104 | | − paddsw m6, m1 ; m6 += round |
| 105 | | − punpckhqdq m1, m1 |
| 106 | | − paddsw m11, m1 ; m11 += round |
| 107 | | − pmulhw m8, m6, m2 ; m8 = m6*q>>16 |
| 108 | | − punpckhqdq m2, m2 |
| 109 | | − pmulhw m13, m11, m2 ; m13 = m11*q>>16 |
| 110 | | − paddw m8, m6 ; m8 += m6 |
| 111 | | − paddw m13, m11 ; m13 += m11 |
| 112 | | − pmulhw m8, m4 ; m8 = m8*qsh>>16 |
| 113 | | − punpckhqdq m4, m4 |
| 114 | | − pmulhw m13, m4 ; m13 = m13*qsh>>16 |
| 115 | | − psignw m8, m9 ; m8 = reinsert sign |
| 116 | | − psignw m13, m10 ; m13 = reinsert sign |
| 117 | | − pand m8, m7 |
| 118 | | − pand m13, m12 |
| 119 | | − |
| 120 | | −%if CONFIG_VP9_HIGHBITDEPTH |
| 121 | | − ; Store 16bit numbers as 32bit numbers in array pointed to by qcoeff |
| 122 | | − pcmpgtw m6, m5, m8 |
| 123 | | − punpckhwd m6, m8, m6 |
| 124 | | − pmovsxwd m11, m8 |
| 125 | | − mova [qcoeffq ], m11 |
| 126 | | − mova [qcoeffq+16], m6 |
| 127 | | − pcmpgtw m6, m5, m13 |
| 128 | | − punpckhwd m6, m13, m6 |
| 129 | | − pmovsxwd m11, m13 |
| 130 | | − mova [qcoeffq+32], m11 |
| 131 | | − mova [qcoeffq+48], m6 |
| 132 | | −%else |
| 133 | | − mova [qcoeffq ], m8 |
| 134 | | − mova [qcoeffq+16], m13 |
| 135 | | −%endif |
| 136 | | − |
| 137 | | − pmullw m8, m3 ; dqc[i] = qc[i] * q |
| 138 | | − punpckhqdq m3, m3 |
| 139 | | − pmullw m13, m3 ; dqc[i] = qc[i] * q |
| 140 | | − |
| 141 | | −%if CONFIG_VP9_HIGHBITDEPTH |
| 142 | | − ; Store 16bit numbers as 32bit numbers in array pointed to by qcoeff |
| 143 | | − pcmpgtw m6, m5, m8 |
| 144 | | − punpckhwd m6, m8, m6 |
| 145 | | − pmovsxwd m11, m8 |
| 146 | | − mova [dqcoeffq ], m11 |
| 147 | | − mova [dqcoeffq+16], m6 |
| 148 | | − pcmpgtw m6, m5, m13 |
| 149 | | − punpckhwd m6, m13, m6 |
| 150 | | − pmovsxwd m11, m13 |
| 151 | | − mova [dqcoeffq+32], m11 |
| 152 | | − mova [dqcoeffq+48], m6 |
| 153 | | −%else |
| 154 | | − mova [dqcoeffq ], m8 |
| 155 | | − mova [dqcoeffq+16], m13 |
| 156 | | −%endif |
| 157 | | − |
| 158 | | − mova m6, [iscanq] ; m6 = scan[i] |
| 159 | | − mova m11, [iscanq+16] ; m11 = scan[i] |
| 160 | | − |
| 161 | | − pcmpeqw m8, m8, m5 ; m8 = c[i] == 0 |
| 162 | | − pcmpeqw m13, m13, m5 ; m13 = c[i] == 0 |
| 163 | | − psubw m6, m6, m7 ; m6 = scan[i] + 1 |
| 164 | | − psubw m11, m11, m12 ; m11 = scan[i] + 1 |
| 165 | | − pandn m8, m8, m6 ; m8 = max(eob) |
| 166 | | − pandn m13, m13, m11 ; m13 = max(eob) |
| 167 | | − pmaxsw m8, m8, m13 |
| 168 | | − |
| 169 | | − ; Horizontally accumulate/max eobs and write into [eob] memory pointer |
| 170 | | − pshufd m7, m8, 0xe |
| 171 | | − pmaxsw m8, m7 |
| 172 | | − pshuflw m7, m8, 0xe |
| 173 | | − pmaxsw m8, m7 |
| 174 | | − pshuflw m7, m8, 0x1 |
| 175 | | − pmaxsw m8, m7 |
| 176 | | − movq rax, m8 |
| 177 | | − mov [eobq], ax |
| 178 | | − |
| 179 | | − vzeroupper |
| 180 | | − RET |
| 181 | | − |
| 182 | | − ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; |
| 183 | | − ;; Generic case of ncoeff != 16 |
| 184 | | − ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; |
| 185 | | − |
| 186 | | −.generic: |
| 187 | | − |
| 188 | | −%endif ; %ifnidn %1, b_32x32 |
| 189 | | − |
| 190 | | −DEFINE_ARGS coeff, ncoeff, skip, zbin, round, quant, shift, \ |
| 191 | | − qcoeff, dqcoeff, dequant, eob, scan, iscan |
| 192 | | − |
| 193 | | − ; Actual quantization loop - setup pointers, rounders, etc. |
| 194 | | − movifnidn coeffq, coeffmp |
| 195 | | − movifnidn ncoeffq, ncoeffmp |
| 196 | | − mov r2, dequantmp |
| 197 | | − movifnidn zbinq, zbinmp |
| 198 | | − movifnidn roundq, roundmp |
| 199 | | − movifnidn quantq, quantmp |
| 200 | | − mova m0, [zbinq] ; m0 = zbin |
| 201 | | − mova m1, [roundq] ; m1 = round |
| 202 | | − mova m2, [quantq] ; m2 = quant |
| 203 | | − mova m3, [r2] ; m3 = dequant |
| 204 | | − pcmpeqw m4, m4 ; All lanes -1 |
| 205 | | −%ifidn %1, b_32x32 |
| 206 | | − psubw m0, m4 |
| 207 | | − psubw m1, m4 |
| 208 | | − psrlw m0, 1 ; m0 = (m0 + 1) / 2 |
| 209 | | − psrlw m1, 1 ; m1 = (m1 + 1) / 2 |
| 210 | | −%endif |
| 211 | | − paddw m0, m4 ; m0 = m0 + 1 |
| 212 | | − |
| 213 | | − mov r2, shiftmp |
| 214 | | − mov r3, qcoeffmp |
| 215 | | − mova m4, [r2] ; m4 = shift |
| 216 | | − mov r4, dqcoeffmp |
| 217 | | − mov r5, iscanmp |
| 218 | | −%ifidn %1, b_32x32 |
| 219 | | − psllw m4, 1 |
| 220 | | −%endif |
| 221 | | − pxor m5, m5 ; m5 = dedicated zero |
| 222 | | − |
| 223 | | − DEFINE_ARGS coeff, ncoeff, d1, qcoeff, dqcoeff, iscan, d2, d3, d4, d5, eob |
| 224 | | − |
| 225 | | −%if CONFIG_VP9_HIGHBITDEPTH |
| 226 | | − lea coeffq, [ coeffq+ncoeffq*4] |
| 227 | | − lea qcoeffq, [ qcoeffq+ncoeffq*4] |
| 228 | | − lea dqcoeffq, [dqcoeffq+ncoeffq*4] |
| 229 | | −%else |
| 230 | | − lea coeffq, [ coeffq+ncoeffq*2] |
| 231 | | − lea qcoeffq, [ qcoeffq+ncoeffq*2] |
| 232 | | − lea dqcoeffq, [dqcoeffq+ncoeffq*2] |
| 233 | | −%endif |
| 234 | | − lea iscanq, [ iscanq+ncoeffq*2] |
| 235 | | − neg ncoeffq |
| 236 | | − |
| 237 | | − ; get DC and first 15 AC coeffs |
| 238 | | −%if CONFIG_VP9_HIGHBITDEPTH |
| 239 | | − ; coeff stored as 32bit numbers & require 16bit numbers |
| 240 | | − mova m9, [coeffq+ncoeffq*4+ 0] |
| 241 | | − packssdw m9, [coeffq+ncoeffq*4+16] |
| 242 | | − mova m10, [coeffq+ncoeffq*4+32] |
| 243 | | − packssdw m10, [coeffq+ncoeffq*4+48] |
| 244 | | −%else |
| 245 | | − mova m9, [coeffq+ncoeffq*2+ 0] ; m9 = c[i] |
| 246 | | − mova m10, [coeffq+ncoeffq*2+16] ; m10 = c[i] |
| 247 | | −%endif |
| 248 | | − |
| 249 | | − pabsw m6, m9 ; m6 = abs(m9) |
| 250 | | − pabsw m11, m10 ; m11 = abs(m10) |
| 251 | | − pcmpgtw m7, m6, m0 ; m7 = c[i] >= zbin |
| 252 | | − punpckhqdq m0, m0 |
| 253 | | − pcmpgtw m12, m11, m0 ; m12 = c[i] >= zbin |
| 254 | | − |
| 255 | | − ; Check if all coeffs are less than zbin. If yes, skip forward quickly. |
| 256 | | − por m14, m7, m12 |
| 257 | | − ptest m14, m14 |
| 258 | | − jnz .first_nonzero |
| 259 | | − |
| 260 | | −%if CONFIG_VP9_HIGHBITDEPTH |
| 261 | | − mova [qcoeffq+ncoeffq*4 ], ymm5 |
| 262 | | − mova [qcoeffq+ncoeffq*4+32], ymm5 |
| 263 | | − mova [dqcoeffq+ncoeffq*4 ], ymm5 |
| 264 | | − mova [dqcoeffq+ncoeffq*4+32], ymm5 |
| 265 | | −%else |
| 266 | | − mova [qcoeffq+ncoeffq*2], ymm5 |
| 267 | | − mova [dqcoeffq+ncoeffq*2], ymm5 |
| 268 | | −%endif |
| 269 | | − |
| 270 | | − add ncoeffq, mmsize |
| 271 | | − |
| 272 | | − punpckhqdq m1, m1 |
| 273 | | − punpckhqdq m2, m2 |
| 274 | | − punpckhqdq m3, m3 |
| 275 | | − punpckhqdq m4, m4 |
| 276 | | − pxor m8, m8 |
| 277 | | − |
| 278 | | − jmp .ac_only_loop |
| 279 | | − |
| 280 | | −.first_nonzero: |
| 281 | | − |
| 282 | | − paddsw m6, m1 ; m6 += round |
| 283 | | − punpckhqdq m1, m1 |
| 284 | | − paddsw m11, m1 ; m11 += round |
| 285 | | − pmulhw m8, m6, m2 ; m8 = m6*q>>16 |
| 286 | | − punpckhqdq m2, m2 |
| 287 | | − pmulhw m13, m11, m2 ; m13 = m11*q>>16 |
| 288 | | − paddw m8, m6 ; m8 += m6 |
| 289 | | − paddw m13, m11 ; m13 += m11 |
| 290 | | − pmulhw m8, m4 ; m8 = m8*qsh>>16 |
| 291 | | − punpckhqdq m4, m4 |
| 292 | | − pmulhw m13, m4 ; m13 = m13*qsh>>16 |
| 293 | | − psignw m8, m9 ; m8 = reinsert sign |
| 294 | | − psignw m13, m10 ; m13 = reinsert sign |
| 295 | | − pand m8, m7 |
| 296 | | − pand m13, m12 |
| 297 | | − |
| 298 | | −%if CONFIG_VP9_HIGHBITDEPTH |
| 299 | | − ; store 16bit numbers as 32bit numbers in array pointed to by qcoeff |
| 300 | | − pcmpgtw m6, m5, m8 |
| 301 | | − punpckhwd m6, m8, m6 |
| 302 | | − pmovsxwd m11, m8 |
| 303 | | − mova [qcoeffq+ncoeffq*4+ 0], m11 |
| 304 | | − mova [qcoeffq+ncoeffq*4+16], m6 |
| 305 | | − pcmpgtw m6, m5, m13 |
| 306 | | − punpckhwd m6, m13, m6 |
| 307 | | − pmovsxwd m11, m13 |
| 308 | | − mova [qcoeffq+ncoeffq*4+32], m11 |
| 309 | | − mova [qcoeffq+ncoeffq*4+48], m6 |
| 310 | | −%else |
| 311 | | − mova [qcoeffq+ncoeffq*2+ 0], m8 |
| 312 | | − mova [qcoeffq+ncoeffq*2+16], m13 |
| 313 | | −%endif |
| 314 | | − |
| 315 | | −%ifidn %1, b_32x32 |
| 316 | | − pabsw m8, m8 |
| 317 | | − pabsw m13, m13 |
| 318 | | −%endif |
| 319 | | − pmullw m8, m3 ; dqc[i] = qc[i] * q |
| 320 | | − punpckhqdq m3, m3 |
| 321 | | − pmullw m13, m3 ; dqc[i] = qc[i] * q |
| 322 | | −%ifidn %1, b_32x32 |
| 323 | | − psrlw m8, 1 |
| 324 | | − psrlw m13, 1 |
| 325 | | − psignw m8, m9 |
| 326 | | − psignw m13, m10 |
| 327 | | −%endif |
| 328 | | − |
| 329 | | −%if CONFIG_VP9_HIGHBITDEPTH |
| 330 | | − ; store 16bit numbers as 32bit numbers in array pointed to by qcoeff |
| 331 | | − pcmpgtw m6, m5, m8 |
| 332 | | − punpckhwd m6, m8, m6 |
| 333 | | − pmovsxwd m11, m8 |
| 334 | | − mova [dqcoeffq+ncoeffq*4+ 0], m11 |
| 335 | | − mova [dqcoeffq+ncoeffq*4+16], m6 |
| 336 | | − pcmpgtw m6, m5, m13 |
| 337 | | − punpckhwd m6, m13, m6 |
| 338 | | − pmovsxwd m11, m13 |
| 339 | | − mova [dqcoeffq+ncoeffq*4+32], m11 |
| 340 | | − mova [dqcoeffq+ncoeffq*4+48], m6 |
| 341 | | −%else |
| 342 | | − mova [dqcoeffq+ncoeffq*2+ 0], m8 |
| 343 | | − mova [dqcoeffq+ncoeffq*2+16], m13 |
| 344 | | −%endif |
| 345 | | − |
| 346 | | − pcmpeqw m8, m5 ; m8 = c[i] == 0 |
| 347 | | − pcmpeqw m13, m5 ; m13 = c[i] == 0 |
| 348 | | − mova m6, [iscanq+ncoeffq*2] ; m6 = scan[i] |
| 349 | | − mova m11, [iscanq+ncoeffq*2+16] ; m11 = scan[i] |
| 350 | | − psubw m6, m7 ; m6 = scan[i] + 1 |
| 351 | | − psubw m11, m12 ; m11 = scan[i] + 1 |
| 352 | | − pandn m8, m6 ; m8 = max(eob) |
| 353 | | − pandn m13, m11 ; m13 = max(eob) |
| 354 | | − pmaxsw m8, m13 |
| 355 | | − add ncoeffq, mmsize |
| 356 | | − |
| 357 | | −.ac_only_loop: |
| 358 | | − |
| 359 | | −%if CONFIG_VP9_HIGHBITDEPTH |
| 360 | | − ; pack coeff from 32bit to 16bit array |
| 361 | | − mova m9, [coeffq+ncoeffq*4+ 0] |
| 362 | | − packssdw m9, [coeffq+ncoeffq*4+16] |
| 363 | | − mova m10, [coeffq+ncoeffq*4+32] |
| 364 | | − packssdw m10, [coeffq+ncoeffq*4+48] |
| 365 | | −%else |
| 366 | | − mova m9, [coeffq+ncoeffq*2+ 0] ; m9 = c[i] |
| 367 | | − mova m10, [coeffq+ncoeffq*2+16] ; m10 = c[i] |
| 368 | | −%endif |
| 369 | | − |
| 370 | | − pabsw m6, m9 ; m6 = abs(m9) |
| 371 | | − pabsw m11, m10 ; m11 = abs(m10) |
| 372 | | − pcmpgtw m7, m6, m0 ; m7 = c[i] >= zbin |
| 373 | | − pcmpgtw m12, m11, m0 ; m12 = c[i] >= zbin |
| 374 | | − |
| 375 | | − ; Check if all coeffs are less than zbin. If yes, skip this itertion. |
| 376 | | − ; And just write zeros as the result would be. |
| 377 | | − por m14, m7, m12 |
| 378 | | − ptest m14, m14 |
| 379 | | − jnz .rest_nonzero |
| 380 | | − |
| 381 | | −%if CONFIG_VP9_HIGHBITDEPTH |
| 382 | | − mova [qcoeffq+ncoeffq*4+ 0], ymm5 |
| 383 | | − mova [qcoeffq+ncoeffq*4+32], ymm5 |
| 384 | | − mova [dqcoeffq+ncoeffq*4+ 0], ymm5 |
| 385 | | − mova [dqcoeffq+ncoeffq*4+32], ymm5 |
| 386 | | −%else |
| 387 | | − mova [qcoeffq+ncoeffq*2+ 0], ymm5 |
| 388 | | − mova [dqcoeffq+ncoeffq*2+ 0], ymm5 |
| 389 | | −%endif |
| 390 | | − add ncoeffq, mmsize |
| 391 | | − jnz .ac_only_loop |
| 392 | | − |
| 393 | | − ; Horizontally accumulate/max eobs and write into [eob] memory pointer |
| 394 | | − mov r2, eobmp |
| 395 | | − pshufd m7, m8, 0xe |
| 396 | | − pmaxsw m8, m7 |
| 397 | | − pshuflw m7, m8, 0xe |
| 398 | | − pmaxsw m8, m7 |
| 399 | | − pshuflw m7, m8, 0x1 |
| 400 | | − pmaxsw m8, m7 |
| 401 | | − movq rax, m8 |
| 402 | | − mov [r2], ax |
| 403 | | − vzeroupper |
| 404 | | − RET |
| 405 | | − |
| 406 | | −.rest_nonzero: |
| 407 | | − paddsw m6, m1 ; m6 += round |
| 408 | | − paddsw m11, m1 ; m11 += round |
| 409 | | − pmulhw m14, m6, m2 ; m14 = m6*q>>16 |
| 410 | | − pmulhw m13, m11, m2 ; m13 = m11*q>>16 |
| 411 | | − paddw m14, m6 ; m14 += m6 |
| 412 | | − paddw m13, m11 ; m13 += m11 |
| 413 | | − pmulhw m14, m4 ; m14 = m14*qsh>>16 |
| 414 | | − pmulhw m13, m4 ; m13 = m13*qsh>>16 |
| 415 | | − psignw m14, m9 ; m14 = reinsert sign |
| 416 | | − psignw m13, m10 ; m13 = reinsert sign |
| 417 | | − pand m14, m7 |
| 418 | | − pand m13, m12 |
| 419 | | − |
| 420 | | −%if CONFIG_VP9_HIGHBITDEPTH |
| 421 | | − ; store 16bit numbers as 32bit numbers in array pointed to by qcoeff |
| 422 | | − pcmpgtw m6, m5, m14 |
| 423 | | − punpckhwd m6, m14, m6 |
| 424 | | − pmovsxwd m11, m14 |
| 425 | | − mova [qcoeffq+ncoeffq*4+ 0], m11 |
| 426 | | − mova [qcoeffq+ncoeffq*4+16], m6 |
| 427 | | − pcmpgtw m6, m5, m13 |
| 428 | | − punpckhwd m6, m13, m6 |
| 429 | | − pmovsxwd m11, m13 |
| 430 | | − mova [qcoeffq+ncoeffq*4+32], m11 |
| 431 | | − mova [qcoeffq+ncoeffq*4+48], m6 |
| 432 | | −%else |
| 433 | | − mova [qcoeffq+ncoeffq*2+ 0], m14 |
| 434 | | − mova [qcoeffq+ncoeffq*2+16], m13 |
| 435 | | −%endif |
| 436 | | − |
| 437 | | −%ifidn %1, b_32x32 |
| 438 | | − pabsw m14, m14 |
| 439 | | − pabsw m13, m13 |
| 440 | | −%endif |
| 441 | | − pmullw m14, m3 ; dqc[i] = qc[i] * q |
| 442 | | − pmullw m13, m3 ; dqc[i] = qc[i] * q |
| 443 | | −%ifidn %1, b_32x32 |
| 444 | | − psrlw m14, 1 |
| 445 | | − psrlw m13, 1 |
| 446 | | − psignw m14, m9 |
| 447 | | − psignw m13, m10 |
| 448 | | −%endif |
| 449 | | − |
| 450 | | −%if CONFIG_VP9_HIGHBITDEPTH |
| 451 | | − ; store 16bit numbers as 32bit numbers in array pointed to by qcoeff |
| 452 | | − pcmpgtw m6, m5, m14 |
| 453 | | − punpckhwd m6, m14, m6 |
| 454 | | − pmovsxwd m11, m14 |
| 455 | | − mova [dqcoeffq+ncoeffq*4+ 0], m11 |
| 456 | | − mova [dqcoeffq+ncoeffq*4+16], m6 |
| 457 | | − pcmpgtw m6, m5, m13 |
| 458 | | − punpckhwd m6, m13, m6 |
| 459 | | − pmovsxwd m11, m13 |
| 460 | | − mova [dqcoeffq+ncoeffq*4+32], m11 |
| 461 | | − mova [dqcoeffq+ncoeffq*4+48], m6 |
| 462 | | −%else |
| 463 | | − mova [dqcoeffq+ncoeffq*2+ 0], m14 |
| 464 | | − mova [dqcoeffq+ncoeffq*2+16], m13 |
| 465 | | −%endif |
| 466 | | − |
| 467 | | − pcmpeqw m14, m5 ; m14 = c[i] == 0 |
| 468 | | − pcmpeqw m13, m5 ; m13 = c[i] == 0 |
| 469 | | − mova m6, [iscanq+ncoeffq*2+ 0] ; m6 = scan[i] |
| 470 | | − mova m11, [iscanq+ncoeffq*2+16] ; m11 = scan[i] |
| 471 | | − psubw m6, m7 ; m6 = scan[i] + 1 |
| 472 | | − psubw m11, m12 ; m11 = scan[i] + 1 |
| 473 | | − pandn m14, m6 ; m14 = max(eob) |
| 474 | | − pandn m13, m11 ; m13 = max(eob) |
| 475 | | − pmaxsw m8, m14 |
| 476 | | − pmaxsw m8, m13 |
| 477 | | − add ncoeffq, mmsize |
| 478 | | − jnz .ac_only_loop |
| 479 | | − |
| 480 | | − ; Horizontally accumulate/max eobs and write into [eob] memory pointer |
| 481 | | − mov r2, eobmp |
| 482 | | − pshufd m7, m8, 0xe |
| 483 | | − pmaxsw m8, m7 |
| 484 | | − pshuflw m7, m8, 0xe |
| 485 | | − pmaxsw m8, m7 |
| 486 | | − pshuflw m7, m8, 0x1 |
| 487 | | − pmaxsw m8, m7 |
| 488 | | − movq rax, m8 |
| 489 | | − mov [r2], ax |
| 490 | | − vzeroupper |
| 491 | | − RET |
| 492 | | −%endmacro |
| 493 | | − |
| 494 | | −INIT_XMM avx |
| 495 | | −QUANTIZE_FN b_32x32, 7 |
| 496 | | − |
| 497 | | −END |