The rounds value is constant and can be one of three hardcoded values, so instead of checking it on every loop, just split the function into three different implementations for each value. Before: aes_decrypt_128_aesni: 93.8 (47.58x) aes_decrypt_192_aesni: 106.9 (49.30x) aes_decrypt_256_aesni: 109.8 (56.50x) aes_encrypt_128_aesni: 93.2 (47.70x) aes_encrypt_192_aesni: 111.1 (48.36x) aes_encrypt_256_aesni: 113.6 (56.27x) After: aes_decrypt_128_aesni: 71.5 (63.31x) aes_decrypt_192_aesni: 96.8 (55.64x) aes_decrypt_256_aesni: 106.1 (58.51x) aes_encrypt_128_aesni: 81.3 (55.92x) aes_encrypt_192_aesni: 91.2 (59.78x) aes_encrypt_256_aesni: 109.0 (58.26x)
Unified · Split
| @@ -234,6 +234,7 @@ int av_aes_init(AVAES *a, const uint8_t *key, int key_bits, int decrypt) · 233 unmodified lines | ||
| 234 | 234 | int KC = key_bits >> 5; |
| 235 | 235 | int rounds = KC + 6; |
| 236 | 236 | |
| 237 | + a->rounds = rounds; | |
| 237 | 238 | a->crypt = decrypt ? aes_decrypt : aes_encrypt; |
| 238 | 239 | if (ARCH_X86) |
| 239 | 240 | ff_init_aes_x86(a, decrypt); |
| @@ -243,8 +244,6 @@ int av_aes_init(AVAES *a, const uint8_t *key, int key_bits, int decrypt) · 3 unmodified lines | ||
| 243 | 244 | if (key_bits != 128 && key_bits != 192 && key_bits != 256) |
| 244 | 245 | return AVERROR(EINVAL); |
| 245 | 246 | |
| 246 | − a->rounds = rounds; | |
| 247 | − | |
| 248 | 247 | memcpy(tk, key, KC * 4); |
| 249 | 248 | memcpy(a->round_key[0].u8, key, KC * 4); |
| 250 | 249 | |
| @@ -26,13 +26,12 @@ SECTION .text · 25 unmodified lines | ||
| 26 | 26 | ; void ff_aes_decrypt(AVAES *a, uint8_t *dst, const uint8_t *src, |
| 27 | 27 | ; int count, uint8_t *iv, int rounds) |
| 28 | 28 | ;----------------------------------------------------------------------------- |
| 29 | −%macro AES_CRYPT 1 | |
| 30 | −cglobal aes_%1rypt, 6,6,2 | |
| 29 | +%macro AES_CRYPT 2 | |
| 30 | +cglobal aes_%1rypt_%2, 5, 5, 2 | |
| 31 | 31 | test r3d, r3d |
| 32 | 32 | je .ret |
| 33 | 33 | shl r3d, 4 |
| 34 | − add r5d, r5d | |
| 35 | − add r0, 0x60 | |
| 34 | + add r0, 0x70 | |
| 36 | 35 | add r2, r3 |
| 37 | 36 | add r1, r3 |
| 38 | 37 | neg r3 |
| @@ -45,17 +44,15 @@ cglobal aes_%1rypt, 6,6,2 · 6 unmodified lines | ||
| 45 | 44 | %ifidn %1, enc |
| 46 | 45 | pxor m0, m1 |
| 47 | 46 | %endif |
| 48 | − pxor m0, [r0+8*r5-0x60] | |
| 49 | − cmp r5d, 24 | |
| 50 | − je .rounds12 | |
| 51 | − jl .rounds10 | |
| 52 | − aes%1 m0, [r0+0x70] | |
| 47 | + pxor m0, [r0+8*2*%2-0x70] | |
| 48 | +%if %2 > 10 | |
| 49 | +%if %2 > 12 | |
| 53 | 50 | aes%1 m0, [r0+0x60] |
| 54 | −.rounds12: | |
| 55 | 51 | aes%1 m0, [r0+0x50] |
| 52 | +%endif | |
| 56 | 53 | aes%1 m0, [r0+0x40] |
| 57 | −.rounds10: | |
| 58 | 54 | aes%1 m0, [r0+0x30] |
| 55 | +%endif | |
| 59 | 56 | aes%1 m0, [r0+0x20] |
| 60 | 57 | aes%1 m0, [r0+0x10] |
| 61 | 58 | aes%1 m0, [r0+0x00] |
| @@ -64,7 +61,8 @@ cglobal aes_%1rypt, 6,6,2 · 2 unmodified lines | ||
| 64 | 61 | aes%1 m0, [r0-0x30] |
| 65 | 62 | aes%1 m0, [r0-0x40] |
| 66 | 63 | aes%1 m0, [r0-0x50] |
| 67 | − aes%1last m0, [r0-0x60] | |
| 64 | + aes%1 m0, [r0-0x60] | |
| 65 | + aes%1last m0, [r0-0x70] | |
| 68 | 66 | test r4, r4 |
| 69 | 67 | je .noiv |
| 70 | 68 | %ifidn %1, enc |
| @@ -90,6 +88,10 @@ cglobal aes_%1rypt, 6,6,2 · 19 unmodified lines | ||
| 90 | 88 | |
| 91 | 89 | %if HAVE_AESNI_EXTERNAL |
| 92 | 90 | INIT_XMM aesni |
| 93 | −AES_CRYPT enc | |
| 94 | −AES_CRYPT dec | |
| 91 | +AES_CRYPT enc, 10 | |
| 92 | +AES_CRYPT enc, 12 | |
| 93 | +AES_CRYPT enc, 14 | |
| 94 | +AES_CRYPT dec, 10 | |
| 95 | +AES_CRYPT dec, 12 | |
| 96 | +AES_CRYPT dec, 14 | |
| 95 | 97 | %endif |
| @@ -22,15 +22,29 @@ · 21 unmodified lines | ||
| 22 | 22 | #include "libavutil/aes_internal.h" |
| 23 | 23 | #include "libavutil/x86/cpu.h" |
| 24 | 24 | |
| 25 | −void ff_aes_decrypt_aesni(AVAES *a, uint8_t *dst, const uint8_t *src, | |
| 26 | − int count, uint8_t *iv, int rounds); | |
| 27 | −void ff_aes_encrypt_aesni(AVAES *a, uint8_t *dst, const uint8_t *src, | |
| 28 | − int count, uint8_t *iv, int rounds); | |
| 25 | +void ff_aes_decrypt_10_aesni(AVAES *a, uint8_t *dst, const uint8_t *src, | |
| 26 | + int count, uint8_t *iv, int rounds); | |
| 27 | +void ff_aes_decrypt_12_aesni(AVAES *a, uint8_t *dst, const uint8_t *src, | |
| 28 | + int count, uint8_t *iv, int rounds); | |
| 29 | +void ff_aes_decrypt_14_aesni(AVAES *a, uint8_t *dst, const uint8_t *src, | |
| 30 | + int count, uint8_t *iv, int rounds); | |
| 31 | +void ff_aes_encrypt_10_aesni(AVAES *a, uint8_t *dst, const uint8_t *src, | |
| 32 | + int count, uint8_t *iv, int rounds); | |
| 33 | +void ff_aes_encrypt_12_aesni(AVAES *a, uint8_t *dst, const uint8_t *src, | |
| 34 | + int count, uint8_t *iv, int rounds); | |
| 35 | +void ff_aes_encrypt_14_aesni(AVAES *a, uint8_t *dst, const uint8_t *src, | |
| 36 | + int count, uint8_t *iv, int rounds); | |
| 29 | 37 | |
| 30 | 38 | void ff_init_aes_x86(AVAES *a, int decrypt) |
| 31 | 39 | { |
| 32 | 40 | int cpu_flags = av_get_cpu_flags(); |
| 33 | 41 | |
| 34 | − if (EXTERNAL_AESNI(cpu_flags)) | |
| 35 | − a->crypt = decrypt ? ff_aes_decrypt_aesni : ff_aes_encrypt_aesni; | |
| 42 | + if (EXTERNAL_AESNI(cpu_flags)) { | |
| 43 | + if (a->rounds == 10) | |
| 44 | + a->crypt = decrypt ? ff_aes_decrypt_10_aesni : ff_aes_encrypt_10_aesni; | |
| 45 | + else if (a->rounds == 12) | |
| 46 | + a->crypt = decrypt ? ff_aes_decrypt_12_aesni : ff_aes_encrypt_12_aesni; | |
| 47 | + else if (a->rounds == 14) | |
| 48 | + a->crypt = decrypt ? ff_aes_decrypt_14_aesni : ff_aes_encrypt_14_aesni; | |
| 49 | + } | |
| 36 | 50 | } |