History

Add AVX FFT implementation.

Vitor Sessak committed 15 years ago

Commit 9d35fa51 parent 13dfce323 files · +450 −207Browse files

Unified · Split

23 files
  1. Changelog +1 −1
  2. libavcodec/aac.h +5 −5
  3. libavcodec/aacenc.h +1 −1
  4. libavcodec/ac3dec.h +5 −5
  5. libavcodec/ac3enc.c +1 −1
  6. libavcodec/atrac1.c +10 −10
  7. libavcodec/atrac3.c +3 −3
  8. libavcodec/binkaudio.c +1 −1
  9. libavcodec/cook.c +1 −1
  10. libavcodec/dca.c +5 −5
  11. libavcodec/fft.c +48 −5
  12. libavcodec/fft.h +2 −1
  13. libavcodec/imc.c +1 −1
  14. libavcodec/nellymoserdec.c +2 −2
  15. libavcodec/nellymoserenc.c +3 −3
  16. libavcodec/qdm2.c +1 −1
  17. libavcodec/wma.h +4 −4
  18. libavcodec/wmaprodec.c +2 −2
  19. libavcodec/wmavoice.c +3 −3
  20. libavcodec/x86/fft.c +8 −1
  21. libavcodec/x86/fft.h +2 −0
  22. libavcodec/x86/fft_mmx.asm +334 −150
  23. libavcodec/x86/fft_sse.c +7 −1
Changelog+1−1
@@ -5,7 +5,7 @@ releases are sorted from youngest to oldest. · 4 unmodified lines
5version <next>:5version <next>:
6 6
7- Lots of deprecated API cruft removed7- Lots of deprecated API cruft removed
8 8- fft and imdct optimizations for AVX (Sandy Bridge) processors
9 9
10version 0.7_beta1:10version 0.7_beta1:
11 11
libavcodec/aac.h+5−5
@@ -223,9 +223,9 @@ typedef struct { · 222 unmodified lines
223 float sf[120]; ///< scalefactors223 float sf[120]; ///< scalefactors
224 int sf_idx[128]; ///< scalefactor indices (used by encoder)224 int sf_idx[128]; ///< scalefactor indices (used by encoder)
225 uint8_t zeroes[128]; ///< band is not coded (used by encoder)225 uint8_t zeroes[128]; ///< band is not coded (used by encoder)
226 DECLARE_ALIGNED(16, float, coeffs)[1024]; ///< coefficients for IMDCT226 DECLARE_ALIGNED(32, float, coeffs)[1024]; ///< coefficients for IMDCT
227 DECLARE_ALIGNED(16, float, saved)[1024]; ///< overlap227 DECLARE_ALIGNED(32, float, saved)[1024]; ///< overlap
228 DECLARE_ALIGNED(16, float, ret)[2048]; ///< PCM output228 DECLARE_ALIGNED(32, float, ret)[2048]; ///< PCM output
229 DECLARE_ALIGNED(16, int16_t, ltp_state)[3072]; ///< time signal for LTP229 DECLARE_ALIGNED(16, int16_t, ltp_state)[3072]; ///< time signal for LTP
230 PredictorState predictor_state[MAX_PREDICTORS];230 PredictorState predictor_state[MAX_PREDICTORS];
231} SingleChannelElement;231} SingleChannelElement;
@@ -272,7 +272,7 @@ typedef struct { · 40 unmodified lines
272 * @defgroup temporary aligned temporary buffers (We do not want to have these on the stack.)272 * @defgroup temporary aligned temporary buffers (We do not want to have these on the stack.)
273 * @{273 * @{
274 */274 */
275 DECLARE_ALIGNED(16, float, buf_mdct)[1024];275 DECLARE_ALIGNED(32, float, buf_mdct)[1024];
276 /** @} */276 /** @} */
277 277
278 /**278 /**
@@ -296,7 +296,7 @@ typedef struct { · 17 unmodified lines
296 int sf_offset; ///< offset into pow2sf_tab as appropriate for dsp.float_to_int16296 int sf_offset; ///< offset into pow2sf_tab as appropriate for dsp.float_to_int16
297 /** @} */297 /** @} */
298 298
299 DECLARE_ALIGNED(16, float, temp)[128];299 DECLARE_ALIGNED(32, float, temp)[128];
300 300
301 enum OCStatus output_configured;301 enum OCStatus output_configured;
302} AACContext;302} AACContext;
libavcodec/aacenc.h+1−1
@@ -64,7 +64,7 @@ typedef struct AACEncContext { · 63 unmodified lines
64 int last_frame;64 int last_frame;
65 float lambda;65 float lambda;
66 DECLARE_ALIGNED(16, int, qcoefs)[96]; ///< quantized coefficients66 DECLARE_ALIGNED(16, int, qcoefs)[96]; ///< quantized coefficients
67 DECLARE_ALIGNED(16, float, scoefs)[1024]; ///< scaled coefficients67 DECLARE_ALIGNED(32, float, scoefs)[1024]; ///< scaled coefficients
68} AACEncContext;68} AACEncContext;
69 69
70#endif /* AVCODEC_AACENC_H */70#endif /* AVCODEC_AACENC_H */
libavcodec/ac3dec.h+5−5
@@ -200,11 +200,11 @@ typedef struct { · 199 unmodified lines
200 200
201///@defgroup arrays aligned arrays201///@defgroup arrays aligned arrays
202 DECLARE_ALIGNED(16, int, fixed_coeffs)[AC3_MAX_CHANNELS][AC3_MAX_COEFS]; ///> fixed-point transform coefficients202 DECLARE_ALIGNED(16, int, fixed_coeffs)[AC3_MAX_CHANNELS][AC3_MAX_COEFS]; ///> fixed-point transform coefficients
203 DECLARE_ALIGNED(16, float, transform_coeffs)[AC3_MAX_CHANNELS][AC3_MAX_COEFS]; ///< transform coefficients203 DECLARE_ALIGNED(32, float, transform_coeffs)[AC3_MAX_CHANNELS][AC3_MAX_COEFS]; ///< transform coefficients
204 DECLARE_ALIGNED(16, float, delay)[AC3_MAX_CHANNELS][AC3_BLOCK_SIZE]; ///< delay - added to the next block204 DECLARE_ALIGNED(32, float, delay)[AC3_MAX_CHANNELS][AC3_BLOCK_SIZE]; ///< delay - added to the next block
205 DECLARE_ALIGNED(16, float, window)[AC3_BLOCK_SIZE]; ///< window coefficients205 DECLARE_ALIGNED(32, float, window)[AC3_BLOCK_SIZE]; ///< window coefficients
206 DECLARE_ALIGNED(16, float, tmp_output)[AC3_BLOCK_SIZE]; ///< temporary storage for output before windowing206 DECLARE_ALIGNED(32, float, tmp_output)[AC3_BLOCK_SIZE]; ///< temporary storage for output before windowing
207 DECLARE_ALIGNED(16, float, output)[AC3_MAX_CHANNELS][AC3_BLOCK_SIZE]; ///< output after imdct transform and windowing207 DECLARE_ALIGNED(32, float, output)[AC3_MAX_CHANNELS][AC3_BLOCK_SIZE]; ///< output after imdct transform and windowing
208///@}208///@}
209} AC3DecodeContext;209} AC3DecodeContext;
210 210
libavcodec/ac3enc.c+1−1
@@ -201,7 +201,7 @@ typedef struct AC3EncodeContext { · 200 unmodified lines
201 201
202 uint8_t exp_strategy[AC3_MAX_CHANNELS][AC3_MAX_BLOCKS]; ///< exponent strategies202 uint8_t exp_strategy[AC3_MAX_CHANNELS][AC3_MAX_BLOCKS]; ///< exponent strategies
203 203
204 DECLARE_ALIGNED(16, SampleType, windowed_samples)[AC3_WINDOW_SIZE];204 DECLARE_ALIGNED(32, SampleType, windowed_samples)[AC3_WINDOW_SIZE];
205} AC3EncodeContext;205} AC3EncodeContext;
206 206
207typedef struct AC3Mant {207typedef struct AC3Mant {
libavcodec/atrac1.c+10−10
@@ -60,11 +60,11 @@ typedef struct { · 59 unmodified lines
60 int log2_block_count[AT1_QMF_BANDS]; ///< log2 number of blocks in a band60 int log2_block_count[AT1_QMF_BANDS]; ///< log2 number of blocks in a band
61 int num_bfus; ///< number of Block Floating Units61 int num_bfus; ///< number of Block Floating Units
62 float* spectrum[2];62 float* spectrum[2];
63 DECLARE_ALIGNED(16, float, spec1)[AT1_SU_SAMPLES]; ///< mdct buffer63 DECLARE_ALIGNED(32, float, spec1)[AT1_SU_SAMPLES]; ///< mdct buffer
64 DECLARE_ALIGNED(16, float, spec2)[AT1_SU_SAMPLES]; ///< mdct buffer64 DECLARE_ALIGNED(32, float, spec2)[AT1_SU_SAMPLES]; ///< mdct buffer
65 DECLARE_ALIGNED(16, float, fst_qmf_delay)[46]; ///< delay line for the 1st stacked QMF filter65 DECLARE_ALIGNED(32, float, fst_qmf_delay)[46]; ///< delay line for the 1st stacked QMF filter
66 DECLARE_ALIGNED(16, float, snd_qmf_delay)[46]; ///< delay line for the 2nd stacked QMF filter66 DECLARE_ALIGNED(32, float, snd_qmf_delay)[46]; ///< delay line for the 2nd stacked QMF filter
67 DECLARE_ALIGNED(16, float, last_qmf_delay)[256+23]; ///< delay line for the last stacked QMF filter67 DECLARE_ALIGNED(32, float, last_qmf_delay)[256+23]; ///< delay line for the last stacked QMF filter
68} AT1SUCtx;68} AT1SUCtx;
69 69
70/**70/**
@@ -72,13 +72,13 @@ typedef struct { · 1 unmodified line
72 */72 */
73typedef struct {73typedef struct {
74 AT1SUCtx SUs[AT1_MAX_CHANNELS]; ///< channel sound unit74 AT1SUCtx SUs[AT1_MAX_CHANNELS]; ///< channel sound unit
75 DECLARE_ALIGNED(16, float, spec)[AT1_SU_SAMPLES]; ///< the mdct spectrum buffer75 DECLARE_ALIGNED(32, float, spec)[AT1_SU_SAMPLES]; ///< the mdct spectrum buffer
76 76
77 DECLARE_ALIGNED(16, float, low)[256];77 DECLARE_ALIGNED(32, float, low)[256];
78 DECLARE_ALIGNED(16, float, mid)[256];78 DECLARE_ALIGNED(32, float, mid)[256];
79 DECLARE_ALIGNED(16, float, high)[512];79 DECLARE_ALIGNED(32, float, high)[512];
80 float* bands[3];80 float* bands[3];
81 DECLARE_ALIGNED(16, float, out_samples)[AT1_MAX_CHANNELS][AT1_SU_SAMPLES];81 DECLARE_ALIGNED(32, float, out_samples)[AT1_MAX_CHANNELS][AT1_SU_SAMPLES];
82 FFTContext mdct_ctx[3];82 FFTContext mdct_ctx[3];
83 int channels;83 int channels;
84 DSPContext dsp;84 DSPContext dsp;
libavcodec/atrac3.c+3−3
@@ -74,8 +74,8 @@ typedef struct { · 73 unmodified lines
74 int gcBlkSwitch;74 int gcBlkSwitch;
75 gain_block gainBlock[2];75 gain_block gainBlock[2];
76 76
77 DECLARE_ALIGNED(16, float, spectrum)[1024];77 DECLARE_ALIGNED(32, float, spectrum)[1024];
78 DECLARE_ALIGNED(16, float, IMDCT_buf)[1024];78 DECLARE_ALIGNED(32, float, IMDCT_buf)[1024];
79 79
80 float delayBuf1[46]; ///<qmf delay buffers80 float delayBuf1[46]; ///<qmf delay buffers
81 float delayBuf2[46];81 float delayBuf2[46];
@@ -122,7 +122,7 @@ typedef struct { · 40 unmodified lines
122 FFTContext mdct_ctx;122 FFTContext mdct_ctx;
123} ATRAC3Context;123} ATRAC3Context;
124 124
125static DECLARE_ALIGNED(16, float,mdct_window)[512];125static DECLARE_ALIGNED(32, float, mdct_window)[512];
126static VLC spectral_coeff_tab[7];126static VLC spectral_coeff_tab[7];
127static float gain_tab1[16];127static float gain_tab1[16];
128static float gain_tab2[31];128static float gain_tab2[31];
libavcodec/binkaudio.c+1−1
@@ -55,7 +55,7 @@ typedef struct { · 54 unmodified lines
55 int num_bands;55 int num_bands;
56 unsigned int *bands;56 unsigned int *bands;
57 float root;57 float root;
58 DECLARE_ALIGNED(16, FFTSample, coeffs)[BINK_BLOCK_MAX_SIZE];58 DECLARE_ALIGNED(32, FFTSample, coeffs)[BINK_BLOCK_MAX_SIZE];
59 DECLARE_ALIGNED(16, short, previous)[BINK_BLOCK_MAX_SIZE / 16]; ///< coeffs from previous audio block59 DECLARE_ALIGNED(16, short, previous)[BINK_BLOCK_MAX_SIZE / 16]; ///< coeffs from previous audio block
60 float *coeffs_ptr[MAX_CHANNELS]; ///< pointers to the coeffs arrays for float_to_int16_interleave60 float *coeffs_ptr[MAX_CHANNELS]; ///< pointers to the coeffs arrays for float_to_int16_interleave
61 union {61 union {
libavcodec/cook.c+1−1
@@ -153,7 +153,7 @@ typedef struct cook { · 152 unmodified lines
153 /* data buffers */153 /* data buffers */
154 154
155 uint8_t* decoded_bytes_buffer;155 uint8_t* decoded_bytes_buffer;
156 DECLARE_ALIGNED(16, float,mono_mdct_output)[2048];156 DECLARE_ALIGNED(32, float, mono_mdct_output)[2048];
157 float decode_buffer_1[1024];157 float decode_buffer_1[1024];
158 float decode_buffer_2[1024];158 float decode_buffer_2[1024];
159 float decode_buffer_0[1060]; /* static allocation for joint decode */159 float decode_buffer_0[1060]; /* static allocation for joint decode */
libavcodec/dca.c+5−5
@@ -321,16 +321,16 @@ typedef struct { · 320 unmodified lines
321 321
322 /* Subband samples history (for ADPCM) */322 /* Subband samples history (for ADPCM) */
323 float subband_samples_hist[DCA_PRIM_CHANNELS_MAX][DCA_SUBBANDS][4];323 float subband_samples_hist[DCA_PRIM_CHANNELS_MAX][DCA_SUBBANDS][4];
324 DECLARE_ALIGNED(16, float, subband_fir_hist)[DCA_PRIM_CHANNELS_MAX][512];324 DECLARE_ALIGNED(32, float, subband_fir_hist)[DCA_PRIM_CHANNELS_MAX][512];
325 DECLARE_ALIGNED(16, float, subband_fir_noidea)[DCA_PRIM_CHANNELS_MAX][32];325 DECLARE_ALIGNED(32, float, subband_fir_noidea)[DCA_PRIM_CHANNELS_MAX][32];
326 int hist_index[DCA_PRIM_CHANNELS_MAX];326 int hist_index[DCA_PRIM_CHANNELS_MAX];
327 DECLARE_ALIGNED(16, float, raXin)[32];327 DECLARE_ALIGNED(32, float, raXin)[32];
328 328
329 int output; ///< type of output329 int output; ///< type of output
330 float scale_bias; ///< output scale330 float scale_bias; ///< output scale
331 331
332 DECLARE_ALIGNED(16, float, subband_samples)[DCA_BLOCKS_MAX][DCA_PRIM_CHANNELS_MAX][DCA_SUBBANDS][8];332 DECLARE_ALIGNED(32, float, subband_samples)[DCA_BLOCKS_MAX][DCA_PRIM_CHANNELS_MAX][DCA_SUBBANDS][8];
333 DECLARE_ALIGNED(16, float, samples)[(DCA_PRIM_CHANNELS_MAX+1)*256];333 DECLARE_ALIGNED(32, float, samples)[(DCA_PRIM_CHANNELS_MAX+1)*256];
334 const float *samples_chanptr[DCA_PRIM_CHANNELS_MAX+1];334 const float *samples_chanptr[DCA_PRIM_CHANNELS_MAX+1];
335 335
336 uint8_t dca_buffer[DCA_MAX_FRAME_SIZE + DCA_MAX_EXSS_HEADER_SIZE + DCA_BUFFER_PADDING_SIZE];336 uint8_t dca_buffer[DCA_MAX_FRAME_SIZE + DCA_MAX_EXSS_HEADER_SIZE + DCA_BUFFER_PADDING_SIZE];
libavcodec/fft.c+48−5
@@ -93,6 +93,44 @@ av_cold void ff_init_ff_cos_tabs(int index) · 92 unmodified lines
93#endif93#endif
94}94}
95 95
96static const int avx_tab[] = {
97 0, 4, 1, 5, 8, 12, 9, 13, 2, 6, 3, 7, 10, 14, 11, 15
98};
99
100static int is_second_half_of_fft32(int i, int n)
101{
102 if (n <= 32)
103 return i >= 16;
104 else if (i < n/2)
105 return is_second_half_of_fft32(i, n/2);
106 else if (i < 3*n/4)
107 return is_second_half_of_fft32(i - n/2, n/4);
108 else
109 return is_second_half_of_fft32(i - 3*n/4, n/4);
110}
111
112static av_cold void fft_perm_avx(FFTContext *s)
113{
114 int i;
115 int n = 1 << s->nbits;
116
117 for (i = 0; i < n; i += 16) {
118 int k;
119 if (is_second_half_of_fft32(i, n)) {
120 for (k = 0; k < 16; k++)
121 s->revtab[-split_radix_permutation(i + k, n, s->inverse) & (n - 1)] =
122 i + avx_tab[k];
123
124 } else {
125 for (k = 0; k < 16; k++) {
126 int j = i + k;
127 j = (j & ~7) | ((j >> 1) & 3) | ((j << 2) & 4);
128 s->revtab[-split_radix_permutation(i + k, n, s->inverse) & (n - 1)] = j;
129 }
130 }
131 }
132}
133
96av_cold int ff_fft_init(FFTContext *s, int nbits, int inverse)134av_cold int ff_fft_init(FFTContext *s, int nbits, int inverse)
97{135{
98 int i, j, n;136 int i, j, n;
@@ -132,11 +170,16 @@ av_cold int ff_fft_init(FFTContext *s, int nbits, int inverse) · 33 unmodified lines
132 for(j=4; j<=nbits; j++) {170 for(j=4; j<=nbits; j++) {
133 ff_init_ff_cos_tabs(j);171 ff_init_ff_cos_tabs(j);
134 }172 }
135 for(i=0; i<n; i++) {173
136 int j = i;174 if (s->fft_permutation == FF_FFT_PERM_AVX) {
137 if (s->fft_permutation == FF_FFT_PERM_SWAP_LSBS)175 fft_perm_avx(s);
138 j = (j&~3) | ((j>>1)&1) | ((j<<1)&2);176 } else {
139 s->revtab[-split_radix_permutation(i, n, s->inverse) & (n-1)] = j;177 for(i=0; i<n; i++) {
178 int j = i;
179 if (s->fft_permutation == FF_FFT_PERM_SWAP_LSBS)
180 j = (j&~3) | ((j>>1)&1) | ((j<<1)&2);
181 s->revtab[-split_radix_permutation(i, n, s->inverse) & (n-1)] = j;
182 }
140 }183 }
141 184
142 return 0;185 return 0;
libavcodec/fft.h+2−1
@@ -85,6 +85,7 @@ struct FFTContext { · 84 unmodified lines
85 int fft_permutation;85 int fft_permutation;
86#define FF_FFT_PERM_DEFAULT 086#define FF_FFT_PERM_DEFAULT 0
87#define FF_FFT_PERM_SWAP_LSBS 187#define FF_FFT_PERM_SWAP_LSBS 1
88#define FF_FFT_PERM_AVX 2
88 int mdct_permutation;89 int mdct_permutation;
89#define FF_MDCT_PERM_NONE 090#define FF_MDCT_PERM_NONE 0
90#define FF_MDCT_PERM_INTERLEAVE 191#define FF_MDCT_PERM_INTERLEAVE 1
@@ -97,7 +98,7 @@ struct FFTContext { · 6 unmodified lines
97#endif98#endif
98 99
99#define COSTABLE(size) \100#define COSTABLE(size) \
100 COSTABLE_CONST DECLARE_ALIGNED(16, FFTSample, FFT_NAME(ff_cos_##size))[size/2]101 COSTABLE_CONST DECLARE_ALIGNED(32, FFTSample, FFT_NAME(ff_cos_##size))[size/2]
101 102
102extern COSTABLE(16);103extern COSTABLE(16);
103extern COSTABLE(32);104extern COSTABLE(32);
libavcodec/imc.c+1−1
@@ -88,7 +88,7 @@ typedef struct { · 87 unmodified lines
88 88
89 DSPContext dsp;89 DSPContext dsp;
90 FFTContext fft;90 FFTContext fft;
91 DECLARE_ALIGNED(16, FFTComplex, samples)[COEFFS/2];91 DECLARE_ALIGNED(32, FFTComplex, samples)[COEFFS/2];
92 float *out_samples;92 float *out_samples;
93} IMCContext;93} IMCContext;
94 94
libavcodec/nellymoserdec.c+2−2
@@ -47,7 +47,7 @@ · 46 unmodified lines
47 47
48typedef struct NellyMoserDecodeContext {48typedef struct NellyMoserDecodeContext {
49 AVCodecContext* avctx;49 AVCodecContext* avctx;
50 DECLARE_ALIGNED(16, float,float_buf)[NELLY_SAMPLES];50 DECLARE_ALIGNED(32, float, float_buf)[NELLY_SAMPLES];
51 float state[128];51 float state[128];
52 AVLFG random_state;52 AVLFG random_state;
53 GetBitContext gb;53 GetBitContext gb;
@@ -55,7 +55,7 @@ typedef struct NellyMoserDecodeContext { · 1 unmodified line
55 DSPContext dsp;55 DSPContext dsp;
56 FFTContext imdct_ctx;56 FFTContext imdct_ctx;
57 FmtConvertContext fmt_conv;57 FmtConvertContext fmt_conv;
58 DECLARE_ALIGNED(16, float,imdct_out)[NELLY_BUF_LEN * 2];58 DECLARE_ALIGNED(32, float, imdct_out)[NELLY_BUF_LEN * 2];
59} NellyMoserDecodeContext;59} NellyMoserDecodeContext;
60 60
61static void overlap_and_window(NellyMoserDecodeContext *s, float *state, float *audio, float *a_in)61static void overlap_and_window(NellyMoserDecodeContext *s, float *state, float *audio, float *a_in)
libavcodec/nellymoserenc.c+3−3
@@ -55,9 +55,9 @@ typedef struct NellyMoserEncodeContext { · 54 unmodified lines
55 int have_saved;55 int have_saved;
56 DSPContext dsp;56 DSPContext dsp;
57 FFTContext mdct_ctx;57 FFTContext mdct_ctx;
58 DECLARE_ALIGNED(16, float, mdct_out)[NELLY_SAMPLES];58 DECLARE_ALIGNED(32, float, mdct_out)[NELLY_SAMPLES];
59 DECLARE_ALIGNED(16, float, in_buff)[NELLY_SAMPLES];59 DECLARE_ALIGNED(32, float, in_buff)[NELLY_SAMPLES];
60 DECLARE_ALIGNED(16, float, buf)[2][3 * NELLY_BUF_LEN]; ///< sample buffer60 DECLARE_ALIGNED(32, float, buf)[2][3 * NELLY_BUF_LEN]; ///< sample buffer
61 float (*opt )[NELLY_BANDS];61 float (*opt )[NELLY_BANDS];
62 uint8_t (*path)[NELLY_BANDS];62 uint8_t (*path)[NELLY_BANDS];
63} NellyMoserEncodeContext;63} NellyMoserEncodeContext;
libavcodec/qdm2.c+1−1
@@ -120,7 +120,7 @@ typedef struct { · 119 unmodified lines
120} FFTCoefficient;120} FFTCoefficient;
121 121
122typedef struct {122typedef struct {
123 DECLARE_ALIGNED(16, QDM2Complex, complex)[MPA_MAX_CHANNELS][256];123 DECLARE_ALIGNED(32, QDM2Complex, complex)[MPA_MAX_CHANNELS][256];
124} QDM2FFT;124} QDM2FFT;
125 125
126/**126/**
libavcodec/wma.h+4−4
@@ -113,15 +113,15 @@ typedef struct WMACodecContext { · 112 unmodified lines
113 uint8_t ms_stereo; ///< true if mid/side stereo mode113 uint8_t ms_stereo; ///< true if mid/side stereo mode
114 uint8_t channel_coded[MAX_CHANNELS]; ///< true if channel is coded114 uint8_t channel_coded[MAX_CHANNELS]; ///< true if channel is coded
115 int exponents_bsize[MAX_CHANNELS]; ///< log2 ratio frame/exp. length115 int exponents_bsize[MAX_CHANNELS]; ///< log2 ratio frame/exp. length
116 DECLARE_ALIGNED(16, float, exponents)[MAX_CHANNELS][BLOCK_MAX_SIZE];116 DECLARE_ALIGNED(32, float, exponents)[MAX_CHANNELS][BLOCK_MAX_SIZE];
117 float max_exponent[MAX_CHANNELS];117 float max_exponent[MAX_CHANNELS];
118 WMACoef coefs1[MAX_CHANNELS][BLOCK_MAX_SIZE];118 WMACoef coefs1[MAX_CHANNELS][BLOCK_MAX_SIZE];
119 DECLARE_ALIGNED(16, float, coefs)[MAX_CHANNELS][BLOCK_MAX_SIZE];119 DECLARE_ALIGNED(32, float, coefs)[MAX_CHANNELS][BLOCK_MAX_SIZE];
120 DECLARE_ALIGNED(16, FFTSample, output)[BLOCK_MAX_SIZE * 2];120 DECLARE_ALIGNED(32, FFTSample, output)[BLOCK_MAX_SIZE * 2];
121 FFTContext mdct_ctx[BLOCK_NB_SIZES];121 FFTContext mdct_ctx[BLOCK_NB_SIZES];
122 float *windows[BLOCK_NB_SIZES];122 float *windows[BLOCK_NB_SIZES];
123 /* output buffer for one frame and the last for IMDCT windowing */123 /* output buffer for one frame and the last for IMDCT windowing */
124 DECLARE_ALIGNED(16, float, frame_out)[MAX_CHANNELS][BLOCK_MAX_SIZE * 2];124 DECLARE_ALIGNED(32, float, frame_out)[MAX_CHANNELS][BLOCK_MAX_SIZE * 2];
125 /* last frame info */125 /* last frame info */
126 uint8_t last_superframe[MAX_CODED_SUPERFRAME_SIZE + 4]; /* padding added */126 uint8_t last_superframe[MAX_CODED_SUPERFRAME_SIZE + 4]; /* padding added */
127 int last_bitoffset;127 int last_bitoffset;
libavcodec/wmaprodec.c+2−2
@@ -145,7 +145,7 @@ typedef struct { · 144 unmodified lines
145 uint8_t table_idx; ///< index in sf_offsets for the scale factor reference block145 uint8_t table_idx; ///< index in sf_offsets for the scale factor reference block
146 float* coeffs; ///< pointer to the subframe decode buffer146 float* coeffs; ///< pointer to the subframe decode buffer
147 uint16_t num_vec_coeffs; ///< number of vector coded coefficients147 uint16_t num_vec_coeffs; ///< number of vector coded coefficients
148 DECLARE_ALIGNED(16, float, out)[WMAPRO_BLOCK_MAX_SIZE + WMAPRO_BLOCK_MAX_SIZE / 2]; ///< output buffer148 DECLARE_ALIGNED(32, float, out)[WMAPRO_BLOCK_MAX_SIZE + WMAPRO_BLOCK_MAX_SIZE / 2]; ///< output buffer
149} WMAProChannelCtx;149} WMAProChannelCtx;
150 150
151/**151/**
@@ -170,7 +170,7 @@ typedef struct WMAProDecodeCtx { · 18 unmodified lines
170 FF_INPUT_BUFFER_PADDING_SIZE];///< compressed frame data170 FF_INPUT_BUFFER_PADDING_SIZE];///< compressed frame data
171 PutBitContext pb; ///< context for filling the frame_data buffer171 PutBitContext pb; ///< context for filling the frame_data buffer
172 FFTContext mdct_ctx[WMAPRO_BLOCK_SIZES]; ///< MDCT context per block size172 FFTContext mdct_ctx[WMAPRO_BLOCK_SIZES]; ///< MDCT context per block size
173 DECLARE_ALIGNED(16, float, tmp)[WMAPRO_BLOCK_MAX_SIZE]; ///< IMDCT output buffer173 DECLARE_ALIGNED(32, float, tmp)[WMAPRO_BLOCK_MAX_SIZE]; ///< IMDCT output buffer
174 float* windows[WMAPRO_BLOCK_SIZES]; ///< windows for the different block sizes174 float* windows[WMAPRO_BLOCK_SIZES]; ///< windows for the different block sizes
175 175
176 /* frame size dependent frame information (set during initialization) */176 /* frame size dependent frame information (set during initialization) */
libavcodec/wmavoice.c+3−3
@@ -275,11 +275,11 @@ typedef struct { · 274 unmodified lines
275 ///< by postfilter275 ///< by postfilter
276 float denoise_filter_cache[MAX_FRAMESIZE];276 float denoise_filter_cache[MAX_FRAMESIZE];
277 int denoise_filter_cache_size; ///< samples in #denoise_filter_cache277 int denoise_filter_cache_size; ///< samples in #denoise_filter_cache
278 DECLARE_ALIGNED(16, float, tilted_lpcs_pf)[0x80];278 DECLARE_ALIGNED(32, float, tilted_lpcs_pf)[0x80];
279 ///< aligned buffer for LPC tilting279 ///< aligned buffer for LPC tilting
280 DECLARE_ALIGNED(16, float, denoise_coeffs_pf)[0x80];280 DECLARE_ALIGNED(32, float, denoise_coeffs_pf)[0x80];
281 ///< aligned buffer for denoise coefficients281 ///< aligned buffer for denoise coefficients
282 DECLARE_ALIGNED(16, float, synth_filter_out_buf)[0x80 + MAX_LSPS_ALIGN16];282 DECLARE_ALIGNED(32, float, synth_filter_out_buf)[0x80 + MAX_LSPS_ALIGN16];
283 ///< aligned buffer for postfilter speech283 ///< aligned buffer for postfilter speech
284 ///< synthesis284 ///< synthesis
285 /**285 /**
libavcodec/x86/fft.c+8−1
@@ -25,7 +25,14 @@ av_cold void ff_fft_init_mmx(FFTContext *s) · 24 unmodified lines
25{25{
26#if HAVE_YASM26#if HAVE_YASM
27 int has_vectors = av_get_cpu_flags();27 int has_vectors = av_get_cpu_flags();
28 if (has_vectors & AV_CPU_FLAG_SSE && HAVE_SSE) {28 if (has_vectors & AV_CPU_FLAG_AVX && HAVE_AVX && s->nbits >= 5) {
29 /* AVX for SB */
30 s->imdct_calc = ff_imdct_calc_sse;
31 s->imdct_half = ff_imdct_half_avx;
32 s->fft_permute = ff_fft_permute_sse;
33 s->fft_calc = ff_fft_calc_avx;
34 s->fft_permutation = FF_FFT_PERM_AVX;
35 } else if (has_vectors & AV_CPU_FLAG_SSE && HAVE_SSE) {
29 /* SSE for P3/P4/K8 */36 /* SSE for P3/P4/K8 */
30 s->imdct_calc = ff_imdct_calc_sse;37 s->imdct_calc = ff_imdct_calc_sse;
31 s->imdct_half = ff_imdct_half_sse;38 s->imdct_half = ff_imdct_half_sse;
libavcodec/x86/fft.h+2−0
@@ -22,6 +22,7 @@ · 21 unmodified lines
22#include "libavcodec/fft.h"22#include "libavcodec/fft.h"
23 23
24void ff_fft_permute_sse(FFTContext *s, FFTComplex *z);24void ff_fft_permute_sse(FFTContext *s, FFTComplex *z);
25void ff_fft_calc_avx(FFTContext *s, FFTComplex *z);
25void ff_fft_calc_sse(FFTContext *s, FFTComplex *z);26void ff_fft_calc_sse(FFTContext *s, FFTComplex *z);
26void ff_fft_calc_3dn(FFTContext *s, FFTComplex *z);27void ff_fft_calc_3dn(FFTContext *s, FFTComplex *z);
27void ff_fft_calc_3dn2(FFTContext *s, FFTComplex *z);28void ff_fft_calc_3dn2(FFTContext *s, FFTComplex *z);
@@ -32,6 +33,7 @@ void ff_imdct_calc_3dn2(FFTContext *s, FFTSample *output, const FFTSample *input · 4 unmodified lines
32void ff_imdct_half_3dn2(FFTContext *s, FFTSample *output, const FFTSample *input);33void ff_imdct_half_3dn2(FFTContext *s, FFTSample *output, const FFTSample *input);
33void ff_imdct_calc_sse(FFTContext *s, FFTSample *output, const FFTSample *input);34void ff_imdct_calc_sse(FFTContext *s, FFTSample *output, const FFTSample *input);
34void ff_imdct_half_sse(FFTContext *s, FFTSample *output, const FFTSample *input);35void ff_imdct_half_sse(FFTContext *s, FFTSample *output, const FFTSample *input);
36void ff_imdct_half_avx(FFTContext *s, FFTSample *output, const FFTSample *input);
35void ff_dct32_float_sse(FFTSample *out, const FFTSample *in);37void ff_dct32_float_sse(FFTSample *out, const FFTSample *in);
36 38
37#endif39#endif
libavcodec/x86/fft_mmx.asm+334−150
@@ -1,6 +1,7 @@
1;******************************************************************************1;******************************************************************************
2;* FFT transform with SSE/3DNow optimizations2;* FFT transform with SSE/3DNow optimizations
3;* Copyright (c) 2008 Loren Merritt3;* Copyright (c) 2008 Loren Merritt
4;* Copyright (c) 2011 Vitor Sessak
4;*5;*
5;* This algorithm (though not any of the implementation details) is6;* This algorithm (though not any of the implementation details) is
6;* based on libdjbfft by D. J. Bernstein.7;* based on libdjbfft by D. J. Bernstein.
@@ -49,9 +50,21 @@ endstruc · 42 unmodified lines
49SECTION_RODATA50SECTION_RODATA
50 51
51%define M_SQRT1_2 0.7071067811865475244052%define M_SQRT1_2 0.70710678118654752440
52ps_root2: times 4 dd M_SQRT1_253%define M_COS_PI_1_8 0.923879532511287
53ps_root2mppm: dd -M_SQRT1_2, M_SQRT1_2, M_SQRT1_2, -M_SQRT1_254%define M_COS_PI_3_8 0.38268343236509
54ps_p1p1m1p1: dd 0, 0, 1<<31, 055
56align 32
57ps_cos16_1: dd 1.0, M_COS_PI_1_8, M_SQRT1_2, M_COS_PI_3_8, 1.0, M_COS_PI_1_8, M_SQRT1_2, M_COS_PI_3_8
58ps_cos16_2: dd 0, M_COS_PI_3_8, M_SQRT1_2, M_COS_PI_1_8, 0, -M_COS_PI_3_8, -M_SQRT1_2, -M_COS_PI_1_8
59
60ps_root2: times 8 dd M_SQRT1_2
61ps_root2mppm: dd -M_SQRT1_2, M_SQRT1_2, M_SQRT1_2, -M_SQRT1_2, -M_SQRT1_2, M_SQRT1_2, M_SQRT1_2, -M_SQRT1_2
62ps_p1p1m1p1: dd 0, 0, 1<<31, 0, 0, 0, 1<<31, 0
63
64perm1: dd 0x00, 0x02, 0x03, 0x01, 0x03, 0x00, 0x02, 0x01
65perm2: dd 0x00, 0x01, 0x02, 0x03, 0x01, 0x00, 0x02, 0x03
66ps_p1p1m1p1root2: dd 1.0, 1.0, -1.0, 1.0, M_SQRT1_2, M_SQRT1_2, M_SQRT1_2, M_SQRT1_2
67ps_m1m1p1m1p1m1m1m1: dd 1<<31, 1<<31, 0, 1<<31, 0, 1<<31, 1<<31, 1<<31
55ps_m1p1: dd 1<<31, 068ps_m1p1: dd 1<<31, 0
56 69
57%assign i 1670%assign i 16
@@ -96,51 +109,80 @@ section .text align=16 · 38 unmodified lines
96 SWAP %3, %6109 SWAP %3, %6
97%endmacro110%endmacro
98 111
112; in: %1 = {r0,i0,r2,i2,r4,i4,r6,i6}
113; %2 = {r1,i1,r3,i3,r5,i5,r7,i7}
114; %3, %4, %5 tmp
115; out: %1 = {r0,r1,r2,r3,i0,i1,i2,i3}
116; %2 = {r4,r5,r6,r7,i4,i5,i6,i7}
117%macro T8_AVX 5
118 vsubps %5, %1, %2 ; v = %1 - %2
119 vaddps %3, %1, %2 ; w = %1 + %2
120 vmulps %2, %5, [ps_p1p1m1p1root2] ; v *= vals1
121 vpermilps %2, %2, [perm1]
122 vblendps %1, %2, %3, 0x33 ; q = {w1,w2,v4,v2,w5,w6,v7,v6}
123 vshufps %5, %3, %2, 0x4e ; r = {w3,w4,v1,v3,w7,w8,v8,v5}
124 vsubps %4, %5, %1 ; s = r - q
125 vaddps %1, %5, %1 ; u = r + q
126 vpermilps %1, %1, [perm2] ; k = {u1,u2,u3,u4,u6,u5,u7,u8}
127 vshufps %5, %4, %1, 0xbb
128 vshufps %3, %4, %1, 0xee
129 vperm2f128 %3, %3, %5, 0x13
130 vxorps %4, %4, [ps_m1m1p1m1p1m1m1m1] ; s *= {1,1,-1,-1,1,-1,-1,-1}
131 vshufps %2, %1, %4, 0xdd
132 vshufps %1, %1, %4, 0x88
133 vperm2f128 %4, %2, %1, 0x02 ; v = {k1,k3,s1,s3,k2,k4,s2,s4}
134 vperm2f128 %1, %1, %2, 0x13 ; w = {k6,k8,s6,s8,k5,k7,s5,s7}
135 vsubps %5, %1, %3
136 vblendps %1, %5, %1, 0x55 ; w -= {0,s7,0,k7,0,s8,0,k8}
137 vsubps %2, %4, %1 ; %2 = v - w
138 vaddps %1, %4, %1 ; %1 = v + w
139%endmacro
140
141; In SSE mode do one fft4 transforms
99; in: %1={r0,i0,r2,i2} %2={r1,i1,r3,i3}142; in: %1={r0,i0,r2,i2} %2={r1,i1,r3,i3}
100; out: %1={r0,r1,r2,r3} %2={i0,i1,i2,i3}143; out: %1={r0,r1,r2,r3} %2={i0,i1,i2,i3}
144;
145; In AVX mode do two fft4 transforms
146; in: %1={r0,i0,r2,i2,r4,i4,r6,i6} %2={r1,i1,r3,i3,r5,i5,r7,i7}
147; out: %1={r0,r1,r2,r3,r4,r5,r6,r7} %2={i0,i1,i2,i3,i4,i5,i6,i7}
101%macro T4_SSE 3148%macro T4_SSE 3
102 mova %3, %1149 subps %3, %1, %2 ; {t3,t4,-t8,t7}
103 addps %1, %2 ; {t1,t2,t6,t5}150 addps %1, %1, %2 ; {t1,t2,t6,t5}
104 subps %3, %2 ; {t3,t4,-t8,t7}151 xorps %3, %3, [ps_p1p1m1p1]
105 xorps %3, [ps_p1p1m1p1]152 shufps %2, %1, %3, 0xbe ; {t6,t5,t7,t8}
106 mova %2, %1153 shufps %1, %1, %3, 0x44 ; {t1,t2,t3,t4}
107 shufps %1, %3, 0x44 ; {t1,t2,t3,t4}154 subps %3, %1, %2 ; {r2,i2,r3,i3}
108 shufps %2, %3, 0xbe ; {t6,t5,t7,t8}155 addps %1, %1, %2 ; {r0,i0,r1,i1}
109 mova %3, %1156 shufps %2, %1, %3, 0xdd ; {i0,i1,i2,i3}
110 addps %1, %2 ; {r0,i0,r1,i1}157 shufps %1, %1, %3, 0x88 ; {r0,r1,r2,r3}
111 subps %3, %2 ; {r2,i2,r3,i3}
112 mova %2, %1
113 shufps %1, %3, 0x88 ; {r0,r1,r2,r3}
114 shufps %2, %3, 0xdd ; {i0,i1,i2,i3}
115%endmacro158%endmacro
116 159
160; In SSE mode do one FFT8
117; in: %1={r0,r1,r2,r3} %2={i0,i1,i2,i3} %3={r4,i4,r6,i6} %4={r5,i5,r7,i7}161; in: %1={r0,r1,r2,r3} %2={i0,i1,i2,i3} %3={r4,i4,r6,i6} %4={r5,i5,r7,i7}
118; out: %1={r0,r1,r2,r3} %2={i0,i1,i2,i3} %1={r4,r5,r6,r7} %2={i4,i5,i6,i7}162; out: %1={r0,r1,r2,r3} %2={i0,i1,i2,i3} %1={r4,r5,r6,r7} %2={i4,i5,i6,i7}
163;
164; In AVX mode do two FFT8
165; in: %1={r0,i0,r2,i2,r8, i8, r10,i10} %2={r1,i1,r3,i3,r9, i9, r11,i11}
166; %3={r4,i4,r6,i6,r12,i12,r14,i14} %4={r5,i5,r7,i7,r13,i13,r15,i15}
167; out: %1={r0,r1,r2,r3,r8, r9, r10,r11} %2={i0,i1,i2,i3,i8, i9, i10,i11}
168; %3={r4,r5,r6,r7,r12,r13,r14,r15} %4={i4,i5,i6,i7,i12,i13,i14,i15}
119%macro T8_SSE 6169%macro T8_SSE 6
120 mova %6, %3170 addps %6, %3, %4 ; {t1,t2,t3,t4}
121 subps %3, %4 ; {r5,i5,r7,i7}171 subps %3, %3, %4 ; {r5,i5,r7,i7}
122 addps %6, %4 ; {t1,t2,t3,t4}172 shufps %4, %3, %3, 0xb1 ; {i5,r5,i7,r7}
123 mova %4, %3173 mulps %3, %3, [ps_root2mppm] ; {-r5,i5,r7,-i7}
124 shufps %4, %4, 0xb1 ; {i5,r5,i7,r7}174 mulps %4, %4, [ps_root2]
125 mulps %3, [ps_root2mppm] ; {-r5,i5,r7,-i7}175 addps %3, %3, %4 ; {t8,t7,ta,t9}
126 mulps %4, [ps_root2]176 shufps %4, %6, %3, 0x9c ; {t1,t4,t7,ta}
127 addps %3, %4 ; {t8,t7,ta,t9}177 shufps %6, %6, %3, 0x36 ; {t3,t2,t9,t8}
128 mova %4, %6178 subps %3, %6, %4 ; {t6,t5,tc,tb}
129 shufps %6, %3, 0x36 ; {t3,t2,t9,t8}179 addps %6, %6, %4 ; {t1,t2,t9,ta}
130 shufps %4, %3, 0x9c ; {t1,t4,t7,ta}180 shufps %5, %6, %3, 0x8d ; {t2,ta,t6,tc}
131 mova %3, %6181 shufps %6, %6, %3, 0xd8 ; {t1,t9,t5,tb}
132 addps %6, %4 ; {t1,t2,t9,ta}182 subps %3, %1, %6 ; {r4,r5,r6,r7}
133 subps %3, %4 ; {t6,t5,tc,tb}183 addps %1, %1, %6 ; {r0,r1,r2,r3}
134 mova %4, %6184 subps %4, %2, %5 ; {i4,i5,i6,i7}
135 shufps %6, %3, 0xd8 ; {t1,t9,t5,tb}185 addps %2, %2, %5 ; {i0,i1,i2,i3}
136 shufps %4, %3, 0x8d ; {t2,ta,t6,tc}
137 mova %3, %1
138 mova %5, %2
139 addps %1, %6 ; {r0,r1,r2,r3}
140 addps %2, %4 ; {i0,i1,i2,i3}
141 subps %3, %6 ; {r4,r5,r6,r7}
142 subps %5, %4 ; {i4,i5,i6,i7}
143 SWAP %4, %5
144%endmacro186%endmacro
145 187
146; scheduled for cpu-bound sizes188; scheduled for cpu-bound sizes
@@ -148,52 +190,44 @@ section .text align=16 · 1 unmodified line
148IF%1 mova m4, Z(4)190IF%1 mova m4, Z(4)
149IF%1 mova m5, Z(5)191IF%1 mova m5, Z(5)
150 mova m0, %2 ; wre192 mova m0, %2 ; wre
151 mova m2, m4
152 mova m1, %3 ; wim193 mova m1, %3 ; wim
153 mova m3, m5194 mulps m2, m4, m0 ; r2*wre
154 mulps m2, m0 ; r2*wre
155IF%1 mova m6, Z2(6)195IF%1 mova m6, Z2(6)
156 mulps m3, m1 ; i2*wim196 mulps m3, m5, m1 ; i2*wim
157IF%1 mova m7, Z2(7)197IF%1 mova m7, Z2(7)
158 mulps m4, m1 ; r2*wim198 mulps m4, m4, m1 ; r2*wim
159 mulps m5, m0 ; i2*wre199 mulps m5, m5, m0 ; i2*wre
160 addps m2, m3 ; r2*wre + i2*wim200 addps m2, m2, m3 ; r2*wre + i2*wim
161 mova m3, m1201 mulps m3, m1, m7 ; i3*wim
162 mulps m1, m6 ; r3*wim202 subps m5, m5, m4 ; i2*wre - r2*wim
163 subps m5, m4 ; i2*wre - r2*wim203 mulps m1, m1, m6 ; r3*wim
164 mova m4, m0204 mulps m4, m0, m6 ; r3*wre
165 mulps m3, m7 ; i3*wim205 mulps m0, m0, m7 ; i3*wre
166 mulps m4, m6 ; r3*wre206 subps m4, m4, m3 ; r3*wre - i3*wim
167 mulps m0, m7 ; i3*wre
168 subps m4, m3 ; r3*wre - i3*wim
169 mova m3, Z(0)207 mova m3, Z(0)
170 addps m0, m1 ; i3*wre + r3*wim208 addps m0, m0, m1 ; i3*wre + r3*wim
171 mova m1, m4209 subps m1, m4, m2 ; t3
172 addps m4, m2 ; t5210 addps m4, m4, m2 ; t5
173 subps m1, m2 ; t3211 subps m3, m3, m4 ; r2
174 subps m3, m4 ; r2212 addps m4, m4, Z(0) ; r0
175 addps m4, Z(0) ; r0
176 mova m6, Z(2)213 mova m6, Z(2)
177 mova Z(4), m3214 mova Z(4), m3
178 mova Z(0), m4215 mova Z(0), m4
179 mova m3, m5216 subps m3, m5, m0 ; t4
180 subps m5, m0 ; t4217 subps m4, m6, m3 ; r3
181 mova m4, m6218 addps m3, m3, m6 ; r1
182 subps m6, m5 ; r3219 mova Z2(6), m4
183 addps m5, m4 ; r1220 mova Z(2), m3
184 mova Z2(6), m6
185 mova Z(2), m5
186 mova m2, Z(3)221 mova m2, Z(3)
187 addps m3, m0 ; t6222 addps m3, m5, m0 ; t6
188 subps m2, m1 ; i3223 subps m2, m2, m1 ; i3
189 mova m7, Z(1)224 mova m7, Z(1)
190 addps m1, Z(3) ; i1225 addps m1, m1, Z(3) ; i1
191 mova Z2(7), m2226 mova Z2(7), m2
192 mova Z(3), m1227 mova Z(3), m1
193 mova m4, m7228 subps m4, m7, m3 ; i2
194 subps m7, m3 ; i2229 addps m3, m3, m7 ; i0
195 addps m3, m4 ; i0230 mova Z(5), m4
196 mova Z(5), m7
197 mova Z(1), m3231 mova Z(1), m3
198%endmacro232%endmacro
199 233
@@ -201,77 +235,55 @@ IF%1 mova m7, Z2(7) · 1 unmodified line
201%macro PASS_BIG 1 ; (!interleave)235%macro PASS_BIG 1 ; (!interleave)
202 mova m4, Z(4) ; r2236 mova m4, Z(4) ; r2
203 mova m5, Z(5) ; i2237 mova m5, Z(5) ; i2
204 mova m2, m4
205 mova m0, [wq] ; wre238 mova m0, [wq] ; wre
206 mova m3, m5
207 mova m1, [wq+o1q] ; wim239 mova m1, [wq+o1q] ; wim
208 mulps m2, m0 ; r2*wre240 mulps m2, m4, m0 ; r2*wre
209 mova m6, Z2(6) ; r3241 mova m6, Z2(6) ; r3
210 mulps m3, m1 ; i2*wim242 mulps m3, m5, m1 ; i2*wim
211 mova m7, Z2(7) ; i3243 mova m7, Z2(7) ; i3
212 mulps m4, m1 ; r2*wim244 mulps m4, m4, m1 ; r2*wim
213 mulps m5, m0 ; i2*wre245 mulps m5, m5, m0 ; i2*wre
214 addps m2, m3 ; r2*wre + i2*wim246 addps m2, m2, m3 ; r2*wre + i2*wim
215 mova m3, m1247 mulps m3, m1, m7 ; i3*wim
216 mulps m1, m6 ; r3*wim248 mulps m1, m1, m6 ; r3*wim
217 subps m5, m4 ; i2*wre - r2*wim249 subps m5, m5, m4 ; i2*wre - r2*wim
218 mova m4, m0250 mulps m4, m0, m6 ; r3*wre
219 mulps m3, m7 ; i3*wim251 mulps m0, m0, m7 ; i3*wre
220 mulps m4, m6 ; r3*wre252 subps m4, m4, m3 ; r3*wre - i3*wim
221 mulps m0, m7 ; i3*wre
222 subps m4, m3 ; r3*wre - i3*wim
223 mova m3, Z(0)253 mova m3, Z(0)
224 addps m0, m1 ; i3*wre + r3*wim254 addps m0, m0, m1 ; i3*wre + r3*wim
225 mova m1, m4255 subps m1, m4, m2 ; t3
226 addps m4, m2 ; t5256 addps m4, m4, m2 ; t5
227 subps m1, m2 ; t3257 subps m3, m3, m4 ; r2
228 subps m3, m4 ; r2258 addps m4, m4, Z(0) ; r0
229 addps m4, Z(0) ; r0
230 mova m6, Z(2)259 mova m6, Z(2)
231 mova Z(4), m3260 mova Z(4), m3
232 mova Z(0), m4261 mova Z(0), m4
233 mova m3, m5262 subps m3, m5, m0 ; t4
234 subps m5, m0 ; t4263 subps m4, m6, m3 ; r3
235 mova m4, m6264 addps m3, m3, m6 ; r1
236 subps m6, m5 ; r3265IF%1 mova Z2(6), m4
237 addps m5, m4 ; r1266IF%1 mova Z(2), m3
238IF%1 mova Z2(6), m6
239IF%1 mova Z(2), m5
240 mova m2, Z(3)267 mova m2, Z(3)
241 addps m3, m0 ; t6268 addps m5, m5, m0 ; t6
242 subps m2, m1 ; i3269 subps m2, m2, m1 ; i3
243 mova m7, Z(1)270 mova m7, Z(1)
244 addps m1, Z(3) ; i1271 addps m1, m1, Z(3) ; i1
245IF%1 mova Z2(7), m2272IF%1 mova Z2(7), m2
246IF%1 mova Z(3), m1273IF%1 mova Z(3), m1
247 mova m4, m7274 subps m6, m7, m5 ; i2
248 subps m7, m3 ; i2275 addps m5, m5, m7 ; i0
249 addps m3, m4 ; i0276IF%1 mova Z(5), m6
250IF%1 mova Z(5), m7277IF%1 mova Z(1), m5
251IF%1 mova Z(1), m3
252%if %1==0278%if %1==0
253 mova m4, m5 ; r1279 INTERL m1, m3, m7, Z, 2
254 mova m0, m6 ; r3280 INTERL m2, m4, m0, Z2, 6
255 unpcklps m5, m1281
256 unpckhps m4, m1
257 unpcklps m6, m2
258 unpckhps m0, m2
259 mova m1, Z(0)282 mova m1, Z(0)
260 mova m2, Z(4)283 mova m2, Z(4)
261 mova Z(2), m5284
262 mova Z(3), m4285 INTERL m5, m1, m3, Z, 0
263 mova Z2(6), m6286 INTERL m6, m2, m7, Z, 4
264 mova Z2(7), m0
265 mova m5, m1 ; r0
266 mova m4, m2 ; r2
267 unpcklps m1, m3
268 unpckhps m5, m3
269 unpcklps m2, m7
270 unpckhps m4, m7
271 mova Z(0), m1
272 mova Z(1), m5
273 mova Z(4), m2
274 mova Z(5), m4
275%endif287%endif
276%endmacro288%endmacro
277 289
@@ -281,13 +293,106 @@ IF%1 mova Z(1), m3 · 3 unmodified lines
281 punpckhdq %3, %2293 punpckhdq %3, %2
282%endmacro294%endmacro
283 295
284INIT_XMM
285%define mova movaps
286
287%define Z(x) [r0+mmsize*x]296%define Z(x) [r0+mmsize*x]
288%define Z2(x) [r0+mmsize*x]297%define Z2(x) [r0+mmsize*x]
298%define ZH(x) [r0+mmsize*x+mmsize/2]
299
300INIT_YMM
301
302align 16
303fft8_avx:
304 mova m0, Z(0)
305 mova m1, Z(1)
306 T8_AVX m0, m1, m2, m3, m4
307 mova Z(0), m0
308 mova Z(1), m1
309 ret
310
311
312align 16
313fft16_avx:
314 mova m2, Z(2)
315 mova m3, Z(3)
316 T4_SSE m2, m3, m7
317
318 mova m0, Z(0)
319 mova m1, Z(1)
320 T8_AVX m0, m1, m4, m5, m7
321
322 mova m4, [ps_cos16_1]
323 mova m5, [ps_cos16_2]
324 vmulps m6, m2, m4
325 vmulps m7, m3, m5
326 vaddps m7, m7, m6
327 vmulps m2, m2, m5
328 vmulps m3, m3, m4
329 vsubps m3, m3, m2
330 vblendps m2, m7, m3, 0xf0
331 vperm2f128 m3, m7, m3, 0x21
332 vaddps m4, m2, m3
333 vsubps m2, m3, m2
334 vperm2f128 m2, m2, m2, 0x01
335 vsubps m3, m1, m2
336 vaddps m1, m1, m2
337 vsubps m5, m0, m4
338 vaddps m0, m0, m4
339 vextractf128 Z(0), m0, 0
340 vextractf128 ZH(0), m1, 0
341 vextractf128 Z(1), m0, 1
342 vextractf128 ZH(1), m1, 1
343 vextractf128 Z(2), m5, 0
344 vextractf128 ZH(2), m3, 0
345 vextractf128 Z(3), m5, 1
346 vextractf128 ZH(3), m3, 1
347 ret
348
349align 16
350fft32_avx:
351 call fft16_avx
352
353 mova m0, Z(4)
354 mova m1, Z(5)
355
356 T4_SSE m0, m1, m4
357
358 mova m2, Z(6)
359 mova m3, Z(7)
360
361 T8_SSE m0, m1, m2, m3, m4, m6
362 ; m0={r0,r1,r2,r3,r8, r9, r10,r11} m1={i0,i1,i2,i3,i8, i9, i10,i11}
363 ; m2={r4,r5,r6,r7,r12,r13,r14,r15} m3={i4,i5,i6,i7,i12,i13,i14,i15}
364
365 vperm2f128 m4, m0, m2, 0x20
366 vperm2f128 m5, m1, m3, 0x20
367 vperm2f128 m6, m0, m2, 0x31
368 vperm2f128 m7, m1, m3, 0x31
369
370 PASS_SMALL 0, [cos_32], [cos_32+32]
371
372 ret
373
374fft32_interleave_avx:
375 call fft32_avx
376 mov r2d, 32
377.deint_loop:
378 mova m2, Z(0)
379 mova m3, Z(1)
380 vunpcklps m0, m2, m3
381 vunpckhps m1, m2, m3
382 vextractf128 Z(0), m0, 0
383 vextractf128 ZH(0), m1, 0
384 vextractf128 Z(1), m0, 1
385 vextractf128 ZH(1), m1, 1
386 add r0, mmsize*2
387 sub r2d, mmsize/4
388 jg .deint_loop
389 ret
390
391INIT_XMM
392%define movdqa movaps
289 393
290align 16394align 16
395fft4_avx:
291fft4_sse:396fft4_sse:
292 mova m0, Z(0)397 mova m0, Z(0)
293 mova m1, Z(1)398 mova m1, Z(1)
@@ -406,6 +511,8 @@ FFT48_3DN _3dn · 112 unmodified lines
406 511
407%define Z(x) [zq + o1q*(x&6) + mmsize*(x&1)]512%define Z(x) [zq + o1q*(x&6) + mmsize*(x&1)]
408%define Z2(x) [zq + o3q + mmsize*(x&1)]513%define Z2(x) [zq + o3q + mmsize*(x&1)]
514%define ZH(x) [zq + o1q*(x&6) + mmsize*(x&1) + mmsize/2]
515%define Z2H(x) [zq + o3q + mmsize*(x&1) + mmsize/2]
409 516
410%macro DECL_PASS 2+ ; name, payload517%macro DECL_PASS 2+ ; name, payload
411align 16518align 16
@@ -423,8 +530,34 @@ DEFINE_ARGS z, w, n, o1, o3 · 11 unmodified lines
423 rep ret530 rep ret
424%endmacro531%endmacro
425 532
533INIT_YMM
534
535%macro INTERL_AVX 5
536 vunpckhps %3, %2, %1
537 vunpcklps %2, %2, %1
538 vextractf128 %4(%5), %2, 0
539 vextractf128 %4 %+ H(%5), %3, 0
540 vextractf128 %4(%5 + 1), %2, 1
541 vextractf128 %4 %+ H(%5 + 1), %3, 1
542%endmacro
543
544%define INTERL INTERL_AVX
545
546DECL_PASS pass_avx, PASS_BIG 1
547DECL_PASS pass_interleave_avx, PASS_BIG 0
548
426INIT_XMM549INIT_XMM
427%define mova movaps550
551%macro INTERL_SSE 5
552 mova %3, %2
553 unpcklps %2, %1
554 unpckhps %3, %1
555 mova %4(%5), %2
556 mova %4(%5+1), %3
557%endmacro
558
559%define INTERL INTERL_SSE
560
428DECL_PASS pass_sse, PASS_BIG 1561DECL_PASS pass_sse, PASS_BIG 1
429DECL_PASS pass_interleave_sse, PASS_BIG 0562DECL_PASS pass_interleave_sse, PASS_BIG 0
430 563
@@ -457,9 +590,12 @@ DECL_PASS pass_interleave_3dn, PASS_BIG 0 · 26 unmodified lines
457 590
458%macro DECL_FFT 2-3 ; nbits, cpu, suffix591%macro DECL_FFT 2-3 ; nbits, cpu, suffix
459%xdefine list_of_fft fft4%2 SECTION_REL, fft8%2 SECTION_REL592%xdefine list_of_fft fft4%2 SECTION_REL, fft8%2 SECTION_REL
460%if %1==5593%if %1>=5
461%xdefine list_of_fft list_of_fft, fft16%2 SECTION_REL594%xdefine list_of_fft list_of_fft, fft16%2 SECTION_REL
462%endif595%endif
596%if %1>=6
597%xdefine list_of_fft list_of_fft, fft32%3%2 SECTION_REL
598%endif
463 599
464%assign n 1<<%1600%assign n 1<<%1
465%rep 17-%1601%rep 17-%1
@@ -492,9 +628,14 @@ section .text · 26 unmodified lines
492; The others pass args in registers and don't spill anything.628; The others pass args in registers and don't spill anything.
493cglobal fft_dispatch%3%2, 2,5,8, z, nbits629cglobal fft_dispatch%3%2, 2,5,8, z, nbits
494 FFT_DISPATCH %3%2, nbits630 FFT_DISPATCH %3%2, nbits
631%ifidn %2, _avx
632 vzeroupper
633%endif
495 RET634 RET
496%endmacro ; DECL_FFT635%endmacro ; DECL_FFT
497 636
637DECL_FFT 6, _avx
638DECL_FFT 6, _avx, _interleave
498DECL_FFT 5, _sse639DECL_FFT 5, _sse
499DECL_FFT 5, _sse, _interleave640DECL_FFT 5, _sse, _interleave
500DECL_FFT 4, _3dn641DECL_FFT 4, _3dn
@@ -533,21 +674,53 @@ INIT_XMM · 32 unmodified lines
533%endmacro674%endmacro
534 675
535%macro CMUL 6 ;j, xmm0, xmm1, 3, 4, 5676%macro CMUL 6 ;j, xmm0, xmm1, 3, 4, 5
536 movaps xmm6, [%4+%1*2]677 mulps m6, %3, [%5+%1]
537 movaps %2, [%4+%1*2+0x10]678 mulps m7, %2, [%5+%1]
538 movaps %3, xmm6679 mulps %2, %2, [%6+%1]
539 movaps xmm7, %2680 mulps %3, %3, [%6+%1]
540 mulps xmm6, [%5+%1]681 subps %2, %2, m6
541 mulps %2, [%6+%1]682 addps %3, %3, m7
542 mulps %3, [%6+%1]683%endmacro
543 mulps xmm7, [%5+%1]684
544 subps %2, xmm6685%macro POSROTATESHUF_AVX 5 ;j, k, z+n8, tcos+n8, tsin+n8
545 addps %3, xmm7686.post:
687 vmovaps ymm1, [%3+%1*2]
688 vmovaps ymm0, [%3+%1*2+0x20]
689 vmovaps ymm3, [%3+%2*2]
690 vmovaps ymm2, [%3+%2*2+0x20]
691
692 CMUL %1, ymm0, ymm1, %3, %4, %5
693 CMUL %2, ymm2, ymm3, %3, %4, %5
694 vshufps ymm1, ymm1, ymm1, 0x1b
695 vshufps ymm3, ymm3, ymm3, 0x1b
696 vperm2f128 ymm1, ymm1, ymm1, 0x01
697 vperm2f128 ymm3, ymm3, ymm3, 0x01
698 vunpcklps ymm6, ymm2, ymm1
699 vunpckhps ymm4, ymm2, ymm1
700 vunpcklps ymm7, ymm0, ymm3
701 vunpckhps ymm5, ymm0, ymm3
702
703 vextractf128 [%3+%1*2], ymm7, 0
704 vextractf128 [%3+%1*2+0x10], ymm5, 0
705 vextractf128 [%3+%1*2+0x20], ymm7, 1
706 vextractf128 [%3+%1*2+0x30], ymm5, 1
707
708 vextractf128 [%3+%2*2], ymm6, 0
709 vextractf128 [%3+%2*2+0x10], ymm4, 0
710 vextractf128 [%3+%2*2+0x20], ymm6, 1
711 vextractf128 [%3+%2*2+0x30], ymm4, 1
712 sub %2, 0x20
713 add %1, 0x20
714 jl .post
546%endmacro715%endmacro
547 716
548%macro POSROTATESHUF 5 ;j, k, z+n8, tcos+n8, tsin+n8717%macro POSROTATESHUF 5 ;j, k, z+n8, tcos+n8, tsin+n8
549.post:718.post:
719 movaps xmm1, [%3+%1*2]
720 movaps xmm0, [%3+%1*2+0x10]
550 CMUL %1, xmm0, xmm1, %3, %4, %5721 CMUL %1, xmm0, xmm1, %3, %4, %5
722 movaps xmm5, [%3+%2*2]
723 movaps xmm4, [%3+%2*2+0x10]
551 CMUL %2, xmm4, xmm5, %3, %4, %5724 CMUL %2, xmm4, xmm5, %3, %4, %5
552 shufps xmm1, xmm1, 0x1b725 shufps xmm1, xmm1, 0x1b
553 shufps xmm5, xmm5, 0x1b726 shufps xmm5, xmm5, 0x1b
@@ -566,7 +739,8 @@ INIT_XMM · 12 unmodified lines
566 jl .post739 jl .post
567%endmacro740%endmacro
568 741
569cglobal imdct_half_sse, 3,7,8; FFTContext *s, FFTSample *output, const FFTSample *input742%macro DECL_IMDCT 2
743cglobal imdct_half%1, 3,7,8; FFTContext *s, FFTSample *output, const FFTSample *input
570%ifdef ARCH_X86_64744%ifdef ARCH_X86_64
571%define rrevtab r10745%define rrevtab r10
572%define rtcos r11746%define rtcos r11
@@ -641,7 +815,7 @@ cglobal imdct_half_sse, 3,7,8; FFTContext *s, FFTSample *output, const FFTSample · 68 unmodified lines
641 mov r0, r1815 mov r0, r1
642 mov r1d, [r5+FFTContext.nbits]816 mov r1d, [r5+FFTContext.nbits]
643 817
644 FFT_DISPATCH _sse, r1818 FFT_DISPATCH %1, r1
645 819
646 mov r0d, [r5+FFTContext.mdctsize]820 mov r0d, [r5+FFTContext.mdctsize]
647 add r6, r0821 add r6, r0
@@ -653,14 +827,24 @@ cglobal imdct_half_sse, 3,7,8; FFTContext *s, FFTSample *output, const FFTSample · 5 unmodified lines
653 mov rtsin, [esp+4]827 mov rtsin, [esp+4]
654%endif828%endif
655 neg r0829 neg r0
656 mov r1, -16830 mov r1, -mmsize
657 sub r1, r0831 sub r1, r0
658 POSROTATESHUF r0, r1, r6, rtcos, rtsin832 %2 r0, r1, r6, rtcos, rtsin
659%ifdef ARCH_X86_64833%ifdef ARCH_X86_64
660 pop r14834 pop r14
661 pop r13835 pop r13
662 pop r12836 pop r12
663%else837%else
664 add esp, 12838 add esp, 12
839%endif
840%ifidn avx_enabled, 1
841 vzeroupper
665%endif842%endif
666 RET843 RET
844%endmacro
845
846DECL_IMDCT _sse, POSROTATESHUF
847
848INIT_YMM
849
850DECL_IMDCT _avx, POSROTATESHUF_AVX
libavcodec/x86/fft_sse.c+7−1
@@ -28,6 +28,12 @@ DECLARE_ASM_CONST(16, int, ff_m1m1m1m1)[4] = · 27 unmodified lines
28 28
29void ff_fft_dispatch_sse(FFTComplex *z, int nbits);29void ff_fft_dispatch_sse(FFTComplex *z, int nbits);
30void ff_fft_dispatch_interleave_sse(FFTComplex *z, int nbits);30void ff_fft_dispatch_interleave_sse(FFTComplex *z, int nbits);
31void ff_fft_dispatch_interleave_avx(FFTComplex *z, int nbits);
32
33void ff_fft_calc_avx(FFTContext *s, FFTComplex *z)
34{
35 ff_fft_dispatch_interleave_avx(z, s->nbits);
36}
31 37
32void ff_fft_calc_sse(FFTContext *s, FFTComplex *z)38void ff_fft_calc_sse(FFTContext *s, FFTComplex *z)
33{39{
@@ -77,7 +83,7 @@ void ff_imdct_calc_sse(FFTContext *s, FFTSample *output, const FFTSample *input) · 43 unmodified lines
77 long n = s->mdct_size;83 long n = s->mdct_size;
78 long n4 = n >> 2;84 long n4 = n >> 2;
79 85
80 ff_imdct_half_sse(s, output+n4, input);86 s->imdct_half(s, output + n4, input);
81 87
82 j = -n;88 j = -n;
83 k = n-16;89 k = n-16;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t