History

ppc: Fix the bug of fft for little endian environment on POWER7 and later

Rong Yan committed 12 years ago

Commit ab123731 parent 8e2ade45 files · +1084 −1Browse files

Unified · Split

5 files
  1. configure +14 −1
  2. libavcodec/ppc/Makefile +1 −0
  3. libavcodec/ppc/fft_altivec.c +12 −0
  4. libavcodec/ppc/fft_vsx.c +227 −0
  5. libavcodec/ppc/fft_vsx.h +830 −0
configure+14−1
@@ -1548,6 +1548,7 @@ ARCH_EXT_LIST_PPC=" · 1547 unmodified lines
15481548 dcbzl
15491549 ldbrx
15501550 ppc4xx
1551+ vsx
15511552 "
15521553
15531554 ARCH_EXT_LIST_X86="
@@ -1933,6 +1934,7 @@ mipsdspr2_deps="mips" · 379 unmodified lines
19331934
19341935 altivec_deps="ppc"
19351936 ppc4xx_deps="ppc"
1937+vsx_deps="ppc"
19361938
19371939 cpunop_deps="i686"
19381940 x86_64_select="i686"
@@ -3691,6 +3693,7 @@ elif enabled mips; then · 1752 unmodified lines
36913693 elif enabled ppc; then
36923694
36933695 disable ldbrx
3696+ disable vsx
36943697
36953698 case $(tolower $cpu) in
36963699 601|ppc601|powerpc601)
@@ -3718,7 +3721,7 @@ elif enabled ppc; then · 21 unmodified lines
37183721 g5|970|ppc970|powerpc970)
37193722 cpuflags="-mcpu=970"
37203723 ;;
3721− power[3-7]*)
3724+ power[3-8]*)
37223725 cpuflags="-mcpu=$cpu"
37233726 ;;
37243727 cell)
@@ -4286,6 +4289,12 @@ unsigned int endian = 'B' << 24 | 'I' << 16 | 'G' << 8 | 'E'; · 561 unmodified lines
42864289 EOF
42874290 od -t x1 $TMPO | grep -q '42 *49 *47 *45' && enable bigendian
42884291
4292+if [ "$cpu" = "power7" ] || [ "$cpu" = "power8" ] ;then
4293+ if ! enabled bigendian ;then
4294+ enable vsx
4295+ fi
4296+fi
4297+
42894298
42904299 if enabled asm; then
42914300 enabled arm && nogas=die
@@ -4415,6 +4424,9 @@ EOF · 123 unmodified lines
44154424 enabled altivec || warn "Altivec disabled, possibly missing --cpu flag"
44164425 fi
44174426
4427+ if enabled vsx; then
4428+ check_cflags -mvsx
4429+ fi
44184430 elif enabled x86; then
44194431
44204432 check_builtin rdtsc intrin.h "__rdtsc()"
@@ -5256,6 +5268,7 @@ fi · 835 unmodified lines
52565268 if enabled ppc; then
52575269 echo "AltiVec enabled ${altivec-no}"
52585270 echo "PPC 4xx optimizations ${ppc4xx-no}"
5271+ echo "PPC VSX optimizations ${vsx-no}"
52595272 echo "dcbzl available ${dcbzl-no}"
52605273 fi
52615274 echo "debug symbols ${debug-no}"
libavcodec/ppc/Makefile+1−0
@@ -26,4 +26,5 @@ ALTIVEC-OBJS-$(CONFIG_DSPUTIL) += ppc/dsputil_altivec.o \ · 25 unmodified lines
2626 ppc/int_altivec.o \
2727
2828 FFT-OBJS-$(HAVE_GNU_AS) += ppc/fft_altivec_s.o
29+FFT-OBJS-$(HAVE_VSX) += ppc/fft_vsx.o
2930 ALTIVEC-OBJS-$(CONFIG_FFT) += $(FFT-OBJS-yes)
libavcodec/ppc/fft_altivec.c+12−0
@@ -36,8 +36,12 @@ · 35 unmodified lines
3636 * It also assumes all FFTComplex are 8 bytes-aligned pairs of floats.
3737 */
3838
39+#if HAVE_VSX
40+#include "fft_vsx.h"
41+#else
3942 void ff_fft_calc_altivec(FFTContext *s, FFTComplex *z);
4043 void ff_fft_calc_interleave_altivec(FFTContext *s, FFTComplex *z);
44+#endif
4145
4246 #if HAVE_GNU_AS && HAVE_ALTIVEC
4347 static void imdct_half_altivec(FFTContext *s, FFTSample *output, const FFTSample *input)
@@ -94,7 +98,11 @@ static void imdct_half_altivec(FFTContext *s, FFTSample *output, const FFTSample · 50 unmodified lines
9498 k--;
9599 } while(k >= 0);
96100
101+#if HAVE_VSX
102+ ff_fft_calc_vsx(s, (FFTComplex*)output);
103+#else
97104 ff_fft_calc_altivec(s, (FFTComplex*)output);
105+#endif
98106
99107 /* post rotation + reordering */
100108 j = -n32;
@@ -147,7 +155,11 @@ av_cold void ff_fft_init_ppc(FFTContext *s) · 46 unmodified lines
147155 if (!PPC_ALTIVEC(av_get_cpu_flags()))
148156 return;
149157
158+#if HAVE_VSX
159+ s->fft_calc = ff_fft_calc_interleave_vsx;
160+#else
150161 s->fft_calc = ff_fft_calc_interleave_altivec;
162+#endif
151163 if (s->mdct_bits >= 5) {
152164 s->imdct_calc = imdct_calc_altivec;
153165 s->imdct_half = imdct_half_altivec;
libavcodec/ppc/fft_vsx.c+227−0
@@ -0,0 +1,227 @@
1+/*
2+ * FFT transform, optimized with VSX built-in functions
3+ * Copyright (c) 2014 Rong Yan
4+ *
5+ * This algorithm (though not any of the implementation details) is
6+ * based on libdjbfft by D. J. Bernstein.
7+ *
8+ * This file is part of FFmpeg.
9+ *
10+ * FFmpeg is free software; you can redistribute it and/or
11+ * modify it under the terms of the GNU Lesser General Public
12+ * License as published by the Free Software Foundation; either
13+ * version 2.1 of the License, or (at your option) any later version.
14+ *
15+ * FFmpeg is distributed in the hope that it will be useful,
16+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
17+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18+ * Lesser General Public License for more details.
19+ *
20+ * You should have received a copy of the GNU Lesser General Public
21+ * License along with FFmpeg; if not, write to the Free Software
22+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23+ */
24+
25+
26+#include "config.h"
27+#include "libavutil/cpu.h"
28+#include "libavutil/ppc/types_altivec.h"
29+#include "libavutil/ppc/util_altivec.h"
30+#include "libavcodec/fft.h"
31+#include "libavcodec/fft-internal.h"
32+#include "fft_vsx.h"
33+
34+#if HAVE_VSX
35+
36+static void fft32_vsx_interleave(FFTComplex *z)
37+{
38+ fft16_vsx_interleave(z);
39+ fft8_vsx_interleave(z+16);
40+ fft8_vsx_interleave(z+24);
41+ pass_vsx_interleave(z,ff_cos_32,4);
42+}
43+
44+static void fft64_vsx_interleave(FFTComplex *z)
45+{
46+ fft32_vsx_interleave(z);
47+ fft16_vsx_interleave(z+32);
48+ fft16_vsx_interleave(z+48);
49+ pass_vsx_interleave(z,ff_cos_64, 8);
50+}
51+static void fft128_vsx_interleave(FFTComplex *z)
52+{
53+ fft64_vsx_interleave(z);
54+ fft32_vsx_interleave(z+64);
55+ fft32_vsx_interleave(z+96);
56+ pass_vsx_interleave(z,ff_cos_128,16);
57+}
58+static void fft256_vsx_interleave(FFTComplex *z)
59+{
60+ fft128_vsx_interleave(z);
61+ fft64_vsx_interleave(z+128);
62+ fft64_vsx_interleave(z+192);
63+ pass_vsx_interleave(z,ff_cos_256,32);
64+}
65+static void fft512_vsx_interleave(FFTComplex *z)
66+{
67+ fft256_vsx_interleave(z);
68+ fft128_vsx_interleave(z+256);
69+ fft128_vsx_interleave(z+384);
70+ pass_vsx_interleave(z,ff_cos_512,64);
71+}
72+static void fft1024_vsx_interleave(FFTComplex *z)
73+{
74+ fft512_vsx_interleave(z);
75+ fft256_vsx_interleave(z+512);
76+ fft256_vsx_interleave(z+768);
77+ pass_vsx_interleave(z,ff_cos_1024,128);
78+
79+}
80+static void fft2048_vsx_interleave(FFTComplex *z)
81+{
82+ fft1024_vsx_interleave(z);
83+ fft512_vsx_interleave(z+1024);
84+ fft512_vsx_interleave(z+1536);
85+ pass_vsx_interleave(z,ff_cos_2048,256);
86+}
87+static void fft4096_vsx_interleave(FFTComplex *z)
88+{
89+ fft2048_vsx_interleave(z);
90+ fft1024_vsx_interleave(z+2048);
91+ fft1024_vsx_interleave(z+3072);
92+ pass_vsx_interleave(z,ff_cos_4096, 512);
93+}
94+static void fft8192_vsx_interleave(FFTComplex *z)
95+{
96+ fft4096_vsx_interleave(z);
97+ fft2048_vsx_interleave(z+4096);
98+ fft2048_vsx_interleave(z+6144);
99+ pass_vsx_interleave(z,ff_cos_8192,1024);
100+}
101+static void fft16384_vsx_interleave(FFTComplex *z)
102+{
103+ fft8192_vsx_interleave(z);
104+ fft4096_vsx_interleave(z+8192);
105+ fft4096_vsx_interleave(z+12288);
106+ pass_vsx_interleave(z,ff_cos_16384,2048);
107+}
108+static void fft32768_vsx_interleave(FFTComplex *z)
109+{
110+ fft16384_vsx_interleave(z);
111+ fft8192_vsx_interleave(z+16384);
112+ fft8192_vsx_interleave(z+24576);
113+ pass_vsx_interleave(z,ff_cos_32768,4096);
114+}
115+static void fft65536_vsx_interleave(FFTComplex *z)
116+{
117+ fft32768_vsx_interleave(z);
118+ fft16384_vsx_interleave(z+32768);
119+ fft16384_vsx_interleave(z+49152);
120+ pass_vsx_interleave(z,ff_cos_65536,8192);
121+}
122+
123+static void fft32_vsx(FFTComplex *z)
124+{
125+ fft16_vsx(z);
126+ fft8_vsx(z+16);
127+ fft8_vsx(z+24);
128+ pass_vsx(z,ff_cos_32,4);
129+}
130+
131+static void fft64_vsx(FFTComplex *z)
132+{
133+ fft32_vsx(z);
134+ fft16_vsx(z+32);
135+ fft16_vsx(z+48);
136+ pass_vsx(z,ff_cos_64, 8);
137+}
138+static void fft128_vsx(FFTComplex *z)
139+{
140+ fft64_vsx(z);
141+ fft32_vsx(z+64);
142+ fft32_vsx(z+96);
143+ pass_vsx(z,ff_cos_128,16);
144+}
145+static void fft256_vsx(FFTComplex *z)
146+{
147+ fft128_vsx(z);
148+ fft64_vsx(z+128);
149+ fft64_vsx(z+192);
150+ pass_vsx(z,ff_cos_256,32);
151+}
152+static void fft512_vsx(FFTComplex *z)
153+{
154+ fft256_vsx(z);
155+ fft128_vsx(z+256);
156+ fft128_vsx(z+384);
157+ pass_vsx(z,ff_cos_512,64);
158+}
159+static void fft1024_vsx(FFTComplex *z)
160+{
161+ fft512_vsx(z);
162+ fft256_vsx(z+512);
163+ fft256_vsx(z+768);
164+ pass_vsx(z,ff_cos_1024,128);
165+
166+}
167+static void fft2048_vsx(FFTComplex *z)
168+{
169+ fft1024_vsx(z);
170+ fft512_vsx(z+1024);
171+ fft512_vsx(z+1536);
172+ pass_vsx(z,ff_cos_2048,256);
173+}
174+static void fft4096_vsx(FFTComplex *z)
175+{
176+ fft2048_vsx(z);
177+ fft1024_vsx(z+2048);
178+ fft1024_vsx(z+3072);
179+ pass_vsx(z,ff_cos_4096, 512);
180+}
181+static void fft8192_vsx(FFTComplex *z)
182+{
183+ fft4096_vsx(z);
184+ fft2048_vsx(z+4096);
185+ fft2048_vsx(z+6144);
186+ pass_vsx(z,ff_cos_8192,1024);
187+}
188+static void fft16384_vsx(FFTComplex *z)
189+{
190+ fft8192_vsx(z);
191+ fft4096_vsx(z+8192);
192+ fft4096_vsx(z+12288);
193+ pass_vsx(z,ff_cos_16384,2048);
194+}
195+static void fft32768_vsx(FFTComplex *z)
196+{
197+ fft16384_vsx(z);
198+ fft8192_vsx(z+16384);
199+ fft8192_vsx(z+24576);
200+ pass_vsx(z,ff_cos_32768,4096);
201+}
202+static void fft65536_vsx(FFTComplex *z)
203+{
204+ fft32768_vsx(z);
205+ fft16384_vsx(z+32768);
206+ fft16384_vsx(z+49152);
207+ pass_vsx(z,ff_cos_65536,8192);
208+}
209+
210+static void (* const fft_dispatch_vsx[])(FFTComplex*) = {
211+ fft4_vsx, fft8_vsx, fft16_vsx, fft32_vsx, fft64_vsx, fft128_vsx, fft256_vsx, fft512_vsx, fft1024_vsx,
212+ fft2048_vsx, fft4096_vsx, fft8192_vsx, fft16384_vsx, fft32768_vsx, fft65536_vsx,
213+};
214+static void (* const fft_dispatch_vsx_interleave[])(FFTComplex*) = {
215+ fft4_vsx_interleave, fft8_vsx_interleave, fft16_vsx_interleave, fft32_vsx_interleave, fft64_vsx_interleave,
216+ fft128_vsx_interleave, fft256_vsx_interleave, fft512_vsx_interleave, fft1024_vsx_interleave,
217+ fft2048_vsx_interleave, fft4096_vsx_interleave, fft8192_vsx_interleave, fft16384_vsx_interleave, fft32768_vsx_interleave, fft65536_vsx_interleave,
218+};
219+void ff_fft_calc_interleave_vsx(FFTContext *s, FFTComplex *z)
220+{
221+ fft_dispatch_vsx_interleave[s->nbits-2](z);
222+}
223+void ff_fft_calc_vsx(FFTContext *s, FFTComplex *z)
224+{
225+ fft_dispatch_vsx[s->nbits-2](z);
226+}
227+#endif /* HAVE_VSX */
libavcodec/ppc/fft_vsx.h+830−0
@@ -0,0 +1,830 @@
1+#ifndef AVCODEC_PPC_FFT_VSX_H
2+#define AVCODEC_PPC_FFT_VSX_H
3+/*
4+ * FFT transform, optimized with VSX built-in functions
5+ * Copyright (c) 2014 Rong Yan Copyright (c) 2009 Loren Merritt
6+ *
7+ * This algorithm (though not any of the implementation details) is
8+ * based on libdjbfft by D. J. Bernstein, and fft_altivec_s.S.
9+ *
10+ * This file is part of FFmpeg.
11+ *
12+ * FFmpeg is free software; you can redistribute it and/or
13+ * modify it under the terms of the GNU Lesser General Public
14+ * License as published by the Free Software Foundation; either
15+ * version 2.1 of the License, or (at your option) any later version.
16+ *
17+ * FFmpeg is distributed in the hope that it will be useful,
18+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
19+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20+ * Lesser General Public License for more details.
21+ *
22+ * You should have received a copy of the GNU Lesser General Public
23+ * License along with FFmpeg; if not, write to the Free Software
24+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
25+ */
26+
27+
28+#include "config.h"
29+#include "libavutil/cpu.h"
30+#include "libavutil/ppc/types_altivec.h"
31+#include "libavutil/ppc/util_altivec.h"
32+#include "libavcodec/fft.h"
33+#include "libavcodec/fft-internal.h"
34+
35+#if HAVE_VSX
36+
37+void ff_fft_calc_interleave_vsx(FFTContext *s, FFTComplex *z);
38+void ff_fft_calc_vsx(FFTContext *s, FFTComplex *z);
39+
40+
41+#define byte_2complex (2*sizeof(FFTComplex))
42+#define byte_4complex (4*sizeof(FFTComplex))
43+#define byte_6complex (6*sizeof(FFTComplex))
44+#define byte_8complex (8*sizeof(FFTComplex))
45+#define byte_10complex (10*sizeof(FFTComplex))
46+#define byte_12complex (12*sizeof(FFTComplex))
47+#define byte_14complex (14*sizeof(FFTComplex))
48+
49+inline static void pass_vsx_interleave(FFTComplex *z, const FFTSample *wre, unsigned int n)
50+{
51+ int o1 = n<<1;
52+ int o2 = n<<2;
53+ int o3 = o1+o2;
54+ int i1, i2, i3;
55+ FFTSample* out = (FFTSample*)z;
56+ const FFTSample *wim = wre+o1;
57+ vec_f vz0, vzo1, vzo2, vzo3;
58+ vec_f x0, x1, x2, x3;
59+ vec_f x4, x5, x6, x7;
60+ vec_f x8, x9, x10, x11;
61+ vec_f x12, x13, x14, x15;
62+ vec_f x16, x17, x18, x19;
63+ vec_f x20, x21, x22, x23;
64+ vec_f vz0plus1, vzo1plus1, vzo2plus1, vzo3plus1;
65+ vec_f y0, y1, y2, y3;
66+ vec_f y4, y5, y8, y9;
67+ vec_f y10, y13, y14, y15;
68+ vec_f y16, y17, y18, y19;
69+ vec_f y20, y21, y22, y23;
70+ vec_f wr1, wi1, wr0, wi0;
71+ vec_f wr2, wi2, wr3, wi3;
72+ vec_f xmulwi0, xmulwi1, ymulwi2, ymulwi3;
73+
74+ n = n-2;
75+ i1 = o1*sizeof(FFTComplex);
76+ i2 = o2*sizeof(FFTComplex);
77+ i3 = o3*sizeof(FFTComplex);
78+ vzo2 = vec_ld(i2, &(out[0])); // zo2.r zo2.i z(o2+1).r z(o2+1).i
79+ vzo2plus1 = vec_ld(i2+16, &(out[0]));
80+ vzo3 = vec_ld(i3, &(out[0])); // zo3.r zo3.i z(o3+1).r z(o3+1).i
81+ vzo3plus1 = vec_ld(i3+16, &(out[0]));
82+ vz0 = vec_ld(0, &(out[0])); // z0.r z0.i z1.r z1.i
83+ vz0plus1 = vec_ld(16, &(out[0]));
84+ vzo1 = vec_ld(i1, &(out[0])); // zo1.r zo1.i z(o1+1).r z(o1+1).i
85+ vzo1plus1 = vec_ld(i1+16, &(out[0]));
86+
87+ x0 = vec_add(vzo2, vzo3);
88+ x1 = vec_sub(vzo2, vzo3);
89+ y0 = vec_add(vzo2plus1, vzo3plus1);
90+ y1 = vec_sub(vzo2plus1, vzo3plus1);
91+
92+ wr1 = vec_splats(wre[1]);
93+ wi1 = vec_splats(wim[-1]);
94+ wi2 = vec_splats(wim[-2]);
95+ wi3 = vec_splats(wim[-3]);
96+ wr2 = vec_splats(wre[2]);
97+ wr3 = vec_splats(wre[3]);
98+
99+ x2 = vec_perm(x0, x1, vcprm(2,s2,3,s3));
100+ x3 = vec_perm(x0, x1, vcprm(s3,3,s2,2));
101+
102+ y4 = vec_perm(y0, y1, vcprm(s1,1,s0,0));
103+ y5 = vec_perm(y0, y1, vcprm(s3,3,s2,2));
104+ y2 = vec_perm(y0, y1, vcprm(0,s0,1,s1));
105+ y3 = vec_perm(y0, y1, vcprm(2,s2,3,s3));
106+
107+ ymulwi2 = vec_mul(y4, wi2);
108+ ymulwi3 = vec_mul(y5, wi3);
109+ x4 = vec_mul(x2, wr1);
110+ x5 = vec_mul(x3, wi1);
111+ y8 = vec_madd(y2, wr2, ymulwi2);
112+ y9 = vec_msub(y2, wr2, ymulwi2);
113+ x6 = vec_add(x4, x5);
114+ x7 = vec_sub(x4, x5);
115+ y13 = vec_madd(y3, wr3, ymulwi3);
116+ y14 = vec_msub(y3, wr3, ymulwi3);
117+
118+ x8 = vec_perm(x6, x7, vcprm(0,1,s2,s3));
119+ y10 = vec_perm(y8, y9, vcprm(0,1,s2,s3));
120+ y15 = vec_perm(y13, y14, vcprm(0,1,s2,s3));
121+
122+ x9 = vec_perm(x0, x8, vcprm(0,1,s0,s2));
123+ x10 = vec_perm(x1, x8, vcprm(1,0,s3,s1));
124+
125+ y16 = vec_perm(y10, y15, vcprm(0,2,s0,s2));
126+ y17 = vec_perm(y10, y15, vcprm(3,1,s3,s1));
127+
128+ x11 = vec_add(vz0, x9);
129+ x12 = vec_sub(vz0, x9);
130+ x13 = vec_add(vzo1, x10);
131+ x14 = vec_sub(vzo1, x10);
132+
133+ y18 = vec_add(vz0plus1, y16);
134+ y19 = vec_sub(vz0plus1, y16);
135+ y20 = vec_add(vzo1plus1, y17);
136+ y21 = vec_sub(vzo1plus1, y17);
137+
138+ x15 = vec_perm(x13, x14, vcprm(0,s1,2,s3));
139+ x16 = vec_perm(x13, x14, vcprm(s0,1,s2,3));
140+ y22 = vec_perm(y20, y21, vcprm(0,s1,2,s3));
141+ y23 = vec_perm(y20, y21, vcprm(s0,1,s2,3));
142+
143+
144+ vec_st(x11, 0, &(out[0]));
145+ vec_st(y18, 16, &(out[0]));
146+ vec_st(x15, i1, &(out[0]));
147+ vec_st(y22, i1+16, &(out[0]));
148+ vec_st(x12, i2, &(out[0]));
149+ vec_st(y19, i2+16, &(out[0]));
150+ vec_st(x16, i3, &(out[0]));
151+ vec_st(y23, i3+16, &(out[0]));
152+
153+ do {
154+ out += 8;
155+ wre += 4;
156+ wim -= 4;
157+ wr0 = vec_splats(wre[0]);
158+ wr1 = vec_splats(wre[1]);
159+ wi0 = vec_splats(wim[0]);
160+ wi1 = vec_splats(wim[-1]);
161+
162+ wr2 = vec_splats(wre[2]);
163+ wr3 = vec_splats(wre[3]);
164+ wi2 = vec_splats(wim[-2]);
165+ wi3 = vec_splats(wim[-3]);
166+
167+ vzo2 = vec_ld(i2, &(out[0])); // zo2.r zo2.i z(o2+1).r z(o2+1).i
168+ vzo2plus1 = vec_ld(i2+16, &(out[0]));
169+ vzo3 = vec_ld(i3, &(out[0])); // zo3.r zo3.i z(o3+1).r z(o3+1).i
170+ vzo3plus1 = vec_ld(i3+16, &(out[0]));
171+ vz0 = vec_ld(0, &(out[0])); // z0.r z0.i z1.r z1.i
172+ vz0plus1 = vec_ld(16, &(out[0]));
173+ vzo1 = vec_ld(i1, &(out[0])); // zo1.r zo1.i z(o1+1).r z(o1+1).i
174+ vzo1plus1 = vec_ld(i1+16, &(out[0]));
175+
176+ x0 = vec_add(vzo2, vzo3);
177+ x1 = vec_sub(vzo2, vzo3);
178+
179+ y0 = vec_add(vzo2plus1, vzo3plus1);
180+ y1 = vec_sub(vzo2plus1, vzo3plus1);
181+
182+ x4 = vec_perm(x0, x1, vcprm(s1,1,s0,0));
183+ x5 = vec_perm(x0, x1, vcprm(s3,3,s2,2));
184+ x2 = vec_perm(x0, x1, vcprm(0,s0,1,s1));
185+ x3 = vec_perm(x0, x1, vcprm(2,s2,3,s3));
186+
187+ y2 = vec_perm(y0, y1, vcprm(0,s0,1,s1));
188+ y3 = vec_perm(y0, y1, vcprm(2,s2,3,s3));
189+ xmulwi0 = vec_mul(x4, wi0);
190+ xmulwi1 = vec_mul(x5, wi1);
191+
192+ y4 = vec_perm(y0, y1, vcprm(s1,1,s0,0));
193+ y5 = vec_perm(y0, y1, vcprm(s3,3,s2,2));
194+
195+ x8 = vec_madd(x2, wr0, xmulwi0);
196+ x9 = vec_msub(x2, wr0, xmulwi0);
197+ ymulwi2 = vec_mul(y4, wi2);
198+ ymulwi3 = vec_mul(y5, wi3);
199+
200+ x13 = vec_madd(x3, wr1, xmulwi1);
201+ x14 = vec_msub(x3, wr1, xmulwi1);
202+
203+ y8 = vec_madd(y2, wr2, ymulwi2);
204+ y9 = vec_msub(y2, wr2, ymulwi2);
205+ y13 = vec_madd(y3, wr3, ymulwi3);
206+ y14 = vec_msub(y3, wr3, ymulwi3);
207+
208+ x10 = vec_perm(x8, x9, vcprm(0,1,s2,s3));
209+ x15 = vec_perm(x13, x14, vcprm(0,1,s2,s3));
210+
211+ y10 = vec_perm(y8, y9, vcprm(0,1,s2,s3));
212+ y15 = vec_perm(y13, y14, vcprm(0,1,s2,s3));
213+
214+ x16 = vec_perm(x10, x15, vcprm(0,2,s0,s2));
215+ x17 = vec_perm(x10, x15, vcprm(3,1,s3,s1));
216+
217+ y16 = vec_perm(y10, y15, vcprm(0,2,s0,s2));
218+ y17 = vec_perm(y10, y15, vcprm(3,1,s3,s1));
219+
220+ x18 = vec_add(vz0, x16);
221+ x19 = vec_sub(vz0, x16);
222+ x20 = vec_add(vzo1, x17);
223+ x21 = vec_sub(vzo1, x17);
224+
225+ y18 = vec_add(vz0plus1, y16);
226+ y19 = vec_sub(vz0plus1, y16);
227+ y20 = vec_add(vzo1plus1, y17);
228+ y21 = vec_sub(vzo1plus1, y17);
229+
230+ x22 = vec_perm(x20, x21, vcprm(0,s1,2,s3));
231+ x23 = vec_perm(x20, x21, vcprm(s0,1,s2,3));
232+
233+ y22 = vec_perm(y20, y21, vcprm(0,s1,2,s3));
234+ y23 = vec_perm(y20, y21, vcprm(s0,1,s2,3));
235+
236+ vec_st(x18, 0, &(out[0]));
237+ vec_st(y18, 16, &(out[0]));
238+ vec_st(x22, i1, &(out[0]));
239+ vec_st(y22, i1+16, &(out[0]));
240+ vec_st(x19, i2, &(out[0]));
241+ vec_st(y19, i2+16, &(out[0]));
242+ vec_st(x23, i3, &(out[0]));
243+ vec_st(y23, i3+16, &(out[0]));
244+ } while (n-=2);
245+}
246+
247+inline static void fft2_vsx_interleave(FFTComplex *z)
248+{
249+ FFTSample r1, i1;
250+
251+ r1 = z[0].re - z[1].re;
252+ z[0].re += z[1].re;
253+ z[1].re = r1;
254+
255+ i1 = z[0].im - z[1].im;
256+ z[0].im += z[1].im;
257+ z[1].im = i1;
258+ }
259+
260+inline static void fft4_vsx_interleave(FFTComplex *z)
261+{
262+ vec_f a, b, c, d;
263+ float* out= (float*)z;
264+ a = vec_ld(0, &(out[0]));
265+ b = vec_ld(byte_2complex, &(out[0]));
266+
267+ c = vec_perm(a, b, vcprm(0,1,s2,s1));
268+ d = vec_perm(a, b, vcprm(2,3,s0,s3));
269+ a = vec_add(c, d);
270+ b = vec_sub(c, d);
271+
272+ c = vec_perm(a, b, vcprm(0,1,s0,s1));
273+ d = vec_perm(a, b, vcprm(2,3,s3,s2));
274+
275+ a = vec_add(c, d);
276+ b = vec_sub(c, d);
277+ vec_st(a, 0, &(out[0]));
278+ vec_st(b, byte_2complex, &(out[0]));
279+}
280+
281+inline static void fft8_vsx_interleave(FFTComplex *z)
282+{
283+ vec_f vz0, vz1, vz2, vz3;
284+ vec_f x0, x1, x2, x3;
285+ vec_f x4, x5, x6, x7;
286+ vec_f x8, x9, x10, x11;
287+ vec_f x12, x13, x14, x15;
288+ vec_f x16, x17, x18, x19;
289+ vec_f x20, x21, x22, x23;
290+ vec_f x24, x25, x26, x27;
291+ vec_f x28, x29, x30, x31;
292+ vec_f x32, x33, x34;
293+
294+ float* out= (float*)z;
295+ vec_f vc1 = {sqrthalf, sqrthalf, sqrthalf, sqrthalf};
296+
297+ vz0 = vec_ld(0, &(out[0]));
298+ vz1 = vec_ld(byte_2complex, &(out[0]));
299+ vz2 = vec_ld(byte_4complex, &(out[0]));
300+ vz3 = vec_ld(byte_6complex, &(out[0]));
301+
302+ x0 = vec_perm(vz0, vz1, vcprm(0,1,s2,s1));
303+ x1 = vec_perm(vz0, vz1, vcprm(2,3,s0,s3));
304+ x2 = vec_perm(vz2, vz3, vcprm(2,1,s0,s1));
305+ x3 = vec_perm(vz2, vz3, vcprm(0,3,s2,s3));
306+
307+ x4 = vec_add(x0, x1);
308+ x5 = vec_sub(x0, x1);
309+ x6 = vec_add(x2, x3);
310+ x7 = vec_sub(x2, x3);
311+
312+ x8 = vec_perm(x4, x5, vcprm(0,1,s0,s1));
313+ x9 = vec_perm(x4, x5, vcprm(2,3,s3,s2));
314+ x10 = vec_perm(x6, x7, vcprm(2,1,s2,s1));
315+ x11 = vec_perm(x6, x7, vcprm(0,3,s0,s3));
316+
317+ x12 = vec_add(x8, x9);
318+ x13 = vec_sub(x8, x9);
319+ x14 = vec_add(x10, x11);
320+ x15 = vec_sub(x10, x11);
321+ x16 = vec_perm(x12, x13, vcprm(0,s0,1,s1));
322+ x17 = vec_perm(x14, x15, vcprm(0,s0,1,s1));
323+ x18 = vec_perm(x16, x17, vcprm(s0,s3,s2,s1));
324+ x19 = vec_add(x16, x18); // z0.r z2.r z0.i z2.i
325+ x20 = vec_sub(x16, x18); // z4.r z6.r z4.i z6.i
326+
327+ x21 = vec_perm(x12, x13, vcprm(2,s2,3,s3));
328+ x22 = vec_perm(x14, x15, vcprm(2,3,s2,s3));
329+ x23 = vec_perm(x14, x15, vcprm(3,2,s3,s2));
330+ x24 = vec_add(x22, x23);
331+ x25 = vec_sub(x22, x23);
332+ x26 = vec_mul( vec_perm(x24, x25, vcprm(2,s2,0,s0)), vc1);
333+
334+ x27 = vec_add(x21, x26); // z1.r z7.r z1.i z3.i
335+ x28 = vec_sub(x21, x26); //z5.r z3.r z5.i z7.i
336+
337+ x29 = vec_perm(x19, x27, vcprm(0,2,s0,s2)); // z0.r z0.i z1.r z1.i
338+ x30 = vec_perm(x19, x27, vcprm(1,3,s1,s3)); // z2.r z2.i z7.r z3.i
339+ x31 = vec_perm(x20, x28, vcprm(0,2,s0,s2)); // z4.r z4.i z5.r z5.i
340+ x32 = vec_perm(x20, x28, vcprm(1,3,s1,s3)); // z6.r z6.i z3.r z7.i
341+ x33 = vec_perm(x30, x32, vcprm(0,1,s2,3)); // z2.r z2.i z3.r z3.i
342+ x34 = vec_perm(x30, x32, vcprm(s0,s1,2,s3)); // z6.r z6.i z7.r z7.i
343+
344+ vec_st(x29, 0, &(out[0]));
345+ vec_st(x33, byte_2complex, &(out[0]));
346+ vec_st(x31, byte_4complex, &(out[0]));
347+ vec_st(x34, byte_6complex, &(out[0]));
348+}
349+
350+inline static void fft16_vsx_interleave(FFTComplex *z)
351+{
352+ float* out= (float*)z;
353+ vec_f vc0 = {sqrthalf, sqrthalf, sqrthalf, sqrthalf};
354+ vec_f vc1 = {ff_cos_16[1], ff_cos_16[1], ff_cos_16[1], ff_cos_16[1]};
355+ vec_f vc2 = {ff_cos_16[3], ff_cos_16[3], ff_cos_16[3], ff_cos_16[3]};
356+ vec_f vz0, vz1, vz2, vz3;
357+ vec_f vz4, vz5, vz6, vz7;
358+ vec_f x0, x1, x2, x3;
359+ vec_f x4, x5, x6, x7;
360+ vec_f x8, x9, x10, x11;
361+ vec_f x12, x13, x14, x15;
362+ vec_f x16, x17, x18, x19;
363+ vec_f x20, x21, x22, x23;
364+ vec_f x24, x25, x26, x27;
365+ vec_f x28, x29, x30, x31;
366+ vec_f x32, x33, x34, x35;
367+ vec_f x36, x37, x38, x39;
368+ vec_f x40, x41, x42, x43;
369+ vec_f x44, x45, x46, x47;
370+ vec_f x48, x49, x50, x51;
371+ vec_f x52, x53, x54, x55;
372+ vec_f x56, x57, x58, x59;
373+ vec_f x60, x61, x62, x63;
374+ vec_f x64, x65, x66, x67;
375+ vec_f x68, x69, x70, x71;
376+ vec_f x72, x73, x74, x75;
377+ vec_f x76, x77, x78, x79;
378+ vec_f x80, x81, x82, x83;
379+ vec_f x84, x85, x86;
380+
381+ vz0 = vec_ld(0, &(out[0]));
382+ vz1 = vec_ld(byte_2complex, &(out[0]));
383+ vz2 = vec_ld(byte_4complex, &(out[0]));
384+ vz3 = vec_ld(byte_6complex, &(out[0]));
385+ vz4 = vec_ld(byte_8complex, &(out[0]));
386+ vz5 = vec_ld(byte_10complex, &(out[0]));
387+ vz6 = vec_ld(byte_12complex, &(out[0]));
388+ vz7 = vec_ld(byte_14complex, &(out[0]));
389+
390+ x0 = vec_perm(vz0, vz1, vcprm(0,1,s2,s1));
391+ x1 = vec_perm(vz0, vz1, vcprm(2,3,s0,s3));
392+ x2 = vec_perm(vz2, vz3, vcprm(0,1,s0,s1));
393+ x3 = vec_perm(vz2, vz3, vcprm(2,3,s2,s3));
394+
395+ x4 = vec_perm(vz4, vz5, vcprm(0,1,s2,s1));
396+ x5 = vec_perm(vz4, vz5, vcprm(2,3,s0,s3));
397+ x6 = vec_perm(vz6, vz7, vcprm(0,1,s2,s1));
398+ x7 = vec_perm(vz6, vz7, vcprm(2,3,s0,s3));
399+
400+ x8 = vec_add(x0, x1);
401+ x9 = vec_sub(x0, x1);
402+ x10 = vec_add(x2, x3);
403+ x11 = vec_sub(x2, x3);
404+
405+ x12 = vec_add(x4, x5);
406+ x13 = vec_sub(x4, x5);
407+ x14 = vec_add(x6, x7);
408+ x15 = vec_sub(x6, x7);
409+
410+ x16 = vec_perm(x8, x9, vcprm(0,1,s0,s1));
411+ x17 = vec_perm(x8, x9, vcprm(2,3,s3,s2));
412+ x18 = vec_perm(x10, x11, vcprm(2,1,s1,s2));
413+ x19 = vec_perm(x10, x11, vcprm(0,3,s0,s3));
414+ x20 = vec_perm(x12, x14, vcprm(0,1,s0, s1));
415+ x21 = vec_perm(x12, x14, vcprm(2,3,s2,s3));
416+ x22 = vec_perm(x13, x15, vcprm(0,1,s0,s1));
417+ x23 = vec_perm(x13, x15, vcprm(3,2,s3,s2));
418+
419+ x24 = vec_add(x16, x17);
420+ x25 = vec_sub(x16, x17);
421+ x26 = vec_add(x18, x19);
422+ x27 = vec_sub(x18, x19);
423+ x28 = vec_add(x20, x21);
424+ x29 = vec_sub(x20, x21);
425+ x30 = vec_add(x22, x23);
426+ x31 = vec_sub(x22, x23);
427+
428+ x32 = vec_add(x24, x26);
429+ x33 = vec_sub(x24, x26);
430+ x34 = vec_perm(x32, x33, vcprm(0,1,s0,s1));
431+
432+ x35 = vec_perm(x28, x29, vcprm(2,1,s1,s2));
433+ x36 = vec_perm(x28, x29, vcprm(0,3,s0,s3));
434+ x37 = vec_add(x35, x36);
435+ x38 = vec_sub(x35, x36);
436+ x39 = vec_perm(x37, x38, vcprm(0,1,s1,s0));
437+
438+ x40 = vec_perm(x27, x38, vcprm(3,2,s2,s3));
439+ x41 = vec_perm(x26, x37, vcprm(2,3,s3,s2));
440+ x42 = vec_add(x40, x41);
441+ x43 = vec_sub(x40, x41);
442+ x44 = vec_mul(x42, vc0);
443+ x45 = vec_mul(x43, vc0);
444+
445+ x46 = vec_add(x34, x39); // z0.r z0.i z4.r z4.i
446+ x47 = vec_sub(x34, x39); // z8.r z8.i z12.r z12.i
447+
448+ x48 = vec_perm(x30, x31, vcprm(2,1,s1,s2));
449+ x49 = vec_perm(x30, x31, vcprm(0,3,s3,s0));
450+ x50 = vec_add(x48, x49);
451+ x51 = vec_sub(x48, x49);
452+ x52 = vec_mul(x50, vc1);
453+ x53 = vec_mul(x50, vc2);
454+ x54 = vec_mul(x51, vc1);
455+ x55 = vec_mul(x51, vc2);
456+
457+ x56 = vec_perm(x24, x25, vcprm(2,3,s2,s3));
458+ x57 = vec_perm(x44, x45, vcprm(0,1,s1,s0));
459+ x58 = vec_add(x56, x57);
460+ x59 = vec_sub(x56, x57);
461+
462+ x60 = vec_perm(x54, x55, vcprm(1,0,3,2));
463+ x61 = vec_perm(x54, x55, vcprm(s1,s0,s3,s2));
464+ x62 = vec_add(x52, x61);
465+ x63 = vec_sub(x52, x61);
466+ x64 = vec_add(x60, x53);
467+ x65 = vec_sub(x60, x53);
468+ x66 = vec_perm(x62, x64, vcprm(0,1,s3,s2));
469+ x67 = vec_perm(x63, x65, vcprm(s0,s1,3,2));
470+
471+ x68 = vec_add(x58, x66); // z1.r z1.i z3.r z3.i
472+ x69 = vec_sub(x58, x66); // z9.r z9.i z11.r z11.i
473+ x70 = vec_add(x59, x67); // z5.r z5.i z15.r z15.i
474+ x71 = vec_sub(x59, x67); // z13.r z13.i z7.r z7.i
475+
476+ x72 = vec_perm(x25, x27, vcprm(s1,s0,s2,s3));
477+ x73 = vec_add(x25, x72);
478+ x74 = vec_sub(x25, x72);
479+ x75 = vec_perm(x73, x74, vcprm(0,1,s0,s1));
480+ x76 = vec_perm(x44, x45, vcprm(3,2,s2,s3));
481+ x77 = vec_add(x75, x76); // z2.r z2.i z6.r z6.i
482+ x78 = vec_sub(x75, x76); // z10.r z10.i z14.r z14.i
483+
484+ x79 = vec_perm(x46, x68, vcprm(0,1,s0,s1)); // z0.r z0.i z1.r z1.i
485+ x80 = vec_perm(x77, x68, vcprm(0,1,s2,s3)); // z2.r z2.i z3.r z3.i
486+ x81 = vec_perm(x46, x70, vcprm(2,3,s0,s1)); // z4.r z4.i z5.r z5.i
487+ x82 = vec_perm(x71, x77, vcprm(s2,s3,2,3)); // z6.r z6.i z7.r z7.i
488+ vec_st(x79, 0, &(out[0]));
489+ vec_st(x80, byte_2complex, &(out[0]));
490+ vec_st(x81, byte_4complex, &(out[0]));
491+ vec_st(x82, byte_6complex, &(out[0]));
492+ x83 = vec_perm(x47, x69, vcprm(0,1,s0,s1)); // z8.r z8.i z9.r z9.i
493+ x84 = vec_perm(x78, x69, vcprm(0,1,s2,s3)); // z10.r z10.i z11.r z11.i
494+ x85 = vec_perm(x47, x71, vcprm(2,3,s0,s1)); // z12.r z12.i z13.r z13.i
495+ x86 = vec_perm(x70, x78, vcprm(s2,s3,2,3)); // z14.r z14.i z15.r z15.i
496+ vec_st(x83, byte_8complex, &(out[0]));
497+ vec_st(x84, byte_10complex, &(out[0]));
498+ vec_st(x85, byte_12complex, &(out[0]));
499+ vec_st(x86, byte_14complex, &(out[0]));
500+}
501+
502+inline static void fft4_vsx(FFTComplex *z)
503+{
504+ vec_f a, b, c, d;
505+ float* out= (float*)z;
506+ a = vec_ld(0, &(out[0]));
507+ b = vec_ld(byte_2complex, &(out[0]));
508+
509+ c = vec_perm(a, b, vcprm(0,1,s2,s1));
510+ d = vec_perm(a, b, vcprm(2,3,s0,s3));
511+ a = vec_add(c, d);
512+ b = vec_sub(c, d);
513+
514+ c = vec_perm(a,b, vcprm(0,s0,1,s1));
515+ d = vec_perm(a, b, vcprm(2,s3,3,s2));
516+
517+ a = vec_add(c, d);
518+ b = vec_sub(c, d);
519+
520+ c = vec_perm(a, b, vcprm(0,1,s0,s1));
521+ d = vec_perm(a, b, vcprm(2,3,s2,s3));
522+
523+ vec_st(c, 0, &(out[0]));
524+ vec_st(d, byte_2complex, &(out[0]));
525+ return;
526+}
527+
528+inline static void fft8_vsx(FFTComplex *z)
529+{
530+ vec_f vz0, vz1, vz2, vz3;
531+ vec_f vz4, vz5, vz6, vz7, vz8;
532+
533+ float* out= (float*)z;
534+ vec_f vc0 = {0.0, 0.0, 0.0, 0.0};
535+ vec_f vc1 = {-sqrthalf, sqrthalf, sqrthalf, -sqrthalf};
536+ vec_f vc2 = {sqrthalf, sqrthalf, sqrthalf, sqrthalf};
537+
538+ vz0 = vec_ld(0, &(out[0]));
539+ vz1 = vec_ld(byte_2complex, &(out[0]));
540+ vz2 = vec_ld(byte_4complex, &(out[0]));
541+ vz3 = vec_ld(byte_6complex, &(out[0]));
542+
543+ vz6 = vec_perm(vz2, vz3, vcprm(0,s0,1,s1));
544+ vz7 = vec_perm(vz2, vz3, vcprm(2,s2,3,s3));
545+ vz4 = vec_perm(vz0, vz1, vcprm(0,1,s2,s1));
546+ vz5 = vec_perm(vz0, vz1, vcprm(2,3,s0,s3));
547+
548+ vz2 = vec_add(vz6, vz7);
549+ vz3 = vec_sub(vz6, vz7);
550+ vz8 = vec_perm(vz3, vz3, vcprm(2,3,0,1));
551+
552+ vz0 = vec_add(vz4, vz5);
553+ vz1 = vec_sub(vz4, vz5);
554+
555+ vz3 = vec_madd(vz3, vc1, vc0);
556+ vz3 = vec_madd(vz8, vc2, vz3);
557+
558+ vz4 = vec_perm(vz0, vz1, vcprm(0,s0,1,s1));
559+ vz5 = vec_perm(vz0, vz1, vcprm(2,s3,3,s2));
560+ vz6 = vec_perm(vz2, vz3, vcprm(1,2,s3,s0));
561+ vz7 = vec_perm(vz2, vz3, vcprm(0,3,s2,s1));
562+
563+ vz0 = vec_add(vz4, vz5);
564+ vz1 = vec_sub(vz4, vz5);
565+ vz2 = vec_add(vz6, vz7);
566+ vz3 = vec_sub(vz6, vz7);
567+
568+ vz4 = vec_perm(vz0, vz1, vcprm(0,1,s0,s1));
569+ vz5 = vec_perm(vz0, vz1, vcprm(2,3,s2,s3));
570+ vz6 = vec_perm(vz2, vz3, vcprm(0,2,s1,s3));
571+ vz7 = vec_perm(vz2, vz3, vcprm(1,3,s0,s2));
572+
573+
574+ vz2 = vec_sub(vz4, vz6);
575+ vz3 = vec_sub(vz5, vz7);
576+
577+ vz0 = vec_add(vz4, vz6);
578+ vz1 = vec_add(vz5, vz7);
579+
580+ vec_st(vz0, 0, &(out[0]));
581+ vec_st(vz1, byte_2complex, &(out[0]));
582+ vec_st(vz2, byte_4complex, &(out[0]));
583+ vec_st(vz3, byte_6complex, &(out[0]));
584+ return;
585+}
586+
587+inline static void fft16_vsx(FFTComplex *z)
588+{
589+ float* out= (float*)z;
590+ vec_f vc0 = {0.0, 0.0, 0.0, 0.0};
591+ vec_f vc1 = {-sqrthalf, sqrthalf, sqrthalf, -sqrthalf};
592+ vec_f vc2 = {sqrthalf, sqrthalf, sqrthalf, sqrthalf};
593+ vec_f vc3 = {1.0, 0.92387953, sqrthalf, 0.38268343};
594+ vec_f vc4 = {0.0, 0.38268343, sqrthalf, 0.92387953};
595+ vec_f vc5 = {-0.0, -0.38268343, -sqrthalf, -0.92387953};
596+
597+ vec_f vz0, vz1, vz2, vz3;
598+ vec_f vz4, vz5, vz6, vz7;
599+ vec_f vz8, vz9, vz10, vz11;
600+ vec_f vz12, vz13;
601+
602+ vz0 = vec_ld(byte_8complex, &(out[0]));
603+ vz1 = vec_ld(byte_10complex, &(out[0]));
604+ vz2 = vec_ld(byte_12complex, &(out[0]));
605+ vz3 = vec_ld(byte_14complex, &(out[0]));
606+
607+ vz4 = vec_perm(vz0, vz1, vcprm(0,1,s2,s1));
608+ vz5 = vec_perm(vz0, vz1, vcprm(2,3,s0,s3));
609+ vz6 = vec_perm(vz2, vz3, vcprm(0,1,s2,s1));
610+ vz7 = vec_perm(vz2, vz3, vcprm(2,3,s0,s3));
611+
612+ vz0 = vec_add(vz4, vz5);
613+ vz1= vec_sub(vz4, vz5);
614+ vz2 = vec_add(vz6, vz7);
615+ vz3 = vec_sub(vz6, vz7);
616+
617+ vz4 = vec_perm(vz0, vz1, vcprm(0,s0,1,s1));
618+ vz5 = vec_perm(vz0, vz1, vcprm(2,s3,3,s2));
619+ vz6 = vec_perm(vz2, vz3, vcprm(0,s0,1,s1));
620+ vz7 = vec_perm(vz2, vz3, vcprm(2,s3,3,s2));
621+
622+ vz0 = vec_add(vz4, vz5);
623+ vz1 = vec_sub(vz4, vz5);
624+ vz2 = vec_add(vz6, vz7);
625+ vz3 = vec_sub(vz6, vz7);
626+
627+ vz4 = vec_perm(vz0, vz1, vcprm(0,1,s0,s1));
628+ vz5 = vec_perm(vz0, vz1, vcprm(2,3,s2,s3));
629+
630+ vz6 = vec_perm(vz2, vz3, vcprm(0,1,s0,s1));
631+ vz7 = vec_perm(vz2, vz3, vcprm(2,3,s2,s3));
632+
633+ vz0 = vec_ld(0, &(out[0]));
634+ vz1 = vec_ld(byte_2complex, &(out[0]));
635+ vz2 = vec_ld(byte_4complex, &(out[0]));
636+ vz3 = vec_ld(byte_6complex, &(out[0]));
637+ vz10 = vec_perm(vz2, vz3, vcprm(0,s0,1,s1));
638+ vz11 = vec_perm(vz2, vz3, vcprm(2,s2,3,s3));
639+ vz8 = vec_perm(vz0, vz1, vcprm(0,1,s2,s1));
640+ vz9 = vec_perm(vz0, vz1, vcprm(2,3,s0,s3));
641+
642+ vz2 = vec_add(vz10, vz11);
643+ vz3 = vec_sub(vz10, vz11);
644+ vz12 = vec_perm(vz3, vz3, vcprm(2,3,0,1));
645+ vz0 = vec_add(vz8, vz9);
646+ vz1 = vec_sub(vz8, vz9);
647+
648+ vz3 = vec_madd(vz3, vc1, vc0);
649+ vz3 = vec_madd(vz12, vc2, vz3);
650+ vz8 = vec_perm(vz0, vz1, vcprm(0,s0,1,s1));
651+ vz9 = vec_perm(vz0, vz1, vcprm(2,s3,3,s2));
652+ vz10 = vec_perm(vz2, vz3, vcprm(1,2,s3,s0));
653+ vz11 = vec_perm(vz2, vz3, vcprm(0,3,s2,s1));
654+
655+ vz0 = vec_add(vz8, vz9);
656+ vz1 = vec_sub(vz8, vz9);
657+ vz2 = vec_add(vz10, vz11);
658+ vz3 = vec_sub(vz10, vz11);
659+
660+ vz8 = vec_perm(vz0, vz1, vcprm(0,1,s0,s1));
661+ vz9 = vec_perm(vz0, vz1, vcprm(2,3,s2,s3));
662+ vz10 = vec_perm(vz2, vz3, vcprm(0,2,s1,s3));
663+ vz11 = vec_perm(vz2, vz3, vcprm(1,3,s0,s2));
664+
665+ vz2 = vec_sub(vz8, vz10);
666+ vz3 = vec_sub(vz9, vz11);
667+ vz0 = vec_add(vz8, vz10);
668+ vz1 = vec_add(vz9, vz11);
669+
670+ vz8 = vec_madd(vz4, vc3, vc0);
671+ vz9 = vec_madd(vz5, vc3, vc0);
672+ vz10 = vec_madd(vz6, vc3, vc0);
673+ vz11 = vec_madd(vz7, vc3, vc0);
674+
675+ vz8 = vec_madd(vz5, vc4, vz8);
676+ vz9 = vec_madd(vz4, vc5, vz9);
677+ vz10 = vec_madd(vz7, vc5, vz10);
678+ vz11 = vec_madd(vz6, vc4, vz11);
679+
680+ vz12 = vec_sub(vz10, vz8);
681+ vz10 = vec_add(vz10, vz8);
682+
683+ vz13 = vec_sub(vz9, vz11);
684+ vz11 = vec_add(vz9, vz11);
685+
686+ vz4 = vec_sub(vz0, vz10);
687+ vz0 = vec_add(vz0, vz10);
688+
689+ vz7= vec_sub(vz3, vz12);
690+ vz3= vec_add(vz3, vz12);
691+
692+ vz5 = vec_sub(vz1, vz11);
693+ vz1 = vec_add(vz1, vz11);
694+
695+ vz6 = vec_sub(vz2, vz13);
696+ vz2 = vec_add(vz2, vz13);
697+
698+ vec_st(vz0, 0, &(out[0]));
699+ vec_st(vz1, byte_2complex, &(out[0]));
700+ vec_st(vz2, byte_4complex, &(out[0]));
701+ vec_st(vz3, byte_6complex, &(out[0]));
702+ vec_st(vz4, byte_8complex, &(out[0]));
703+ vec_st(vz5, byte_10complex, &(out[0]));
704+ vec_st(vz6, byte_12complex, &(out[0]));
705+ vec_st(vz7, byte_14complex, &(out[0]));
706+ return;
707+
708+}
709+inline static void pass_vsx(FFTComplex * z, const FFTSample * wre, unsigned int n)
710+{
711+ int o1 = n<<1;
712+ int o2 = n<<2;
713+ int o3 = o1+o2;
714+ int i1, i2, i3;
715+ FFTSample* out = (FFTSample*)z;
716+ const FFTSample *wim = wre+o1;
717+ vec_f v0, v1, v2, v3;
718+ vec_f v4, v5, v6, v7;
719+ vec_f v8, v9, v10, v11;
720+ vec_f v12, v13;
721+
722+ n = n-2;
723+ i1 = o1*sizeof(FFTComplex);
724+ i2 = o2*sizeof(FFTComplex);
725+ i3 = o3*sizeof(FFTComplex);
726+
727+ v8 = vec_ld(0, &(wre[0]));
728+ v10 = vec_ld(0, &(wim[0]));
729+ v9 = vec_ld(0, &(wim[-4]));
730+ v9 = vec_perm(v9, v10, vcprm(s0,3,2,1));
731+
732+ v4 = vec_ld(i2, &(out[0]));
733+ v5 = vec_ld(i2+16, &(out[0]));
734+ v6 = vec_ld(i3, &(out[0]));
735+ v7 = vec_ld(i3+16, &(out[0]));
736+ v10 = vec_mul(v4, v8); // r2*wre
737+ v11 = vec_mul(v5, v8); // i2*wre
738+ v12 = vec_mul(v6, v8); // r3*wre
739+ v13 = vec_mul(v7, v8); // i3*wre
740+
741+ v0 = vec_ld(0, &(out[0])); // r0
742+ v3 = vec_ld(i1+16, &(out[0])); // i1
743+ v10 = vec_madd(v5, v9, v10); // r2*wim
744+ v11 = vec_nmsub(v4, v9, v11); // i2*wim
745+ v12 = vec_nmsub(v7, v9, v12); // r3*wim
746+ v13 = vec_madd(v6, v9, v13); // i3*wim
747+
748+ v1 = vec_ld(16, &(out[0])); // i0
749+ v2 = vec_ld(i1, &(out[0])); // r1
750+ v8 = vec_sub(v12, v10);
751+ v12 = vec_add(v12, v10);
752+ v9 = vec_sub(v11, v13);
753+ v13 = vec_add(v11, v13);
754+ v4 = vec_sub(v0, v12);
755+ v0 = vec_add(v0, v12);
756+ v7 = vec_sub(v3, v8);
757+ v3 = vec_add(v3, v8);
758+
759+ vec_st(v0, 0, &(out[0])); // r0
760+ vec_st(v3, i1+16, &(out[0])); // i1
761+ vec_st(v4, i2, &(out[0])); // r2
762+ vec_st(v7, i3+16, &(out[0]));// i3
763+
764+ v5 = vec_sub(v1, v13);
765+ v1 = vec_add(v1, v13);
766+ v6 = vec_sub(v2, v9);
767+ v2 = vec_add(v2, v9);
768+
769+ vec_st(v1, 16, &(out[0])); // i0
770+ vec_st(v2, i1, &(out[0])); // r1
771+ vec_st(v5, i2+16, &(out[0])); // i2
772+ vec_st(v6, i3, &(out[0])); // r3
773+
774+ do {
775+ out += 8;
776+ wre += 4;
777+ wim -= 4;
778+
779+ v8 = vec_ld(0, &(wre[0]));
780+ v10 = vec_ld(0, &(wim[0]));
781+ v9 = vec_ld(0, &(wim[-4]));
782+ v9 = vec_perm(v9, v10, vcprm(s0,3,2,1));
783+
784+ v4 = vec_ld(i2, &(out[0])); // r2
785+ v5 = vec_ld(i2+16, &(out[0])); // i2
786+ v6 = vec_ld(i3, &(out[0])); // r3
787+ v7 = vec_ld(i3+16, &(out[0]));// i3
788+ v10 = vec_mul(v4, v8); // r2*wre
789+ v11 = vec_mul(v5, v8); // i2*wre
790+ v12 = vec_mul(v6, v8); // r3*wre
791+ v13 = vec_mul(v7, v8); // i3*wre
792+
793+ v0 = vec_ld(0, &(out[0])); // r0
794+ v3 = vec_ld(i1+16, &(out[0])); // i1
795+ v10 = vec_madd(v5, v9, v10); // r2*wim
796+ v11 = vec_nmsub(v4, v9, v11); // i2*wim
797+ v12 = vec_nmsub(v7, v9, v12); // r3*wim
798+ v13 = vec_madd(v6, v9, v13); // i3*wim
799+
800+ v1 = vec_ld(16, &(out[0])); // i0
801+ v2 = vec_ld(i1, &(out[0])); // r1
802+ v8 = vec_sub(v12, v10);
803+ v12 = vec_add(v12, v10);
804+ v9 = vec_sub(v11, v13);
805+ v13 = vec_add(v11, v13);
806+ v4 = vec_sub(v0, v12);
807+ v0 = vec_add(v0, v12);
808+ v7 = vec_sub(v3, v8);
809+ v3 = vec_add(v3, v8);
810+
811+ vec_st(v0, 0, &(out[0])); // r0
812+ vec_st(v3, i1+16, &(out[0])); // i1
813+ vec_st(v4, i2, &(out[0])); // r2
814+ vec_st(v7, i3+16, &(out[0])); // i3
815+
816+ v5 = vec_sub(v1, v13);
817+ v1 = vec_add(v1, v13);
818+ v6 = vec_sub(v2, v9);
819+ v2 = vec_add(v2, v9);
820+
821+ vec_st(v1, 16, &(out[0])); // i0
822+ vec_st(v2, i1, &(out[0])); // r1
823+ vec_st(v5, i2+16, &(out[0])); // i2
824+ vec_st(v6, i3, &(out[0])); // r3
825+ } while (n-=2);
826+}
827+
828+#endif
829+
830+#endif /* AVCODEC_PPC_FFT_VSX_H */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t