ppc: Fix the bug of fft for little endian environment on POWER7 and later

Rong Yan committed 12 years ago

Commit ab123731 parent 8e2ade45 files · +1084 −1Browse files

Unified · Split

5 files

configure +14 −1
libavcodec/ppc/Makefile +1 −0
libavcodec/ppc/fft_altivec.c +12 −0
libavcodec/ppc/fft_vsx.c +227 −0
libavcodec/ppc/fft_vsx.h +830 −0

     dcbzl
     ldbrx
     ppc4xx
+    vsx
 "
 ARCH_EXT_LIST_X86="
 altivec_deps="ppc"
 ppc4xx_deps="ppc"
+vsx_deps="ppc"
 cpunop_deps="i686"
 x86_64_select="i686"
 elif enabled ppc; then
     disable ldbrx
+    disable vsx
     case $(tolower $cpu) in
         601|ppc601|powerpc601)
         g5|970|ppc970|powerpc970)
             cpuflags="-mcpu=970"
         ;;
-        power[3-7]*)
+        power[3-8]*)
             cpuflags="-mcpu=$cpu"
         ;;
         cell)
 EOF
 od -t x1 $TMPO | grep -q '42 *49 *47 *45' && enable bigendian
+if  [ "$cpu" = "power7" ] || [ "$cpu" = "power8" ] ;then
+    if ! enabled bigendian ;then
+        enable vsx
+    fi
+fi
 if enabled asm; then
     enabled     arm         && nogas=die
         enabled altivec || warn "Altivec disabled, possibly missing --cpu flag"
     fi
+    if enabled vsx; then
+        check_cflags -mvsx
+    fi
 elif enabled x86; then
     check_builtin rdtsc    intrin.h   "__rdtsc()"
 if enabled ppc; then
     echo "AltiVec enabled           ${altivec-no}"
     echo "PPC 4xx optimizations     ${ppc4xx-no}"
+    echo "PPC VSX optimizations     ${vsx-no}"
     echo "dcbzl available           ${dcbzl-no}"
 fi
 echo "debug symbols             ${debug-no}"

libavcodec/ppc/Makefile+1−0

@@ -26,4 +26,5 @@ ALTIVEC-OBJS-$(CONFIG_DSPUTIL) += ppc/dsputil_altivec.o \ · 25 unmodified lines
26	ppc/int_altivec.o \	26	ppc/int_altivec.o \
27		27
28	FFT-OBJS-$(HAVE_GNU_AS) += ppc/fft_altivec_s.o	28	FFT-OBJS-$(HAVE_GNU_AS) += ppc/fft_altivec_s.o
		29	FFT-OBJS-$(HAVE_VSX) += ppc/fft_vsx.o
29	ALTIVEC-OBJS-$(CONFIG_FFT) += $(FFT-OBJS-yes)	30	ALTIVEC-OBJS-$(CONFIG_FFT) += $(FFT-OBJS-yes)

libavcodec/ppc/fft_altivec.c+12−0

  * It also assumes all FFTComplex are 8 bytes-aligned pairs of floats.
  */
+#if HAVE_VSX
+#include "fft_vsx.h"
+#else
 void ff_fft_calc_altivec(FFTContext *s, FFTComplex *z);
 void ff_fft_calc_interleave_altivec(FFTContext *s, FFTComplex *z);
+#endif
 #if HAVE_GNU_AS && HAVE_ALTIVEC
 static void imdct_half_altivec(FFTContext *s, FFTSample *output, const FFTSample *input)
         k--;
     } while(k >= 0);
+#if HAVE_VSX
+    ff_fft_calc_vsx(s, (FFTComplex*)output);
+#else
     ff_fft_calc_altivec(s, (FFTComplex*)output);
+#endif
     /* post rotation + reordering */
     j = -n32;
     if (!PPC_ALTIVEC(av_get_cpu_flags()))
         return;
+#if HAVE_VSX
+    s->fft_calc = ff_fft_calc_interleave_vsx;
+#else
     s->fft_calc   = ff_fft_calc_interleave_altivec;
+#endif
     if (s->mdct_bits >= 5) {
         s->imdct_calc = imdct_calc_altivec;
         s->imdct_half = imdct_half_altivec;

libavcodec/ppc/fft_vsx.c+227−0

@@ -0,0 +1,227 @@
	1	/*
	2	* FFT transform, optimized with VSX built-in functions
	3	* Copyright (c) 2014 Rong Yan
	4	*
	5	* This algorithm (though not any of the implementation details) is
	6	* based on libdjbfft by D. J. Bernstein.
	7	*
	8	* This file is part of FFmpeg.
	9	*
	10	* FFmpeg is free software; you can redistribute it and/or
	11	* modify it under the terms of the GNU Lesser General Public
	12	* License as published by the Free Software Foundation; either
	13	* version 2.1 of the License, or (at your option) any later version.
	14	*
	15	* FFmpeg is distributed in the hope that it will be useful,
	16	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	17	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	18	* Lesser General Public License for more details.
	19	*
	20	* You should have received a copy of the GNU Lesser General Public
	21	* License along with FFmpeg; if not, write to the Free Software
	22	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	23	*/
	24
	25
	26	#include "config.h"
	27	#include "libavutil/cpu.h"
	28	#include "libavutil/ppc/types_altivec.h"
	29	#include "libavutil/ppc/util_altivec.h"
	30	#include "libavcodec/fft.h"
	31	#include "libavcodec/fft-internal.h"
	32	#include "fft_vsx.h"
	33
	34	#if HAVE_VSX
	35
	36	static void fft32_vsx_interleave(FFTComplex *z)
	37	{
	38	fft16_vsx_interleave(z);
	39	fft8_vsx_interleave(z+16);
	40	fft8_vsx_interleave(z+24);
	41	pass_vsx_interleave(z,ff_cos_32,4);
	42	}
	43
	44	static void fft64_vsx_interleave(FFTComplex *z)
	45	{
	46	fft32_vsx_interleave(z);
	47	fft16_vsx_interleave(z+32);
	48	fft16_vsx_interleave(z+48);
	49	pass_vsx_interleave(z,ff_cos_64, 8);
	50	}
	51	static void fft128_vsx_interleave(FFTComplex *z)
	52	{
	53	fft64_vsx_interleave(z);
	54	fft32_vsx_interleave(z+64);
	55	fft32_vsx_interleave(z+96);
	56	pass_vsx_interleave(z,ff_cos_128,16);
	57	}
	58	static void fft256_vsx_interleave(FFTComplex *z)
	59	{
	60	fft128_vsx_interleave(z);
	61	fft64_vsx_interleave(z+128);
	62	fft64_vsx_interleave(z+192);
	63	pass_vsx_interleave(z,ff_cos_256,32);
	64	}
	65	static void fft512_vsx_interleave(FFTComplex *z)
	66	{
	67	fft256_vsx_interleave(z);
	68	fft128_vsx_interleave(z+256);
	69	fft128_vsx_interleave(z+384);
	70	pass_vsx_interleave(z,ff_cos_512,64);
	71	}
	72	static void fft1024_vsx_interleave(FFTComplex *z)
	73	{
	74	fft512_vsx_interleave(z);
	75	fft256_vsx_interleave(z+512);
	76	fft256_vsx_interleave(z+768);
	77	pass_vsx_interleave(z,ff_cos_1024,128);
	78
	79	}
	80	static void fft2048_vsx_interleave(FFTComplex *z)
	81	{
	82	fft1024_vsx_interleave(z);
	83	fft512_vsx_interleave(z+1024);
	84	fft512_vsx_interleave(z+1536);
	85	pass_vsx_interleave(z,ff_cos_2048,256);
	86	}
	87	static void fft4096_vsx_interleave(FFTComplex *z)
	88	{
	89	fft2048_vsx_interleave(z);
	90	fft1024_vsx_interleave(z+2048);
	91	fft1024_vsx_interleave(z+3072);
	92	pass_vsx_interleave(z,ff_cos_4096, 512);
	93	}
	94	static void fft8192_vsx_interleave(FFTComplex *z)
	95	{
	96	fft4096_vsx_interleave(z);
	97	fft2048_vsx_interleave(z+4096);
	98	fft2048_vsx_interleave(z+6144);
	99	pass_vsx_interleave(z,ff_cos_8192,1024);
	100	}
	101	static void fft16384_vsx_interleave(FFTComplex *z)
	102	{
	103	fft8192_vsx_interleave(z);
	104	fft4096_vsx_interleave(z+8192);
	105	fft4096_vsx_interleave(z+12288);
	106	pass_vsx_interleave(z,ff_cos_16384,2048);
	107	}
	108	static void fft32768_vsx_interleave(FFTComplex *z)
	109	{
	110	fft16384_vsx_interleave(z);
	111	fft8192_vsx_interleave(z+16384);
	112	fft8192_vsx_interleave(z+24576);
	113	pass_vsx_interleave(z,ff_cos_32768,4096);
	114	}
	115	static void fft65536_vsx_interleave(FFTComplex *z)
	116	{
	117	fft32768_vsx_interleave(z);
	118	fft16384_vsx_interleave(z+32768);
	119	fft16384_vsx_interleave(z+49152);
	120	pass_vsx_interleave(z,ff_cos_65536,8192);
	121	}
	122
	123	static void fft32_vsx(FFTComplex *z)
	124	{
	125	fft16_vsx(z);
	126	fft8_vsx(z+16);
	127	fft8_vsx(z+24);
	128	pass_vsx(z,ff_cos_32,4);
	129	}
	130
	131	static void fft64_vsx(FFTComplex *z)
	132	{
	133	fft32_vsx(z);
	134	fft16_vsx(z+32);
	135	fft16_vsx(z+48);
	136	pass_vsx(z,ff_cos_64, 8);
	137	}
	138	static void fft128_vsx(FFTComplex *z)
	139	{
	140	fft64_vsx(z);
	141	fft32_vsx(z+64);
	142	fft32_vsx(z+96);
	143	pass_vsx(z,ff_cos_128,16);
	144	}
	145	static void fft256_vsx(FFTComplex *z)
	146	{
	147	fft128_vsx(z);
	148	fft64_vsx(z+128);
	149	fft64_vsx(z+192);
	150	pass_vsx(z,ff_cos_256,32);
	151	}
	152	static void fft512_vsx(FFTComplex *z)
	153	{
	154	fft256_vsx(z);
	155	fft128_vsx(z+256);
	156	fft128_vsx(z+384);
	157	pass_vsx(z,ff_cos_512,64);
	158	}
	159	static void fft1024_vsx(FFTComplex *z)
	160	{
	161	fft512_vsx(z);
	162	fft256_vsx(z+512);
	163	fft256_vsx(z+768);
	164	pass_vsx(z,ff_cos_1024,128);
	165
	166	}
	167	static void fft2048_vsx(FFTComplex *z)
	168	{
	169	fft1024_vsx(z);
	170	fft512_vsx(z+1024);
	171	fft512_vsx(z+1536);
	172	pass_vsx(z,ff_cos_2048,256);
	173	}
	174	static void fft4096_vsx(FFTComplex *z)
	175	{
	176	fft2048_vsx(z);
	177	fft1024_vsx(z+2048);
	178	fft1024_vsx(z+3072);
	179	pass_vsx(z,ff_cos_4096, 512);
	180	}
	181	static void fft8192_vsx(FFTComplex *z)
	182	{
	183	fft4096_vsx(z);
	184	fft2048_vsx(z+4096);
	185	fft2048_vsx(z+6144);
	186	pass_vsx(z,ff_cos_8192,1024);
	187	}
	188	static void fft16384_vsx(FFTComplex *z)
	189	{
	190	fft8192_vsx(z);
	191	fft4096_vsx(z+8192);
	192	fft4096_vsx(z+12288);
	193	pass_vsx(z,ff_cos_16384,2048);
	194	}
	195	static void fft32768_vsx(FFTComplex *z)
	196	{
	197	fft16384_vsx(z);
	198	fft8192_vsx(z+16384);
	199	fft8192_vsx(z+24576);
	200	pass_vsx(z,ff_cos_32768,4096);
	201	}
	202	static void fft65536_vsx(FFTComplex *z)
	203	{
	204	fft32768_vsx(z);
	205	fft16384_vsx(z+32768);
	206	fft16384_vsx(z+49152);
	207	pass_vsx(z,ff_cos_65536,8192);
	208	}
	209
	210	static void (* const fft_dispatch_vsx[])(FFTComplex*) = {
	211	fft4_vsx, fft8_vsx, fft16_vsx, fft32_vsx, fft64_vsx, fft128_vsx, fft256_vsx, fft512_vsx, fft1024_vsx,
	212	fft2048_vsx, fft4096_vsx, fft8192_vsx, fft16384_vsx, fft32768_vsx, fft65536_vsx,
	213	};
	214	static void (* const fft_dispatch_vsx_interleave[])(FFTComplex*) = {
	215	fft4_vsx_interleave, fft8_vsx_interleave, fft16_vsx_interleave, fft32_vsx_interleave, fft64_vsx_interleave,
	216	fft128_vsx_interleave, fft256_vsx_interleave, fft512_vsx_interleave, fft1024_vsx_interleave,
	217	fft2048_vsx_interleave, fft4096_vsx_interleave, fft8192_vsx_interleave, fft16384_vsx_interleave, fft32768_vsx_interleave, fft65536_vsx_interleave,
	218	};
	219	void ff_fft_calc_interleave_vsx(FFTContext s, FFTComplex z)
	220	{
	221	fft_dispatch_vsx_interleave[s->nbits-2](z);
	222	}
	223	void ff_fft_calc_vsx(FFTContext s, FFTComplex z)
	224	{
	225	fft_dispatch_vsx[s->nbits-2](z);
	226	}
	227	#endif /* HAVE_VSX */

libavcodec/ppc/fft_vsx.h+830−0

@@ -0,0 +1,830 @@
	1	#ifndef AVCODEC_PPC_FFT_VSX_H
	2	#define AVCODEC_PPC_FFT_VSX_H
	3	/*
	4	* FFT transform, optimized with VSX built-in functions
	5	* Copyright (c) 2014 Rong Yan Copyright (c) 2009 Loren Merritt
	6	*
	7	* This algorithm (though not any of the implementation details) is
	8	* based on libdjbfft by D. J. Bernstein, and fft_altivec_s.S.
	9	*
	10	* This file is part of FFmpeg.
	11	*
	12	* FFmpeg is free software; you can redistribute it and/or
	13	* modify it under the terms of the GNU Lesser General Public
	14	* License as published by the Free Software Foundation; either
	15	* version 2.1 of the License, or (at your option) any later version.
	16	*
	17	* FFmpeg is distributed in the hope that it will be useful,
	18	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	20	* Lesser General Public License for more details.
	21	*
	22	* You should have received a copy of the GNU Lesser General Public
	23	* License along with FFmpeg; if not, write to the Free Software
	24	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	25	*/
	26
	27
	28	#include "config.h"
	29	#include "libavutil/cpu.h"
	30	#include "libavutil/ppc/types_altivec.h"
	31	#include "libavutil/ppc/util_altivec.h"
	32	#include "libavcodec/fft.h"
	33	#include "libavcodec/fft-internal.h"
	34
	35	#if HAVE_VSX
	36
	37	void ff_fft_calc_interleave_vsx(FFTContext s, FFTComplex z);
	38	void ff_fft_calc_vsx(FFTContext s, FFTComplex z);
	39
	40
	41	#define byte_2complex (2*sizeof(FFTComplex))
	42	#define byte_4complex (4*sizeof(FFTComplex))
	43	#define byte_6complex (6*sizeof(FFTComplex))
	44	#define byte_8complex (8*sizeof(FFTComplex))
	45	#define byte_10complex (10*sizeof(FFTComplex))
	46	#define byte_12complex (12*sizeof(FFTComplex))
	47	#define byte_14complex (14*sizeof(FFTComplex))
	48
	49	inline static void pass_vsx_interleave(FFTComplex z, const FFTSample wre, unsigned int n)
	50	{
	51	int o1 = n<<1;
	52	int o2 = n<<2;
	53	int o3 = o1+o2;
	54	int i1, i2, i3;
	55	FFTSample* out = (FFTSample*)z;
	56	const FFTSample *wim = wre+o1;
	57	vec_f vz0, vzo1, vzo2, vzo3;
	58	vec_f x0, x1, x2, x3;
	59	vec_f x4, x5, x6, x7;
	60	vec_f x8, x9, x10, x11;
	61	vec_f x12, x13, x14, x15;
	62	vec_f x16, x17, x18, x19;
	63	vec_f x20, x21, x22, x23;
	64	vec_f vz0plus1, vzo1plus1, vzo2plus1, vzo3plus1;
	65	vec_f y0, y1, y2, y3;
	66	vec_f y4, y5, y8, y9;
	67	vec_f y10, y13, y14, y15;
	68	vec_f y16, y17, y18, y19;
	69	vec_f y20, y21, y22, y23;
	70	vec_f wr1, wi1, wr0, wi0;
	71	vec_f wr2, wi2, wr3, wi3;
	72	vec_f xmulwi0, xmulwi1, ymulwi2, ymulwi3;
	73
	74	n = n-2;
	75	i1 = o1*sizeof(FFTComplex);
	76	i2 = o2*sizeof(FFTComplex);
	77	i3 = o3*sizeof(FFTComplex);
	78	vzo2 = vec_ld(i2, &(out[0])); // zo2.r zo2.i z(o2+1).r z(o2+1).i
	79	vzo2plus1 = vec_ld(i2+16, &(out[0]));
	80	vzo3 = vec_ld(i3, &(out[0])); // zo3.r zo3.i z(o3+1).r z(o3+1).i
	81	vzo3plus1 = vec_ld(i3+16, &(out[0]));
	82	vz0 = vec_ld(0, &(out[0])); // z0.r z0.i z1.r z1.i
	83	vz0plus1 = vec_ld(16, &(out[0]));
	84	vzo1 = vec_ld(i1, &(out[0])); // zo1.r zo1.i z(o1+1).r z(o1+1).i
	85	vzo1plus1 = vec_ld(i1+16, &(out[0]));
	86
	87	x0 = vec_add(vzo2, vzo3);
	88	x1 = vec_sub(vzo2, vzo3);
	89	y0 = vec_add(vzo2plus1, vzo3plus1);
	90	y1 = vec_sub(vzo2plus1, vzo3plus1);
	91
	92	wr1 = vec_splats(wre[1]);
	93	wi1 = vec_splats(wim[-1]);
	94	wi2 = vec_splats(wim[-2]);
	95	wi3 = vec_splats(wim[-3]);
	96	wr2 = vec_splats(wre[2]);
	97	wr3 = vec_splats(wre[3]);
	98
	99	x2 = vec_perm(x0, x1, vcprm(2,s2,3,s3));
	100	x3 = vec_perm(x0, x1, vcprm(s3,3,s2,2));
	101
	102	y4 = vec_perm(y0, y1, vcprm(s1,1,s0,0));
	103	y5 = vec_perm(y0, y1, vcprm(s3,3,s2,2));
	104	y2 = vec_perm(y0, y1, vcprm(0,s0,1,s1));
	105	y3 = vec_perm(y0, y1, vcprm(2,s2,3,s3));
	106
	107	ymulwi2 = vec_mul(y4, wi2);
	108	ymulwi3 = vec_mul(y5, wi3);
	109	x4 = vec_mul(x2, wr1);
	110	x5 = vec_mul(x3, wi1);
	111	y8 = vec_madd(y2, wr2, ymulwi2);
	112	y9 = vec_msub(y2, wr2, ymulwi2);
	113	x6 = vec_add(x4, x5);
	114	x7 = vec_sub(x4, x5);
	115	y13 = vec_madd(y3, wr3, ymulwi3);
	116	y14 = vec_msub(y3, wr3, ymulwi3);
	117
	118	x8 = vec_perm(x6, x7, vcprm(0,1,s2,s3));
	119	y10 = vec_perm(y8, y9, vcprm(0,1,s2,s3));
	120	y15 = vec_perm(y13, y14, vcprm(0,1,s2,s3));
	121
	122	x9 = vec_perm(x0, x8, vcprm(0,1,s0,s2));
	123	x10 = vec_perm(x1, x8, vcprm(1,0,s3,s1));
	124
	125	y16 = vec_perm(y10, y15, vcprm(0,2,s0,s2));
	126	y17 = vec_perm(y10, y15, vcprm(3,1,s3,s1));
	127
	128	x11 = vec_add(vz0, x9);
	129	x12 = vec_sub(vz0, x9);
	130	x13 = vec_add(vzo1, x10);
	131	x14 = vec_sub(vzo1, x10);
	132
	133	y18 = vec_add(vz0plus1, y16);
	134	y19 = vec_sub(vz0plus1, y16);
	135	y20 = vec_add(vzo1plus1, y17);
	136	y21 = vec_sub(vzo1plus1, y17);
	137
	138	x15 = vec_perm(x13, x14, vcprm(0,s1,2,s3));
	139	x16 = vec_perm(x13, x14, vcprm(s0,1,s2,3));
	140	y22 = vec_perm(y20, y21, vcprm(0,s1,2,s3));
	141	y23 = vec_perm(y20, y21, vcprm(s0,1,s2,3));
	142
	143
	144	vec_st(x11, 0, &(out[0]));
	145	vec_st(y18, 16, &(out[0]));
	146	vec_st(x15, i1, &(out[0]));
	147	vec_st(y22, i1+16, &(out[0]));
	148	vec_st(x12, i2, &(out[0]));
	149	vec_st(y19, i2+16, &(out[0]));
	150	vec_st(x16, i3, &(out[0]));
	151	vec_st(y23, i3+16, &(out[0]));
	152
	153	do {
	154	out += 8;
	155	wre += 4;
	156	wim -= 4;
	157	wr0 = vec_splats(wre[0]);
	158	wr1 = vec_splats(wre[1]);
	159	wi0 = vec_splats(wim[0]);
	160	wi1 = vec_splats(wim[-1]);
	161
	162	wr2 = vec_splats(wre[2]);
	163	wr3 = vec_splats(wre[3]);
	164	wi2 = vec_splats(wim[-2]);
	165	wi3 = vec_splats(wim[-3]);
	166
	167	vzo2 = vec_ld(i2, &(out[0])); // zo2.r zo2.i z(o2+1).r z(o2+1).i
	168	vzo2plus1 = vec_ld(i2+16, &(out[0]));
	169	vzo3 = vec_ld(i3, &(out[0])); // zo3.r zo3.i z(o3+1).r z(o3+1).i
	170	vzo3plus1 = vec_ld(i3+16, &(out[0]));
	171	vz0 = vec_ld(0, &(out[0])); // z0.r z0.i z1.r z1.i
	172	vz0plus1 = vec_ld(16, &(out[0]));
	173	vzo1 = vec_ld(i1, &(out[0])); // zo1.r zo1.i z(o1+1).r z(o1+1).i
	174	vzo1plus1 = vec_ld(i1+16, &(out[0]));
	175
	176	x0 = vec_add(vzo2, vzo3);
	177	x1 = vec_sub(vzo2, vzo3);
	178
	179	y0 = vec_add(vzo2plus1, vzo3plus1);
	180	y1 = vec_sub(vzo2plus1, vzo3plus1);
	181
	182	x4 = vec_perm(x0, x1, vcprm(s1,1,s0,0));
	183	x5 = vec_perm(x0, x1, vcprm(s3,3,s2,2));
	184	x2 = vec_perm(x0, x1, vcprm(0,s0,1,s1));
	185	x3 = vec_perm(x0, x1, vcprm(2,s2,3,s3));
	186
	187	y2 = vec_perm(y0, y1, vcprm(0,s0,1,s1));
	188	y3 = vec_perm(y0, y1, vcprm(2,s2,3,s3));
	189	xmulwi0 = vec_mul(x4, wi0);
	190	xmulwi1 = vec_mul(x5, wi1);
	191
	192	y4 = vec_perm(y0, y1, vcprm(s1,1,s0,0));
	193	y5 = vec_perm(y0, y1, vcprm(s3,3,s2,2));
	194
	195	x8 = vec_madd(x2, wr0, xmulwi0);
	196	x9 = vec_msub(x2, wr0, xmulwi0);
	197	ymulwi2 = vec_mul(y4, wi2);
	198	ymulwi3 = vec_mul(y5, wi3);
	199
	200	x13 = vec_madd(x3, wr1, xmulwi1);
	201	x14 = vec_msub(x3, wr1, xmulwi1);
	202
	203	y8 = vec_madd(y2, wr2, ymulwi2);
	204	y9 = vec_msub(y2, wr2, ymulwi2);
	205	y13 = vec_madd(y3, wr3, ymulwi3);
	206	y14 = vec_msub(y3, wr3, ymulwi3);
	207
	208	x10 = vec_perm(x8, x9, vcprm(0,1,s2,s3));
	209	x15 = vec_perm(x13, x14, vcprm(0,1,s2,s3));
	210
	211	y10 = vec_perm(y8, y9, vcprm(0,1,s2,s3));
	212	y15 = vec_perm(y13, y14, vcprm(0,1,s2,s3));
	213
	214	x16 = vec_perm(x10, x15, vcprm(0,2,s0,s2));
	215	x17 = vec_perm(x10, x15, vcprm(3,1,s3,s1));
	216
	217	y16 = vec_perm(y10, y15, vcprm(0,2,s0,s2));
	218	y17 = vec_perm(y10, y15, vcprm(3,1,s3,s1));
	219
	220	x18 = vec_add(vz0, x16);
	221	x19 = vec_sub(vz0, x16);
	222	x20 = vec_add(vzo1, x17);
	223	x21 = vec_sub(vzo1, x17);
	224
	225	y18 = vec_add(vz0plus1, y16);
	226	y19 = vec_sub(vz0plus1, y16);
	227	y20 = vec_add(vzo1plus1, y17);
	228	y21 = vec_sub(vzo1plus1, y17);
	229
	230	x22 = vec_perm(x20, x21, vcprm(0,s1,2,s3));
	231	x23 = vec_perm(x20, x21, vcprm(s0,1,s2,3));
	232
	233	y22 = vec_perm(y20, y21, vcprm(0,s1,2,s3));
	234	y23 = vec_perm(y20, y21, vcprm(s0,1,s2,3));
	235
	236	vec_st(x18, 0, &(out[0]));
	237	vec_st(y18, 16, &(out[0]));
	238	vec_st(x22, i1, &(out[0]));
	239	vec_st(y22, i1+16, &(out[0]));
	240	vec_st(x19, i2, &(out[0]));
	241	vec_st(y19, i2+16, &(out[0]));
	242	vec_st(x23, i3, &(out[0]));
	243	vec_st(y23, i3+16, &(out[0]));
	244	} while (n-=2);
	245	}
	246
	247	inline static void fft2_vsx_interleave(FFTComplex *z)
	248	{
	249	FFTSample r1, i1;
	250
	251	r1 = z[0].re - z[1].re;
	252	z[0].re += z[1].re;
	253	z[1].re = r1;
	254
	255	i1 = z[0].im - z[1].im;
	256	z[0].im += z[1].im;
	257	z[1].im = i1;
	258	}
	259
	260	inline static void fft4_vsx_interleave(FFTComplex *z)
	261	{
	262	vec_f a, b, c, d;
	263	float* out= (float*)z;
	264	a = vec_ld(0, &(out[0]));
	265	b = vec_ld(byte_2complex, &(out[0]));
	266
	267	c = vec_perm(a, b, vcprm(0,1,s2,s1));
	268	d = vec_perm(a, b, vcprm(2,3,s0,s3));
	269	a = vec_add(c, d);
	270	b = vec_sub(c, d);
	271
	272	c = vec_perm(a, b, vcprm(0,1,s0,s1));
	273	d = vec_perm(a, b, vcprm(2,3,s3,s2));
	274
	275	a = vec_add(c, d);
	276	b = vec_sub(c, d);
	277	vec_st(a, 0, &(out[0]));
	278	vec_st(b, byte_2complex, &(out[0]));
	279	}
	280
	281	inline static void fft8_vsx_interleave(FFTComplex *z)
	282	{
	283	vec_f vz0, vz1, vz2, vz3;
	284	vec_f x0, x1, x2, x3;
	285	vec_f x4, x5, x6, x7;
	286	vec_f x8, x9, x10, x11;
	287	vec_f x12, x13, x14, x15;
	288	vec_f x16, x17, x18, x19;
	289	vec_f x20, x21, x22, x23;
	290	vec_f x24, x25, x26, x27;
	291	vec_f x28, x29, x30, x31;
	292	vec_f x32, x33, x34;
	293
	294	float* out= (float*)z;
	295	vec_f vc1 = {sqrthalf, sqrthalf, sqrthalf, sqrthalf};
	296
	297	vz0 = vec_ld(0, &(out[0]));
	298	vz1 = vec_ld(byte_2complex, &(out[0]));
	299	vz2 = vec_ld(byte_4complex, &(out[0]));
	300	vz3 = vec_ld(byte_6complex, &(out[0]));
	301
	302	x0 = vec_perm(vz0, vz1, vcprm(0,1,s2,s1));
	303	x1 = vec_perm(vz0, vz1, vcprm(2,3,s0,s3));
	304	x2 = vec_perm(vz2, vz3, vcprm(2,1,s0,s1));
	305	x3 = vec_perm(vz2, vz3, vcprm(0,3,s2,s3));
	306
	307	x4 = vec_add(x0, x1);
	308	x5 = vec_sub(x0, x1);
	309	x6 = vec_add(x2, x3);
	310	x7 = vec_sub(x2, x3);
	311
	312	x8 = vec_perm(x4, x5, vcprm(0,1,s0,s1));
	313	x9 = vec_perm(x4, x5, vcprm(2,3,s3,s2));
	314	x10 = vec_perm(x6, x7, vcprm(2,1,s2,s1));
	315	x11 = vec_perm(x6, x7, vcprm(0,3,s0,s3));
	316
	317	x12 = vec_add(x8, x9);
	318	x13 = vec_sub(x8, x9);
	319	x14 = vec_add(x10, x11);
	320	x15 = vec_sub(x10, x11);
	321	x16 = vec_perm(x12, x13, vcprm(0,s0,1,s1));
	322	x17 = vec_perm(x14, x15, vcprm(0,s0,1,s1));
	323	x18 = vec_perm(x16, x17, vcprm(s0,s3,s2,s1));
	324	x19 = vec_add(x16, x18); // z0.r z2.r z0.i z2.i
	325	x20 = vec_sub(x16, x18); // z4.r z6.r z4.i z6.i
	326
	327	x21 = vec_perm(x12, x13, vcprm(2,s2,3,s3));
	328	x22 = vec_perm(x14, x15, vcprm(2,3,s2,s3));
	329	x23 = vec_perm(x14, x15, vcprm(3,2,s3,s2));
	330	x24 = vec_add(x22, x23);
	331	x25 = vec_sub(x22, x23);
	332	x26 = vec_mul( vec_perm(x24, x25, vcprm(2,s2,0,s0)), vc1);
	333
	334	x27 = vec_add(x21, x26); // z1.r z7.r z1.i z3.i
	335	x28 = vec_sub(x21, x26); //z5.r z3.r z5.i z7.i
	336
	337	x29 = vec_perm(x19, x27, vcprm(0,2,s0,s2)); // z0.r z0.i z1.r z1.i
	338	x30 = vec_perm(x19, x27, vcprm(1,3,s1,s3)); // z2.r z2.i z7.r z3.i
	339	x31 = vec_perm(x20, x28, vcprm(0,2,s0,s2)); // z4.r z4.i z5.r z5.i
	340	x32 = vec_perm(x20, x28, vcprm(1,3,s1,s3)); // z6.r z6.i z3.r z7.i
	341	x33 = vec_perm(x30, x32, vcprm(0,1,s2,3)); // z2.r z2.i z3.r z3.i
	342	x34 = vec_perm(x30, x32, vcprm(s0,s1,2,s3)); // z6.r z6.i z7.r z7.i
	343
	344	vec_st(x29, 0, &(out[0]));
	345	vec_st(x33, byte_2complex, &(out[0]));
	346	vec_st(x31, byte_4complex, &(out[0]));
	347	vec_st(x34, byte_6complex, &(out[0]));
	348	}
	349
	350	inline static void fft16_vsx_interleave(FFTComplex *z)
	351	{
	352	float* out= (float*)z;
	353	vec_f vc0 = {sqrthalf, sqrthalf, sqrthalf, sqrthalf};
	354	vec_f vc1 = {ff_cos_16[1], ff_cos_16[1], ff_cos_16[1], ff_cos_16[1]};
	355	vec_f vc2 = {ff_cos_16[3], ff_cos_16[3], ff_cos_16[3], ff_cos_16[3]};
	356	vec_f vz0, vz1, vz2, vz3;
	357	vec_f vz4, vz5, vz6, vz7;
	358	vec_f x0, x1, x2, x3;
	359	vec_f x4, x5, x6, x7;
	360	vec_f x8, x9, x10, x11;
	361	vec_f x12, x13, x14, x15;
	362	vec_f x16, x17, x18, x19;
	363	vec_f x20, x21, x22, x23;
	364	vec_f x24, x25, x26, x27;
	365	vec_f x28, x29, x30, x31;
	366	vec_f x32, x33, x34, x35;
	367	vec_f x36, x37, x38, x39;
	368	vec_f x40, x41, x42, x43;
	369	vec_f x44, x45, x46, x47;
	370	vec_f x48, x49, x50, x51;
	371	vec_f x52, x53, x54, x55;
	372	vec_f x56, x57, x58, x59;
	373	vec_f x60, x61, x62, x63;
	374	vec_f x64, x65, x66, x67;
	375	vec_f x68, x69, x70, x71;
	376	vec_f x72, x73, x74, x75;
	377	vec_f x76, x77, x78, x79;
	378	vec_f x80, x81, x82, x83;
	379	vec_f x84, x85, x86;
	380
	381	vz0 = vec_ld(0, &(out[0]));
	382	vz1 = vec_ld(byte_2complex, &(out[0]));
	383	vz2 = vec_ld(byte_4complex, &(out[0]));
	384	vz3 = vec_ld(byte_6complex, &(out[0]));
	385	vz4 = vec_ld(byte_8complex, &(out[0]));
	386	vz5 = vec_ld(byte_10complex, &(out[0]));
	387	vz6 = vec_ld(byte_12complex, &(out[0]));
	388	vz7 = vec_ld(byte_14complex, &(out[0]));
	389
	390	x0 = vec_perm(vz0, vz1, vcprm(0,1,s2,s1));
	391	x1 = vec_perm(vz0, vz1, vcprm(2,3,s0,s3));
	392	x2 = vec_perm(vz2, vz3, vcprm(0,1,s0,s1));
	393	x3 = vec_perm(vz2, vz3, vcprm(2,3,s2,s3));
	394
	395	x4 = vec_perm(vz4, vz5, vcprm(0,1,s2,s1));
	396	x5 = vec_perm(vz4, vz5, vcprm(2,3,s0,s3));
	397	x6 = vec_perm(vz6, vz7, vcprm(0,1,s2,s1));
	398	x7 = vec_perm(vz6, vz7, vcprm(2,3,s0,s3));
	399
	400	x8 = vec_add(x0, x1);
	401	x9 = vec_sub(x0, x1);
	402	x10 = vec_add(x2, x3);
	403	x11 = vec_sub(x2, x3);
	404
	405	x12 = vec_add(x4, x5);
	406	x13 = vec_sub(x4, x5);
	407	x14 = vec_add(x6, x7);
	408	x15 = vec_sub(x6, x7);
	409
	410	x16 = vec_perm(x8, x9, vcprm(0,1,s0,s1));
	411	x17 = vec_perm(x8, x9, vcprm(2,3,s3,s2));
	412	x18 = vec_perm(x10, x11, vcprm(2,1,s1,s2));
	413	x19 = vec_perm(x10, x11, vcprm(0,3,s0,s3));
	414	x20 = vec_perm(x12, x14, vcprm(0,1,s0, s1));
	415	x21 = vec_perm(x12, x14, vcprm(2,3,s2,s3));
	416	x22 = vec_perm(x13, x15, vcprm(0,1,s0,s1));
	417	x23 = vec_perm(x13, x15, vcprm(3,2,s3,s2));
	418
	419	x24 = vec_add(x16, x17);
	420	x25 = vec_sub(x16, x17);
	421	x26 = vec_add(x18, x19);
	422	x27 = vec_sub(x18, x19);
	423	x28 = vec_add(x20, x21);
	424	x29 = vec_sub(x20, x21);
	425	x30 = vec_add(x22, x23);
	426	x31 = vec_sub(x22, x23);
	427
	428	x32 = vec_add(x24, x26);
	429	x33 = vec_sub(x24, x26);
	430	x34 = vec_perm(x32, x33, vcprm(0,1,s0,s1));
	431
	432	x35 = vec_perm(x28, x29, vcprm(2,1,s1,s2));
	433	x36 = vec_perm(x28, x29, vcprm(0,3,s0,s3));
	434	x37 = vec_add(x35, x36);
	435	x38 = vec_sub(x35, x36);
	436	x39 = vec_perm(x37, x38, vcprm(0,1,s1,s0));
	437
	438	x40 = vec_perm(x27, x38, vcprm(3,2,s2,s3));
	439	x41 = vec_perm(x26, x37, vcprm(2,3,s3,s2));
	440	x42 = vec_add(x40, x41);
	441	x43 = vec_sub(x40, x41);
	442	x44 = vec_mul(x42, vc0);
	443	x45 = vec_mul(x43, vc0);
	444
	445	x46 = vec_add(x34, x39); // z0.r z0.i z4.r z4.i
	446	x47 = vec_sub(x34, x39); // z8.r z8.i z12.r z12.i
	447
	448	x48 = vec_perm(x30, x31, vcprm(2,1,s1,s2));
	449	x49 = vec_perm(x30, x31, vcprm(0,3,s3,s0));
	450	x50 = vec_add(x48, x49);
	451	x51 = vec_sub(x48, x49);
	452	x52 = vec_mul(x50, vc1);
	453	x53 = vec_mul(x50, vc2);
	454	x54 = vec_mul(x51, vc1);
	455	x55 = vec_mul(x51, vc2);
	456
	457	x56 = vec_perm(x24, x25, vcprm(2,3,s2,s3));
	458	x57 = vec_perm(x44, x45, vcprm(0,1,s1,s0));
	459	x58 = vec_add(x56, x57);
	460	x59 = vec_sub(x56, x57);
	461
	462	x60 = vec_perm(x54, x55, vcprm(1,0,3,2));
	463	x61 = vec_perm(x54, x55, vcprm(s1,s0,s3,s2));
	464	x62 = vec_add(x52, x61);
	465	x63 = vec_sub(x52, x61);
	466	x64 = vec_add(x60, x53);
	467	x65 = vec_sub(x60, x53);
	468	x66 = vec_perm(x62, x64, vcprm(0,1,s3,s2));
	469	x67 = vec_perm(x63, x65, vcprm(s0,s1,3,2));
	470
	471	x68 = vec_add(x58, x66); // z1.r z1.i z3.r z3.i
	472	x69 = vec_sub(x58, x66); // z9.r z9.i z11.r z11.i
	473	x70 = vec_add(x59, x67); // z5.r z5.i z15.r z15.i
	474	x71 = vec_sub(x59, x67); // z13.r z13.i z7.r z7.i
	475
	476	x72 = vec_perm(x25, x27, vcprm(s1,s0,s2,s3));
	477	x73 = vec_add(x25, x72);
	478	x74 = vec_sub(x25, x72);
	479	x75 = vec_perm(x73, x74, vcprm(0,1,s0,s1));
	480	x76 = vec_perm(x44, x45, vcprm(3,2,s2,s3));
	481	x77 = vec_add(x75, x76); // z2.r z2.i z6.r z6.i
	482	x78 = vec_sub(x75, x76); // z10.r z10.i z14.r z14.i
	483
	484	x79 = vec_perm(x46, x68, vcprm(0,1,s0,s1)); // z0.r z0.i z1.r z1.i
	485	x80 = vec_perm(x77, x68, vcprm(0,1,s2,s3)); // z2.r z2.i z3.r z3.i
	486	x81 = vec_perm(x46, x70, vcprm(2,3,s0,s1)); // z4.r z4.i z5.r z5.i
	487	x82 = vec_perm(x71, x77, vcprm(s2,s3,2,3)); // z6.r z6.i z7.r z7.i
	488	vec_st(x79, 0, &(out[0]));
	489	vec_st(x80, byte_2complex, &(out[0]));
	490	vec_st(x81, byte_4complex, &(out[0]));
	491	vec_st(x82, byte_6complex, &(out[0]));
	492	x83 = vec_perm(x47, x69, vcprm(0,1,s0,s1)); // z8.r z8.i z9.r z9.i
	493	x84 = vec_perm(x78, x69, vcprm(0,1,s2,s3)); // z10.r z10.i z11.r z11.i
	494	x85 = vec_perm(x47, x71, vcprm(2,3,s0,s1)); // z12.r z12.i z13.r z13.i
	495	x86 = vec_perm(x70, x78, vcprm(s2,s3,2,3)); // z14.r z14.i z15.r z15.i
	496	vec_st(x83, byte_8complex, &(out[0]));
	497	vec_st(x84, byte_10complex, &(out[0]));
	498	vec_st(x85, byte_12complex, &(out[0]));
	499	vec_st(x86, byte_14complex, &(out[0]));
	500	}
	501
	502	inline static void fft4_vsx(FFTComplex *z)
	503	{
	504	vec_f a, b, c, d;
	505	float* out= (float*)z;
	506	a = vec_ld(0, &(out[0]));
	507	b = vec_ld(byte_2complex, &(out[0]));
	508
	509	c = vec_perm(a, b, vcprm(0,1,s2,s1));
	510	d = vec_perm(a, b, vcprm(2,3,s0,s3));
	511	a = vec_add(c, d);
	512	b = vec_sub(c, d);
	513
	514	c = vec_perm(a,b, vcprm(0,s0,1,s1));
	515	d = vec_perm(a, b, vcprm(2,s3,3,s2));
	516
	517	a = vec_add(c, d);
	518	b = vec_sub(c, d);
	519
	520	c = vec_perm(a, b, vcprm(0,1,s0,s1));
	521	d = vec_perm(a, b, vcprm(2,3,s2,s3));
	522
	523	vec_st(c, 0, &(out[0]));
	524	vec_st(d, byte_2complex, &(out[0]));
	525	return;
	526	}
	527
	528	inline static void fft8_vsx(FFTComplex *z)
	529	{
	530	vec_f vz0, vz1, vz2, vz3;
	531	vec_f vz4, vz5, vz6, vz7, vz8;
	532
	533	float* out= (float*)z;
	534	vec_f vc0 = {0.0, 0.0, 0.0, 0.0};
	535	vec_f vc1 = {-sqrthalf, sqrthalf, sqrthalf, -sqrthalf};
	536	vec_f vc2 = {sqrthalf, sqrthalf, sqrthalf, sqrthalf};
	537
	538	vz0 = vec_ld(0, &(out[0]));
	539	vz1 = vec_ld(byte_2complex, &(out[0]));
	540	vz2 = vec_ld(byte_4complex, &(out[0]));
	541	vz3 = vec_ld(byte_6complex, &(out[0]));
	542
	543	vz6 = vec_perm(vz2, vz3, vcprm(0,s0,1,s1));
	544	vz7 = vec_perm(vz2, vz3, vcprm(2,s2,3,s3));
	545	vz4 = vec_perm(vz0, vz1, vcprm(0,1,s2,s1));
	546	vz5 = vec_perm(vz0, vz1, vcprm(2,3,s0,s3));
	547
	548	vz2 = vec_add(vz6, vz7);
	549	vz3 = vec_sub(vz6, vz7);
	550	vz8 = vec_perm(vz3, vz3, vcprm(2,3,0,1));
	551
	552	vz0 = vec_add(vz4, vz5);
	553	vz1 = vec_sub(vz4, vz5);
	554
	555	vz3 = vec_madd(vz3, vc1, vc0);
	556	vz3 = vec_madd(vz8, vc2, vz3);
	557
	558	vz4 = vec_perm(vz0, vz1, vcprm(0,s0,1,s1));
	559	vz5 = vec_perm(vz0, vz1, vcprm(2,s3,3,s2));
	560	vz6 = vec_perm(vz2, vz3, vcprm(1,2,s3,s0));
	561	vz7 = vec_perm(vz2, vz3, vcprm(0,3,s2,s1));
	562
	563	vz0 = vec_add(vz4, vz5);
	564	vz1 = vec_sub(vz4, vz5);
	565	vz2 = vec_add(vz6, vz7);
	566	vz3 = vec_sub(vz6, vz7);
	567
	568	vz4 = vec_perm(vz0, vz1, vcprm(0,1,s0,s1));
	569	vz5 = vec_perm(vz0, vz1, vcprm(2,3,s2,s3));
	570	vz6 = vec_perm(vz2, vz3, vcprm(0,2,s1,s3));
	571	vz7 = vec_perm(vz2, vz3, vcprm(1,3,s0,s2));
	572
	573
	574	vz2 = vec_sub(vz4, vz6);
	575	vz3 = vec_sub(vz5, vz7);
	576
	577	vz0 = vec_add(vz4, vz6);
	578	vz1 = vec_add(vz5, vz7);
	579
	580	vec_st(vz0, 0, &(out[0]));
	581	vec_st(vz1, byte_2complex, &(out[0]));
	582	vec_st(vz2, byte_4complex, &(out[0]));
	583	vec_st(vz3, byte_6complex, &(out[0]));
	584	return;
	585	}
	586
	587	inline static void fft16_vsx(FFTComplex *z)
	588	{
	589	float* out= (float*)z;
	590	vec_f vc0 = {0.0, 0.0, 0.0, 0.0};
	591	vec_f vc1 = {-sqrthalf, sqrthalf, sqrthalf, -sqrthalf};
	592	vec_f vc2 = {sqrthalf, sqrthalf, sqrthalf, sqrthalf};
	593	vec_f vc3 = {1.0, 0.92387953, sqrthalf, 0.38268343};
	594	vec_f vc4 = {0.0, 0.38268343, sqrthalf, 0.92387953};
	595	vec_f vc5 = {-0.0, -0.38268343, -sqrthalf, -0.92387953};
	596
	597	vec_f vz0, vz1, vz2, vz3;
	598	vec_f vz4, vz5, vz6, vz7;
	599	vec_f vz8, vz9, vz10, vz11;
	600	vec_f vz12, vz13;
	601
	602	vz0 = vec_ld(byte_8complex, &(out[0]));
	603	vz1 = vec_ld(byte_10complex, &(out[0]));
	604	vz2 = vec_ld(byte_12complex, &(out[0]));
	605	vz3 = vec_ld(byte_14complex, &(out[0]));
	606
	607	vz4 = vec_perm(vz0, vz1, vcprm(0,1,s2,s1));
	608	vz5 = vec_perm(vz0, vz1, vcprm(2,3,s0,s3));
	609	vz6 = vec_perm(vz2, vz3, vcprm(0,1,s2,s1));
	610	vz7 = vec_perm(vz2, vz3, vcprm(2,3,s0,s3));
	611
	612	vz0 = vec_add(vz4, vz5);
	613	vz1= vec_sub(vz4, vz5);
	614	vz2 = vec_add(vz6, vz7);
	615	vz3 = vec_sub(vz6, vz7);
	616
	617	vz4 = vec_perm(vz0, vz1, vcprm(0,s0,1,s1));
	618	vz5 = vec_perm(vz0, vz1, vcprm(2,s3,3,s2));
	619	vz6 = vec_perm(vz2, vz3, vcprm(0,s0,1,s1));
	620	vz7 = vec_perm(vz2, vz3, vcprm(2,s3,3,s2));
	621
	622	vz0 = vec_add(vz4, vz5);
	623	vz1 = vec_sub(vz4, vz5);
	624	vz2 = vec_add(vz6, vz7);
	625	vz3 = vec_sub(vz6, vz7);
	626
	627	vz4 = vec_perm(vz0, vz1, vcprm(0,1,s0,s1));
	628	vz5 = vec_perm(vz0, vz1, vcprm(2,3,s2,s3));
	629
	630	vz6 = vec_perm(vz2, vz3, vcprm(0,1,s0,s1));
	631	vz7 = vec_perm(vz2, vz3, vcprm(2,3,s2,s3));
	632
	633	vz0 = vec_ld(0, &(out[0]));
	634	vz1 = vec_ld(byte_2complex, &(out[0]));
	635	vz2 = vec_ld(byte_4complex, &(out[0]));
	636	vz3 = vec_ld(byte_6complex, &(out[0]));
	637	vz10 = vec_perm(vz2, vz3, vcprm(0,s0,1,s1));
	638	vz11 = vec_perm(vz2, vz3, vcprm(2,s2,3,s3));
	639	vz8 = vec_perm(vz0, vz1, vcprm(0,1,s2,s1));
	640	vz9 = vec_perm(vz0, vz1, vcprm(2,3,s0,s3));
	641
	642	vz2 = vec_add(vz10, vz11);
	643	vz3 = vec_sub(vz10, vz11);
	644	vz12 = vec_perm(vz3, vz3, vcprm(2,3,0,1));
	645	vz0 = vec_add(vz8, vz9);
	646	vz1 = vec_sub(vz8, vz9);
	647
	648	vz3 = vec_madd(vz3, vc1, vc0);
	649	vz3 = vec_madd(vz12, vc2, vz3);
	650	vz8 = vec_perm(vz0, vz1, vcprm(0,s0,1,s1));
	651	vz9 = vec_perm(vz0, vz1, vcprm(2,s3,3,s2));
	652	vz10 = vec_perm(vz2, vz3, vcprm(1,2,s3,s0));
	653	vz11 = vec_perm(vz2, vz3, vcprm(0,3,s2,s1));
	654
	655	vz0 = vec_add(vz8, vz9);
	656	vz1 = vec_sub(vz8, vz9);
	657	vz2 = vec_add(vz10, vz11);
	658	vz3 = vec_sub(vz10, vz11);
	659
	660	vz8 = vec_perm(vz0, vz1, vcprm(0,1,s0,s1));
	661	vz9 = vec_perm(vz0, vz1, vcprm(2,3,s2,s3));
	662	vz10 = vec_perm(vz2, vz3, vcprm(0,2,s1,s3));
	663	vz11 = vec_perm(vz2, vz3, vcprm(1,3,s0,s2));
	664
	665	vz2 = vec_sub(vz8, vz10);
	666	vz3 = vec_sub(vz9, vz11);
	667	vz0 = vec_add(vz8, vz10);
	668	vz1 = vec_add(vz9, vz11);
	669
	670	vz8 = vec_madd(vz4, vc3, vc0);
	671	vz9 = vec_madd(vz5, vc3, vc0);
	672	vz10 = vec_madd(vz6, vc3, vc0);
	673	vz11 = vec_madd(vz7, vc3, vc0);
	674
	675	vz8 = vec_madd(vz5, vc4, vz8);
	676	vz9 = vec_madd(vz4, vc5, vz9);
	677	vz10 = vec_madd(vz7, vc5, vz10);
	678	vz11 = vec_madd(vz6, vc4, vz11);
	679
	680	vz12 = vec_sub(vz10, vz8);
	681	vz10 = vec_add(vz10, vz8);
	682
	683	vz13 = vec_sub(vz9, vz11);
	684	vz11 = vec_add(vz9, vz11);
	685
	686	vz4 = vec_sub(vz0, vz10);
	687	vz0 = vec_add(vz0, vz10);
	688
	689	vz7= vec_sub(vz3, vz12);
	690	vz3= vec_add(vz3, vz12);
	691
	692	vz5 = vec_sub(vz1, vz11);
	693	vz1 = vec_add(vz1, vz11);
	694
	695	vz6 = vec_sub(vz2, vz13);
	696	vz2 = vec_add(vz2, vz13);
	697
	698	vec_st(vz0, 0, &(out[0]));
	699	vec_st(vz1, byte_2complex, &(out[0]));
	700	vec_st(vz2, byte_4complex, &(out[0]));
	701	vec_st(vz3, byte_6complex, &(out[0]));
	702	vec_st(vz4, byte_8complex, &(out[0]));
	703	vec_st(vz5, byte_10complex, &(out[0]));
	704	vec_st(vz6, byte_12complex, &(out[0]));
	705	vec_st(vz7, byte_14complex, &(out[0]));
	706	return;
	707
	708	}
	709	inline static void pass_vsx(FFTComplex * z, const FFTSample * wre, unsigned int n)
	710	{
	711	int o1 = n<<1;
	712	int o2 = n<<2;
	713	int o3 = o1+o2;
	714	int i1, i2, i3;
	715	FFTSample* out = (FFTSample*)z;
	716	const FFTSample *wim = wre+o1;
	717	vec_f v0, v1, v2, v3;
	718	vec_f v4, v5, v6, v7;
	719	vec_f v8, v9, v10, v11;
	720	vec_f v12, v13;
	721
	722	n = n-2;
	723	i1 = o1*sizeof(FFTComplex);
	724	i2 = o2*sizeof(FFTComplex);
	725	i3 = o3*sizeof(FFTComplex);
	726
	727	v8 = vec_ld(0, &(wre[0]));
	728	v10 = vec_ld(0, &(wim[0]));
	729	v9 = vec_ld(0, &(wim[-4]));
	730	v9 = vec_perm(v9, v10, vcprm(s0,3,2,1));
	731
	732	v4 = vec_ld(i2, &(out[0]));
	733	v5 = vec_ld(i2+16, &(out[0]));
	734	v6 = vec_ld(i3, &(out[0]));
	735	v7 = vec_ld(i3+16, &(out[0]));
	736	v10 = vec_mul(v4, v8); // r2*wre
	737	v11 = vec_mul(v5, v8); // i2*wre
	738	v12 = vec_mul(v6, v8); // r3*wre
	739	v13 = vec_mul(v7, v8); // i3*wre
	740
	741	v0 = vec_ld(0, &(out[0])); // r0
	742	v3 = vec_ld(i1+16, &(out[0])); // i1
	743	v10 = vec_madd(v5, v9, v10); // r2*wim
	744	v11 = vec_nmsub(v4, v9, v11); // i2*wim
	745	v12 = vec_nmsub(v7, v9, v12); // r3*wim
	746	v13 = vec_madd(v6, v9, v13); // i3*wim
	747
	748	v1 = vec_ld(16, &(out[0])); // i0
	749	v2 = vec_ld(i1, &(out[0])); // r1
	750	v8 = vec_sub(v12, v10);
	751	v12 = vec_add(v12, v10);
	752	v9 = vec_sub(v11, v13);
	753	v13 = vec_add(v11, v13);
	754	v4 = vec_sub(v0, v12);
	755	v0 = vec_add(v0, v12);
	756	v7 = vec_sub(v3, v8);
	757	v3 = vec_add(v3, v8);
	758
	759	vec_st(v0, 0, &(out[0])); // r0
	760	vec_st(v3, i1+16, &(out[0])); // i1
	761	vec_st(v4, i2, &(out[0])); // r2
	762	vec_st(v7, i3+16, &(out[0]));// i3
	763
	764	v5 = vec_sub(v1, v13);
	765	v1 = vec_add(v1, v13);
	766	v6 = vec_sub(v2, v9);
	767	v2 = vec_add(v2, v9);
	768
	769	vec_st(v1, 16, &(out[0])); // i0
	770	vec_st(v2, i1, &(out[0])); // r1
	771	vec_st(v5, i2+16, &(out[0])); // i2
	772	vec_st(v6, i3, &(out[0])); // r3
	773
	774	do {
	775	out += 8;
	776	wre += 4;
	777	wim -= 4;
	778
	779	v8 = vec_ld(0, &(wre[0]));
	780	v10 = vec_ld(0, &(wim[0]));
	781	v9 = vec_ld(0, &(wim[-4]));
	782	v9 = vec_perm(v9, v10, vcprm(s0,3,2,1));
	783
	784	v4 = vec_ld(i2, &(out[0])); // r2
	785	v5 = vec_ld(i2+16, &(out[0])); // i2
	786	v6 = vec_ld(i3, &(out[0])); // r3
	787	v7 = vec_ld(i3+16, &(out[0]));// i3
	788	v10 = vec_mul(v4, v8); // r2*wre
	789	v11 = vec_mul(v5, v8); // i2*wre
	790	v12 = vec_mul(v6, v8); // r3*wre
	791	v13 = vec_mul(v7, v8); // i3*wre
	792
	793	v0 = vec_ld(0, &(out[0])); // r0
	794	v3 = vec_ld(i1+16, &(out[0])); // i1
	795	v10 = vec_madd(v5, v9, v10); // r2*wim
	796	v11 = vec_nmsub(v4, v9, v11); // i2*wim
	797	v12 = vec_nmsub(v7, v9, v12); // r3*wim
	798	v13 = vec_madd(v6, v9, v13); // i3*wim
	799
	800	v1 = vec_ld(16, &(out[0])); // i0
	801	v2 = vec_ld(i1, &(out[0])); // r1
	802	v8 = vec_sub(v12, v10);
	803	v12 = vec_add(v12, v10);
	804	v9 = vec_sub(v11, v13);
	805	v13 = vec_add(v11, v13);
	806	v4 = vec_sub(v0, v12);
	807	v0 = vec_add(v0, v12);
	808	v7 = vec_sub(v3, v8);
	809	v3 = vec_add(v3, v8);
	810
	811	vec_st(v0, 0, &(out[0])); // r0
	812	vec_st(v3, i1+16, &(out[0])); // i1
	813	vec_st(v4, i2, &(out[0])); // r2
	814	vec_st(v7, i3+16, &(out[0])); // i3
	815
	816	v5 = vec_sub(v1, v13);
	817	v1 = vec_add(v1, v13);
	818	v6 = vec_sub(v2, v9);
	819	v2 = vec_add(v2, v9);
	820
	821	vec_st(v1, 16, &(out[0])); // i0
	822	vec_st(v2, i1, &(out[0])); // r1
	823	vec_st(v5, i2+16, &(out[0])); // i2
	824	vec_st(v6, i3, &(out[0])); // r3
	825	} while (n-=2);
	826	}
	827
	828	#endif
	829
	830	#endif /* AVCODEC_PPC_FFT_VSX_H */

ppc: Fix the bug of fft for little endian environment on POWER7 and later

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code