History

[Processing/x86] DyadicBilinearDownsample optimizations

Average vertically before horizontally; horizontal averaging is more worksome. Doing the vertical averaging first reduces the number of horizontal averages by half. Use pmaddubsw and pavgw to do the horizontal averaging for a slight performance improvement. Minor tweaks. Improve the SSSE3 dyadic downsample routines and drop the SSE4 routines. The non-temporal loads used in the SSE4 routines do nothing for cache- backed memory AFAIK. Adjust tests because averaging vertically first gives slightly different output. ~2.39x speedup for the widthx32 routine on Haswell when not memory-bound. ~2.20x speedup for the widthx16 routine on Haswell when not memory-bound. Note that the widthx16 routine can be unrolled for further speedup.

Sindre Aamås committed 10 years ago

Commit 8a0af4a1 parent 7cbb75e7 files · +95 −372Browse files

Showing the changes to codec/processing (3 of 7 files). Show all

Unified · Split

3 files
  1. codec/processing/src/downsample/downsample.cpp +0 −2
  2. codec/processing/src/downsample/downsample.h +0 −4
  3. codec/processing/src/x86/downsample_bilinear.asm +46 −356
codec/processing/src/downsample/downsample.cpp+0−2
@@ -102,8 +102,6 @@ void CDownsampling::InitDownsampleFuncs (SDownsampleFuncs& sDownsampleFunc, int · 101 unmodified lines
102102 sDownsampleFunc.pfGeneralRatioLuma = GeneralBilinearFastDownsamplerWrap_ssse3;
103103 }
104104 if (iCpuFlag & WELS_CPU_SSE41) {
105− sDownsampleFunc.pfHalfAverageWidthx32 = DyadicBilinearDownsamplerWidthx32_sse4;
106− sDownsampleFunc.pfHalfAverageWidthx16 = DyadicBilinearDownsamplerWidthx16_sse4;
107105 sDownsampleFunc.pfOneThirdDownsampler = DyadicBilinearOneThirdDownsampler_sse4;
108106 sDownsampleFunc.pfQuarterDownsampler = DyadicBilinearQuarterDownsampler_sse4;
109107 sDownsampleFunc.pfGeneralRatioChroma = GeneralBilinearAccurateDownsamplerWrap_sse41;
codec/processing/src/downsample/downsample.h+0−4
@@ -94,10 +94,6 @@ HalveDownsampleFunc DyadicBilinearDownsamplerWidthx32_sse; · 93 unmodified lines
9494 HalveDownsampleFunc DyadicBilinearDownsamplerWidthx16_ssse3;
9595 // iSrcWidth= x32 pixels
9696 HalveDownsampleFunc DyadicBilinearDownsamplerWidthx32_ssse3;
97−// iSrcWidth= x16 pixels
98−HalveDownsampleFunc DyadicBilinearDownsamplerWidthx16_sse4;
99−// iSrcWidth= x32 pixels
100−HalveDownsampleFunc DyadicBilinearDownsamplerWidthx32_sse4;
10197
10298 GeneralDownsampleFunc GeneralBilinearFastDownsamplerWrap_sse2;
10399 GeneralDownsampleFunc GeneralBilinearAccurateDownsamplerWrap_sse2;
codec/processing/src/x86/downsample_bilinear.asm+46−356
@@ -40,6 +40,10 @@ · 39 unmodified lines
4040 ;*************************************************************************/
4141 %include "asm_inc.asm"
4242
43+%ifdef __NASM_VER__
44+ %use smartalign
45+%endif
46+
4347 ;***********************************************************************
4448 ; Macros and other preprocessor constants
4549 ;***********************************************************************
@@ -471,24 +475,12 @@ WELS_EXTERN DyadicBilinearDownsamplerWidthx8_sse · 425 unmodified lines
471475
472476
473477
474−; got about 50% improvement over DyadicBilinearDownsamplerWidthx32_sse
475478 ;***********************************************************************
476479 ; void DyadicBilinearDownsamplerWidthx32_ssse3( unsigned char* pDst, const int iDstStride,
477480 ; unsigned char* pSrc, const int iSrcStride,
478481 ; const int iSrcWidth, const int iSrcHeight );
479482 ;***********************************************************************
480483 WELS_EXTERN DyadicBilinearDownsamplerWidthx32_ssse3
481− ;push ebx
482− ;push edx
483− ;push esi
484− ;push edi
485− ;push ebp
486−
487− ;mov edi, [esp+24] ; pDst
488− ;mov edx, [esp+28] ; iDstStride
489− ;mov esi, [esp+32] ; pSrc
490− ;mov ecx, [esp+36] ; iSrcStride
491− ;mov ebp, [esp+44] ; iSrcHeight
492484 %ifdef X86_32
493485 push r6
494486 %assign push_num 1
@@ -496,7 +488,7 @@ WELS_EXTERN DyadicBilinearDownsamplerWidthx32_ssse3 · 1 unmodified line
496488 %assign push_num 0
497489 %endif
498490 LOAD_6_PARA
499− PUSH_XMM 8
491+ PUSH_XMM 4
500492 SIGN_EXTENSION r1, r1d
501493 SIGN_EXTENSION r3, r3d
502494 SIGN_EXTENSION r4, r4d
@@ -508,96 +500,44 @@ WELS_EXTERN DyadicBilinearDownsamplerWidthx32_ssse3 · 5 unmodified lines
508500 %endif
509501 sar r5, $01 ; iSrcHeight >> 1
510502
511− movdqa xmm7, [shufb_mask_low] ; mask low
512− movdqa xmm6, [shufb_mask_high] ; mask high
503+ WELS_DB1 xmm3
504+ WELS_Zero xmm2
505+ sar r4, $01 ; iSrcWidth >> 1
506+ add r0, r4 ; pDst += iSrcWidth >> 1
513507
514508 .yloops4:
515− ;mov eax, [esp+40] ; iSrcWidth
516− ;sar eax, $01 ; iSrcWidth >> 1
517− ;mov ebx, eax ; iDstWidth restored at ebx
518− ;sar eax, $04 ; (iSrcWidth >> 1) / 16 ; loop count = num_of_mb
519− ;neg ebx ; - (iSrcWidth >> 1)
520509 %ifdef X86_32
521510 mov r4, arg5
522511 %else
523512 mov r4, r12
524513 %endif
525514 sar r4, $01 ; iSrcWidth >> 1
526− mov r6, r4 ; iDstWidth restored at ebx
527− sar r4, $04 ; (iSrcWidth >> 1) / 16 ; loop count = num_of_mb
528− neg r6 ; - (iSrcWidth >> 1)
515+ neg r4 ; -(iSrcWidth >> 1)
516+ mov r6, r4
517+ align 16
529518 ; each loop = source bandwidth: 32 bytes
530519 .xloops4:
531− ; 1st part horizonal loop: x16 bytes
532− ; mem hi<- ->lo
533− ;1st Line Src: xmm0: h H g G f F e E d D c C b B a A
534− ; xmm1: p P o O n N m M l L k K j J i I
535− ;2nd Line Src: xmm2: h H g G f F e E d D c C b B a A
536− ; xmm3: p P o O n N m M l L k K j J i I
537− ;=> target:
538− ;: P O N M L K J I H G F E D C B A
539− ;: p o n m l k j i h g f e d c b a
540− ;: P .. A
541− ;: p .. a
542−
543− ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
544− movdqa xmm0, [r2] ; 1st_src_line
545− movdqa xmm1, [r2+16] ; 1st_src_line + 16
546− movdqa xmm2, [r2+r3] ; 2nd_src_line
547− movdqa xmm3, [r2+r3+16] ; 2nd_src_line + 16
548−
549− ; packing & avg
550− movdqa xmm4, xmm0 ; h H g G f F e E d D c C b B a A
551− pshufb xmm0, xmm7 ; 0 H 0 G 0 F 0 E 0 D 0 C 0 B 0 A
552− pshufb xmm4, xmm6 ; 0 h 0 g 0 f 0 e 0 d 0 c 0 b 0 a
553− ; another implementation for xmm4 high bits
554−; psubb xmm4, xmm0 ; h 0 g 0 f 0 e 0 d 0 c 0 b 0 a 0
555−; psrlw xmm4, 8 ; 0 h 0 g 0 f 0 e 0 d 0 c 0 b 0 a
556− pavgb xmm0, xmm4
557−
558− movdqa xmm5, xmm1
559− pshufb xmm1, xmm7
560− pshufb xmm5, xmm6
561−; psubb xmm5, xmm1
562−; psrlw xmm5, 8
563− pavgb xmm1, xmm5
564−
565− movdqa xmm4, xmm2
566− pshufb xmm2, xmm7
567− pshufb xmm4, xmm6
568−; psubb xmm4, xmm2
569−; psrlw xmm4, 8
570− pavgb xmm2, xmm4
571−
572− movdqa xmm5, xmm3
573− pshufb xmm3, xmm7
574− pshufb xmm5, xmm6
575−; psubb xmm5, xmm3
576−; psrlw xmm5, 8
577− pavgb xmm3, xmm5
578−
579− packuswb xmm0, xmm1
580− packuswb xmm2, xmm3
581− pavgb xmm0, xmm2
582−
583− ; write pDst
584− movdqa [r0], xmm0
585−
586− ; next SMB
587− lea r2, [r2+32]
588− lea r0, [r0+16]
589−
590− dec r4
591− jg near .xloops4
520+ movdqa xmm0, [r2+r3]
521+ movdqa xmm1, [r2+r3+16]
522+ pavgb xmm0, [r2] ; avg vertical pixels 0-15
523+ pavgb xmm1, [r2+16] ; avg vertical pixels 16-31
524+ add r2, 32 ; pSrc += 32
525+ pmaddubsw xmm0, xmm3 ; pairwise horizontal sum neighboring pixels 0-15
526+ pmaddubsw xmm1, xmm3 ; pairwise horizontal sum neighboring pixels 16-31
527+ pavgw xmm0, xmm2 ; (sum + 1) >> 1
528+ pavgw xmm1, xmm2 ; (sum + 1) >> 1
529+ packuswb xmm0, xmm1 ; pack words to bytes
530+ movdqa [r0+r4], xmm0 ; store results
531+ add r4, 16
532+ jl .xloops4
592533
593534 ; next line
594535 lea r2, [r2+2*r3] ; next end of lines
595536 lea r2, [r2+2*r6] ; reset to base 0 [- 2 * iDstWidth]
596537 lea r0, [r0+r1]
597− lea r0, [r0+r6] ; reset to base 0 [- iDstWidth]
598538
599− dec r5
600− jg near .yloops4
539+ sub r5, 1
540+ jg .yloops4
601541
602542 %ifndef X86_32
603543 pop r12
@@ -623,7 +563,7 @@ WELS_EXTERN DyadicBilinearDownsamplerWidthx16_ssse3 · 19 unmodified lines
623563 %assign push_num 0
624564 %endif
625565 LOAD_6_PARA
626− PUSH_XMM 6
566+ PUSH_XMM 4
627567 SIGN_EXTENSION r1, r1d
628568 SIGN_EXTENSION r3, r3d
629569 SIGN_EXTENSION r4, r4d
@@ -634,8 +574,11 @@ WELS_EXTERN DyadicBilinearDownsamplerWidthx16_ssse3 · 4 unmodified lines
634574 mov r12, r4
635575 %endif
636576 sar r5, $01 ; iSrcHeight >> 1
637− movdqa xmm5, [shufb_mask_low] ; mask low
638− movdqa xmm4, [shufb_mask_high] ; mask high
577+ WELS_DB1 xmm3
578+ WELS_Zero xmm2
579+ add r2, r4 ; pSrc += iSrcWidth
580+ sar r4, $01 ; iSrcWidth >> 1
581+ add r0, r4 ; pDst += iSrcWidth >> 1
639582
640583 .yloops5:
641584 %ifdef X86_32
@@ -644,279 +587,26 @@ WELS_EXTERN DyadicBilinearDownsamplerWidthx16_ssse3 · 2 unmodified lines
644587 mov r4, r12
645588 %endif
646589 sar r4, $01 ; iSrcWidth >> 1
647− mov r6, r4 ; iDstWidth restored at ebx
648− sar r4, $03 ; (iSrcWidth >> 1) / 8 ; loop count = num_of_mb
649− neg r6 ; - (iSrcWidth >> 1)
590+ neg r4 ; -(iSrcWidth >> 1)
591+ lea r6, [r2+r3] ; pSrc + iSrcStride
592+ align 16
650593 ; each loop = source bandwidth: 16 bytes
651594 .xloops5:
652− ; horizonal loop: x16 bytes by source
653− ; mem hi<- ->lo
654− ;1st line pSrc: xmm0: h H g G f F e E d D c C b B a A
655− ;2nd line pSrc: xmm1: p P o O n N m M l L k K j J i I
656− ;=> target:
657− ;: H G F E D C B A, P O N M L K J I
658− ;: h g f e d c b a, p o n m l k j i
659−
660− ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
661− movdqa xmm0, [r2] ; 1st_src_line
662− movdqa xmm1, [r2+r3] ; 2nd_src_line
663−
664− ; packing & avg
665− movdqa xmm2, xmm0 ; h H g G f F e E d D c C b B a A
666− pshufb xmm0, xmm5 ; 0 H 0 G 0 F 0 E 0 D 0 C 0 B 0 A
667− pshufb xmm2, xmm4 ; 0 h 0 g 0 f 0 e 0 d 0 c 0 b 0 a
668− ; another implementation for xmm2 high bits
669−; psubb xmm2, xmm0 ; h 0 g 0 f 0 e 0 d 0 c 0 b 0 a 0
670−; psrlw xmm2, 8 ; 0 h 0 g 0 f 0 e 0 d 0 c 0 b 0 a
671− pavgb xmm0, xmm2
672−
673− movdqa xmm3, xmm1
674− pshufb xmm1, xmm5
675− pshufb xmm3, xmm4
676−; psubb xmm3, xmm1
677−; psrlw xmm3, 8
678− pavgb xmm1, xmm3
679−
680− pavgb xmm0, xmm1
681− packuswb xmm0, xmm1
682−
683− ; write pDst
684− movq [r0], xmm0
685−
686− ; next SMB
687− lea r2, [r2+16]
688− lea r0, [r0+8]
689−
690− dec r4
691− jg near .xloops5
692−
693− lea r2, [r2+2*r3] ; next end of lines
694− lea r2, [r2+2*r6] ; reset to base 0 [- 2 * iDstWidth]
695− lea r0, [r0+r1]
696− lea r0, [r0+r6] ; reset to base 0 [- iDstWidth]
697−
698− dec r5
699− jg near .yloops5
700−
701−%ifndef X86_32
702− pop r12
703−%endif
704−
705− POP_XMM
706− LOAD_6_PARA_POP
707−%ifdef X86_32
708− pop r6
709−%endif
710− ret
711−
712−; got about 65% improvement over DyadicBilinearDownsamplerWidthx32_sse
713−;***********************************************************************
714−; void DyadicBilinearDownsamplerWidthx32_sse4( unsigned char* pDst, const int iDstStride,
715−; unsigned char* pSrc, const int iSrcStride,
716−; const int iSrcWidth, const int iSrcHeight );
717−;***********************************************************************
718−WELS_EXTERN DyadicBilinearDownsamplerWidthx32_sse4
719−%ifdef X86_32
720− push r6
721− %assign push_num 1
722−%else
723− %assign push_num 0
724−%endif
725− LOAD_6_PARA
726− PUSH_XMM 8
727− SIGN_EXTENSION r1, r1d
728− SIGN_EXTENSION r3, r3d
729− SIGN_EXTENSION r4, r4d
730− SIGN_EXTENSION r5, r5d
731−
732−%ifndef X86_32
733− push r12
734− mov r12, r4
735−%endif
736− sar r5, $01 ; iSrcHeight >> 1
737−
738− movdqa xmm7, [shufb_mask_low] ; mask low
739− movdqa xmm6, [shufb_mask_high] ; mask high
740−
741−.yloops6:
742−%ifdef X86_32
743− mov r4, arg5
744−%else
745− mov r4, r12
746−%endif
747− sar r4, $01 ; iSrcWidth >> 1
748− mov r6, r4 ; iDstWidth restored at ebx
749− sar r4, $04 ; (iSrcWidth >> 1) / 16 ; loop count = num_of_mb
750− neg r6 ; - (iSrcWidth >> 1)
751− ; each loop = source bandwidth: 32 bytes
752−.xloops6:
753− ; 1st part horizonal loop: x16 bytes
754− ; mem hi<- ->lo
755− ;1st Line Src: xmm0: h H g G f F e E d D c C b B a A
756− ; xmm1: p P o O n N m M l L k K j J i I
757− ;2nd Line Src: xmm2: h H g G f F e E d D c C b B a A
758− ; xmm3: p P o O n N m M l L k K j J i I
759− ;=> target:
760− ;: P O N M L K J I H G F E D C B A
761− ;: p o n m l k j i h g f e d c b a
762− ;: P .. A
763− ;: p .. a
764−
765− ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
766− movntdqa xmm0, [r2] ; 1st_src_line
767− movntdqa xmm1, [r2+16] ; 1st_src_line + 16
768− movntdqa xmm2, [r2+r3] ; 2nd_src_line
769− movntdqa xmm3, [r2+r3+16] ; 2nd_src_line + 16
770−
771− ; packing & avg
772− movdqa xmm4, xmm0 ; h H g G f F e E d D c C b B a A
773− pshufb xmm0, xmm7 ; 0 H 0 G 0 F 0 E 0 D 0 C 0 B 0 A
774− pshufb xmm4, xmm6 ; 0 h 0 g 0 f 0 e 0 d 0 c 0 b 0 a
775−; psubb xmm4, xmm0 ; h 0 g 0 f 0 e 0 d 0 c 0 b 0 a 0
776−; psrlw xmm4, 8 ; 0 h 0 g 0 f 0 e 0 d 0 c 0 b 0 a
777− pavgb xmm0, xmm4
778−
779− movdqa xmm5, xmm1
780− pshufb xmm1, xmm7
781− pshufb xmm5, xmm6
782−; psubb xmm5, xmm1
783−; psrlw xmm5, 8
784− pavgb xmm1, xmm5
785−
786− movdqa xmm4, xmm2
787− pshufb xmm2, xmm7
788− pshufb xmm4, xmm6
789−; psubb xmm4, xmm2
790−; psrlw xmm4, 8
791− pavgb xmm2, xmm4
792−
793− movdqa xmm5, xmm3
794− pshufb xmm3, xmm7
795− pshufb xmm5, xmm6
796−; psubb xmm5, xmm3
797−; psrlw xmm5, 8
798− pavgb xmm3, xmm5
799−
800− packuswb xmm0, xmm1
801− packuswb xmm2, xmm3
802− pavgb xmm0, xmm2
803−
804− ; write pDst
805− movdqa [r0], xmm0
806−
807− ; next SMB
808− lea r2, [r2+32]
809− lea r0, [r0+16]
810−
811− dec r4
812− jg near .xloops6
813−
814− lea r2, [r2+2*r3] ; next end of lines
815− lea r2, [r2+2*r6] ; reset to base 0 [- 2 * iDstWidth]
816− lea r0, [r0+r1]
817− lea r0, [r0+r6] ; reset to base 0 [- iDstWidth]
818−
819− dec r5
820− jg near .yloops6
821−
822−%ifndef X86_32
823− pop r12
824−%endif
825−
826− POP_XMM
827− LOAD_6_PARA_POP
828−%ifdef X86_32
829− pop r6
830−%endif
831− ret
832−
833−;***********************************************************************
834−; void DyadicBilinearDownsamplerWidthx16_sse4( unsigned char* pDst, const int iDstStride,
835−; unsigned char* pSrc, const int iSrcStride,
836−; const int iSrcWidth, const int iSrcHeight );
837−;***********************************************************************
838−WELS_EXTERN DyadicBilinearDownsamplerWidthx16_sse4
839−%ifdef X86_32
840− push r6
841− %assign push_num 1
842−%else
843− %assign push_num 0
844−%endif
845− LOAD_6_PARA
846− PUSH_XMM 6
847− SIGN_EXTENSION r1, r1d
848− SIGN_EXTENSION r3, r3d
849− SIGN_EXTENSION r4, r4d
850− SIGN_EXTENSION r5, r5d
851−
852−%ifndef X86_32
853− push r12
854− mov r12, r4
855−%endif
856− sar r5, $01 ; iSrcHeight >> 1
857− movdqa xmm5, [shufb_mask_low] ; mask low
858− movdqa xmm4, [shufb_mask_high] ; mask high
859−
860−.yloops7:
861−%ifdef X86_32
862− mov r4, arg5
863−%else
864− mov r4, r12
865−%endif
866− sar r4, $01 ; iSrcWidth >> 1
867− mov r6, r4 ; iDstWidth restored at ebx
868− sar r4, $03 ; (iSrcWidth >> 1) / 8 ; loop count = num_of_mb
869− neg r6 ; - (iSrcWidth >> 1)
870− ; each loop = source bandwidth: 16 bytes
871−.xloops7:
872− ; horizonal loop: x16 bytes by source
873− ; mem hi<- ->lo
874− ;1st line pSrc: xmm0: h H g G f F e E d D c C b B a A
875− ;2nd line pSrc: xmm1: p P o O n N m M l L k K j J i I
876− ;=> target:
877− ;: H G F E D C B A, P O N M L K J I
878− ;: h g f e d c b a, p o n m l k j i
879−
880− ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
881− movntdqa xmm0, [r2] ; 1st_src_line
882− movntdqa xmm1, [r2+r3] ; 2nd_src_line
883−
884− ; packing & avg
885− movdqa xmm2, xmm0 ; h H g G f F e E d D c C b B a A
886− pshufb xmm0, xmm5 ; 0 H 0 G 0 F 0 E 0 D 0 C 0 B 0 A
887− pshufb xmm2, xmm4 ; 0 h 0 g 0 f 0 e 0 d 0 c 0 b 0 a
888−; psubb xmm2, xmm0 ; h 0 g 0 f 0 e 0 d 0 c 0 b 0 a 0
889−; psrlw xmm2, 8 ; 0 h 0 g 0 f 0 e 0 d 0 c 0 b 0 a
890− pavgb xmm0, xmm2
891−
892− movdqa xmm3, xmm1
893− pshufb xmm1, xmm5
894− pshufb xmm3, xmm4
895−; psubb xmm3, xmm1
896−; psrlw xmm3, 8
897− pavgb xmm1, xmm3
898−
899− pavgb xmm0, xmm1
900− packuswb xmm0, xmm1
901−
902− ; write pDst
903− movq [r0], xmm0
904−
905− ; next SMB
906− lea r2, [r2+16]
907− lea r0, [r0+8]
908−
909− dec r4
910− jg near .xloops7
595+ movdqa xmm0, [r2+2*r4]
596+ pavgb xmm0, [r6+2*r4] ; avg vertical pixels
597+ pmaddubsw xmm0, xmm3 ; pairwise horizontal sum neighboring pixels
598+ pavgw xmm0, xmm2 ; (sum + 1) >> 1
599+ packuswb xmm0, xmm0 ; pack words to bytes
600+ movlps [r0+r4], xmm0 ; store results
601+ add r4, 8
602+ jl .xloops5
911603
912604 ; next line
913605 lea r2, [r2+2*r3] ; next end of lines
914− lea r2, [r2+2*r6] ; reset to base 0 [- 2 * iDstWidth]
915606 lea r0, [r0+r1]
916− lea r0, [r0+r6] ; reset to base 0 [- iDstWidth]
917607
918− dec r5
919− jg near .yloops7
608+ sub r5, 1
609+ jg .yloops5
920610
921611 %ifndef X86_32
922612 pop r12

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t