/* *  Copyright 2013 The LibYuv Project Authors. All rights reserved. * *  Use of this source code is governed by a BSD-style license *  that can be found in the LICENSE file in the root of the source *  tree. An additional intellectual property rights grant can be found *  in the file PATENTS. All contributing project authors may *  be found in the AUTHORS file in the root of the source tree. */ #include "libyuv/row.h"#include "libyuv/scale_row.h" #ifdef __cplusplusnamespace libyuv {extern "C" {#endif // This module is for GCC x86 and x64.#if !defined(LIBYUV_DISABLE_X86) && \    (defined(__x86_64__) || (defined(__i386__) && !defined(_MSC_VER))) // Offsets for source bytes 0 to 9static const uvec8 kShuf0 = {0,   1,   3,   4,   5,   7,   8,   9,                             128, 128, 128, 128, 128, 128, 128, 128}; // Offsets for source bytes 11 to 20 with 8 subtracted = 3 to 12.static const uvec8 kShuf1 = {3,   4,   5,   7,   8,   9,   11,  12,                             128, 128, 128, 128, 128, 128, 128, 128}; // Offsets for source bytes 21 to 31 with 16 subtracted = 5 to 31.static const uvec8 kShuf2 = {5,   7,   8,   9,   11,  12,  13,  15,                             128, 128, 128, 128, 128, 128, 128, 128}; // Offsets for source bytes 0 to 10static const uvec8 kShuf01 = {0, 1, 1, 2, 2, 3, 4, 5, 5, 6, 6, 7, 8, 9, 9, 10}; // Offsets for source bytes 10 to 21 with 8 subtracted = 3 to 13.static const uvec8 kShuf11 = {2, 3, 4, 5,  5,  6,  6,  7,                              8, 9, 9, 10, 10, 11, 12, 13}; // Offsets for source bytes 21 to 31 with 16 subtracted = 5 to 31.static const uvec8 kShuf21 = {5,  6,  6,  7,  8,  9,  9,  10,                              10, 11, 12, 13, 13, 14, 14, 15}; // Coefficients for source bytes 0 to 10static const uvec8 kMadd01 = {3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2}; // Coefficients for source bytes 10 to 21static const uvec8 kMadd11 = {1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1}; // Coefficients for source bytes 21 to 31static const uvec8 kMadd21 = {2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3}; // Coefficients for source bytes 21 to 31static const vec16 kRound34 = {2, 2, 2, 2, 2, 2, 2, 2}; static const uvec8 kShuf38a = {0,   3,   6,   8,   11,  14,  128, 128,                               128, 128, 128, 128, 128, 128, 128, 128}; static const uvec8 kShuf38b = {128, 128, 128, 128, 128, 128, 0,   3,                               6,   8,   11,  14,  128, 128, 128, 128}; // Arrange words 0,3,6 into 0,1,2static const uvec8 kShufAc = {0,   1,   6,   7,   12,  13,  128, 128,                              128, 128, 128, 128, 128, 128, 128, 128}; // Arrange words 0,3,6 into 3,4,5static const uvec8 kShufAc3 = {128, 128, 128, 128, 128, 128, 0,   1,                               6,   7,   12,  13,  128, 128, 128, 128}; // Scaling values for boxes of 3x3 and 2x3static const uvec16 kScaleAc33 = {65536 / 9, 65536 / 9, 65536 / 6, 65536 / 9,                                  65536 / 9, 65536 / 6, 0,         0}; // Arrange first value for pixels 0,1,2,3,4,5static const uvec8 kShufAb0 = {0,  128, 3,  128, 6,   128, 8,   128,                               11, 128, 14, 128, 128, 128, 128, 128}; // Arrange second value for pixels 0,1,2,3,4,5static const uvec8 kShufAb1 = {1,  128, 4,  128, 7,   128, 9,   128,                               12, 128, 15, 128, 128, 128, 128, 128}; // Arrange third value for pixels 0,1,2,3,4,5static const uvec8 kShufAb2 = {2,  128, 5,   128, 128, 128, 10,  128,                               13, 128, 128, 128, 128, 128, 128, 128}; // Scaling values for boxes of 3x2 and 2x2static const uvec16 kScaleAb2 = {65536 / 3, 65536 / 3, 65536 / 2, 65536 / 3,                                 65536 / 3, 65536 / 2, 0,         0}; // GCC versions of row functions are verbatim conversions from Visual C.// Generated using gcc disassembly on Visual C object file:// objdump -D yuvscaler.obj >yuvscaler.txt void ScaleRowDown2_SSSE3(const uint8_t* src_ptr,                         ptrdiff_t src_stride,                         uint8_t* dst_ptr,                         int dst_width) {  (void)src_stride;  asm volatile(      // 16 pixel loop.      LABELALIGN      "1:                                        \n"      "movdqu    (%0),%%xmm0                     \n"      "movdqu    0x10(%0),%%xmm1                 \n"      "lea       0x20(%0),%0                     \n"      "psrlw     $0x8,%%xmm0                     \n"      "psrlw     $0x8,%%xmm1                     \n"      "packuswb  %%xmm1,%%xmm0                   \n"      "movdqu    %%xmm0,(%1)                     \n"      "lea       0x10(%1),%1                     \n"      "sub       $0x10,%2                        \n"      "jg        1b                              \n"      : "+r"(src_ptr),   // %0        "+r"(dst_ptr),   // %1        "+r"(dst_width)  // %2        ::"memory",        "cc", "xmm0", "xmm1");} void ScaleRowDown2Linear_SSSE3(const uint8_t* src_ptr,                               ptrdiff_t src_stride,                               uint8_t* dst_ptr,                               int dst_width) {  (void)src_stride;  asm volatile(      "pcmpeqb    %%xmm4,%%xmm4                  \n"      "psrlw      $0xf,%%xmm4                    \n"      "packuswb   %%xmm4,%%xmm4                  \n"      "pxor       %%xmm5,%%xmm5                  \n"       LABELALIGN      "1:                                        \n"      "movdqu    (%0),%%xmm0                     \n"      "movdqu    0x10(%0),%%xmm1                 \n"      "lea       0x20(%0),%0                     \n"      "pmaddubsw  %%xmm4,%%xmm0                  \n"      "pmaddubsw  %%xmm4,%%xmm1                  \n"      "pavgw      %%xmm5,%%xmm0                  \n"      "pavgw      %%xmm5,%%xmm1                  \n"      "packuswb   %%xmm1,%%xmm0                  \n"      "movdqu    %%xmm0,(%1)                     \n"      "lea       0x10(%1),%1                     \n"      "sub       $0x10,%2                        \n"      "jg        1b                              \n"      : "+r"(src_ptr),   // %0        "+r"(dst_ptr),   // %1        "+r"(dst_width)  // %2        ::"memory",        "cc", "xmm0", "xmm1", "xmm4", "xmm5");} void ScaleRowDown2Box_SSSE3(const uint8_t* src_ptr,                            ptrdiff_t src_stride,                            uint8_t* dst_ptr,                            int dst_width) {  asm volatile(      "pcmpeqb    %%xmm4,%%xmm4                  \n"      "psrlw      $0xf,%%xmm4                    \n"      "packuswb   %%xmm4,%%xmm4                  \n"      "pxor       %%xmm5,%%xmm5                  \n"       LABELALIGN      "1:                                        \n"      "movdqu    (%0),%%xmm0                     \n"      "movdqu    0x10(%0),%%xmm1                 \n"      "movdqu    0x00(%0,%3,1),%%xmm2            \n"      "movdqu    0x10(%0,%3,1),%%xmm3            \n"      "lea       0x20(%0),%0                     \n"      "pmaddubsw  %%xmm4,%%xmm0                  \n"      "pmaddubsw  %%xmm4,%%xmm1                  \n"      "pmaddubsw  %%xmm4,%%xmm2                  \n"      "pmaddubsw  %%xmm4,%%xmm3                  \n"      "paddw      %%xmm2,%%xmm0                  \n"      "paddw      %%xmm3,%%xmm1                  \n"      "psrlw      $0x1,%%xmm0                    \n"      "psrlw      $0x1,%%xmm1                    \n"      "pavgw      %%xmm5,%%xmm0                  \n"      "pavgw      %%xmm5,%%xmm1                  \n"      "packuswb   %%xmm1,%%xmm0                  \n"      "movdqu    %%xmm0,(%1)                     \n"      "lea       0x10(%1),%1                     \n"      "sub       $0x10,%2                        \n"      "jg        1b                              \n"      : "+r"(src_ptr),               // %0        "+r"(dst_ptr),               // %1        "+r"(dst_width)              // %2      : "r"((intptr_t)(src_stride))  // %3      : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5");} #ifdef HAS_SCALEROWDOWN2_AVX2void ScaleRowDown2_AVX2(const uint8_t* src_ptr,                        ptrdiff_t src_stride,                        uint8_t* dst_ptr,                        int dst_width) {  (void)src_stride;  asm volatile(       LABELALIGN      "1:                                        \n"      "vmovdqu    (%0),%%ymm0                    \n"      "vmovdqu    0x20(%0),%%ymm1                \n"      "lea        0x40(%0),%0                    \n"      "vpsrlw     $0x8,%%ymm0,%%ymm0             \n"      "vpsrlw     $0x8,%%ymm1,%%ymm1             \n"      "vpackuswb  %%ymm1,%%ymm0,%%ymm0           \n"      "vpermq     $0xd8,%%ymm0,%%ymm0            \n"      "vmovdqu    %%ymm0,(%1)                    \n"      "lea        0x20(%1),%1                    \n"      "sub        $0x20,%2                       \n"      "jg         1b                             \n"      "vzeroupper                                \n"      : "+r"(src_ptr),   // %0        "+r"(dst_ptr),   // %1        "+r"(dst_width)  // %2        ::"memory",        "cc", "xmm0", "xmm1");} void ScaleRowDown2Linear_AVX2(const uint8_t* src_ptr,                              ptrdiff_t src_stride,                              uint8_t* dst_ptr,                              int dst_width) {  (void)src_stride;  asm volatile(      "vpcmpeqb   %%ymm4,%%ymm4,%%ymm4           \n"      "vpsrlw     $0xf,%%ymm4,%%ymm4             \n"      "vpackuswb  %%ymm4,%%ymm4,%%ymm4           \n"      "vpxor      %%ymm5,%%ymm5,%%ymm5           \n"       LABELALIGN      "1:                                        \n"      "vmovdqu    (%0),%%ymm0                    \n"      "vmovdqu    0x20(%0),%%ymm1                \n"      "lea        0x40(%0),%0                    \n"      "vpmaddubsw %%ymm4,%%ymm0,%%ymm0           \n"      "vpmaddubsw %%ymm4,%%ymm1,%%ymm1           \n"      "vpavgw     %%ymm5,%%ymm0,%%ymm0           \n"      "vpavgw     %%ymm5,%%ymm1,%%ymm1           \n"      "vpackuswb  %%ymm1,%%ymm0,%%ymm0           \n"      "vpermq     $0xd8,%%ymm0,%%ymm0            \n"      "vmovdqu    %%ymm0,(%1)                    \n"      "lea        0x20(%1),%1                    \n"      "sub        $0x20,%2                       \n"      "jg         1b                             \n"      "vzeroupper                                \n"      : "+r"(src_ptr),   // %0        "+r"(dst_ptr),   // %1        "+r"(dst_width)  // %2        ::"memory",        "cc", "xmm0", "xmm1", "xmm4", "xmm5");} void ScaleRowDown2Box_AVX2(const uint8_t* src_ptr,                           ptrdiff_t src_stride,                           uint8_t* dst_ptr,                           int dst_width) {  asm volatile(      "vpcmpeqb   %%ymm4,%%ymm4,%%ymm4           \n"      "vpsrlw     $0xf,%%ymm4,%%ymm4             \n"      "vpackuswb  %%ymm4,%%ymm4,%%ymm4           \n"      "vpxor      %%ymm5,%%ymm5,%%ymm5           \n"       LABELALIGN      "1:                                        \n"      "vmovdqu    (%0),%%ymm0                    \n"      "vmovdqu    0x20(%0),%%ymm1                \n"      "vmovdqu    0x00(%0,%3,1),%%ymm2           \n"      "vmovdqu    0x20(%0,%3,1),%%ymm3           \n"      "lea        0x40(%0),%0                    \n"      "vpmaddubsw %%ymm4,%%ymm0,%%ymm0           \n"      "vpmaddubsw %%ymm4,%%ymm1,%%ymm1           \n"      "vpmaddubsw %%ymm4,%%ymm2,%%ymm2           \n"      "vpmaddubsw %%ymm4,%%ymm3,%%ymm3           \n"      "vpaddw     %%ymm2,%%ymm0,%%ymm0           \n"      "vpaddw     %%ymm3,%%ymm1,%%ymm1           \n"      "vpsrlw     $0x1,%%ymm0,%%ymm0             \n"      "vpsrlw     $0x1,%%ymm1,%%ymm1             \n"      "vpavgw     %%ymm5,%%ymm0,%%ymm0           \n"      "vpavgw     %%ymm5,%%ymm1,%%ymm1           \n"      "vpackuswb  %%ymm1,%%ymm0,%%ymm0           \n"      "vpermq     $0xd8,%%ymm0,%%ymm0            \n"      "vmovdqu    %%ymm0,(%1)                    \n"      "lea        0x20(%1),%1                    \n"      "sub        $0x20,%2                       \n"      "jg         1b                             \n"      "vzeroupper                                \n"      : "+r"(src_ptr),               // %0        "+r"(dst_ptr),               // %1        "+r"(dst_width)              // %2      : "r"((intptr_t)(src_stride))  // %3      : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5");}#endif  // HAS_SCALEROWDOWN2_AVX2 void ScaleRowDown4_SSSE3(const uint8_t* src_ptr,                         ptrdiff_t src_stride,                         uint8_t* dst_ptr,                         int dst_width) {  (void)src_stride;  asm volatile(      "pcmpeqb   %%xmm5,%%xmm5                   \n"      "psrld     $0x18,%%xmm5                    \n"      "pslld     $0x10,%%xmm5                    \n"       LABELALIGN      "1:                                        \n"      "movdqu    (%0),%%xmm0                     \n"      "movdqu    0x10(%0),%%xmm1                 \n"      "lea       0x20(%0),%0                     \n"      "pand      %%xmm5,%%xmm0                   \n"      "pand      %%xmm5,%%xmm1                   \n"      "packuswb  %%xmm1,%%xmm0                   \n"      "psrlw     $0x8,%%xmm0                     \n"      "packuswb  %%xmm0,%%xmm0                   \n"      "movq      %%xmm0,(%1)                     \n"      "lea       0x8(%1),%1                      \n"      "sub       $0x8,%2                         \n"      "jg        1b                              \n"      : "+r"(src_ptr),   // %0        "+r"(dst_ptr),   // %1        "+r"(dst_width)  // %2        ::"memory",        "cc", "xmm0", "xmm1", "xmm5");} void ScaleRowDown4Box_SSSE3(const uint8_t* src_ptr,                            ptrdiff_t src_stride,                            uint8_t* dst_ptr,                            int dst_width) {  intptr_t stridex3;  asm volatile(      "pcmpeqb    %%xmm4,%%xmm4                  \n"      "psrlw      $0xf,%%xmm4                    \n"      "movdqa     %%xmm4,%%xmm5                  \n"      "packuswb   %%xmm4,%%xmm4                  \n"      "psllw      $0x3,%%xmm5                    \n"      "lea       0x00(%4,%4,2),%3                \n"       LABELALIGN      "1:                                        \n"      "movdqu    (%0),%%xmm0                     \n"      "movdqu    0x10(%0),%%xmm1                 \n"      "movdqu    0x00(%0,%4,1),%%xmm2            \n"      "movdqu    0x10(%0,%4,1),%%xmm3            \n"      "pmaddubsw  %%xmm4,%%xmm0                  \n"      "pmaddubsw  %%xmm4,%%xmm1                  \n"      "pmaddubsw  %%xmm4,%%xmm2                  \n"      "pmaddubsw  %%xmm4,%%xmm3                  \n"      "paddw      %%xmm2,%%xmm0                  \n"      "paddw      %%xmm3,%%xmm1                  \n"      "movdqu    0x00(%0,%4,2),%%xmm2            \n"      "movdqu    0x10(%0,%4,2),%%xmm3            \n"      "pmaddubsw  %%xmm4,%%xmm2                  \n"      "pmaddubsw  %%xmm4,%%xmm3                  \n"      "paddw      %%xmm2,%%xmm0                  \n"      "paddw      %%xmm3,%%xmm1                  \n"      "movdqu    0x00(%0,%3,1),%%xmm2            \n"      "movdqu    0x10(%0,%3,1),%%xmm3            \n"      "lea       0x20(%0),%0                     \n"      "pmaddubsw  %%xmm4,%%xmm2                  \n"      "pmaddubsw  %%xmm4,%%xmm3                  \n"      "paddw      %%xmm2,%%xmm0                  \n"      "paddw      %%xmm3,%%xmm1                  \n"      "phaddw     %%xmm1,%%xmm0                  \n"      "paddw      %%xmm5,%%xmm0                  \n"      "psrlw      $0x4,%%xmm0                    \n"      "packuswb   %%xmm0,%%xmm0                  \n"      "movq      %%xmm0,(%1)                     \n"      "lea       0x8(%1),%1                      \n"      "sub       $0x8,%2                         \n"      "jg        1b                              \n"      : "+r"(src_ptr),               // %0        "+r"(dst_ptr),               // %1        "+r"(dst_width),             // %2        "=&r"(stridex3)              // %3      : "r"((intptr_t)(src_stride))  // %4      : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");} #ifdef HAS_SCALEROWDOWN4_AVX2void ScaleRowDown4_AVX2(const uint8_t* src_ptr,                        ptrdiff_t src_stride,                        uint8_t* dst_ptr,                        int dst_width) {  (void)src_stride;  asm volatile(      "vpcmpeqb   %%ymm5,%%ymm5,%%ymm5           \n"      "vpsrld     $0x18,%%ymm5,%%ymm5            \n"      "vpslld     $0x10,%%ymm5,%%ymm5            \n"       LABELALIGN      "1:                                        \n"      "vmovdqu    (%0),%%ymm0                    \n"      "vmovdqu    0x20(%0),%%ymm1                \n"      "lea        0x40(%0),%0                    \n"      "vpand      %%ymm5,%%ymm0,%%ymm0           \n"      "vpand      %%ymm5,%%ymm1,%%ymm1           \n"      "vpackuswb  %%ymm1,%%ymm0,%%ymm0           \n"      "vpermq     $0xd8,%%ymm0,%%ymm0            \n"      "vpsrlw     $0x8,%%ymm0,%%ymm0             \n"      "vpackuswb  %%ymm0,%%ymm0,%%ymm0           \n"      "vpermq     $0xd8,%%ymm0,%%ymm0            \n"      "vmovdqu    %%xmm0,(%1)                    \n"      "lea        0x10(%1),%1                    \n"      "sub        $0x10,%2                       \n"      "jg         1b                             \n"      "vzeroupper                                \n"      : "+r"(src_ptr),   // %0        "+r"(dst_ptr),   // %1        "+r"(dst_width)  // %2        ::"memory",        "cc", "xmm0", "xmm1", "xmm5");} void ScaleRowDown4Box_AVX2(const uint8_t* src_ptr,                           ptrdiff_t src_stride,                           uint8_t* dst_ptr,                           int dst_width) {  asm volatile(      "vpcmpeqb   %%ymm4,%%ymm4,%%ymm4           \n"      "vpsrlw     $0xf,%%ymm4,%%ymm4             \n"      "vpsllw     $0x3,%%ymm4,%%ymm5             \n"      "vpackuswb  %%ymm4,%%ymm4,%%ymm4           \n"       LABELALIGN      "1:                                        \n"      "vmovdqu    (%0),%%ymm0                    \n"      "vmovdqu    0x20(%0),%%ymm1                \n"      "vmovdqu    0x00(%0,%3,1),%%ymm2           \n"      "vmovdqu    0x20(%0,%3,1),%%ymm3           \n"      "vpmaddubsw %%ymm4,%%ymm0,%%ymm0           \n"      "vpmaddubsw %%ymm4,%%ymm1,%%ymm1           \n"      "vpmaddubsw %%ymm4,%%ymm2,%%ymm2           \n"      "vpmaddubsw %%ymm4,%%ymm3,%%ymm3           \n"      "vpaddw     %%ymm2,%%ymm0,%%ymm0           \n"      "vpaddw     %%ymm3,%%ymm1,%%ymm1           \n"      "vmovdqu    0x00(%0,%3,2),%%ymm2           \n"      "vmovdqu    0x20(%0,%3,2),%%ymm3           \n"      "vpmaddubsw %%ymm4,%%ymm2,%%ymm2           \n"      "vpmaddubsw %%ymm4,%%ymm3,%%ymm3           \n"      "vpaddw     %%ymm2,%%ymm0,%%ymm0           \n"      "vpaddw     %%ymm3,%%ymm1,%%ymm1           \n"      "vmovdqu    0x00(%0,%4,1),%%ymm2           \n"      "vmovdqu    0x20(%0,%4,1),%%ymm3           \n"      "lea        0x40(%0),%0                    \n"      "vpmaddubsw %%ymm4,%%ymm2,%%ymm2           \n"      "vpmaddubsw %%ymm4,%%ymm3,%%ymm3           \n"      "vpaddw     %%ymm2,%%ymm0,%%ymm0           \n"      "vpaddw     %%ymm3,%%ymm1,%%ymm1           \n"      "vphaddw    %%ymm1,%%ymm0,%%ymm0           \n"      "vpermq     $0xd8,%%ymm0,%%ymm0            \n"      "vpaddw     %%ymm5,%%ymm0,%%ymm0           \n"      "vpsrlw     $0x4,%%ymm0,%%ymm0             \n"      "vpackuswb  %%ymm0,%%ymm0,%%ymm0           \n"      "vpermq     $0xd8,%%ymm0,%%ymm0            \n"      "vmovdqu    %%xmm0,(%1)                    \n"      "lea        0x10(%1),%1                    \n"      "sub        $0x10,%2                       \n"      "jg         1b                             \n"      "vzeroupper                                \n"      : "+r"(src_ptr),                   // %0        "+r"(dst_ptr),                   // %1        "+r"(dst_width)                  // %2      : "r"((intptr_t)(src_stride)),     // %3        "r"((intptr_t)(src_stride * 3))  // %4      : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");}#endif  // HAS_SCALEROWDOWN4_AVX2 void ScaleRowDown34_SSSE3(const uint8_t* src_ptr,                          ptrdiff_t src_stride,                          uint8_t* dst_ptr,                          int dst_width) {  (void)src_stride;  asm volatile(      "movdqa    %0,%%xmm3                       \n"      "movdqa    %1,%%xmm4                       \n"      "movdqa    %2,%%xmm5                       \n"      :      : "m"(kShuf0),  // %0        "m"(kShuf1),  // %1        "m"(kShuf2)   // %2      );  asm volatile(       LABELALIGN      "1:                                        \n"      "movdqu    (%0),%%xmm0                     \n"      "movdqu    0x10(%0),%%xmm2                 \n"      "lea       0x20(%0),%0                     \n"      "movdqa    %%xmm2,%%xmm1                   \n"      "palignr   $0x8,%%xmm0,%%xmm1              \n"      "pshufb    %%xmm3,%%xmm0                   \n"      "pshufb    %%xmm4,%%xmm1                   \n"      "pshufb    %%xmm5,%%xmm2                   \n"      "movq      %%xmm0,(%1)                     \n"      "movq      %%xmm1,0x8(%1)                  \n"

Lines 1–500 of 1374. Load more