/* * Copyright © 2018-2019, VideoLAN and dav1d authors * Copyright © 2018-2019, Two Orioles, LLC * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, this * list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "config.h" #include <stddef.h>#include <stdint.h> #include "common/intops.h" #include "src/itx_1d.h" #define CLIP(a) iclip(a, min, max) /* * In some places, we use the pattern like this: * t2 = ((in1 * 1567 - in3 * (3784 - 4096) + 2048) >> 12) - in3; * even though the reference code might use something like: * t2 = (in1 * 1567 - in3 * 3784 + 2048) >> 12; * * The reason for this is that for 12 bits/component bitstreams (corrupt/ * invalid ones, but they are codable nonetheless), each coefficient or * input can be 19(+sign) bits, and therefore if the combination of the * two multipliers (each 12 bits) is >= 4096, the result of the add/sub * after the pair of multiplies will exceed the 31+sign bit range. Signed * integer overflows are UB in C, and we'd like to prevent that. * * To workaround this, we invert one of the two coefficients (or, if both are * multiples of 2, we reduce their magnitude by one bit). It should be noted * that SIMD implementations do not have to follow this exact behaviour. The * AV1 spec clearly states that the result of the multiply/add pairs should * fit in 31+sign bit intermediates, and that streams violating this convention * are not AV1-compliant. So, as long as we don't trigger UB (which some people * would consider a security vulnerability), we're fine. So, SIMD can simply * use the faster implementation, even if that might in some cases result in * integer overflows, since these are not considered valid AV1 anyway, and in * e.g. x86 assembly, integer overflows are not considered UB, but they merely * wrap around. */ static NOINLINE voidinv_dct4_1d_internal_c(int32_t *const c, const ptrdiff_t stride, const int min, const int max, const int tx64){ assert(stride > 0); const int in0 = c[0 * stride], in1 = c[1 * stride]; int t0, t1, t2, t3; if (tx64) { t0 = t1 = (in0 * 181 + 128) >> 8; t2 = (in1 * 1567 + 2048) >> 12; t3 = (in1 * 3784 + 2048) >> 12; } else { const int in2 = c[2 * stride], in3 = c[3 * stride]; t0 = ((in0 + in2) * 181 + 128) >> 8; t1 = ((in0 - in2) * 181 + 128) >> 8; t2 = ((in1 * 1567 - in3 * (3784 - 4096) + 2048) >> 12) - in3; t3 = ((in1 * (3784 - 4096) + in3 * 1567 + 2048) >> 12) + in1; } c[0 * stride] = CLIP(t0 + t3); c[1 * stride] = CLIP(t1 + t2); c[2 * stride] = CLIP(t1 - t2); c[3 * stride] = CLIP(t0 - t3);} void dav1d_inv_dct4_1d_c(int32_t *const c, const ptrdiff_t stride, const int min, const int max){ inv_dct4_1d_internal_c(c, stride, min, max, 0);} static NOINLINE voidinv_dct8_1d_internal_c(int32_t *const c, const ptrdiff_t stride, const int min, const int max, const int tx64){ assert(stride > 0); inv_dct4_1d_internal_c(c, stride << 1, min, max, tx64); const int in1 = c[1 * stride], in3 = c[3 * stride]; int t4a, t5a, t6a, t7a; if (tx64) { t4a = (in1 * 799 + 2048) >> 12; t5a = (in3 * -2276 + 2048) >> 12; t6a = (in3 * 3406 + 2048) >> 12; t7a = (in1 * 4017 + 2048) >> 12; } else { const int in5 = c[5 * stride], in7 = c[7 * stride]; t4a = ((in1 * 799 - in7 * (4017 - 4096) + 2048) >> 12) - in7; t5a = (in5 * 1703 - in3 * 1138 + 1024) >> 11; t6a = (in5 * 1138 + in3 * 1703 + 1024) >> 11; t7a = ((in1 * (4017 - 4096) + in7 * 799 + 2048) >> 12) + in1; } const int t4 = CLIP(t4a + t5a); t5a = CLIP(t4a - t5a); const int t7 = CLIP(t7a + t6a); t6a = CLIP(t7a - t6a); const int t5 = ((t6a - t5a) * 181 + 128) >> 8; const int t6 = ((t6a + t5a) * 181 + 128) >> 8; const int t0 = c[0 * stride]; const int t1 = c[2 * stride]; const int t2 = c[4 * stride]; const int t3 = c[6 * stride]; c[0 * stride] = CLIP(t0 + t7); c[1 * stride] = CLIP(t1 + t6); c[2 * stride] = CLIP(t2 + t5); c[3 * stride] = CLIP(t3 + t4); c[4 * stride] = CLIP(t3 - t4); c[5 * stride] = CLIP(t2 - t5); c[6 * stride] = CLIP(t1 - t6); c[7 * stride] = CLIP(t0 - t7);} void dav1d_inv_dct8_1d_c(int32_t *const c, const ptrdiff_t stride, const int min, const int max){ inv_dct8_1d_internal_c(c, stride, min, max, 0);} static NOINLINE voidinv_dct16_1d_internal_c(int32_t *const c, const ptrdiff_t stride, const int min, const int max, int tx64){ assert(stride > 0); inv_dct8_1d_internal_c(c, stride << 1, min, max, tx64); const int in1 = c[1 * stride], in3 = c[3 * stride]; const int in5 = c[5 * stride], in7 = c[7 * stride]; int t8a, t9a, t10a, t11a, t12a, t13a, t14a, t15a; if (tx64) { t8a = (in1 * 401 + 2048) >> 12; t9a = (in7 * -2598 + 2048) >> 12; t10a = (in5 * 1931 + 2048) >> 12; t11a = (in3 * -1189 + 2048) >> 12; t12a = (in3 * 3920 + 2048) >> 12; t13a = (in5 * 3612 + 2048) >> 12; t14a = (in7 * 3166 + 2048) >> 12; t15a = (in1 * 4076 + 2048) >> 12; } else { const int in9 = c[ 9 * stride], in11 = c[11 * stride]; const int in13 = c[13 * stride], in15 = c[15 * stride]; t8a = ((in1 * 401 - in15 * (4076 - 4096) + 2048) >> 12) - in15; t9a = (in9 * 1583 - in7 * 1299 + 1024) >> 11; t10a = ((in5 * 1931 - in11 * (3612 - 4096) + 2048) >> 12) - in11; t11a = ((in13 * (3920 - 4096) - in3 * 1189 + 2048) >> 12) + in13; t12a = ((in13 * 1189 + in3 * (3920 - 4096) + 2048) >> 12) + in3; t13a = ((in5 * (3612 - 4096) + in11 * 1931 + 2048) >> 12) + in5; t14a = (in9 * 1299 + in7 * 1583 + 1024) >> 11; t15a = ((in1 * (4076 - 4096) + in15 * 401 + 2048) >> 12) + in1; } int t8 = CLIP(t8a + t9a); int t9 = CLIP(t8a - t9a); int t10 = CLIP(t11a - t10a); int t11 = CLIP(t11a + t10a); int t12 = CLIP(t12a + t13a); int t13 = CLIP(t12a - t13a); int t14 = CLIP(t15a - t14a); int t15 = CLIP(t15a + t14a); t9a = (( t14 * 1567 - t9 * (3784 - 4096) + 2048) >> 12) - t9; t14a = (( t14 * (3784 - 4096) + t9 * 1567 + 2048) >> 12) + t14; t10a = ((-(t13 * (3784 - 4096) + t10 * 1567) + 2048) >> 12) - t13; t13a = (( t13 * 1567 - t10 * (3784 - 4096) + 2048) >> 12) - t10; t8a = CLIP(t8 + t11); t9 = CLIP(t9a + t10a); t10 = CLIP(t9a - t10a); t11a = CLIP(t8 - t11); t12a = CLIP(t15 - t12); t13 = CLIP(t14a - t13a); t14 = CLIP(t14a + t13a); t15a = CLIP(t15 + t12); t10a = ((t13 - t10) * 181 + 128) >> 8; t13a = ((t13 + t10) * 181 + 128) >> 8; t11 = ((t12a - t11a) * 181 + 128) >> 8; t12 = ((t12a + t11a) * 181 + 128) >> 8; const int t0 = c[ 0 * stride]; const int t1 = c[ 2 * stride]; const int t2 = c[ 4 * stride]; const int t3 = c[ 6 * stride]; const int t4 = c[ 8 * stride]; const int t5 = c[10 * stride]; const int t6 = c[12 * stride]; const int t7 = c[14 * stride]; c[ 0 * stride] = CLIP(t0 + t15a); c[ 1 * stride] = CLIP(t1 + t14); c[ 2 * stride] = CLIP(t2 + t13a); c[ 3 * stride] = CLIP(t3 + t12); c[ 4 * stride] = CLIP(t4 + t11); c[ 5 * stride] = CLIP(t5 + t10a); c[ 6 * stride] = CLIP(t6 + t9); c[ 7 * stride] = CLIP(t7 + t8a); c[ 8 * stride] = CLIP(t7 - t8a); c[ 9 * stride] = CLIP(t6 - t9); c[10 * stride] = CLIP(t5 - t10a); c[11 * stride] = CLIP(t4 - t11); c[12 * stride] = CLIP(t3 - t12); c[13 * stride] = CLIP(t2 - t13a); c[14 * stride] = CLIP(t1 - t14); c[15 * stride] = CLIP(t0 - t15a);} void dav1d_inv_dct16_1d_c(int32_t *const c, const ptrdiff_t stride, const int min, const int max){ inv_dct16_1d_internal_c(c, stride, min, max, 0);} static NOINLINE voidinv_dct32_1d_internal_c(int32_t *const c, const ptrdiff_t stride, const int min, const int max, const int tx64){ assert(stride > 0); inv_dct16_1d_internal_c(c, stride << 1, min, max, tx64); const int in1 = c[ 1 * stride], in3 = c[ 3 * stride]; const int in5 = c[ 5 * stride], in7 = c[ 7 * stride]; const int in9 = c[ 9 * stride], in11 = c[11 * stride]; const int in13 = c[13 * stride], in15 = c[15 * stride]; int t16a, t17a, t18a, t19a, t20a, t21a, t22a, t23a; int t24a, t25a, t26a, t27a, t28a, t29a, t30a, t31a; if (tx64) { t16a = (in1 * 201 + 2048) >> 12; t17a = (in15 * -2751 + 2048) >> 12; t18a = (in9 * 1751 + 2048) >> 12; t19a = (in7 * -1380 + 2048) >> 12; t20a = (in5 * 995 + 2048) >> 12; t21a = (in11 * -2106 + 2048) >> 12; t22a = (in13 * 2440 + 2048) >> 12; t23a = (in3 * -601 + 2048) >> 12; t24a = (in3 * 4052 + 2048) >> 12; t25a = (in13 * 3290 + 2048) >> 12; t26a = (in11 * 3513 + 2048) >> 12; t27a = (in5 * 3973 + 2048) >> 12; t28a = (in7 * 3857 + 2048) >> 12; t29a = (in9 * 3703 + 2048) >> 12; t30a = (in15 * 3035 + 2048) >> 12; t31a = (in1 * 4091 + 2048) >> 12; } else { const int in17 = c[17 * stride], in19 = c[19 * stride]; const int in21 = c[21 * stride], in23 = c[23 * stride]; const int in25 = c[25 * stride], in27 = c[27 * stride]; const int in29 = c[29 * stride], in31 = c[31 * stride]; t16a = ((in1 * 201 - in31 * (4091 - 4096) + 2048) >> 12) - in31; t17a = ((in17 * (3035 - 4096) - in15 * 2751 + 2048) >> 12) + in17; t18a = ((in9 * 1751 - in23 * (3703 - 4096) + 2048) >> 12) - in23; t19a = ((in25 * (3857 - 4096) - in7 * 1380 + 2048) >> 12) + in25; t20a = ((in5 * 995 - in27 * (3973 - 4096) + 2048) >> 12) - in27; t21a = ((in21 * (3513 - 4096) - in11 * 2106 + 2048) >> 12) + in21; t22a = (in13 * 1220 - in19 * 1645 + 1024) >> 11; t23a = ((in29 * (4052 - 4096) - in3 * 601 + 2048) >> 12) + in29; t24a = ((in29 * 601 + in3 * (4052 - 4096) + 2048) >> 12) + in3; t25a = (in13 * 1645 + in19 * 1220 + 1024) >> 11; t26a = ((in21 * 2106 + in11 * (3513 - 4096) + 2048) >> 12) + in11; t27a = ((in5 * (3973 - 4096) + in27 * 995 + 2048) >> 12) + in5; t28a = ((in25 * 1380 + in7 * (3857 - 4096) + 2048) >> 12) + in7; t29a = ((in9 * (3703 - 4096) + in23 * 1751 + 2048) >> 12) + in9; t30a = ((in17 * 2751 + in15 * (3035 - 4096) + 2048) >> 12) + in15; t31a = ((in1 * (4091 - 4096) + in31 * 201 + 2048) >> 12) + in1; } int t16 = CLIP(t16a + t17a); int t17 = CLIP(t16a - t17a); int t18 = CLIP(t19a - t18a); int t19 = CLIP(t19a + t18a); int t20 = CLIP(t20a + t21a); int t21 = CLIP(t20a - t21a); int t22 = CLIP(t23a - t22a); int t23 = CLIP(t23a + t22a); int t24 = CLIP(t24a + t25a); int t25 = CLIP(t24a - t25a); int t26 = CLIP(t27a - t26a); int t27 = CLIP(t27a + t26a); int t28 = CLIP(t28a + t29a); int t29 = CLIP(t28a - t29a); int t30 = CLIP(t31a - t30a); int t31 = CLIP(t31a + t30a); t17a = (( t30 * 799 - t17 * (4017 - 4096) + 2048) >> 12) - t17; t30a = (( t30 * (4017 - 4096) + t17 * 799 + 2048) >> 12) + t30; t18a = ((-(t29 * (4017 - 4096) + t18 * 799) + 2048) >> 12) - t29; t29a = (( t29 * 799 - t18 * (4017 - 4096) + 2048) >> 12) - t18; t21a = ( t26 * 1703 - t21 * 1138 + 1024) >> 11; t26a = ( t26 * 1138 + t21 * 1703 + 1024) >> 11; t22a = (-(t25 * 1138 + t22 * 1703 ) + 1024) >> 11; t25a = ( t25 * 1703 - t22 * 1138 + 1024) >> 11; t16a = CLIP(t16 + t19); t17 = CLIP(t17a + t18a); t18 = CLIP(t17a - t18a); t19a = CLIP(t16 - t19); t20a = CLIP(t23 - t20); t21 = CLIP(t22a - t21a); t22 = CLIP(t22a + t21a); t23a = CLIP(t23 + t20); t24a = CLIP(t24 + t27); t25 = CLIP(t25a + t26a); t26 = CLIP(t25a - t26a); t27a = CLIP(t24 - t27); t28a = CLIP(t31 - t28); t29 = CLIP(t30a - t29a); t30 = CLIP(t30a + t29a); t31a = CLIP(t31 + t28); t18a = (( t29 * 1567 - t18 * (3784 - 4096) + 2048) >> 12) - t18; t29a = (( t29 * (3784 - 4096) + t18 * 1567 + 2048) >> 12) + t29; t19 = (( t28a * 1567 - t19a * (3784 - 4096) + 2048) >> 12) - t19a; t28 = (( t28a * (3784 - 4096) + t19a * 1567 + 2048) >> 12) + t28a; t20 = ((-(t27a * (3784 - 4096) + t20a * 1567) + 2048) >> 12) - t27a; t27 = (( t27a * 1567 - t20a * (3784 - 4096) + 2048) >> 12) - t20a; t21a = ((-(t26 * (3784 - 4096) + t21 * 1567) + 2048) >> 12) - t26; t26a = (( t26 * 1567 - t21 * (3784 - 4096) + 2048) >> 12) - t21; t16 = CLIP(t16a + t23a); t17a = CLIP(t17 + t22); t18 = CLIP(t18a + t21a); t19a = CLIP(t19 + t20); t20a = CLIP(t19 - t20); t21 = CLIP(t18a - t21a); t22a = CLIP(t17 - t22); t23 = CLIP(t16a - t23a); t24 = CLIP(t31a - t24a); t25a = CLIP(t30 - t25); t26 = CLIP(t29a - t26a); t27a = CLIP(t28 - t27); t28a = CLIP(t28 + t27); t29 = CLIP(t29a + t26a); t30a = CLIP(t30 + t25); t31 = CLIP(t31a + t24a); t20 = ((t27a - t20a) * 181 + 128) >> 8; t27 = ((t27a + t20a) * 181 + 128) >> 8; t21a = ((t26 - t21 ) * 181 + 128) >> 8; t26a = ((t26 + t21 ) * 181 + 128) >> 8; t22 = ((t25a - t22a) * 181 + 128) >> 8; t25 = ((t25a + t22a) * 181 + 128) >> 8; t23a = ((t24 - t23 ) * 181 + 128) >> 8; t24a = ((t24 + t23 ) * 181 + 128) >> 8; const int t0 = c[ 0 * stride]; const int t1 = c[ 2 * stride]; const int t2 = c[ 4 * stride]; const int t3 = c[ 6 * stride]; const int t4 = c[ 8 * stride]; const int t5 = c[10 * stride]; const int t6 = c[12 * stride]; const int t7 = c[14 * stride]; const int t8 = c[16 * stride]; const int t9 = c[18 * stride]; const int t10 = c[20 * stride]; const int t11 = c[22 * stride]; const int t12 = c[24 * stride]; const int t13 = c[26 * stride]; const int t14 = c[28 * stride]; const int t15 = c[30 * stride]; c[ 0 * stride] = CLIP(t0 + t31); c[ 1 * stride] = CLIP(t1 + t30a); c[ 2 * stride] = CLIP(t2 + t29); c[ 3 * stride] = CLIP(t3 + t28a); c[ 4 * stride] = CLIP(t4 + t27); c[ 5 * stride] = CLIP(t5 + t26a); c[ 6 * stride] = CLIP(t6 + t25); c[ 7 * stride] = CLIP(t7 + t24a); c[ 8 * stride] = CLIP(t8 + t23a); c[ 9 * stride] = CLIP(t9 + t22); c[10 * stride] = CLIP(t10 + t21a); c[11 * stride] = CLIP(t11 + t20); c[12 * stride] = CLIP(t12 + t19a); c[13 * stride] = CLIP(t13 + t18); c[14 * stride] = CLIP(t14 + t17a); c[15 * stride] = CLIP(t15 + t16); c[16 * stride] = CLIP(t15 - t16); c[17 * stride] = CLIP(t14 - t17a); c[18 * stride] = CLIP(t13 - t18); c[19 * stride] = CLIP(t12 - t19a); c[20 * stride] = CLIP(t11 - t20); c[21 * stride] = CLIP(t10 - t21a); c[22 * stride] = CLIP(t9 - t22); c[23 * stride] = CLIP(t8 - t23a); c[24 * stride] = CLIP(t7 - t24a); c[25 * stride] = CLIP(t6 - t25); c[26 * stride] = CLIP(t5 - t26a); c[27 * stride] = CLIP(t4 - t27); c[28 * stride] = CLIP(t3 - t28a); c[29 * stride] = CLIP(t2 - t29); c[30 * stride] = CLIP(t1 - t30a); c[31 * stride] = CLIP(t0 - t31);} void dav1d_inv_dct32_1d_c(int32_t *const c, const ptrdiff_t stride, const int min, const int max){ inv_dct32_1d_internal_c(c, stride, min, max, 0);} void dav1d_inv_dct64_1d_c(int32_t *const c, const ptrdiff_t stride, const int min, const int max){ assert(stride > 0); inv_dct32_1d_internal_c(c, stride << 1, min, max, 1); const int in1 = c[ 1 * stride], in3 = c[ 3 * stride]; const int in5 = c[ 5 * stride], in7 = c[ 7 * stride]; const int in9 = c[ 9 * stride], in11 = c[11 * stride]; const int in13 = c[13 * stride], in15 = c[15 * stride]; const int in17 = c[17 * stride], in19 = c[19 * stride]; const int in21 = c[21 * stride], in23 = c[23 * stride]; const int in25 = c[25 * stride], in27 = c[27 * stride]; const int in29 = c[29 * stride], in31 = c[31 * stride]; int t32a = (in1 * 101 + 2048) >> 12; int t33a = (in31 * -2824 + 2048) >> 12; int t34a = (in17 * 1660 + 2048) >> 12; int t35a = (in15 * -1474 + 2048) >> 12; int t36a = (in9 * 897 + 2048) >> 12; int t37a = (in23 * -2191 + 2048) >> 12; int t38a = (in25 * 2359 + 2048) >> 12; int t39a = (in7 * -700 + 2048) >> 12; int t40a = (in5 * 501 + 2048) >> 12; int t41a = (in27 * -2520 + 2048) >> 12; int t42a = (in21 * 2019 + 2048) >> 12; int t43a = (in11 * -1092 + 2048) >> 12; int t44a = (in13 * 1285 + 2048) >> 12; int t45a = (in19 * -1842 + 2048) >> 12; int t46a = (in29 * 2675 + 2048) >> 12; int t47a = (in3 * -301 + 2048) >> 12; int t48a = (in3 * 4085 + 2048) >> 12; int t49a = (in29 * 3102 + 2048) >> 12; int t50a = (in19 * 3659 + 2048) >> 12; int t51a = (in13 * 3889 + 2048) >> 12; int t52a = (in11 * 3948 + 2048) >> 12; int t53a = (in21 * 3564 + 2048) >> 12; int t54a = (in27 * 3229 + 2048) >> 12; int t55a = (in5 * 4065 + 2048) >> 12; int t56a = (in7 * 4036 + 2048) >> 12; int t57a = (in25 * 3349 + 2048) >> 12; int t58a = (in23 * 3461 + 2048) >> 12; int t59a = (in9 * 3996 + 2048) >> 12; int t60a = (in15 * 3822 + 2048) >> 12; int t61a = (in17 * 3745 + 2048) >> 12; int t62a = (in31 * 2967 + 2048) >> 12; int t63a = (in1 * 4095 + 2048) >> 12; int t32 = CLIP(t32a + t33a); int t33 = CLIP(t32a - t33a); int t34 = CLIP(t35a - t34a); int t35 = CLIP(t35a + t34a); int t36 = CLIP(t36a + t37a); int t37 = CLIP(t36a - t37a); int t38 = CLIP(t39a - t38a); int t39 = CLIP(t39a + t38a); int t40 = CLIP(t40a + t41a); int t41 = CLIP(t40a - t41a); int t42 = CLIP(t43a - t42a); int t43 = CLIP(t43a + t42a); int t44 = CLIP(t44a + t45a); int t45 = CLIP(t44a - t45a); int t46 = CLIP(t47a - t46a); int t47 = CLIP(t47a + t46a); int t48 = CLIP(t48a + t49a);Lines 1–500 of 1034. Load more
/* * Copyright © 2018-2019, VideoLAN and dav1d authors * Copyright © 2018-2019, Two Orioles, LLC * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, this * list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "config.h" #include <stddef.h> #include <stdint.h> #include "common/intops.h" #include "src/itx_1d.h" #define CLIP(a) iclip(a, min, max) /* * In some places, we use the pattern like this: * t2 = ((in1 * 1567 - in3 * (3784 - 4096) + 2048) >> 12) - in3; * even though the reference code might use something like: * t2 = (in1 * 1567 - in3 * 3784 + 2048) >> 12; * * The reason for this is that for 12 bits/component bitstreams (corrupt/ * invalid ones, but they are codable nonetheless), each coefficient or * input can be 19(+sign) bits, and therefore if the combination of the * two multipliers (each 12 bits) is >= 4096, the result of the add/sub * after the pair of multiplies will exceed the 31+sign bit range. Signed * integer overflows are UB in C, and we'd like to prevent that. * * To workaround this, we invert one of the two coefficients (or, if both are * multiples of 2, we reduce their magnitude by one bit). It should be noted * that SIMD implementations do not have to follow this exact behaviour. The * AV1 spec clearly states that the result of the multiply/add pairs should * fit in 31+sign bit intermediates, and that streams violating this convention * are not AV1-compliant. So, as long as we don't trigger UB (which some people * would consider a security vulnerability), we're fine. So, SIMD can simply * use the faster implementation, even if that might in some cases result in * integer overflows, since these are not considered valid AV1 anyway, and in * e.g. x86 assembly, integer overflows are not considered UB, but they merely * wrap around. */ static NOINLINE void inv_dct4_1d_internal_c(int32_t *const c, const ptrdiff_t stride, const int min, const int max, const int tx64) { assert(stride > 0); const int in0 = c[0 * stride], in1 = c[1 * stride]; int t0, t1, t2, t3; if (tx64) { t0 = t1 = (in0 * 181 + 128) >> 8; t2 = (in1 * 1567 + 2048) >> 12; t3 = (in1 * 3784 + 2048) >> 12; } else { const int in2 = c[2 * stride], in3 = c[3 * stride]; t0 = ((in0 + in2) * 181 + 128) >> 8; t1 = ((in0 - in2) * 181 + 128) >> 8; t2 = ((in1 * 1567 - in3 * (3784 - 4096) + 2048) >> 12) - in3; t3 = ((in1 * (3784 - 4096) + in3 * 1567 + 2048) >> 12) + in1; } c[0 * stride] = CLIP(t0 + t3); c[1 * stride] = CLIP(t1 + t2); c[2 * stride] = CLIP(t1 - t2); c[3 * stride] = CLIP(t0 - t3); } void dav1d_inv_dct4_1d_c(int32_t *const c, const ptrdiff_t stride, const int min, const int max) { inv_dct4_1d_internal_c(c, stride, min, max, 0); } static NOINLINE void inv_dct8_1d_internal_c(int32_t *const c, const ptrdiff_t stride, const int min, const int max, const int tx64) { assert(stride > 0); inv_dct4_1d_internal_c(c, stride << 1, min, max, tx64); const int in1 = c[1 * stride], in3 = c[3 * stride]; int t4a, t5a, t6a, t7a; if (tx64) { t4a = (in1 * 799 + 2048) >> 12; t5a = (in3 * -2276 + 2048) >> 12; t6a = (in3 * 3406 + 2048) >> 12; t7a = (in1 * 4017 + 2048) >> 12; } else { const int in5 = c[5 * stride], in7 = c[7 * stride]; t4a = ((in1 * 799 - in7 * (4017 - 4096) + 2048) >> 12) - in7; t5a = (in5 * 1703 - in3 * 1138 + 1024) >> 11; t6a = (in5 * 1138 + in3 * 1703 + 1024) >> 11; t7a = ((in1 * (4017 - 4096) + in7 * 799 + 2048) >> 12) + in1; } const int t4 = CLIP(t4a + t5a); t5a = CLIP(t4a - t5a); const int t7 = CLIP(t7a + t6a); t6a = CLIP(t7a - t6a); const int t5 = ((t6a - t5a) * 181 + 128) >> 8; const int t6 = ((t6a + t5a) * 181 + 128) >> 8; const int t0 = c[0 * stride]; const int t1 = c[2 * stride]; const int t2 = c[4 * stride]; const int t3 = c[6 * stride]; c[0 * stride] = CLIP(t0 + t7); c[1 * stride] = CLIP(t1 + t6); c[2 * stride] = CLIP(t2 + t5); c[3 * stride] = CLIP(t3 + t4); c[4 * stride] = CLIP(t3 - t4); c[5 * stride] = CLIP(t2 - t5); c[6 * stride] = CLIP(t1 - t6); c[7 * stride] = CLIP(t0 - t7); } void dav1d_inv_dct8_1d_c(int32_t *const c, const ptrdiff_t stride, const int min, const int max) { inv_dct8_1d_internal_c(c, stride, min, max, 0); } static NOINLINE void inv_dct16_1d_internal_c(int32_t *const c, const ptrdiff_t stride, const int min, const int max, int tx64) { assert(stride > 0); inv_dct8_1d_internal_c(c, stride << 1, min, max, tx64); const int in1 = c[1 * stride], in3 = c[3 * stride]; const int in5 = c[5 * stride], in7 = c[7 * stride]; int t8a, t9a, t10a, t11a, t12a, t13a, t14a, t15a; if (tx64) { t8a = (in1 * 401 + 2048) >> 12; t9a = (in7 * -2598 + 2048) >> 12; t10a = (in5 * 1931 + 2048) >> 12; t11a = (in3 * -1189 + 2048) >> 12; t12a = (in3 * 3920 + 2048) >> 12; t13a = (in5 * 3612 + 2048) >> 12; t14a = (in7 * 3166 + 2048) >> 12; t15a = (in1 * 4076 + 2048) >> 12; } else { const int in9 = c[ 9 * stride], in11 = c[11 * stride]; const int in13 = c[13 * stride], in15 = c[15 * stride]; t8a = ((in1 * 401 - in15 * (4076 - 4096) + 2048) >> 12) - in15; t9a = (in9 * 1583 - in7 * 1299 + 1024) >> 11; t10a = ((in5 * 1931 - in11 * (3612 - 4096) + 2048) >> 12) - in11; t11a = ((in13 * (3920 - 4096) - in3 * 1189 + 2048) >> 12) + in13; t12a = ((in13 * 1189 + in3 * (3920 - 4096) + 2048) >> 12) + in3; t13a = ((in5 * (3612 - 4096) + in11 * 1931 + 2048) >> 12) + in5; t14a = (in9 * 1299 + in7 * 1583 + 1024) >> 11; t15a = ((in1 * (4076 - 4096) + in15 * 401 + 2048) >> 12) + in1; } int t8 = CLIP(t8a + t9a); int t9 = CLIP(t8a - t9a); int t10 = CLIP(t11a - t10a); int t11 = CLIP(t11a + t10a); int t12 = CLIP(t12a + t13a); int t13 = CLIP(t12a - t13a); int t14 = CLIP(t15a - t14a); int t15 = CLIP(t15a + t14a); t9a = (( t14 * 1567 - t9 * (3784 - 4096) + 2048) >> 12) - t9; t14a = (( t14 * (3784 - 4096) + t9 * 1567 + 2048) >> 12) + t14; t10a = ((-(t13 * (3784 - 4096) + t10 * 1567) + 2048) >> 12) - t13; t13a = (( t13 * 1567 - t10 * (3784 - 4096) + 2048) >> 12) - t10; t8a = CLIP(t8 + t11); t9 = CLIP(t9a + t10a); t10 = CLIP(t9a - t10a); t11a = CLIP(t8 - t11); t12a = CLIP(t15 - t12); t13 = CLIP(t14a - t13a); t14 = CLIP(t14a + t13a); t15a = CLIP(t15 + t12); t10a = ((t13 - t10) * 181 + 128) >> 8; t13a = ((t13 + t10) * 181 + 128) >> 8; t11 = ((t12a - t11a) * 181 + 128) >> 8; t12 = ((t12a + t11a) * 181 + 128) >> 8; const int t0 = c[ 0 * stride]; const int t1 = c[ 2 * stride]; const int t2 = c[ 4 * stride]; const int t3 = c[ 6 * stride]; const int t4 = c[ 8 * stride]; const int t5 = c[10 * stride]; const int t6 = c[12 * stride]; const int t7 = c[14 * stride]; c[ 0 * stride] = CLIP(t0 + t15a); c[ 1 * stride] = CLIP(t1 + t14); c[ 2 * stride] = CLIP(t2 + t13a); c[ 3 * stride] = CLIP(t3 + t12); c[ 4 * stride] = CLIP(t4 + t11); c[ 5 * stride] = CLIP(t5 + t10a); c[ 6 * stride] = CLIP(t6 + t9); c[ 7 * stride] = CLIP(t7 + t8a); c[ 8 * stride] = CLIP(t7 - t8a); c[ 9 * stride] = CLIP(t6 - t9); c[10 * stride] = CLIP(t5 - t10a); c[11 * stride] = CLIP(t4 - t11); c[12 * stride] = CLIP(t3 - t12); c[13 * stride] = CLIP(t2 - t13a); c[14 * stride] = CLIP(t1 - t14); c[15 * stride] = CLIP(t0 - t15a); } void dav1d_inv_dct16_1d_c(int32_t *const c, const ptrdiff_t stride, const int min, const int max) { inv_dct16_1d_internal_c(c, stride, min, max, 0); } static NOINLINE void inv_dct32_1d_internal_c(int32_t *const c, const ptrdiff_t stride, const int min, const int max, const int tx64) { assert(stride > 0); inv_dct16_1d_internal_c(c, stride << 1, min, max, tx64); const int in1 = c[ 1 * stride], in3 = c[ 3 * stride]; const int in5 = c[ 5 * stride], in7 = c[ 7 * stride]; const int in9 = c[ 9 * stride], in11 = c[11 * stride]; const int in13 = c[13 * stride], in15 = c[15 * stride]; int t16a, t17a, t18a, t19a, t20a, t21a, t22a, t23a; int t24a, t25a, t26a, t27a, t28a, t29a, t30a, t31a; if (tx64) { t16a = (in1 * 201 + 2048) >> 12; t17a = (in15 * -2751 + 2048) >> 12; t18a = (in9 * 1751 + 2048) >> 12; t19a = (in7 * -1380 + 2048) >> 12; t20a = (in5 * 995 + 2048) >> 12; t21a = (in11 * -2106 + 2048) >> 12; t22a = (in13 * 2440 + 2048) >> 12; t23a = (in3 * -601 + 2048) >> 12; t24a = (in3 * 4052 + 2048) >> 12; t25a = (in13 * 3290 + 2048) >> 12; t26a = (in11 * 3513 + 2048) >> 12; t27a = (in5 * 3973 + 2048) >> 12; t28a = (in7 * 3857 + 2048) >> 12; t29a = (in9 * 3703 + 2048) >> 12; t30a = (in15 * 3035 + 2048) >> 12; t31a = (in1 * 4091 + 2048) >> 12; } else { const int in17 = c[17 * stride], in19 = c[19 * stride]; const int in21 = c[21 * stride], in23 = c[23 * stride]; const int in25 = c[25 * stride], in27 = c[27 * stride]; const int in29 = c[29 * stride], in31 = c[31 * stride]; t16a = ((in1 * 201 - in31 * (4091 - 4096) + 2048) >> 12) - in31; t17a = ((in17 * (3035 - 4096) - in15 * 2751 + 2048) >> 12) + in17; t18a = ((in9 * 1751 - in23 * (3703 - 4096) + 2048) >> 12) - in23; t19a = ((in25 * (3857 - 4096) - in7 * 1380 + 2048) >> 12) + in25; t20a = ((in5 * 995 - in27 * (3973 - 4096) + 2048) >> 12) - in27; t21a = ((in21 * (3513 - 4096) - in11 * 2106 + 2048) >> 12) + in21; t22a = (in13 * 1220 - in19 * 1645 + 1024) >> 11; t23a = ((in29 * (4052 - 4096) - in3 * 601 + 2048) >> 12) + in29; t24a = ((in29 * 601 + in3 * (4052 - 4096) + 2048) >> 12) + in3; t25a = (in13 * 1645 + in19 * 1220 + 1024) >> 11; t26a = ((in21 * 2106 + in11 * (3513 - 4096) + 2048) >> 12) + in11; t27a = ((in5 * (3973 - 4096) + in27 * 995 + 2048) >> 12) + in5; t28a = ((in25 * 1380 + in7 * (3857 - 4096) + 2048) >> 12) + in7; t29a = ((in9 * (3703 - 4096) + in23 * 1751 + 2048) >> 12) + in9; t30a = ((in17 * 2751 + in15 * (3035 - 4096) + 2048) >> 12) + in15; t31a = ((in1 * (4091 - 4096) + in31 * 201 + 2048) >> 12) + in1; } int t16 = CLIP(t16a + t17a); int t17 = CLIP(t16a - t17a); int t18 = CLIP(t19a - t18a); int t19 = CLIP(t19a + t18a); int t20 = CLIP(t20a + t21a); int t21 = CLIP(t20a - t21a); int t22 = CLIP(t23a - t22a); int t23 = CLIP(t23a + t22a); int t24 = CLIP(t24a + t25a); int t25 = CLIP(t24a - t25a); int t26 = CLIP(t27a - t26a); int t27 = CLIP(t27a + t26a); int t28 = CLIP(t28a + t29a); int t29 = CLIP(t28a - t29a); int t30 = CLIP(t31a - t30a); int t31 = CLIP(t31a + t30a); t17a = (( t30 * 799 - t17 * (4017 - 4096) + 2048) >> 12) - t17; t30a = (( t30 * (4017 - 4096) + t17 * 799 + 2048) >> 12) + t30; t18a = ((-(t29 * (4017 - 4096) + t18 * 799) + 2048) >> 12) - t29; t29a = (( t29 * 799 - t18 * (4017 - 4096) + 2048) >> 12) - t18; t21a = ( t26 * 1703 - t21 * 1138 + 1024) >> 11; t26a = ( t26 * 1138 + t21 * 1703 + 1024) >> 11; t22a = (-(t25 * 1138 + t22 * 1703 ) + 1024) >> 11; t25a = ( t25 * 1703 - t22 * 1138 + 1024) >> 11; t16a = CLIP(t16 + t19); t17 = CLIP(t17a + t18a); t18 = CLIP(t17a - t18a); t19a = CLIP(t16 - t19); t20a = CLIP(t23 - t20); t21 = CLIP(t22a - t21a); t22 = CLIP(t22a + t21a); t23a = CLIP(t23 + t20); t24a = CLIP(t24 + t27); t25 = CLIP(t25a + t26a); t26 = CLIP(t25a - t26a); t27a = CLIP(t24 - t27); t28a = CLIP(t31 - t28); t29 = CLIP(t30a - t29a); t30 = CLIP(t30a + t29a); t31a = CLIP(t31 + t28); t18a = (( t29 * 1567 - t18 * (3784 - 4096) + 2048) >> 12) - t18; t29a = (( t29 * (3784 - 4096) + t18 * 1567 + 2048) >> 12) + t29; t19 = (( t28a * 1567 - t19a * (3784 - 4096) + 2048) >> 12) - t19a; t28 = (( t28a * (3784 - 4096) + t19a * 1567 + 2048) >> 12) + t28a; t20 = ((-(t27a * (3784 - 4096) + t20a * 1567) + 2048) >> 12) - t27a; t27 = (( t27a * 1567 - t20a * (3784 - 4096) + 2048) >> 12) - t20a; t21a = ((-(t26 * (3784 - 4096) + t21 * 1567) + 2048) >> 12) - t26; t26a = (( t26 * 1567 - t21 * (3784 - 4096) + 2048) >> 12) - t21; t16 = CLIP(t16a + t23a); t17a = CLIP(t17 + t22); t18 = CLIP(t18a + t21a); t19a = CLIP(t19 + t20); t20a = CLIP(t19 - t20); t21 = CLIP(t18a - t21a); t22a = CLIP(t17 - t22); t23 = CLIP(t16a - t23a); t24 = CLIP(t31a - t24a); t25a = CLIP(t30 - t25); t26 = CLIP(t29a - t26a); t27a = CLIP(t28 - t27); t28a = CLIP(t28 + t27); t29 = CLIP(t29a + t26a); t30a = CLIP(t30 + t25); t31 = CLIP(t31a + t24a); t20 = ((t27a - t20a) * 181 + 128) >> 8; t27 = ((t27a + t20a) * 181 + 128) >> 8; t21a = ((t26 - t21 ) * 181 + 128) >> 8; t26a = ((t26 + t21 ) * 181 + 128) >> 8; t22 = ((t25a - t22a) * 181 + 128) >> 8; t25 = ((t25a + t22a) * 181 + 128) >> 8; t23a = ((t24 - t23 ) * 181 + 128) >> 8; t24a = ((t24 + t23 ) * 181 + 128) >> 8; const int t0 = c[ 0 * stride]; const int t1 = c[ 2 * stride]; const int t2 = c[ 4 * stride]; const int t3 = c[ 6 * stride]; const int t4 = c[ 8 * stride]; const int t5 = c[10 * stride]; const int t6 = c[12 * stride]; const int t7 = c[14 * stride]; const int t8 = c[16 * stride]; const int t9 = c[18 * stride]; const int t10 = c[20 * stride]; const int t11 = c[22 * stride]; const int t12 = c[24 * stride]; const int t13 = c[26 * stride]; const int t14 = c[28 * stride]; const int t15 = c[30 * stride]; c[ 0 * stride] = CLIP(t0 + t31); c[ 1 * stride] = CLIP(t1 + t30a); c[ 2 * stride] = CLIP(t2 + t29); c[ 3 * stride] = CLIP(t3 + t28a); c[ 4 * stride] = CLIP(t4 + t27); c[ 5 * stride] = CLIP(t5 + t26a); c[ 6 * stride] = CLIP(t6 + t25); c[ 7 * stride] = CLIP(t7 + t24a); c[ 8 * stride] = CLIP(t8 + t23a); c[ 9 * stride] = CLIP(t9 + t22); c[10 * stride] = CLIP(t10 + t21a); c[11 * stride] = CLIP(t11 + t20); c[12 * stride] = CLIP(t12 + t19a); c[13 * stride] = CLIP(t13 + t18); c[14 * stride] = CLIP(t14 + t17a); c[15 * stride] = CLIP(t15 + t16); c[16 * stride] = CLIP(t15 - t16); c[17 * stride] = CLIP(t14 - t17a); c[18 * stride] = CLIP(t13 - t18); c[19 * stride] = CLIP(t12 - t19a); c[20 * stride] = CLIP(t11 - t20); c[21 * stride] = CLIP(t10 - t21a); c[22 * stride] = CLIP(t9 - t22); c[23 * stride] = CLIP(t8 - t23a); c[24 * stride] = CLIP(t7 - t24a); c[25 * stride] = CLIP(t6 - t25); c[26 * stride] = CLIP(t5 - t26a); c[27 * stride] = CLIP(t4 - t27); c[28 * stride] = CLIP(t3 - t28a); c[29 * stride] = CLIP(t2 - t29); c[30 * stride] = CLIP(t1 - t30a); c[31 * stride] = CLIP(t0 - t31); } void dav1d_inv_dct32_1d_c(int32_t *const c, const ptrdiff_t stride, const int min, const int max) { inv_dct32_1d_internal_c(c, stride, min, max, 0); } void dav1d_inv_dct64_1d_c(int32_t *const c, const ptrdiff_t stride, const int min, const int max) { assert(stride > 0); inv_dct32_1d_internal_c(c, stride << 1, min, max, 1); const int in1 = c[ 1 * stride], in3 = c[ 3 * stride]; const int in5 = c[ 5 * stride], in7 = c[ 7 * stride]; const int in9 = c[ 9 * stride], in11 = c[11 * stride]; const int in13 = c[13 * stride], in15 = c[15 * stride]; const int in17 = c[17 * stride], in19 = c[19 * stride]; const int in21 = c[21 * stride], in23 = c[23 * stride]; const int in25 = c[25 * stride], in27 = c[27 * stride]; const int in29 = c[29 * stride], in31 = c[31 * stride]; int t32a = (in1 * 101 + 2048) >> 12; int t33a = (in31 * -2824 + 2048) >> 12; int t34a = (in17 * 1660 + 2048) >> 12; int t35a = (in15 * -1474 + 2048) >> 12; int t36a = (in9 * 897 + 2048) >> 12; int t37a = (in23 * -2191 + 2048) >> 12; int t38a = (in25 * 2359 + 2048) >> 12; int t39a = (in7 * -700 + 2048) >> 12; int t40a = (in5 * 501 + 2048) >> 12; int t41a = (in27 * -2520 + 2048) >> 12; int t42a = (in21 * 2019 + 2048) >> 12; int t43a = (in11 * -1092 + 2048) >> 12; int t44a = (in13 * 1285 + 2048) >> 12; int t45a = (in19 * -1842 + 2048) >> 12; int t46a = (in29 * 2675 + 2048) >> 12; int t47a = (in3 * -301 + 2048) >> 12; int t48a = (in3 * 4085 + 2048) >> 12; int t49a = (in29 * 3102 + 2048) >> 12; int t50a = (in19 * 3659 + 2048) >> 12; int t51a = (in13 * 3889 + 2048) >> 12; int t52a = (in11 * 3948 + 2048) >> 12; int t53a = (in21 * 3564 + 2048) >> 12; int t54a = (in27 * 3229 + 2048) >> 12; int t55a = (in5 * 4065 + 2048) >> 12; int t56a = (in7 * 4036 + 2048) >> 12; int t57a = (in25 * 3349 + 2048) >> 12; int t58a = (in23 * 3461 + 2048) >> 12; int t59a = (in9 * 3996 + 2048) >> 12; int t60a = (in15 * 3822 + 2048) >> 12; int t61a = (in17 * 3745 + 2048) >> 12; int t62a = (in31 * 2967 + 2048) >> 12; int t63a = (in1 * 4095 + 2048) >> 12; int t32 = CLIP(t32a + t33a); int t33 = CLIP(t32a - t33a); int t34 = CLIP(t35a - t34a); int t35 = CLIP(t35a + t34a); int t36 = CLIP(t36a + t37a); int t37 = CLIP(t36a - t37a); int t38 = CLIP(t39a - t38a); int t39 = CLIP(t39a + t38a); int t40 = CLIP(t40a + t41a); int t41 = CLIP(t40a - t41a); int t42 = CLIP(t43a - t42a); int t43 = CLIP(t43a + t42a); int t44 = CLIP(t44a + t45a); int t45 = CLIP(t44a - t45a); int t46 = CLIP(t47a - t46a); int t47 = CLIP(t47a + t46a); int t48 = CLIP(t48a + t49a);