Line data Source code
1 : /*
2 : * Copyright (c) 2016, Alliance for Open Media. All rights reserved
3 : *
4 : * This source code is subject to the terms of the BSD 2 Clause License and
5 : * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 : * was not distributed with this source code in the LICENSE file, you can
7 : * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 : * Media Patent License 1.0 was not distributed with this source code in the
9 : * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 : */
11 :
12 : #include <assert.h>
13 : #if defined(_MSC_VER) && _MSC_VER <= 1500
14 : // Need to include math.h before calling tmmintrin.h/intrin.h
15 : // in certain versions of MSVS.
16 : #include <math.h>
17 : #endif
18 : #include <tmmintrin.h> // SSSE3
19 :
20 : #include "./av1_rtcd.h"
21 : #include "aom_dsp/x86/inv_txfm_sse2.h"
22 : #include "aom_dsp/x86/txfm_common_sse2.h"
23 :
24 0 : void av1_fdct8x8_quant_ssse3(
25 : const int16_t *input, int stride, int16_t *coeff_ptr, intptr_t n_coeffs,
26 : int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr,
27 : const int16_t *quant_ptr, const int16_t *quant_shift_ptr,
28 : int16_t *qcoeff_ptr, int16_t *dqcoeff_ptr, const int16_t *dequant_ptr,
29 : uint16_t *eob_ptr, const int16_t *scan_ptr, const int16_t *iscan_ptr) {
30 : __m128i zero;
31 : int pass;
32 : // Constants
33 : // When we use them, in one case, they are all the same. In all others
34 : // it's a pair of them that we need to repeat four times. This is done
35 : // by constructing the 32 bit constant corresponding to that pair.
36 0 : const __m128i k__dual_p16_p16 = dual_set_epi16(23170, 23170);
37 0 : const __m128i k__cospi_p16_p16 = _mm_set1_epi16((int16_t)cospi_16_64);
38 0 : const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64);
39 0 : const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64);
40 0 : const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64);
41 0 : const __m128i k__cospi_p28_p04 = pair_set_epi16(cospi_28_64, cospi_4_64);
42 0 : const __m128i k__cospi_m04_p28 = pair_set_epi16(-cospi_4_64, cospi_28_64);
43 0 : const __m128i k__cospi_p12_p20 = pair_set_epi16(cospi_12_64, cospi_20_64);
44 0 : const __m128i k__cospi_m20_p12 = pair_set_epi16(-cospi_20_64, cospi_12_64);
45 0 : const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING);
46 : // Load input
47 0 : __m128i in0 = _mm_load_si128((const __m128i *)(input + 0 * stride));
48 0 : __m128i in1 = _mm_load_si128((const __m128i *)(input + 1 * stride));
49 0 : __m128i in2 = _mm_load_si128((const __m128i *)(input + 2 * stride));
50 0 : __m128i in3 = _mm_load_si128((const __m128i *)(input + 3 * stride));
51 0 : __m128i in4 = _mm_load_si128((const __m128i *)(input + 4 * stride));
52 0 : __m128i in5 = _mm_load_si128((const __m128i *)(input + 5 * stride));
53 0 : __m128i in6 = _mm_load_si128((const __m128i *)(input + 6 * stride));
54 0 : __m128i in7 = _mm_load_si128((const __m128i *)(input + 7 * stride));
55 : __m128i *in[8];
56 0 : int index = 0;
57 :
58 : (void)scan_ptr;
59 : (void)zbin_ptr;
60 : (void)quant_shift_ptr;
61 : (void)coeff_ptr;
62 :
63 : // Pre-condition input (shift by two)
64 0 : in0 = _mm_slli_epi16(in0, 2);
65 0 : in1 = _mm_slli_epi16(in1, 2);
66 0 : in2 = _mm_slli_epi16(in2, 2);
67 0 : in3 = _mm_slli_epi16(in3, 2);
68 0 : in4 = _mm_slli_epi16(in4, 2);
69 0 : in5 = _mm_slli_epi16(in5, 2);
70 0 : in6 = _mm_slli_epi16(in6, 2);
71 0 : in7 = _mm_slli_epi16(in7, 2);
72 :
73 0 : in[0] = &in0;
74 0 : in[1] = &in1;
75 0 : in[2] = &in2;
76 0 : in[3] = &in3;
77 0 : in[4] = &in4;
78 0 : in[5] = &in5;
79 0 : in[6] = &in6;
80 0 : in[7] = &in7;
81 :
82 : // We do two passes, first the columns, then the rows. The results of the
83 : // first pass are transposed so that the same column code can be reused. The
84 : // results of the second pass are also transposed so that the rows (processed
85 : // as columns) are put back in row positions.
86 0 : for (pass = 0; pass < 2; pass++) {
87 : // To store results of each pass before the transpose.
88 : __m128i res0, res1, res2, res3, res4, res5, res6, res7;
89 : // Add/subtract
90 0 : const __m128i q0 = _mm_add_epi16(in0, in7);
91 0 : const __m128i q1 = _mm_add_epi16(in1, in6);
92 0 : const __m128i q2 = _mm_add_epi16(in2, in5);
93 0 : const __m128i q3 = _mm_add_epi16(in3, in4);
94 0 : const __m128i q4 = _mm_sub_epi16(in3, in4);
95 0 : const __m128i q5 = _mm_sub_epi16(in2, in5);
96 0 : const __m128i q6 = _mm_sub_epi16(in1, in6);
97 0 : const __m128i q7 = _mm_sub_epi16(in0, in7);
98 : // Work on first four results
99 : {
100 : // Add/subtract
101 0 : const __m128i r0 = _mm_add_epi16(q0, q3);
102 0 : const __m128i r1 = _mm_add_epi16(q1, q2);
103 0 : const __m128i r2 = _mm_sub_epi16(q1, q2);
104 0 : const __m128i r3 = _mm_sub_epi16(q0, q3);
105 : // Interleave to do the multiply by constants which gets us into 32bits
106 0 : const __m128i t0 = _mm_unpacklo_epi16(r0, r1);
107 0 : const __m128i t1 = _mm_unpackhi_epi16(r0, r1);
108 0 : const __m128i t2 = _mm_unpacklo_epi16(r2, r3);
109 0 : const __m128i t3 = _mm_unpackhi_epi16(r2, r3);
110 :
111 0 : const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p16_p16);
112 0 : const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p16_p16);
113 0 : const __m128i u2 = _mm_madd_epi16(t0, k__cospi_p16_m16);
114 0 : const __m128i u3 = _mm_madd_epi16(t1, k__cospi_p16_m16);
115 :
116 0 : const __m128i u4 = _mm_madd_epi16(t2, k__cospi_p24_p08);
117 0 : const __m128i u5 = _mm_madd_epi16(t3, k__cospi_p24_p08);
118 0 : const __m128i u6 = _mm_madd_epi16(t2, k__cospi_m08_p24);
119 0 : const __m128i u7 = _mm_madd_epi16(t3, k__cospi_m08_p24);
120 : // dct_const_round_shift
121 :
122 0 : const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING);
123 0 : const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING);
124 0 : const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING);
125 0 : const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING);
126 :
127 0 : const __m128i v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING);
128 0 : const __m128i v5 = _mm_add_epi32(u5, k__DCT_CONST_ROUNDING);
129 0 : const __m128i v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING);
130 0 : const __m128i v7 = _mm_add_epi32(u7, k__DCT_CONST_ROUNDING);
131 :
132 0 : const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
133 0 : const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS);
134 0 : const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
135 0 : const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS);
136 :
137 0 : const __m128i w4 = _mm_srai_epi32(v4, DCT_CONST_BITS);
138 0 : const __m128i w5 = _mm_srai_epi32(v5, DCT_CONST_BITS);
139 0 : const __m128i w6 = _mm_srai_epi32(v6, DCT_CONST_BITS);
140 0 : const __m128i w7 = _mm_srai_epi32(v7, DCT_CONST_BITS);
141 : // Combine
142 :
143 0 : res0 = _mm_packs_epi32(w0, w1);
144 0 : res4 = _mm_packs_epi32(w2, w3);
145 0 : res2 = _mm_packs_epi32(w4, w5);
146 0 : res6 = _mm_packs_epi32(w6, w7);
147 : }
148 : // Work on next four results
149 : {
150 : // Interleave to do the multiply by constants which gets us into 32bits
151 0 : const __m128i d0 = _mm_sub_epi16(q6, q5);
152 0 : const __m128i d1 = _mm_add_epi16(q6, q5);
153 0 : const __m128i r0 = _mm_mulhrs_epi16(d0, k__dual_p16_p16);
154 0 : const __m128i r1 = _mm_mulhrs_epi16(d1, k__dual_p16_p16);
155 :
156 : // Add/subtract
157 0 : const __m128i x0 = _mm_add_epi16(q4, r0);
158 0 : const __m128i x1 = _mm_sub_epi16(q4, r0);
159 0 : const __m128i x2 = _mm_sub_epi16(q7, r1);
160 0 : const __m128i x3 = _mm_add_epi16(q7, r1);
161 : // Interleave to do the multiply by constants which gets us into 32bits
162 0 : const __m128i t0 = _mm_unpacklo_epi16(x0, x3);
163 0 : const __m128i t1 = _mm_unpackhi_epi16(x0, x3);
164 0 : const __m128i t2 = _mm_unpacklo_epi16(x1, x2);
165 0 : const __m128i t3 = _mm_unpackhi_epi16(x1, x2);
166 0 : const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p28_p04);
167 0 : const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p28_p04);
168 0 : const __m128i u2 = _mm_madd_epi16(t0, k__cospi_m04_p28);
169 0 : const __m128i u3 = _mm_madd_epi16(t1, k__cospi_m04_p28);
170 0 : const __m128i u4 = _mm_madd_epi16(t2, k__cospi_p12_p20);
171 0 : const __m128i u5 = _mm_madd_epi16(t3, k__cospi_p12_p20);
172 0 : const __m128i u6 = _mm_madd_epi16(t2, k__cospi_m20_p12);
173 0 : const __m128i u7 = _mm_madd_epi16(t3, k__cospi_m20_p12);
174 : // dct_const_round_shift
175 0 : const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING);
176 0 : const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING);
177 0 : const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING);
178 0 : const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING);
179 0 : const __m128i v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING);
180 0 : const __m128i v5 = _mm_add_epi32(u5, k__DCT_CONST_ROUNDING);
181 0 : const __m128i v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING);
182 0 : const __m128i v7 = _mm_add_epi32(u7, k__DCT_CONST_ROUNDING);
183 0 : const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
184 0 : const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS);
185 0 : const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
186 0 : const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS);
187 0 : const __m128i w4 = _mm_srai_epi32(v4, DCT_CONST_BITS);
188 0 : const __m128i w5 = _mm_srai_epi32(v5, DCT_CONST_BITS);
189 0 : const __m128i w6 = _mm_srai_epi32(v6, DCT_CONST_BITS);
190 0 : const __m128i w7 = _mm_srai_epi32(v7, DCT_CONST_BITS);
191 : // Combine
192 0 : res1 = _mm_packs_epi32(w0, w1);
193 0 : res7 = _mm_packs_epi32(w2, w3);
194 0 : res5 = _mm_packs_epi32(w4, w5);
195 0 : res3 = _mm_packs_epi32(w6, w7);
196 : }
197 : // Transpose the 8x8.
198 : {
199 : // 00 01 02 03 04 05 06 07
200 : // 10 11 12 13 14 15 16 17
201 : // 20 21 22 23 24 25 26 27
202 : // 30 31 32 33 34 35 36 37
203 : // 40 41 42 43 44 45 46 47
204 : // 50 51 52 53 54 55 56 57
205 : // 60 61 62 63 64 65 66 67
206 : // 70 71 72 73 74 75 76 77
207 0 : const __m128i tr0_0 = _mm_unpacklo_epi16(res0, res1);
208 0 : const __m128i tr0_1 = _mm_unpacklo_epi16(res2, res3);
209 0 : const __m128i tr0_2 = _mm_unpackhi_epi16(res0, res1);
210 0 : const __m128i tr0_3 = _mm_unpackhi_epi16(res2, res3);
211 0 : const __m128i tr0_4 = _mm_unpacklo_epi16(res4, res5);
212 0 : const __m128i tr0_5 = _mm_unpacklo_epi16(res6, res7);
213 0 : const __m128i tr0_6 = _mm_unpackhi_epi16(res4, res5);
214 0 : const __m128i tr0_7 = _mm_unpackhi_epi16(res6, res7);
215 : // 00 10 01 11 02 12 03 13
216 : // 20 30 21 31 22 32 23 33
217 : // 04 14 05 15 06 16 07 17
218 : // 24 34 25 35 26 36 27 37
219 : // 40 50 41 51 42 52 43 53
220 : // 60 70 61 71 62 72 63 73
221 : // 54 54 55 55 56 56 57 57
222 : // 64 74 65 75 66 76 67 77
223 0 : const __m128i tr1_0 = _mm_unpacklo_epi32(tr0_0, tr0_1);
224 0 : const __m128i tr1_1 = _mm_unpacklo_epi32(tr0_2, tr0_3);
225 0 : const __m128i tr1_2 = _mm_unpackhi_epi32(tr0_0, tr0_1);
226 0 : const __m128i tr1_3 = _mm_unpackhi_epi32(tr0_2, tr0_3);
227 0 : const __m128i tr1_4 = _mm_unpacklo_epi32(tr0_4, tr0_5);
228 0 : const __m128i tr1_5 = _mm_unpacklo_epi32(tr0_6, tr0_7);
229 0 : const __m128i tr1_6 = _mm_unpackhi_epi32(tr0_4, tr0_5);
230 0 : const __m128i tr1_7 = _mm_unpackhi_epi32(tr0_6, tr0_7);
231 : // 00 10 20 30 01 11 21 31
232 : // 40 50 60 70 41 51 61 71
233 : // 02 12 22 32 03 13 23 33
234 : // 42 52 62 72 43 53 63 73
235 : // 04 14 24 34 05 15 21 36
236 : // 44 54 64 74 45 55 61 76
237 : // 06 16 26 36 07 17 27 37
238 : // 46 56 66 76 47 57 67 77
239 0 : in0 = _mm_unpacklo_epi64(tr1_0, tr1_4);
240 0 : in1 = _mm_unpackhi_epi64(tr1_0, tr1_4);
241 0 : in2 = _mm_unpacklo_epi64(tr1_2, tr1_6);
242 0 : in3 = _mm_unpackhi_epi64(tr1_2, tr1_6);
243 0 : in4 = _mm_unpacklo_epi64(tr1_1, tr1_5);
244 0 : in5 = _mm_unpackhi_epi64(tr1_1, tr1_5);
245 0 : in6 = _mm_unpacklo_epi64(tr1_3, tr1_7);
246 0 : in7 = _mm_unpackhi_epi64(tr1_3, tr1_7);
247 : // 00 10 20 30 40 50 60 70
248 : // 01 11 21 31 41 51 61 71
249 : // 02 12 22 32 42 52 62 72
250 : // 03 13 23 33 43 53 63 73
251 : // 04 14 24 34 44 54 64 74
252 : // 05 15 25 35 45 55 65 75
253 : // 06 16 26 36 46 56 66 76
254 : // 07 17 27 37 47 57 67 77
255 : }
256 : }
257 : // Post-condition output and store it
258 : {
259 : // Post-condition (division by two)
260 : // division of two 16 bits signed numbers using shifts
261 : // n / 2 = (n - (n >> 15)) >> 1
262 0 : const __m128i sign_in0 = _mm_srai_epi16(in0, 15);
263 0 : const __m128i sign_in1 = _mm_srai_epi16(in1, 15);
264 0 : const __m128i sign_in2 = _mm_srai_epi16(in2, 15);
265 0 : const __m128i sign_in3 = _mm_srai_epi16(in3, 15);
266 0 : const __m128i sign_in4 = _mm_srai_epi16(in4, 15);
267 0 : const __m128i sign_in5 = _mm_srai_epi16(in5, 15);
268 0 : const __m128i sign_in6 = _mm_srai_epi16(in6, 15);
269 0 : const __m128i sign_in7 = _mm_srai_epi16(in7, 15);
270 0 : in0 = _mm_sub_epi16(in0, sign_in0);
271 0 : in1 = _mm_sub_epi16(in1, sign_in1);
272 0 : in2 = _mm_sub_epi16(in2, sign_in2);
273 0 : in3 = _mm_sub_epi16(in3, sign_in3);
274 0 : in4 = _mm_sub_epi16(in4, sign_in4);
275 0 : in5 = _mm_sub_epi16(in5, sign_in5);
276 0 : in6 = _mm_sub_epi16(in6, sign_in6);
277 0 : in7 = _mm_sub_epi16(in7, sign_in7);
278 0 : in0 = _mm_srai_epi16(in0, 1);
279 0 : in1 = _mm_srai_epi16(in1, 1);
280 0 : in2 = _mm_srai_epi16(in2, 1);
281 0 : in3 = _mm_srai_epi16(in3, 1);
282 0 : in4 = _mm_srai_epi16(in4, 1);
283 0 : in5 = _mm_srai_epi16(in5, 1);
284 0 : in6 = _mm_srai_epi16(in6, 1);
285 0 : in7 = _mm_srai_epi16(in7, 1);
286 : }
287 :
288 0 : iscan_ptr += n_coeffs;
289 0 : qcoeff_ptr += n_coeffs;
290 0 : dqcoeff_ptr += n_coeffs;
291 0 : n_coeffs = -n_coeffs;
292 0 : zero = _mm_setzero_si128();
293 :
294 0 : if (!skip_block) {
295 : __m128i eob;
296 : __m128i round, quant, dequant, thr;
297 : int16_t nzflag;
298 : {
299 : __m128i coeff0, coeff1;
300 :
301 : // Setup global values
302 : {
303 0 : round = _mm_load_si128((const __m128i *)round_ptr);
304 0 : quant = _mm_load_si128((const __m128i *)quant_ptr);
305 0 : dequant = _mm_load_si128((const __m128i *)dequant_ptr);
306 : }
307 :
308 : {
309 : __m128i coeff0_sign, coeff1_sign;
310 : __m128i qcoeff0, qcoeff1;
311 : __m128i qtmp0, qtmp1;
312 : // Do DC and first 15 AC
313 0 : coeff0 = *in[0];
314 0 : coeff1 = *in[1];
315 :
316 : // Poor man's sign extract
317 0 : coeff0_sign = _mm_srai_epi16(coeff0, 15);
318 0 : coeff1_sign = _mm_srai_epi16(coeff1, 15);
319 0 : qcoeff0 = _mm_xor_si128(coeff0, coeff0_sign);
320 0 : qcoeff1 = _mm_xor_si128(coeff1, coeff1_sign);
321 0 : qcoeff0 = _mm_sub_epi16(qcoeff0, coeff0_sign);
322 0 : qcoeff1 = _mm_sub_epi16(qcoeff1, coeff1_sign);
323 :
324 0 : qcoeff0 = _mm_adds_epi16(qcoeff0, round);
325 0 : round = _mm_unpackhi_epi64(round, round);
326 0 : qcoeff1 = _mm_adds_epi16(qcoeff1, round);
327 0 : qtmp0 = _mm_mulhi_epi16(qcoeff0, quant);
328 0 : quant = _mm_unpackhi_epi64(quant, quant);
329 0 : qtmp1 = _mm_mulhi_epi16(qcoeff1, quant);
330 :
331 : // Reinsert signs
332 0 : qcoeff0 = _mm_xor_si128(qtmp0, coeff0_sign);
333 0 : qcoeff1 = _mm_xor_si128(qtmp1, coeff1_sign);
334 0 : qcoeff0 = _mm_sub_epi16(qcoeff0, coeff0_sign);
335 0 : qcoeff1 = _mm_sub_epi16(qcoeff1, coeff1_sign);
336 :
337 0 : _mm_store_si128((__m128i *)(qcoeff_ptr + n_coeffs), qcoeff0);
338 0 : _mm_store_si128((__m128i *)(qcoeff_ptr + n_coeffs) + 1, qcoeff1);
339 :
340 0 : coeff0 = _mm_mullo_epi16(qcoeff0, dequant);
341 0 : dequant = _mm_unpackhi_epi64(dequant, dequant);
342 0 : coeff1 = _mm_mullo_epi16(qcoeff1, dequant);
343 :
344 0 : _mm_store_si128((__m128i *)(dqcoeff_ptr + n_coeffs), coeff0);
345 0 : _mm_store_si128((__m128i *)(dqcoeff_ptr + n_coeffs) + 1, coeff1);
346 : }
347 :
348 : {
349 : // Scan for eob
350 : __m128i zero_coeff0, zero_coeff1;
351 : __m128i nzero_coeff0, nzero_coeff1;
352 : __m128i iscan0, iscan1;
353 : __m128i eob1;
354 0 : zero_coeff0 = _mm_cmpeq_epi16(coeff0, zero);
355 0 : zero_coeff1 = _mm_cmpeq_epi16(coeff1, zero);
356 0 : nzero_coeff0 = _mm_cmpeq_epi16(zero_coeff0, zero);
357 0 : nzero_coeff1 = _mm_cmpeq_epi16(zero_coeff1, zero);
358 0 : iscan0 = _mm_load_si128((const __m128i *)(iscan_ptr + n_coeffs));
359 0 : iscan1 = _mm_load_si128((const __m128i *)(iscan_ptr + n_coeffs) + 1);
360 : // Add one to convert from indices to counts
361 0 : iscan0 = _mm_sub_epi16(iscan0, nzero_coeff0);
362 0 : iscan1 = _mm_sub_epi16(iscan1, nzero_coeff1);
363 0 : eob = _mm_and_si128(iscan0, nzero_coeff0);
364 0 : eob1 = _mm_and_si128(iscan1, nzero_coeff1);
365 0 : eob = _mm_max_epi16(eob, eob1);
366 : }
367 0 : n_coeffs += 8 * 2;
368 : }
369 :
370 : // AC only loop
371 0 : index = 2;
372 0 : thr = _mm_srai_epi16(dequant, 1);
373 0 : while (n_coeffs < 0) {
374 : __m128i coeff0, coeff1;
375 : {
376 : __m128i coeff0_sign, coeff1_sign;
377 : __m128i qcoeff0, qcoeff1;
378 : __m128i qtmp0, qtmp1;
379 :
380 0 : assert(index < (int)(sizeof(in) / sizeof(in[0])) - 1);
381 0 : coeff0 = *in[index];
382 0 : coeff1 = *in[index + 1];
383 :
384 : // Poor man's sign extract
385 0 : coeff0_sign = _mm_srai_epi16(coeff0, 15);
386 0 : coeff1_sign = _mm_srai_epi16(coeff1, 15);
387 0 : qcoeff0 = _mm_xor_si128(coeff0, coeff0_sign);
388 0 : qcoeff1 = _mm_xor_si128(coeff1, coeff1_sign);
389 0 : qcoeff0 = _mm_sub_epi16(qcoeff0, coeff0_sign);
390 0 : qcoeff1 = _mm_sub_epi16(qcoeff1, coeff1_sign);
391 :
392 0 : nzflag = _mm_movemask_epi8(_mm_cmpgt_epi16(qcoeff0, thr)) |
393 0 : _mm_movemask_epi8(_mm_cmpgt_epi16(qcoeff1, thr));
394 :
395 0 : if (nzflag) {
396 0 : qcoeff0 = _mm_adds_epi16(qcoeff0, round);
397 0 : qcoeff1 = _mm_adds_epi16(qcoeff1, round);
398 0 : qtmp0 = _mm_mulhi_epi16(qcoeff0, quant);
399 0 : qtmp1 = _mm_mulhi_epi16(qcoeff1, quant);
400 :
401 : // Reinsert signs
402 0 : qcoeff0 = _mm_xor_si128(qtmp0, coeff0_sign);
403 0 : qcoeff1 = _mm_xor_si128(qtmp1, coeff1_sign);
404 0 : qcoeff0 = _mm_sub_epi16(qcoeff0, coeff0_sign);
405 0 : qcoeff1 = _mm_sub_epi16(qcoeff1, coeff1_sign);
406 :
407 0 : _mm_store_si128((__m128i *)(qcoeff_ptr + n_coeffs), qcoeff0);
408 0 : _mm_store_si128((__m128i *)(qcoeff_ptr + n_coeffs) + 1, qcoeff1);
409 :
410 0 : coeff0 = _mm_mullo_epi16(qcoeff0, dequant);
411 0 : coeff1 = _mm_mullo_epi16(qcoeff1, dequant);
412 :
413 0 : _mm_store_si128((__m128i *)(dqcoeff_ptr + n_coeffs), coeff0);
414 0 : _mm_store_si128((__m128i *)(dqcoeff_ptr + n_coeffs) + 1, coeff1);
415 : } else {
416 0 : _mm_store_si128((__m128i *)(qcoeff_ptr + n_coeffs), zero);
417 0 : _mm_store_si128((__m128i *)(qcoeff_ptr + n_coeffs) + 1, zero);
418 :
419 0 : _mm_store_si128((__m128i *)(dqcoeff_ptr + n_coeffs), zero);
420 0 : _mm_store_si128((__m128i *)(dqcoeff_ptr + n_coeffs) + 1, zero);
421 : }
422 : }
423 :
424 0 : if (nzflag) {
425 : // Scan for eob
426 : __m128i zero_coeff0, zero_coeff1;
427 : __m128i nzero_coeff0, nzero_coeff1;
428 : __m128i iscan0, iscan1;
429 : __m128i eob0, eob1;
430 0 : zero_coeff0 = _mm_cmpeq_epi16(coeff0, zero);
431 0 : zero_coeff1 = _mm_cmpeq_epi16(coeff1, zero);
432 0 : nzero_coeff0 = _mm_cmpeq_epi16(zero_coeff0, zero);
433 0 : nzero_coeff1 = _mm_cmpeq_epi16(zero_coeff1, zero);
434 0 : iscan0 = _mm_load_si128((const __m128i *)(iscan_ptr + n_coeffs));
435 0 : iscan1 = _mm_load_si128((const __m128i *)(iscan_ptr + n_coeffs) + 1);
436 : // Add one to convert from indices to counts
437 0 : iscan0 = _mm_sub_epi16(iscan0, nzero_coeff0);
438 0 : iscan1 = _mm_sub_epi16(iscan1, nzero_coeff1);
439 0 : eob0 = _mm_and_si128(iscan0, nzero_coeff0);
440 0 : eob1 = _mm_and_si128(iscan1, nzero_coeff1);
441 0 : eob0 = _mm_max_epi16(eob0, eob1);
442 0 : eob = _mm_max_epi16(eob, eob0);
443 : }
444 0 : n_coeffs += 8 * 2;
445 0 : index += 2;
446 : }
447 :
448 : // Accumulate EOB
449 : {
450 : __m128i eob_shuffled;
451 0 : eob_shuffled = _mm_shuffle_epi32(eob, 0xe);
452 0 : eob = _mm_max_epi16(eob, eob_shuffled);
453 0 : eob_shuffled = _mm_shufflelo_epi16(eob, 0xe);
454 0 : eob = _mm_max_epi16(eob, eob_shuffled);
455 0 : eob_shuffled = _mm_shufflelo_epi16(eob, 0x1);
456 0 : eob = _mm_max_epi16(eob, eob_shuffled);
457 0 : *eob_ptr = _mm_extract_epi16(eob, 1);
458 : }
459 : } else {
460 : do {
461 0 : _mm_store_si128((__m128i *)(dqcoeff_ptr + n_coeffs), zero);
462 0 : _mm_store_si128((__m128i *)(dqcoeff_ptr + n_coeffs) + 1, zero);
463 0 : _mm_store_si128((__m128i *)(qcoeff_ptr + n_coeffs), zero);
464 0 : _mm_store_si128((__m128i *)(qcoeff_ptr + n_coeffs) + 1, zero);
465 0 : n_coeffs += 8 * 2;
466 0 : } while (n_coeffs < 0);
467 0 : *eob_ptr = 0;
468 : }
469 0 : }
|
