Line data Source code
1 : /*
2 : * Copyright (c) 2015 The WebM project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 :
11 : #if defined(_MSC_VER)
12 : #include <intrin.h>
13 : #endif
14 : #include <emmintrin.h>
15 : #include <smmintrin.h>
16 :
17 : #include "vpx_dsp/vpx_dsp_common.h"
18 : #include "vp9/encoder/vp9_encoder.h"
19 : #include "vpx_ports/mem.h"
20 :
21 : #ifdef __GNUC__
22 : #define LIKELY(v) __builtin_expect(v, 1)
23 : #define UNLIKELY(v) __builtin_expect(v, 0)
24 : #else
25 : #define LIKELY(v) (v)
26 : #define UNLIKELY(v) (v)
27 : #endif
28 :
29 0 : static INLINE int_mv pack_int_mv(int16_t row, int16_t col) {
30 : int_mv result;
31 0 : result.as_mv.row = row;
32 0 : result.as_mv.col = col;
33 0 : return result;
34 : }
35 :
36 0 : static INLINE MV_JOINT_TYPE get_mv_joint(const int_mv mv) {
37 : // This is simplified from the C implementation to utilise that
38 : // x->nmvjointsadcost[1] == x->nmvjointsadcost[2] and
39 : // x->nmvjointsadcost[1] == x->nmvjointsadcost[3]
40 0 : return mv.as_int == 0 ? 0 : 1;
41 : }
42 :
43 0 : static INLINE int mv_cost(const int_mv mv, const int *joint_cost,
44 : int *const comp_cost[2]) {
45 0 : return joint_cost[get_mv_joint(mv)] + comp_cost[0][mv.as_mv.row] +
46 0 : comp_cost[1][mv.as_mv.col];
47 : }
48 :
49 0 : static int mvsad_err_cost(const MACROBLOCK *x, const int_mv mv, const MV *ref,
50 : int sad_per_bit) {
51 0 : const int_mv diff =
52 0 : pack_int_mv(mv.as_mv.row - ref->row, mv.as_mv.col - ref->col);
53 0 : return ROUND_POWER_OF_TWO(
54 : (unsigned)mv_cost(diff, x->nmvjointsadcost, x->nmvsadcost) * sad_per_bit,
55 : VP9_PROB_COST_SHIFT);
56 : }
57 :
58 : /*****************************************************************************
59 : * This function utilizes 3 properties of the cost function lookup tables, *
60 : * constructed in using 'cal_nmvjointsadcost' and 'cal_nmvsadcosts' in *
61 : * vp9_encoder.c. *
62 : * For the joint cost: *
63 : * - mvjointsadcost[1] == mvjointsadcost[2] == mvjointsadcost[3] *
64 : * For the component costs: *
65 : * - For all i: mvsadcost[0][i] == mvsadcost[1][i] *
66 : * (Equal costs for both components) *
67 : * - For all i: mvsadcost[0][i] == mvsadcost[0][-i] *
68 : * (Cost function is even) *
69 : * If these do not hold, then this function cannot be used without *
70 : * modification, in which case you can revert to using the C implementation, *
71 : * which does not rely on these properties. *
72 : *****************************************************************************/
73 0 : int vp9_diamond_search_sad_avx(const MACROBLOCK *x,
74 : const search_site_config *cfg, MV *ref_mv,
75 : MV *best_mv, int search_param, int sad_per_bit,
76 : int *num00, const vp9_variance_fn_ptr_t *fn_ptr,
77 : const MV *center_mv) {
78 0 : const int_mv maxmv = pack_int_mv(x->mv_limits.row_max, x->mv_limits.col_max);
79 0 : const __m128i v_max_mv_w = _mm_set1_epi32(maxmv.as_int);
80 0 : const int_mv minmv = pack_int_mv(x->mv_limits.row_min, x->mv_limits.col_min);
81 0 : const __m128i v_min_mv_w = _mm_set1_epi32(minmv.as_int);
82 :
83 0 : const __m128i v_spb_d = _mm_set1_epi32(sad_per_bit);
84 :
85 0 : const __m128i v_joint_cost_0_d = _mm_set1_epi32(x->nmvjointsadcost[0]);
86 0 : const __m128i v_joint_cost_1_d = _mm_set1_epi32(x->nmvjointsadcost[1]);
87 :
88 : // search_param determines the length of the initial step and hence the number
89 : // of iterations.
90 : // 0 = initial step (MAX_FIRST_STEP) pel
91 : // 1 = (MAX_FIRST_STEP/2) pel,
92 : // 2 = (MAX_FIRST_STEP/4) pel...
93 0 : const MV *ss_mv = &cfg->ss_mv[cfg->searches_per_step * search_param];
94 0 : const intptr_t *ss_os = &cfg->ss_os[cfg->searches_per_step * search_param];
95 0 : const int tot_steps = cfg->total_steps - search_param;
96 :
97 0 : const int_mv fcenter_mv =
98 0 : pack_int_mv(center_mv->row >> 3, center_mv->col >> 3);
99 0 : const __m128i vfcmv = _mm_set1_epi32(fcenter_mv.as_int);
100 :
101 0 : const int ref_row = clamp(ref_mv->row, minmv.as_mv.row, maxmv.as_mv.row);
102 0 : const int ref_col = clamp(ref_mv->col, minmv.as_mv.col, maxmv.as_mv.col);
103 :
104 0 : int_mv bmv = pack_int_mv(ref_row, ref_col);
105 0 : int_mv new_bmv = bmv;
106 0 : __m128i v_bmv_w = _mm_set1_epi32(bmv.as_int);
107 :
108 0 : const int what_stride = x->plane[0].src.stride;
109 0 : const int in_what_stride = x->e_mbd.plane[0].pre[0].stride;
110 0 : const uint8_t *const what = x->plane[0].src.buf;
111 0 : const uint8_t *const in_what =
112 0 : x->e_mbd.plane[0].pre[0].buf + ref_row * in_what_stride + ref_col;
113 :
114 : // Work out the start point for the search
115 0 : const uint8_t *best_address = in_what;
116 0 : const uint8_t *new_best_address = best_address;
117 : #if ARCH_X86_64
118 0 : __m128i v_ba_q = _mm_set1_epi64x((intptr_t)best_address);
119 : #else
120 : __m128i v_ba_d = _mm_set1_epi32((intptr_t)best_address);
121 : #endif
122 :
123 : unsigned int best_sad;
124 : int i, j, step;
125 :
126 : // Check the prerequisite cost function properties that are easy to check
127 : // in an assert. See the function-level documentation for details on all
128 : // prerequisites.
129 0 : assert(x->nmvjointsadcost[1] == x->nmvjointsadcost[2]);
130 0 : assert(x->nmvjointsadcost[1] == x->nmvjointsadcost[3]);
131 :
132 : // Check the starting position
133 0 : best_sad = fn_ptr->sdf(what, what_stride, in_what, in_what_stride);
134 0 : best_sad += mvsad_err_cost(x, bmv, &fcenter_mv.as_mv, sad_per_bit);
135 :
136 0 : *num00 = 0;
137 :
138 0 : for (i = 0, step = 0; step < tot_steps; step++) {
139 0 : for (j = 0; j < cfg->searches_per_step; j += 4, i += 4) {
140 : __m128i v_sad_d, v_cost_d, v_outside_d, v_inside_d, v_diff_mv_w;
141 : #if ARCH_X86_64
142 : __m128i v_blocka[2];
143 : #else
144 : __m128i v_blocka[1];
145 : #endif
146 :
147 : // Compute the candidate motion vectors
148 0 : const __m128i v_ss_mv_w = _mm_loadu_si128((const __m128i *)&ss_mv[i]);
149 0 : const __m128i v_these_mv_w = _mm_add_epi16(v_bmv_w, v_ss_mv_w);
150 : // Clamp them to the search bounds
151 0 : __m128i v_these_mv_clamp_w = v_these_mv_w;
152 0 : v_these_mv_clamp_w = _mm_min_epi16(v_these_mv_clamp_w, v_max_mv_w);
153 0 : v_these_mv_clamp_w = _mm_max_epi16(v_these_mv_clamp_w, v_min_mv_w);
154 : // The ones that did not change are inside the search area
155 0 : v_inside_d = _mm_cmpeq_epi32(v_these_mv_clamp_w, v_these_mv_w);
156 :
157 : // If none of them are inside, then move on
158 0 : if (LIKELY(_mm_test_all_zeros(v_inside_d, v_inside_d))) {
159 0 : continue;
160 : }
161 :
162 : // The inverse mask indicates which of the MVs are outside
163 0 : v_outside_d = _mm_xor_si128(v_inside_d, _mm_set1_epi8(0xff));
164 : // Shift right to keep the sign bit clear, we will use this later
165 : // to set the cost to the maximum value.
166 0 : v_outside_d = _mm_srli_epi32(v_outside_d, 1);
167 :
168 : // Compute the difference MV
169 0 : v_diff_mv_w = _mm_sub_epi16(v_these_mv_clamp_w, vfcmv);
170 : // We utilise the fact that the cost function is even, and use the
171 : // absolute difference. This allows us to use unsigned indexes later
172 : // and reduces cache pressure somewhat as only a half of the table
173 : // is ever referenced.
174 0 : v_diff_mv_w = _mm_abs_epi16(v_diff_mv_w);
175 :
176 : // Compute the SIMD pointer offsets.
177 : {
178 : #if ARCH_X86_64 // sizeof(intptr_t) == 8
179 : // Load the offsets
180 0 : __m128i v_bo10_q = _mm_loadu_si128((const __m128i *)&ss_os[i + 0]);
181 0 : __m128i v_bo32_q = _mm_loadu_si128((const __m128i *)&ss_os[i + 2]);
182 : // Set the ones falling outside to zero
183 0 : v_bo10_q = _mm_and_si128(v_bo10_q, _mm_cvtepi32_epi64(v_inside_d));
184 0 : v_bo32_q =
185 0 : _mm_and_si128(v_bo32_q, _mm_unpackhi_epi32(v_inside_d, v_inside_d));
186 : // Compute the candidate addresses
187 0 : v_blocka[0] = _mm_add_epi64(v_ba_q, v_bo10_q);
188 0 : v_blocka[1] = _mm_add_epi64(v_ba_q, v_bo32_q);
189 : #else // ARCH_X86 // sizeof(intptr_t) == 4
190 : __m128i v_bo_d = _mm_loadu_si128((const __m128i *)&ss_os[i]);
191 : v_bo_d = _mm_and_si128(v_bo_d, v_inside_d);
192 : v_blocka[0] = _mm_add_epi32(v_ba_d, v_bo_d);
193 : #endif
194 : }
195 :
196 0 : fn_ptr->sdx4df(what, what_stride, (const uint8_t **)&v_blocka[0],
197 : in_what_stride, (uint32_t *)&v_sad_d);
198 :
199 : // Look up the component cost of the residual motion vector
200 : {
201 0 : const int32_t row0 = _mm_extract_epi16(v_diff_mv_w, 0);
202 0 : const int32_t col0 = _mm_extract_epi16(v_diff_mv_w, 1);
203 0 : const int32_t row1 = _mm_extract_epi16(v_diff_mv_w, 2);
204 0 : const int32_t col1 = _mm_extract_epi16(v_diff_mv_w, 3);
205 0 : const int32_t row2 = _mm_extract_epi16(v_diff_mv_w, 4);
206 0 : const int32_t col2 = _mm_extract_epi16(v_diff_mv_w, 5);
207 0 : const int32_t row3 = _mm_extract_epi16(v_diff_mv_w, 6);
208 0 : const int32_t col3 = _mm_extract_epi16(v_diff_mv_w, 7);
209 :
210 : // Note: This is a use case for vpgather in AVX2
211 0 : const uint32_t cost0 = x->nmvsadcost[0][row0] + x->nmvsadcost[0][col0];
212 0 : const uint32_t cost1 = x->nmvsadcost[0][row1] + x->nmvsadcost[0][col1];
213 0 : const uint32_t cost2 = x->nmvsadcost[0][row2] + x->nmvsadcost[0][col2];
214 0 : const uint32_t cost3 = x->nmvsadcost[0][row3] + x->nmvsadcost[0][col3];
215 :
216 : __m128i v_cost_10_d, v_cost_32_d;
217 0 : v_cost_10_d = _mm_cvtsi32_si128(cost0);
218 0 : v_cost_10_d = _mm_insert_epi32(v_cost_10_d, cost1, 1);
219 0 : v_cost_32_d = _mm_cvtsi32_si128(cost2);
220 0 : v_cost_32_d = _mm_insert_epi32(v_cost_32_d, cost3, 1);
221 0 : v_cost_d = _mm_unpacklo_epi64(v_cost_10_d, v_cost_32_d);
222 : }
223 :
224 : // Now add in the joint cost
225 : {
226 0 : const __m128i v_sel_d =
227 0 : _mm_cmpeq_epi32(v_diff_mv_w, _mm_setzero_si128());
228 0 : const __m128i v_joint_cost_d =
229 : _mm_blendv_epi8(v_joint_cost_1_d, v_joint_cost_0_d, v_sel_d);
230 0 : v_cost_d = _mm_add_epi32(v_cost_d, v_joint_cost_d);
231 : }
232 :
233 : // Multiply by sad_per_bit
234 0 : v_cost_d = _mm_mullo_epi32(v_cost_d, v_spb_d);
235 : // ROUND_POWER_OF_TWO(v_cost_d, VP9_PROB_COST_SHIFT)
236 0 : v_cost_d = _mm_add_epi32(v_cost_d,
237 : _mm_set1_epi32(1 << (VP9_PROB_COST_SHIFT - 1)));
238 0 : v_cost_d = _mm_srai_epi32(v_cost_d, VP9_PROB_COST_SHIFT);
239 : // Add the cost to the sad
240 0 : v_sad_d = _mm_add_epi32(v_sad_d, v_cost_d);
241 :
242 : // Make the motion vectors outside the search area have max cost
243 : // by or'ing in the comparison mask, this way the minimum search won't
244 : // pick them.
245 0 : v_sad_d = _mm_or_si128(v_sad_d, v_outside_d);
246 :
247 : // Find the minimum value and index horizontally in v_sad_d
248 : {
249 : // Try speculatively on 16 bits, so we can use the minpos intrinsic
250 0 : const __m128i v_sad_w = _mm_packus_epi32(v_sad_d, v_sad_d);
251 0 : const __m128i v_minp_w = _mm_minpos_epu16(v_sad_w);
252 :
253 0 : uint32_t local_best_sad = _mm_extract_epi16(v_minp_w, 0);
254 0 : uint32_t local_best_idx = _mm_extract_epi16(v_minp_w, 1);
255 :
256 : // If the local best value is not saturated, just use it, otherwise
257 : // find the horizontal minimum again the hard way on 32 bits.
258 : // This is executed rarely.
259 0 : if (UNLIKELY(local_best_sad == 0xffff)) {
260 : __m128i v_loval_d, v_hival_d, v_loidx_d, v_hiidx_d, v_sel_d;
261 :
262 0 : v_loval_d = v_sad_d;
263 0 : v_loidx_d = _mm_set_epi32(3, 2, 1, 0);
264 0 : v_hival_d = _mm_srli_si128(v_loval_d, 8);
265 0 : v_hiidx_d = _mm_srli_si128(v_loidx_d, 8);
266 :
267 0 : v_sel_d = _mm_cmplt_epi32(v_hival_d, v_loval_d);
268 :
269 0 : v_loval_d = _mm_blendv_epi8(v_loval_d, v_hival_d, v_sel_d);
270 0 : v_loidx_d = _mm_blendv_epi8(v_loidx_d, v_hiidx_d, v_sel_d);
271 0 : v_hival_d = _mm_srli_si128(v_loval_d, 4);
272 0 : v_hiidx_d = _mm_srli_si128(v_loidx_d, 4);
273 :
274 0 : v_sel_d = _mm_cmplt_epi32(v_hival_d, v_loval_d);
275 :
276 0 : v_loval_d = _mm_blendv_epi8(v_loval_d, v_hival_d, v_sel_d);
277 0 : v_loidx_d = _mm_blendv_epi8(v_loidx_d, v_hiidx_d, v_sel_d);
278 :
279 0 : local_best_sad = _mm_extract_epi32(v_loval_d, 0);
280 0 : local_best_idx = _mm_extract_epi32(v_loidx_d, 0);
281 : }
282 :
283 : // Update the global minimum if the local minimum is smaller
284 0 : if (LIKELY(local_best_sad < best_sad)) {
285 0 : new_bmv = ((const int_mv *)&v_these_mv_w)[local_best_idx];
286 0 : new_best_address = ((const uint8_t **)v_blocka)[local_best_idx];
287 :
288 0 : best_sad = local_best_sad;
289 : }
290 : }
291 : }
292 :
293 0 : bmv = new_bmv;
294 0 : best_address = new_best_address;
295 :
296 0 : v_bmv_w = _mm_set1_epi32(bmv.as_int);
297 : #if ARCH_X86_64
298 0 : v_ba_q = _mm_set1_epi64x((intptr_t)best_address);
299 : #else
300 : v_ba_d = _mm_set1_epi32((intptr_t)best_address);
301 : #endif
302 :
303 0 : if (UNLIKELY(best_address == in_what)) {
304 0 : (*num00)++;
305 : }
306 : }
307 :
308 0 : *best_mv = bmv.as_mv;
309 0 : return best_sad;
310 : }
|
