Line data Source code
1 : /*
2 : * Copyright (c) 2016, Alliance for Open Media. All rights reserved
3 : *
4 : * This source code is subject to the terms of the BSD 2 Clause License and
5 : * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 : * was not distributed with this source code in the LICENSE file, you can
7 : * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 : * Media Patent License 1.0 was not distributed with this source code in the
9 : * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 : */
11 :
12 : #include <stdio.h>
13 : #include <stdlib.h>
14 : #include <memory.h>
15 : #include <math.h>
16 : #include <assert.h>
17 :
18 : #include "av1/encoder/global_motion.h"
19 :
20 : #include "av1/common/warped_motion.h"
21 :
22 : #include "av1/encoder/segmentation.h"
23 : #include "av1/encoder/corner_detect.h"
24 : #include "av1/encoder/corner_match.h"
25 : #include "av1/encoder/ransac.h"
26 :
27 : #define MAX_CORNERS 4096
28 : #define MIN_INLIER_PROB 0.1
29 :
30 : #define MIN_TRANS_THRESH (1 * GM_TRANS_DECODE_FACTOR)
31 :
32 : // Border over which to compute the global motion
33 : #define ERRORADV_BORDER 0
34 :
35 : #define ERRORADV_MAX_THRESH 0.995
36 : #define ERRORADV_COST_PRODUCT_THRESH 26000
37 :
38 0 : int is_enough_erroradvantage(double best_erroradvantage, int params_cost) {
39 0 : return best_erroradvantage < ERRORADV_MAX_THRESH &&
40 0 : best_erroradvantage * params_cost < ERRORADV_COST_PRODUCT_THRESH;
41 : }
42 :
43 0 : static void convert_to_params(const double *params, int32_t *model) {
44 : int i;
45 0 : int alpha_present = 0;
46 0 : model[0] = (int32_t)floor(params[0] * (1 << GM_TRANS_PREC_BITS) + 0.5);
47 0 : model[1] = (int32_t)floor(params[1] * (1 << GM_TRANS_PREC_BITS) + 0.5);
48 0 : model[0] = (int32_t)clamp(model[0], GM_TRANS_MIN, GM_TRANS_MAX) *
49 : GM_TRANS_DECODE_FACTOR;
50 0 : model[1] = (int32_t)clamp(model[1], GM_TRANS_MIN, GM_TRANS_MAX) *
51 : GM_TRANS_DECODE_FACTOR;
52 :
53 0 : for (i = 2; i < 6; ++i) {
54 0 : const int diag_value = ((i == 2 || i == 5) ? (1 << GM_ALPHA_PREC_BITS) : 0);
55 0 : model[i] = (int32_t)floor(params[i] * (1 << GM_ALPHA_PREC_BITS) + 0.5);
56 0 : model[i] =
57 0 : (int32_t)clamp(model[i] - diag_value, GM_ALPHA_MIN, GM_ALPHA_MAX);
58 0 : alpha_present |= (model[i] != 0);
59 0 : model[i] = (model[i] + diag_value) * GM_ALPHA_DECODE_FACTOR;
60 : }
61 0 : for (; i < 8; ++i) {
62 0 : model[i] = (int32_t)floor(params[i] * (1 << GM_ROW3HOMO_PREC_BITS) + 0.5);
63 0 : model[i] = (int32_t)clamp(model[i], GM_ROW3HOMO_MIN, GM_ROW3HOMO_MAX) *
64 : GM_ROW3HOMO_DECODE_FACTOR;
65 0 : alpha_present |= (model[i] != 0);
66 : }
67 :
68 0 : if (!alpha_present) {
69 0 : if (abs(model[0]) < MIN_TRANS_THRESH && abs(model[1]) < MIN_TRANS_THRESH) {
70 0 : model[0] = 0;
71 0 : model[1] = 0;
72 : }
73 : }
74 0 : }
75 :
76 0 : void convert_model_to_params(const double *params, WarpedMotionParams *model) {
77 0 : convert_to_params(params, model->wmmat);
78 0 : model->wmtype = get_gmtype(model);
79 0 : }
80 :
81 : // Adds some offset to a global motion parameter and handles
82 : // all of the necessary precision shifts, clamping, and
83 : // zero-centering.
84 0 : static int32_t add_param_offset(int param_index, int32_t param_value,
85 : int32_t offset) {
86 0 : const int scale_vals[3] = { GM_TRANS_PREC_DIFF, GM_ALPHA_PREC_DIFF,
87 : GM_ROW3HOMO_PREC_DIFF };
88 0 : const int clamp_vals[3] = { GM_TRANS_MAX, GM_ALPHA_MAX, GM_ROW3HOMO_MAX };
89 : // type of param: 0 - translation, 1 - affine, 2 - homography
90 0 : const int param_type = (param_index < 2 ? 0 : (param_index < 6 ? 1 : 2));
91 0 : const int is_one_centered = (param_index == 2 || param_index == 5);
92 :
93 : // Make parameter zero-centered and offset the shift that was done to make
94 : // it compatible with the warped model
95 0 : param_value = (param_value - (is_one_centered << WARPEDMODEL_PREC_BITS)) >>
96 0 : scale_vals[param_type];
97 : // Add desired offset to the rescaled/zero-centered parameter
98 0 : param_value += offset;
99 : // Clamp the parameter so it does not overflow the number of bits allotted
100 : // to it in the bitstream
101 0 : param_value = (int32_t)clamp(param_value, -clamp_vals[param_type],
102 : clamp_vals[param_type]);
103 : // Rescale the parameter to WARPEDMODEL_PRECISION_BITS so it is compatible
104 : // with the warped motion library
105 0 : param_value *= (1 << scale_vals[param_type]);
106 :
107 : // Undo the zero-centering step if necessary
108 0 : return param_value + (is_one_centered << WARPEDMODEL_PREC_BITS);
109 : }
110 :
111 0 : static void force_wmtype(WarpedMotionParams *wm, TransformationType wmtype) {
112 0 : switch (wmtype) {
113 0 : case IDENTITY: wm->wmmat[0] = 0; wm->wmmat[1] = 0;
114 : case TRANSLATION:
115 0 : wm->wmmat[2] = 1 << WARPEDMODEL_PREC_BITS;
116 0 : wm->wmmat[3] = 0;
117 0 : case ROTZOOM: wm->wmmat[4] = -wm->wmmat[3]; wm->wmmat[5] = wm->wmmat[2];
118 0 : case AFFINE: wm->wmmat[6] = wm->wmmat[7] = 0; break;
119 0 : case HORTRAPEZOID: wm->wmmat[6] = wm->wmmat[4] = 0; break;
120 0 : case VERTRAPEZOID: wm->wmmat[7] = wm->wmmat[3] = 0; break;
121 0 : case HOMOGRAPHY: break;
122 0 : default: assert(0);
123 : }
124 0 : wm->wmtype = wmtype;
125 0 : }
126 :
127 0 : int64_t refine_integerized_param(WarpedMotionParams *wm,
128 : TransformationType wmtype,
129 : #if CONFIG_HIGHBITDEPTH
130 : int use_hbd, int bd,
131 : #endif // CONFIG_HIGHBITDEPTH
132 : uint8_t *ref, int r_width, int r_height,
133 : int r_stride, uint8_t *dst, int d_width,
134 : int d_height, int d_stride,
135 : int n_refinements) {
136 : static const int max_trans_model_params[TRANS_TYPES] = {
137 : 0, 2, 4, 6, 8, 8, 8
138 : };
139 0 : const int border = ERRORADV_BORDER;
140 0 : int i = 0, p;
141 0 : int n_params = max_trans_model_params[wmtype];
142 0 : int32_t *param_mat = wm->wmmat;
143 : int64_t step_error, best_error;
144 : int32_t step;
145 : int32_t *param;
146 : int32_t curr_param;
147 : int32_t best_param;
148 :
149 0 : force_wmtype(wm, wmtype);
150 0 : best_error = av1_warp_error(wm,
151 : #if CONFIG_HIGHBITDEPTH
152 : use_hbd, bd,
153 : #endif // CONFIG_HIGHBITDEPTH
154 : ref, r_width, r_height, r_stride,
155 0 : dst + border * d_stride + border, border, border,
156 0 : d_width - 2 * border, d_height - 2 * border,
157 : d_stride, 0, 0, 16, 16);
158 0 : step = 1 << (n_refinements + 1);
159 0 : for (i = 0; i < n_refinements; i++, step >>= 1) {
160 0 : for (p = 0; p < n_params; ++p) {
161 0 : int step_dir = 0;
162 : // Skip searches for parameters that are forced to be 0
163 0 : if (wmtype == HORTRAPEZOID && (p == 4 || p == 6)) continue;
164 0 : if (wmtype == VERTRAPEZOID && (p == 3 || p == 7)) continue;
165 0 : param = param_mat + p;
166 0 : curr_param = *param;
167 0 : best_param = curr_param;
168 : // look to the left
169 0 : *param = add_param_offset(p, curr_param, -step);
170 0 : step_error = av1_warp_error(
171 : wm,
172 : #if CONFIG_HIGHBITDEPTH
173 : use_hbd, bd,
174 : #endif // CONFIG_HIGHBITDEPTH
175 0 : ref, r_width, r_height, r_stride, dst + border * d_stride + border,
176 0 : border, border, d_width - 2 * border, d_height - 2 * border, d_stride,
177 : 0, 0, 16, 16);
178 0 : if (step_error < best_error) {
179 0 : best_error = step_error;
180 0 : best_param = *param;
181 0 : step_dir = -1;
182 : }
183 :
184 : // look to the right
185 0 : *param = add_param_offset(p, curr_param, step);
186 0 : step_error = av1_warp_error(
187 : wm,
188 : #if CONFIG_HIGHBITDEPTH
189 : use_hbd, bd,
190 : #endif // CONFIG_HIGHBITDEPTH
191 0 : ref, r_width, r_height, r_stride, dst + border * d_stride + border,
192 0 : border, border, d_width - 2 * border, d_height - 2 * border, d_stride,
193 : 0, 0, 16, 16);
194 0 : if (step_error < best_error) {
195 0 : best_error = step_error;
196 0 : best_param = *param;
197 0 : step_dir = 1;
198 : }
199 0 : *param = best_param;
200 :
201 : // look to the direction chosen above repeatedly until error increases
202 : // for the biggest step size
203 0 : while (step_dir) {
204 0 : *param = add_param_offset(p, best_param, step * step_dir);
205 0 : step_error = av1_warp_error(
206 : wm,
207 : #if CONFIG_HIGHBITDEPTH
208 : use_hbd, bd,
209 : #endif // CONFIG_HIGHBITDEPTH
210 0 : ref, r_width, r_height, r_stride, dst + border * d_stride + border,
211 0 : border, border, d_width - 2 * border, d_height - 2 * border,
212 : d_stride, 0, 0, 16, 16);
213 0 : if (step_error < best_error) {
214 0 : best_error = step_error;
215 0 : best_param = *param;
216 : } else {
217 0 : *param = best_param;
218 0 : step_dir = 0;
219 : }
220 : }
221 : }
222 : }
223 0 : force_wmtype(wm, wmtype);
224 0 : wm->wmtype = get_gmtype(wm);
225 0 : return best_error;
226 : }
227 :
228 0 : static INLINE RansacFunc get_ransac_type(TransformationType type) {
229 0 : switch (type) {
230 0 : case HOMOGRAPHY: return ransac_homography;
231 0 : case HORTRAPEZOID: return ransac_hortrapezoid;
232 0 : case VERTRAPEZOID: return ransac_vertrapezoid;
233 0 : case AFFINE: return ransac_affine;
234 0 : case ROTZOOM: return ransac_rotzoom;
235 0 : case TRANSLATION: return ransac_translation;
236 0 : default: assert(0); return NULL;
237 : }
238 : }
239 :
240 : #if CONFIG_HIGHBITDEPTH
241 0 : static unsigned char *downconvert_frame(YV12_BUFFER_CONFIG *frm,
242 : int bit_depth) {
243 : int i, j;
244 0 : uint16_t *orig_buf = CONVERT_TO_SHORTPTR(frm->y_buffer);
245 0 : uint8_t *buf = malloc(frm->y_height * frm->y_stride * sizeof(*buf));
246 :
247 0 : for (i = 0; i < frm->y_height; ++i)
248 0 : for (j = 0; j < frm->y_width; ++j)
249 0 : buf[i * frm->y_stride + j] =
250 0 : orig_buf[i * frm->y_stride + j] >> (bit_depth - 8);
251 :
252 0 : return buf;
253 : }
254 : #endif
255 :
256 0 : int compute_global_motion_feature_based(
257 : TransformationType type, YV12_BUFFER_CONFIG *frm, YV12_BUFFER_CONFIG *ref,
258 : #if CONFIG_HIGHBITDEPTH
259 : int bit_depth,
260 : #endif
261 : int *num_inliers_by_motion, double *params_by_motion, int num_motions) {
262 : int i;
263 : int num_frm_corners, num_ref_corners;
264 : int num_correspondences;
265 : int *correspondences;
266 : int frm_corners[2 * MAX_CORNERS], ref_corners[2 * MAX_CORNERS];
267 0 : unsigned char *frm_buffer = frm->y_buffer;
268 0 : unsigned char *ref_buffer = ref->y_buffer;
269 0 : RansacFunc ransac = get_ransac_type(type);
270 :
271 : #if CONFIG_HIGHBITDEPTH
272 0 : if (frm->flags & YV12_FLAG_HIGHBITDEPTH) {
273 : // The frame buffer is 16-bit, so we need to convert to 8 bits for the
274 : // following code. We cache the result until the frame is released.
275 0 : if (frm->y_buffer_8bit)
276 0 : frm_buffer = frm->y_buffer_8bit;
277 : else
278 0 : frm_buffer = frm->y_buffer_8bit = downconvert_frame(frm, bit_depth);
279 : }
280 0 : if (ref->flags & YV12_FLAG_HIGHBITDEPTH) {
281 0 : if (ref->y_buffer_8bit)
282 0 : ref_buffer = ref->y_buffer_8bit;
283 : else
284 0 : ref_buffer = ref->y_buffer_8bit = downconvert_frame(ref, bit_depth);
285 : }
286 : #endif
287 :
288 : // compute interest points in images using FAST features
289 0 : num_frm_corners = fast_corner_detect(frm_buffer, frm->y_width, frm->y_height,
290 : frm->y_stride, frm_corners, MAX_CORNERS);
291 0 : num_ref_corners = fast_corner_detect(ref_buffer, ref->y_width, ref->y_height,
292 : ref->y_stride, ref_corners, MAX_CORNERS);
293 :
294 : // find correspondences between the two images
295 0 : correspondences =
296 0 : (int *)malloc(num_frm_corners * 4 * sizeof(*correspondences));
297 0 : num_correspondences = determine_correspondence(
298 : frm_buffer, (int *)frm_corners, num_frm_corners, ref_buffer,
299 : (int *)ref_corners, num_ref_corners, frm->y_width, frm->y_height,
300 : frm->y_stride, ref->y_stride, correspondences);
301 :
302 0 : ransac(correspondences, num_correspondences, num_inliers_by_motion,
303 : params_by_motion, num_motions);
304 :
305 0 : free(correspondences);
306 :
307 : // Set num_inliers = 0 for motions with too few inliers so they are ignored.
308 0 : for (i = 0; i < num_motions; ++i) {
309 0 : if (num_inliers_by_motion[i] < MIN_INLIER_PROB * num_correspondences) {
310 0 : num_inliers_by_motion[i] = 0;
311 : }
312 : }
313 :
314 : // Return true if any one of the motions has inliers.
315 0 : for (i = 0; i < num_motions; ++i) {
316 0 : if (num_inliers_by_motion[i] > 0) return 1;
317 : }
318 0 : return 0;
319 : }
|
