Line data Source code
1 : /*
2 : * Copyright (c) 2016, Alliance for Open Media. All rights reserved
3 : *
4 : * This source code is subject to the terms of the BSD 2 Clause License and
5 : * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 : * was not distributed with this source code in the LICENSE file, you can
7 : * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 : * Media Patent License 1.0 was not distributed with this source code in the
9 : * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 : */
11 :
12 : #include "./aom_config.h"
13 : #include "aom_dsp/aom_dsp_common.h"
14 : #include "aom_mem/aom_mem.h"
15 : #include "av1/common/entropymode.h"
16 : #include "av1/common/thread_common.h"
17 : #include "av1/common/reconinter.h"
18 :
19 : #if CONFIG_MULTITHREAD
20 0 : static INLINE void mutex_lock(pthread_mutex_t *const mutex) {
21 0 : const int kMaxTryLocks = 4000;
22 0 : int locked = 0;
23 : int i;
24 :
25 0 : for (i = 0; i < kMaxTryLocks; ++i) {
26 0 : if (!pthread_mutex_trylock(mutex)) {
27 0 : locked = 1;
28 0 : break;
29 : }
30 : }
31 :
32 0 : if (!locked) pthread_mutex_lock(mutex);
33 0 : }
34 : #endif // CONFIG_MULTITHREAD
35 :
36 0 : static INLINE void sync_read(AV1LfSync *const lf_sync, int r, int c) {
37 : #if CONFIG_MULTITHREAD
38 0 : const int nsync = lf_sync->sync_range;
39 :
40 0 : if (r && !(c & (nsync - 1))) {
41 0 : pthread_mutex_t *const mutex = &lf_sync->mutex_[r - 1];
42 0 : mutex_lock(mutex);
43 :
44 0 : while (c > lf_sync->cur_sb_col[r - 1] - nsync) {
45 0 : pthread_cond_wait(&lf_sync->cond_[r - 1], mutex);
46 : }
47 0 : pthread_mutex_unlock(mutex);
48 : }
49 : #else
50 : (void)lf_sync;
51 : (void)r;
52 : (void)c;
53 : #endif // CONFIG_MULTITHREAD
54 0 : }
55 :
56 0 : static INLINE void sync_write(AV1LfSync *const lf_sync, int r, int c,
57 : const int sb_cols) {
58 : #if CONFIG_MULTITHREAD
59 0 : const int nsync = lf_sync->sync_range;
60 : int cur;
61 : // Only signal when there are enough filtered SB for next row to run.
62 0 : int sig = 1;
63 :
64 0 : if (c < sb_cols - 1) {
65 0 : cur = c;
66 0 : if (c % nsync) sig = 0;
67 : } else {
68 0 : cur = sb_cols + nsync;
69 : }
70 :
71 0 : if (sig) {
72 0 : mutex_lock(&lf_sync->mutex_[r]);
73 :
74 0 : lf_sync->cur_sb_col[r] = cur;
75 :
76 0 : pthread_cond_signal(&lf_sync->cond_[r]);
77 0 : pthread_mutex_unlock(&lf_sync->mutex_[r]);
78 : }
79 : #else
80 : (void)lf_sync;
81 : (void)r;
82 : (void)c;
83 : (void)sb_cols;
84 : #endif // CONFIG_MULTITHREAD
85 0 : }
86 :
87 : #if !CONFIG_EXT_PARTITION_TYPES
88 0 : static INLINE enum lf_path get_loop_filter_path(
89 : int y_only, struct macroblockd_plane planes[MAX_MB_PLANE]) {
90 0 : if (y_only)
91 0 : return LF_PATH_444;
92 0 : else if (planes[1].subsampling_y == 1 && planes[1].subsampling_x == 1)
93 0 : return LF_PATH_420;
94 0 : else if (planes[1].subsampling_y == 0 && planes[1].subsampling_x == 0)
95 0 : return LF_PATH_444;
96 : else
97 0 : return LF_PATH_SLOW;
98 : }
99 :
100 0 : static INLINE void loop_filter_block_plane_ver(
101 : AV1_COMMON *cm, struct macroblockd_plane planes[MAX_MB_PLANE], int plane,
102 : MODE_INFO **mi, int mi_row, int mi_col, enum lf_path path,
103 : LOOP_FILTER_MASK *lfm) {
104 0 : if (plane == 0) {
105 0 : av1_filter_block_plane_ss00_ver(cm, &planes[0], mi_row, lfm);
106 : } else {
107 0 : switch (path) {
108 : case LF_PATH_420:
109 0 : av1_filter_block_plane_ss11_ver(cm, &planes[plane], mi_row, lfm);
110 0 : break;
111 : case LF_PATH_444:
112 0 : av1_filter_block_plane_ss00_ver(cm, &planes[plane], mi_row, lfm);
113 0 : break;
114 : case LF_PATH_SLOW:
115 0 : av1_filter_block_plane_non420_ver(cm, &planes[plane], mi, mi_row,
116 : mi_col, plane);
117 0 : break;
118 : }
119 : }
120 0 : }
121 :
122 0 : static INLINE void loop_filter_block_plane_hor(
123 : AV1_COMMON *cm, struct macroblockd_plane planes[MAX_MB_PLANE], int plane,
124 : MODE_INFO **mi, int mi_row, int mi_col, enum lf_path path,
125 : LOOP_FILTER_MASK *lfm) {
126 0 : if (plane == 0) {
127 0 : av1_filter_block_plane_ss00_hor(cm, &planes[0], mi_row, lfm);
128 : } else {
129 0 : switch (path) {
130 : case LF_PATH_420:
131 0 : av1_filter_block_plane_ss11_hor(cm, &planes[plane], mi_row, lfm);
132 0 : break;
133 : case LF_PATH_444:
134 0 : av1_filter_block_plane_ss00_hor(cm, &planes[plane], mi_row, lfm);
135 0 : break;
136 : case LF_PATH_SLOW:
137 0 : av1_filter_block_plane_non420_hor(cm, &planes[plane], mi, mi_row,
138 : mi_col, plane);
139 0 : break;
140 : }
141 : }
142 0 : }
143 : #endif
144 : // Row-based multi-threaded loopfilter hook
145 : #if CONFIG_PARALLEL_DEBLOCKING
146 : static int loop_filter_ver_row_worker(AV1LfSync *const lf_sync,
147 : LFWorkerData *const lf_data) {
148 : const int num_planes = lf_data->y_only ? 1 : MAX_MB_PLANE;
149 : int mi_row, mi_col;
150 : #if !CONFIG_EXT_PARTITION_TYPES
151 : enum lf_path path = get_loop_filter_path(lf_data->y_only, lf_data->planes);
152 : #endif
153 : for (mi_row = lf_data->start; mi_row < lf_data->stop;
154 : mi_row += lf_sync->num_workers * lf_data->cm->mib_size) {
155 : MODE_INFO **const mi =
156 : lf_data->cm->mi_grid_visible + mi_row * lf_data->cm->mi_stride;
157 :
158 : for (mi_col = 0; mi_col < lf_data->cm->mi_cols;
159 : mi_col += lf_data->cm->mib_size) {
160 : LOOP_FILTER_MASK lfm;
161 : int plane;
162 :
163 : av1_setup_dst_planes(lf_data->planes, lf_data->cm->sb_size,
164 : lf_data->frame_buffer, mi_row, mi_col);
165 : av1_setup_mask(lf_data->cm, mi_row, mi_col, mi + mi_col,
166 : lf_data->cm->mi_stride, &lfm);
167 :
168 : #if CONFIG_EXT_PARTITION_TYPES
169 : for (plane = 0; plane < num_planes; ++plane)
170 : av1_filter_block_plane_non420_ver(lf_data->cm, &lf_data->planes[plane],
171 : mi + mi_col, mi_row, mi_col, plane);
172 : #else
173 :
174 : for (plane = 0; plane < num_planes; ++plane)
175 : loop_filter_block_plane_ver(lf_data->cm, lf_data->planes, plane,
176 : mi + mi_col, mi_row, mi_col, path, &lfm);
177 : #endif
178 : }
179 : }
180 : return 1;
181 : }
182 :
183 : static int loop_filter_hor_row_worker(AV1LfSync *const lf_sync,
184 : LFWorkerData *const lf_data) {
185 : const int num_planes = lf_data->y_only ? 1 : MAX_MB_PLANE;
186 : const int sb_cols =
187 : mi_cols_aligned_to_sb(lf_data->cm) >> lf_data->cm->mib_size_log2;
188 : int mi_row, mi_col;
189 : #if !CONFIG_EXT_PARTITION_TYPES
190 : enum lf_path path = get_loop_filter_path(lf_data->y_only, lf_data->planes);
191 : #endif
192 :
193 : for (mi_row = lf_data->start; mi_row < lf_data->stop;
194 : mi_row += lf_sync->num_workers * lf_data->cm->mib_size) {
195 : MODE_INFO **const mi =
196 : lf_data->cm->mi_grid_visible + mi_row * lf_data->cm->mi_stride;
197 :
198 : for (mi_col = 0; mi_col < lf_data->cm->mi_cols;
199 : mi_col += lf_data->cm->mib_size) {
200 : const int r = mi_row >> lf_data->cm->mib_size_log2;
201 : const int c = mi_col >> lf_data->cm->mib_size_log2;
202 : LOOP_FILTER_MASK lfm;
203 : int plane;
204 :
205 : // TODO(wenhao.zhang@intel.com): For better parallelization, reorder
206 : // the outer loop to column-based and remove the synchronizations here.
207 : sync_read(lf_sync, r, c);
208 :
209 : av1_setup_dst_planes(lf_data->planes, lf_data->cm->sb_size,
210 : lf_data->frame_buffer, mi_row, mi_col);
211 : av1_setup_mask(lf_data->cm, mi_row, mi_col, mi + mi_col,
212 : lf_data->cm->mi_stride, &lfm);
213 : #if CONFIG_EXT_PARTITION_TYPES
214 : for (plane = 0; plane < num_planes; ++plane)
215 : av1_filter_block_plane_non420_hor(lf_data->cm, &lf_data->planes[plane],
216 : mi + mi_col, mi_row, mi_col, plane);
217 : #else
218 : for (plane = 0; plane < num_planes; ++plane)
219 : loop_filter_block_plane_hor(lf_data->cm, lf_data->planes, plane,
220 : mi + mi_col, mi_row, mi_col, path, &lfm);
221 : #endif
222 : sync_write(lf_sync, r, c, sb_cols);
223 : }
224 : }
225 : return 1;
226 : }
227 : #else // CONFIG_PARALLEL_DEBLOCKING
228 0 : static int loop_filter_row_worker(AV1LfSync *const lf_sync,
229 : LFWorkerData *const lf_data) {
230 0 : const int num_planes = lf_data->y_only ? 1 : MAX_MB_PLANE;
231 0 : const int sb_cols =
232 0 : mi_cols_aligned_to_sb(lf_data->cm) >> lf_data->cm->mib_size_log2;
233 : int mi_row, mi_col;
234 : #if !CONFIG_EXT_PARTITION_TYPES
235 0 : enum lf_path path = get_loop_filter_path(lf_data->y_only, lf_data->planes);
236 : #endif // !CONFIG_EXT_PARTITION_TYPES
237 :
238 : #if CONFIG_EXT_PARTITION
239 : printf(
240 : "STOPPING: This code has not been modified to work with the "
241 : "extended coding unit size experiment");
242 : exit(EXIT_FAILURE);
243 : #endif // CONFIG_EXT_PARTITION
244 :
245 0 : for (mi_row = lf_data->start; mi_row < lf_data->stop;
246 0 : mi_row += lf_sync->num_workers * lf_data->cm->mib_size) {
247 0 : MODE_INFO **const mi =
248 0 : lf_data->cm->mi_grid_visible + mi_row * lf_data->cm->mi_stride;
249 :
250 0 : for (mi_col = 0; mi_col < lf_data->cm->mi_cols;
251 0 : mi_col += lf_data->cm->mib_size) {
252 0 : const int r = mi_row >> lf_data->cm->mib_size_log2;
253 0 : const int c = mi_col >> lf_data->cm->mib_size_log2;
254 : #if !CONFIG_EXT_PARTITION_TYPES
255 : LOOP_FILTER_MASK lfm;
256 : #endif
257 : int plane;
258 :
259 0 : sync_read(lf_sync, r, c);
260 :
261 0 : av1_setup_dst_planes(lf_data->planes, lf_data->cm->sb_size,
262 0 : lf_data->frame_buffer, mi_row, mi_col);
263 : #if CONFIG_EXT_PARTITION_TYPES
264 : for (plane = 0; plane < num_planes; ++plane) {
265 : av1_filter_block_plane_non420_ver(lf_data->cm, &lf_data->planes[plane],
266 : mi + mi_col, mi_row, mi_col, plane);
267 : av1_filter_block_plane_non420_hor(lf_data->cm, &lf_data->planes[plane],
268 : mi + mi_col, mi_row, mi_col, plane);
269 : }
270 : #else
271 0 : av1_setup_mask(lf_data->cm, mi_row, mi_col, mi + mi_col,
272 0 : lf_data->cm->mi_stride, &lfm);
273 :
274 0 : for (plane = 0; plane < num_planes; ++plane) {
275 0 : loop_filter_block_plane_ver(lf_data->cm, lf_data->planes, plane,
276 0 : mi + mi_col, mi_row, mi_col, path, &lfm);
277 0 : loop_filter_block_plane_hor(lf_data->cm, lf_data->planes, plane,
278 0 : mi + mi_col, mi_row, mi_col, path, &lfm);
279 : }
280 : #endif // CONFIG_EXT_PARTITION_TYPES
281 0 : sync_write(lf_sync, r, c, sb_cols);
282 : }
283 : }
284 0 : return 1;
285 : }
286 : #endif // CONFIG_PARALLEL_DEBLOCKING
287 :
288 0 : static void loop_filter_rows_mt(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm,
289 : struct macroblockd_plane planes[MAX_MB_PLANE],
290 : int start, int stop, int y_only,
291 : AVxWorker *workers, int nworkers,
292 : AV1LfSync *lf_sync) {
293 0 : const AVxWorkerInterface *const winterface = aom_get_worker_interface();
294 : // Number of superblock rows and cols
295 0 : const int sb_rows = mi_rows_aligned_to_sb(cm) >> cm->mib_size_log2;
296 : // Decoder may allocate more threads than number of tiles based on user's
297 : // input.
298 0 : const int tile_cols = cm->tile_cols;
299 0 : const int num_workers = AOMMIN(nworkers, tile_cols);
300 : int i;
301 :
302 : #if CONFIG_EXT_PARTITION
303 : printf(
304 : "STOPPING: This code has not been modified to work with the "
305 : "extended coding unit size experiment");
306 : exit(EXIT_FAILURE);
307 : #endif // CONFIG_EXT_PARTITION
308 :
309 0 : if (!lf_sync->sync_range || sb_rows != lf_sync->rows ||
310 0 : num_workers > lf_sync->num_workers) {
311 0 : av1_loop_filter_dealloc(lf_sync);
312 0 : av1_loop_filter_alloc(lf_sync, cm, sb_rows, cm->width, num_workers);
313 : }
314 :
315 : // Set up loopfilter thread data.
316 : // The decoder is capping num_workers because it has been observed that using
317 : // more threads on the loopfilter than there are cores will hurt performance
318 : // on Android. This is because the system will only schedule the tile decode
319 : // workers on cores equal to the number of tile columns. Then if the decoder
320 : // tries to use more threads for the loopfilter, it will hurt performance
321 : // because of contention. If the multithreading code changes in the future
322 : // then the number of workers used by the loopfilter should be revisited.
323 :
324 : #if CONFIG_PARALLEL_DEBLOCKING
325 : // Initialize cur_sb_col to -1 for all SB rows.
326 : memset(lf_sync->cur_sb_col, -1, sizeof(*lf_sync->cur_sb_col) * sb_rows);
327 :
328 : // Filter all the vertical edges in the whole frame
329 : for (i = 0; i < num_workers; ++i) {
330 : AVxWorker *const worker = &workers[i];
331 : LFWorkerData *const lf_data = &lf_sync->lfdata[i];
332 :
333 : worker->hook = (AVxWorkerHook)loop_filter_ver_row_worker;
334 : worker->data1 = lf_sync;
335 : worker->data2 = lf_data;
336 :
337 : // Loopfilter data
338 : av1_loop_filter_data_reset(lf_data, frame, cm, planes);
339 : lf_data->start = start + i * cm->mib_size;
340 : lf_data->stop = stop;
341 : lf_data->y_only = y_only;
342 :
343 : // Start loopfiltering
344 : if (i == num_workers - 1) {
345 : winterface->execute(worker);
346 : } else {
347 : winterface->launch(worker);
348 : }
349 : }
350 :
351 : // Wait till all rows are finished
352 : for (i = 0; i < num_workers; ++i) {
353 : winterface->sync(&workers[i]);
354 : }
355 :
356 : memset(lf_sync->cur_sb_col, -1, sizeof(*lf_sync->cur_sb_col) * sb_rows);
357 : // Filter all the horizontal edges in the whole frame
358 : for (i = 0; i < num_workers; ++i) {
359 : AVxWorker *const worker = &workers[i];
360 : LFWorkerData *const lf_data = &lf_sync->lfdata[i];
361 :
362 : worker->hook = (AVxWorkerHook)loop_filter_hor_row_worker;
363 : worker->data1 = lf_sync;
364 : worker->data2 = lf_data;
365 :
366 : // Loopfilter data
367 : av1_loop_filter_data_reset(lf_data, frame, cm, planes);
368 : lf_data->start = start + i * cm->mib_size;
369 : lf_data->stop = stop;
370 : lf_data->y_only = y_only;
371 :
372 : // Start loopfiltering
373 : if (i == num_workers - 1) {
374 : winterface->execute(worker);
375 : } else {
376 : winterface->launch(worker);
377 : }
378 : }
379 :
380 : // Wait till all rows are finished
381 : for (i = 0; i < num_workers; ++i) {
382 : winterface->sync(&workers[i]);
383 : }
384 : #else // CONFIG_PARALLEL_DEBLOCKING
385 : // Initialize cur_sb_col to -1 for all SB rows.
386 0 : memset(lf_sync->cur_sb_col, -1, sizeof(*lf_sync->cur_sb_col) * sb_rows);
387 :
388 0 : for (i = 0; i < num_workers; ++i) {
389 0 : AVxWorker *const worker = &workers[i];
390 0 : LFWorkerData *const lf_data = &lf_sync->lfdata[i];
391 :
392 0 : worker->hook = (AVxWorkerHook)loop_filter_row_worker;
393 0 : worker->data1 = lf_sync;
394 0 : worker->data2 = lf_data;
395 :
396 : // Loopfilter data
397 0 : av1_loop_filter_data_reset(lf_data, frame, cm, planes);
398 0 : lf_data->start = start + i * cm->mib_size;
399 0 : lf_data->stop = stop;
400 0 : lf_data->y_only = y_only;
401 :
402 : // Start loopfiltering
403 0 : if (i == num_workers - 1) {
404 0 : winterface->execute(worker);
405 : } else {
406 0 : winterface->launch(worker);
407 : }
408 : }
409 :
410 : // Wait till all rows are finished
411 0 : for (i = 0; i < num_workers; ++i) {
412 0 : winterface->sync(&workers[i]);
413 : }
414 : #endif // CONFIG_PARALLEL_DEBLOCKING
415 0 : }
416 :
417 0 : void av1_loop_filter_frame_mt(YV12_BUFFER_CONFIG *frame, AV1_COMMON *cm,
418 : struct macroblockd_plane planes[MAX_MB_PLANE],
419 : int frame_filter_level, int y_only,
420 : int partial_frame, AVxWorker *workers,
421 : int num_workers, AV1LfSync *lf_sync) {
422 : int start_mi_row, end_mi_row, mi_rows_to_filter;
423 :
424 0 : if (!frame_filter_level) return;
425 :
426 0 : start_mi_row = 0;
427 0 : mi_rows_to_filter = cm->mi_rows;
428 0 : if (partial_frame && cm->mi_rows > 8) {
429 0 : start_mi_row = cm->mi_rows >> 1;
430 0 : start_mi_row &= 0xfffffff8;
431 0 : mi_rows_to_filter = AOMMAX(cm->mi_rows / 8, 8);
432 : }
433 0 : end_mi_row = start_mi_row + mi_rows_to_filter;
434 0 : av1_loop_filter_frame_init(cm, frame_filter_level);
435 :
436 0 : loop_filter_rows_mt(frame, cm, planes, start_mi_row, end_mi_row, y_only,
437 : workers, num_workers, lf_sync);
438 : }
439 :
440 : // Set up nsync by width.
441 0 : static INLINE int get_sync_range(int width) {
442 : // nsync numbers are picked by testing. For example, for 4k
443 : // video, using 4 gives best performance.
444 0 : if (width < 640)
445 0 : return 1;
446 0 : else if (width <= 1280)
447 0 : return 2;
448 0 : else if (width <= 4096)
449 0 : return 4;
450 : else
451 0 : return 8;
452 : }
453 :
454 : // Allocate memory for lf row synchronization
455 0 : void av1_loop_filter_alloc(AV1LfSync *lf_sync, AV1_COMMON *cm, int rows,
456 : int width, int num_workers) {
457 0 : lf_sync->rows = rows;
458 : #if CONFIG_MULTITHREAD
459 : {
460 : int i;
461 :
462 0 : CHECK_MEM_ERROR(cm, lf_sync->mutex_,
463 : aom_malloc(sizeof(*lf_sync->mutex_) * rows));
464 0 : if (lf_sync->mutex_) {
465 0 : for (i = 0; i < rows; ++i) {
466 0 : pthread_mutex_init(&lf_sync->mutex_[i], NULL);
467 : }
468 : }
469 :
470 0 : CHECK_MEM_ERROR(cm, lf_sync->cond_,
471 : aom_malloc(sizeof(*lf_sync->cond_) * rows));
472 0 : if (lf_sync->cond_) {
473 0 : for (i = 0; i < rows; ++i) {
474 0 : pthread_cond_init(&lf_sync->cond_[i], NULL);
475 : }
476 : }
477 : }
478 : #endif // CONFIG_MULTITHREAD
479 :
480 0 : CHECK_MEM_ERROR(cm, lf_sync->lfdata,
481 : aom_malloc(num_workers * sizeof(*lf_sync->lfdata)));
482 0 : lf_sync->num_workers = num_workers;
483 :
484 0 : CHECK_MEM_ERROR(cm, lf_sync->cur_sb_col,
485 : aom_malloc(sizeof(*lf_sync->cur_sb_col) * rows));
486 :
487 : // Set up nsync.
488 0 : lf_sync->sync_range = get_sync_range(width);
489 0 : }
490 :
491 : // Deallocate lf synchronization related mutex and data
492 0 : void av1_loop_filter_dealloc(AV1LfSync *lf_sync) {
493 0 : if (lf_sync != NULL) {
494 : #if CONFIG_MULTITHREAD
495 : int i;
496 :
497 0 : if (lf_sync->mutex_ != NULL) {
498 0 : for (i = 0; i < lf_sync->rows; ++i) {
499 0 : pthread_mutex_destroy(&lf_sync->mutex_[i]);
500 : }
501 0 : aom_free(lf_sync->mutex_);
502 : }
503 0 : if (lf_sync->cond_ != NULL) {
504 0 : for (i = 0; i < lf_sync->rows; ++i) {
505 0 : pthread_cond_destroy(&lf_sync->cond_[i]);
506 : }
507 0 : aom_free(lf_sync->cond_);
508 : }
509 : #endif // CONFIG_MULTITHREAD
510 0 : aom_free(lf_sync->lfdata);
511 0 : aom_free(lf_sync->cur_sb_col);
512 : // clear the structure as the source of this call may be a resize in which
513 : // case this call will be followed by an _alloc() which may fail.
514 0 : av1_zero(*lf_sync);
515 : }
516 0 : }
517 :
518 : // Accumulate frame counts. FRAME_COUNTS consist solely of 'unsigned int'
519 : // members, so we treat it as an array, and sum over the whole length.
520 0 : void av1_accumulate_frame_counts(FRAME_COUNTS *acc_counts,
521 : FRAME_COUNTS *counts) {
522 0 : unsigned int *const acc = (unsigned int *)acc_counts;
523 0 : const unsigned int *const cnt = (unsigned int *)counts;
524 :
525 0 : const unsigned int n_counts = sizeof(FRAME_COUNTS) / sizeof(unsigned int);
526 : unsigned int i;
527 :
528 0 : for (i = 0; i < n_counts; i++) acc[i] += cnt[i];
529 0 : }
|
