Line data Source code
1 : /*
2 : * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 :
11 : #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
12 : #include "webrtc/modules/video_processing/video_denoiser.h"
13 : #include "libyuv/planar_functions.h"
14 :
15 : namespace webrtc {
16 :
17 : #if DISPLAY || DISPLAYNEON
18 : static void CopyMem8x8(const uint8_t* src,
19 : int src_stride,
20 : uint8_t* dst,
21 : int dst_stride) {
22 : for (int i = 0; i < 8; i++) {
23 : memcpy(dst, src, 8);
24 : src += src_stride;
25 : dst += dst_stride;
26 : }
27 : }
28 :
29 : static void ShowRect(const std::unique_ptr<DenoiserFilter>& filter,
30 : const std::unique_ptr<uint8_t[]>& d_status,
31 : const std::unique_ptr<uint8_t[]>& moving_edge_red,
32 : const std::unique_ptr<uint8_t[]>& x_density,
33 : const std::unique_ptr<uint8_t[]>& y_density,
34 : const uint8_t* u_src, int stride_u_src,
35 : const uint8_t* v_src, int stride_v_src,
36 : uint8_t* u_dst, int stride_u_dst,
37 : uint8_t* v_dst, int stride_v_dst,
38 : int mb_rows_,
39 : int mb_cols_) {
40 : for (int mb_row = 0; mb_row < mb_rows_; ++mb_row) {
41 : for (int mb_col = 0; mb_col < mb_cols_; ++mb_col) {
42 : int mb_index = mb_row * mb_cols_ + mb_col;
43 : const uint8_t* mb_src_u =
44 : u_src + (mb_row << 3) * stride_u_src + (mb_col << 3);
45 : const uint8_t* mb_src_v =
46 : v_src + (mb_row << 3) * stride_v_src + (mb_col << 3);
47 : uint8_t* mb_dst_u = u_dst + (mb_row << 3) * stride_u_dst + (mb_col << 3);
48 : uint8_t* mb_dst_v = v_dst + (mb_row << 3) * stride_v_dst + (mb_col << 3);
49 : uint8_t uv_tmp[8 * 8];
50 : memset(uv_tmp, 200, 8 * 8);
51 : if (d_status[mb_index] == 1) {
52 : // Paint to red.
53 : CopyMem8x8(mb_src_u, stride_u_src, mb_dst_u, stride_u_dst);
54 : CopyMem8x8(uv_tmp, 8, mb_dst_v, stride_v_dst);
55 : } else if (moving_edge_red[mb_row * mb_cols_ + mb_col] &&
56 : x_density[mb_col] * y_density[mb_row]) {
57 : // Paint to blue.
58 : CopyMem8x8(uv_tmp, 8, mb_dst_u, stride_u_dst);
59 : CopyMem8x8(mb_src_v, stride_v_src, mb_dst_v, stride_v_dst);
60 : } else {
61 : CopyMem8x8(mb_src_u, stride_u_src, mb_dst_u, stride_u_dst);
62 : CopyMem8x8(mb_src_v, stride_v_src, mb_dst_v, stride_v_dst);
63 : }
64 : }
65 : }
66 : }
67 : #endif
68 :
69 0 : VideoDenoiser::VideoDenoiser(bool runtime_cpu_detection)
70 : : width_(0),
71 : height_(0),
72 : filter_(DenoiserFilter::Create(runtime_cpu_detection, &cpu_type_)),
73 0 : ne_(new NoiseEstimation()) {}
74 :
75 0 : void VideoDenoiser::DenoiserReset(rtc::scoped_refptr<VideoFrameBuffer> frame) {
76 0 : width_ = frame->width();
77 0 : height_ = frame->height();
78 0 : mb_cols_ = width_ >> 4;
79 0 : mb_rows_ = height_ >> 4;
80 :
81 : // Init noise estimator and allocate buffers.
82 0 : ne_->Init(width_, height_, cpu_type_);
83 0 : moving_edge_.reset(new uint8_t[mb_cols_ * mb_rows_]);
84 0 : mb_filter_decision_.reset(new DenoiserDecision[mb_cols_ * mb_rows_]);
85 0 : x_density_.reset(new uint8_t[mb_cols_]);
86 0 : y_density_.reset(new uint8_t[mb_rows_]);
87 0 : moving_object_.reset(new uint8_t[mb_cols_ * mb_rows_]);
88 0 : }
89 :
90 0 : int VideoDenoiser::PositionCheck(int mb_row, int mb_col, int noise_level) {
91 0 : if (noise_level == 0)
92 0 : return 1;
93 0 : if ((mb_row <= (mb_rows_ >> 4)) || (mb_col <= (mb_cols_ >> 4)) ||
94 0 : (mb_col >= (15 * mb_cols_ >> 4)))
95 0 : return 3;
96 0 : else if ((mb_row <= (mb_rows_ >> 3)) || (mb_col <= (mb_cols_ >> 3)) ||
97 0 : (mb_col >= (7 * mb_cols_ >> 3)))
98 0 : return 2;
99 : else
100 0 : return 1;
101 : }
102 :
103 0 : void VideoDenoiser::ReduceFalseDetection(
104 : const std::unique_ptr<uint8_t[]>& d_status,
105 : std::unique_ptr<uint8_t[]>* moving_edge_red,
106 : int noise_level) {
107 : // From up left corner.
108 0 : int mb_col_stop = mb_cols_ - 1;
109 0 : for (int mb_row = 0; mb_row <= mb_rows_ - 1; ++mb_row) {
110 0 : for (int mb_col = 0; mb_col <= mb_col_stop; ++mb_col) {
111 0 : if (d_status[mb_row * mb_cols_ + mb_col]) {
112 0 : mb_col_stop = mb_col - 1;
113 0 : break;
114 : }
115 0 : (*moving_edge_red)[mb_row * mb_cols_ + mb_col] = 0;
116 : }
117 : }
118 : // From bottom left corner.
119 0 : mb_col_stop = mb_cols_ - 1;
120 0 : for (int mb_row = mb_rows_ - 1; mb_row >= 0; --mb_row) {
121 0 : for (int mb_col = 0; mb_col <= mb_col_stop; ++mb_col) {
122 0 : if (d_status[mb_row * mb_cols_ + mb_col]) {
123 0 : mb_col_stop = mb_col - 1;
124 0 : break;
125 : }
126 0 : (*moving_edge_red)[mb_row * mb_cols_ + mb_col] = 0;
127 : }
128 : }
129 : // From up right corner.
130 0 : mb_col_stop = 0;
131 0 : for (int mb_row = 0; mb_row <= mb_rows_ - 1; ++mb_row) {
132 0 : for (int mb_col = mb_cols_ - 1; mb_col >= mb_col_stop; --mb_col) {
133 0 : if (d_status[mb_row * mb_cols_ + mb_col]) {
134 0 : mb_col_stop = mb_col + 1;
135 0 : break;
136 : }
137 0 : (*moving_edge_red)[mb_row * mb_cols_ + mb_col] = 0;
138 : }
139 : }
140 : // From bottom right corner.
141 0 : mb_col_stop = 0;
142 0 : for (int mb_row = mb_rows_ - 1; mb_row >= 0; --mb_row) {
143 0 : for (int mb_col = mb_cols_ - 1; mb_col >= mb_col_stop; --mb_col) {
144 0 : if (d_status[mb_row * mb_cols_ + mb_col]) {
145 0 : mb_col_stop = mb_col + 1;
146 0 : break;
147 : }
148 0 : (*moving_edge_red)[mb_row * mb_cols_ + mb_col] = 0;
149 : }
150 : }
151 0 : }
152 :
153 0 : bool VideoDenoiser::IsTrailingBlock(const std::unique_ptr<uint8_t[]>& d_status,
154 : int mb_row,
155 : int mb_col) {
156 0 : bool ret = false;
157 0 : int mb_index = mb_row * mb_cols_ + mb_col;
158 0 : if (!mb_row || !mb_col || mb_row == mb_rows_ - 1 || mb_col == mb_cols_ - 1)
159 0 : ret = false;
160 : else
161 0 : ret = d_status[mb_index + 1] || d_status[mb_index - 1] ||
162 0 : d_status[mb_index + mb_cols_] || d_status[mb_index - mb_cols_];
163 0 : return ret;
164 : }
165 :
166 0 : void VideoDenoiser::CopySrcOnMOB(const uint8_t* y_src,
167 : int stride_src,
168 : uint8_t* y_dst,
169 : int stride_dst) {
170 : // Loop over to copy src block if the block is marked as moving object block
171 : // or if the block may cause trailing artifacts.
172 0 : for (int mb_row = 0; mb_row < mb_rows_; ++mb_row) {
173 0 : const int mb_index_base = mb_row * mb_cols_;
174 0 : const uint8_t* mb_src_base = y_src + (mb_row << 4) * stride_src;
175 0 : uint8_t* mb_dst_base = y_dst + (mb_row << 4) * stride_dst;
176 0 : for (int mb_col = 0; mb_col < mb_cols_; ++mb_col) {
177 0 : const int mb_index = mb_index_base + mb_col;
178 0 : const uint32_t offset_col = mb_col << 4;
179 0 : const uint8_t* mb_src = mb_src_base + offset_col;
180 0 : uint8_t* mb_dst = mb_dst_base + offset_col;
181 : // Check if the block is a moving object block or may cause a trailing
182 : // artifacts.
183 0 : if (mb_filter_decision_[mb_index] != FILTER_BLOCK ||
184 0 : IsTrailingBlock(moving_edge_, mb_row, mb_col) ||
185 0 : (x_density_[mb_col] * y_density_[mb_row] &&
186 0 : moving_object_[mb_row * mb_cols_ + mb_col])) {
187 : // Copy y source.
188 0 : filter_->CopyMem16x16(mb_src, stride_src, mb_dst, stride_dst);
189 : }
190 : }
191 : }
192 0 : }
193 :
194 0 : void VideoDenoiser::CopyLumaOnMargin(const uint8_t* y_src,
195 : int stride_src,
196 : uint8_t* y_dst,
197 : int stride_dst) {
198 0 : int height_margin = height_ - (mb_rows_ << 4);
199 0 : if (height_margin > 0) {
200 0 : const uint8_t* margin_y_src = y_src + (mb_rows_ << 4) * stride_src;
201 0 : uint8_t* margin_y_dst = y_dst + (mb_rows_ << 4) * stride_dst;
202 0 : libyuv::CopyPlane(margin_y_src, stride_src, margin_y_dst, stride_dst,
203 0 : width_, height_margin);
204 : }
205 0 : int width_margin = width_ - (mb_cols_ << 4);
206 0 : if (width_margin > 0) {
207 0 : const uint8_t* margin_y_src = y_src + (mb_cols_ << 4);
208 0 : uint8_t* margin_y_dst = y_dst + (mb_cols_ << 4);
209 0 : libyuv::CopyPlane(margin_y_src, stride_src, margin_y_dst, stride_dst,
210 0 : width_ - (mb_cols_ << 4), mb_rows_ << 4);
211 : }
212 0 : }
213 :
214 0 : rtc::scoped_refptr<VideoFrameBuffer> VideoDenoiser::DenoiseFrame(
215 : rtc::scoped_refptr<VideoFrameBuffer> frame,
216 : bool noise_estimation_enabled) {
217 : // If previous width and height are different from current frame's, need to
218 : // reallocate the buffers and no denoising for the current frame.
219 0 : if (!prev_buffer_ || width_ != frame->width() || height_ != frame->height()) {
220 0 : DenoiserReset(frame);
221 0 : prev_buffer_ = frame;
222 0 : return frame;
223 : }
224 :
225 : // Set buffer pointers.
226 0 : const uint8_t* y_src = frame->DataY();
227 0 : int stride_y_src = frame->StrideY();
228 : rtc::scoped_refptr<I420Buffer> dst =
229 0 : buffer_pool_.CreateBuffer(width_, height_);
230 :
231 0 : uint8_t* y_dst = dst->MutableDataY();
232 0 : int stride_y_dst = dst->StrideY();
233 :
234 0 : const uint8_t* y_dst_prev = prev_buffer_->DataY();
235 0 : int stride_prev = prev_buffer_->StrideY();
236 :
237 0 : memset(x_density_.get(), 0, mb_cols_);
238 0 : memset(y_density_.get(), 0, mb_rows_);
239 0 : memset(moving_object_.get(), 1, mb_cols_ * mb_rows_);
240 :
241 0 : uint8_t noise_level = noise_estimation_enabled ? ne_->GetNoiseLevel() : 0;
242 0 : int thr_var_base = 16 * 16 * 2;
243 : // Loop over blocks to accumulate/extract noise level and update x/y_density
244 : // factors for moving object detection.
245 0 : for (int mb_row = 0; mb_row < mb_rows_; ++mb_row) {
246 0 : const int mb_index_base = mb_row * mb_cols_;
247 0 : const uint8_t* mb_src_base = y_src + (mb_row << 4) * stride_y_src;
248 0 : uint8_t* mb_dst_base = y_dst + (mb_row << 4) * stride_y_dst;
249 0 : const uint8_t* mb_dst_prev_base = y_dst_prev + (mb_row << 4) * stride_prev;
250 0 : for (int mb_col = 0; mb_col < mb_cols_; ++mb_col) {
251 0 : const int mb_index = mb_index_base + mb_col;
252 0 : const bool ne_enable = (mb_index % NOISE_SUBSAMPLE_INTERVAL == 0);
253 0 : const int pos_factor = PositionCheck(mb_row, mb_col, noise_level);
254 0 : const uint32_t thr_var_adp = thr_var_base * pos_factor;
255 0 : const uint32_t offset_col = mb_col << 4;
256 0 : const uint8_t* mb_src = mb_src_base + offset_col;
257 0 : uint8_t* mb_dst = mb_dst_base + offset_col;
258 0 : const uint8_t* mb_dst_prev = mb_dst_prev_base + offset_col;
259 :
260 : // TODO(jackychen): Need SSE2/NEON opt.
261 0 : int luma = 0;
262 0 : if (ne_enable) {
263 0 : for (int i = 4; i < 12; ++i) {
264 0 : for (int j = 4; j < 12; ++j) {
265 0 : luma += mb_src[i * stride_y_src + j];
266 : }
267 : }
268 : }
269 :
270 : // Get the filtered block and filter_decision.
271 0 : mb_filter_decision_[mb_index] =
272 0 : filter_->MbDenoise(mb_dst_prev, stride_prev, mb_dst, stride_y_dst,
273 0 : mb_src, stride_y_src, 0, noise_level);
274 :
275 : // If filter decision is FILTER_BLOCK, no need to check moving edge.
276 : // It is unlikely for a moving edge block to be filtered in current
277 : // setting.
278 0 : if (mb_filter_decision_[mb_index] == FILTER_BLOCK) {
279 0 : uint32_t sse_t = 0;
280 0 : if (ne_enable) {
281 : // The variance used in noise estimation is based on the src block in
282 : // time t (mb_src) and filtered block in time t-1 (mb_dist_prev).
283 0 : uint32_t noise_var = filter_->Variance16x8(
284 0 : mb_dst_prev, stride_y_dst, mb_src, stride_y_src, &sse_t);
285 0 : ne_->GetNoise(mb_index, noise_var, luma);
286 : }
287 0 : moving_edge_[mb_index] = 0; // Not a moving edge block.
288 : } else {
289 0 : uint32_t sse_t = 0;
290 : // The variance used in MOD is based on the filtered blocks in time
291 : // T (mb_dst) and T-1 (mb_dst_prev).
292 0 : uint32_t noise_var = filter_->Variance16x8(
293 0 : mb_dst_prev, stride_prev, mb_dst, stride_y_dst, &sse_t);
294 0 : if (noise_var > thr_var_adp) { // Moving edge checking.
295 0 : if (ne_enable) {
296 0 : ne_->ResetConsecLowVar(mb_index);
297 : }
298 0 : moving_edge_[mb_index] = 1; // Mark as moving edge block.
299 0 : x_density_[mb_col] += (pos_factor < 3);
300 0 : y_density_[mb_row] += (pos_factor < 3);
301 : } else {
302 0 : moving_edge_[mb_index] = 0;
303 0 : if (ne_enable) {
304 : // The variance used in noise estimation is based on the src block
305 : // in time t (mb_src) and filtered block in time t-1 (mb_dist_prev).
306 0 : uint32_t noise_var = filter_->Variance16x8(
307 0 : mb_dst_prev, stride_prev, mb_src, stride_y_src, &sse_t);
308 0 : ne_->GetNoise(mb_index, noise_var, luma);
309 : }
310 : }
311 : }
312 : } // End of for loop
313 : } // End of for loop
314 :
315 0 : ReduceFalseDetection(moving_edge_, &moving_object_, noise_level);
316 :
317 0 : CopySrcOnMOB(y_src, stride_y_src, y_dst, stride_y_dst);
318 :
319 : // When frame width/height not divisible by 16, copy the margin to
320 : // denoised_frame.
321 0 : if ((mb_rows_ << 4) != height_ || (mb_cols_ << 4) != width_)
322 0 : CopyLumaOnMargin(y_src, stride_y_src, y_dst, stride_y_dst);
323 :
324 : // Copy u/v planes.
325 0 : libyuv::CopyPlane(frame->DataU(), frame->StrideU(),
326 0 : dst->MutableDataU(), dst->StrideU(),
327 0 : (width_ + 1) >> 1, (height_ + 1) >> 1);
328 0 : libyuv::CopyPlane(frame->DataV(), frame->StrideV(),
329 0 : dst->MutableDataV(), dst->StrideV(),
330 0 : (width_ + 1) >> 1, (height_ + 1) >> 1);
331 :
332 : #if DISPLAY || DISPLAYNEON
333 : // Show rectangular region
334 : ShowRect(filter_, moving_edge_, moving_object_, x_density_, y_density_,
335 : frame->DataU(), frame->StrideU(), frame->DataV(), frame->StrideV(),
336 : dst->MutableDataU(), dst->StrideU(),
337 : dst->MutableDataV(), dst->StrideV(),
338 : mb_rows_, mb_cols_);
339 : #endif
340 0 : prev_buffer_ = dst;
341 0 : return dst;
342 : }
343 :
344 : } // namespace webrtc
|
