Line data Source code
1 : /*
2 : * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 : #include "webrtc/modules/audio_processing/aec3/echo_canceller3.h"
11 :
12 : #include <sstream>
13 :
14 : #include "webrtc/base/atomicops.h"
15 : #include "webrtc/modules/audio_processing/logging/apm_data_dumper.h"
16 :
17 : namespace webrtc {
18 :
19 : namespace {
20 :
21 0 : bool DetectSaturation(rtc::ArrayView<const float> y) {
22 0 : for (auto y_k : y) {
23 0 : if (y_k >= 32767.0f || y_k <= -32768.0f) {
24 0 : return true;
25 : }
26 : }
27 0 : return false;
28 : }
29 :
30 0 : void FillSubFrameView(AudioBuffer* frame,
31 : size_t sub_frame_index,
32 : std::vector<rtc::ArrayView<float>>* sub_frame_view) {
33 0 : RTC_DCHECK_GE(1, sub_frame_index);
34 0 : RTC_DCHECK_LE(0, sub_frame_index);
35 0 : RTC_DCHECK_EQ(frame->num_bands(), sub_frame_view->size());
36 0 : for (size_t k = 0; k < sub_frame_view->size(); ++k) {
37 0 : (*sub_frame_view)[k] = rtc::ArrayView<float>(
38 0 : &frame->split_bands_f(0)[k][sub_frame_index * kSubFrameLength],
39 : kSubFrameLength);
40 : }
41 0 : }
42 :
43 0 : void FillSubFrameView(std::vector<std::vector<float>>* frame,
44 : size_t sub_frame_index,
45 : std::vector<rtc::ArrayView<float>>* sub_frame_view) {
46 0 : RTC_DCHECK_GE(1, sub_frame_index);
47 0 : RTC_DCHECK_EQ(frame->size(), sub_frame_view->size());
48 0 : for (size_t k = 0; k < frame->size(); ++k) {
49 0 : (*sub_frame_view)[k] = rtc::ArrayView<float>(
50 0 : &(*frame)[k][sub_frame_index * kSubFrameLength], kSubFrameLength);
51 : }
52 0 : }
53 :
54 0 : void ProcessCaptureFrameContent(
55 : AudioBuffer* capture,
56 : bool known_echo_path_change,
57 : bool saturated_microphone_signal,
58 : size_t sub_frame_index,
59 : FrameBlocker* capture_blocker,
60 : BlockFramer* output_framer,
61 : BlockProcessor* block_processor,
62 : std::vector<std::vector<float>>* block,
63 : std::vector<rtc::ArrayView<float>>* sub_frame_view) {
64 0 : FillSubFrameView(capture, sub_frame_index, sub_frame_view);
65 0 : capture_blocker->InsertSubFrameAndExtractBlock(*sub_frame_view, block);
66 0 : block_processor->ProcessCapture(known_echo_path_change,
67 0 : saturated_microphone_signal, block);
68 0 : output_framer->InsertBlockAndExtractSubFrame(*block, sub_frame_view);
69 0 : }
70 :
71 0 : void ProcessRemainingCaptureFrameContent(
72 : bool known_echo_path_change,
73 : bool saturated_microphone_signal,
74 : FrameBlocker* capture_blocker,
75 : BlockFramer* output_framer,
76 : BlockProcessor* block_processor,
77 : std::vector<std::vector<float>>* block) {
78 0 : if (!capture_blocker->IsBlockAvailable()) {
79 0 : return;
80 : }
81 :
82 0 : capture_blocker->ExtractBlock(block);
83 0 : block_processor->ProcessCapture(known_echo_path_change,
84 0 : saturated_microphone_signal, block);
85 0 : output_framer->InsertBlock(*block);
86 : }
87 :
88 0 : bool BufferRenderFrameContent(
89 : std::vector<std::vector<float>>* render_frame,
90 : size_t sub_frame_index,
91 : FrameBlocker* render_blocker,
92 : BlockProcessor* block_processor,
93 : std::vector<std::vector<float>>* block,
94 : std::vector<rtc::ArrayView<float>>* sub_frame_view) {
95 0 : FillSubFrameView(render_frame, sub_frame_index, sub_frame_view);
96 0 : render_blocker->InsertSubFrameAndExtractBlock(*sub_frame_view, block);
97 0 : return block_processor->BufferRender(block);
98 : }
99 :
100 0 : bool BufferRemainingRenderFrameContent(FrameBlocker* render_blocker,
101 : BlockProcessor* block_processor,
102 : std::vector<std::vector<float>>* block) {
103 0 : if (!render_blocker->IsBlockAvailable()) {
104 0 : return false;
105 : }
106 0 : render_blocker->ExtractBlock(block);
107 0 : return block_processor->BufferRender(block);
108 : }
109 :
110 0 : void CopyAudioBufferIntoFrame(AudioBuffer* buffer,
111 : size_t num_bands,
112 : size_t frame_length,
113 : std::vector<std::vector<float>>* frame) {
114 0 : RTC_DCHECK_EQ(num_bands, frame->size());
115 0 : for (size_t i = 0; i < num_bands; ++i) {
116 0 : rtc::ArrayView<float> buffer_view(&buffer->split_bands_f(0)[i][0],
117 0 : frame_length);
118 0 : std::copy(buffer_view.begin(), buffer_view.end(), (*frame)[i].begin());
119 : }
120 0 : }
121 :
122 : // [B,A] = butter(2,100/4000,'high')
123 : const CascadedBiQuadFilter::BiQuadCoefficients
124 : kHighPassFilterCoefficients_8kHz = {{0.94598f, -1.89195f, 0.94598f},
125 : {-1.88903f, 0.89487f}};
126 : const int kNumberOfHighPassBiQuads_8kHz = 1;
127 :
128 : // [B,A] = butter(2,100/8000,'high')
129 : const CascadedBiQuadFilter::BiQuadCoefficients
130 : kHighPassFilterCoefficients_16kHz = {{0.97261f, -1.94523f, 0.97261f},
131 : {-1.94448f, 0.94598f}};
132 : const int kNumberOfHighPassBiQuads_16kHz = 1;
133 :
134 : static constexpr size_t kRenderTransferQueueSize = 30;
135 :
136 : } // namespace
137 :
138 0 : class EchoCanceller3::RenderWriter {
139 : public:
140 : RenderWriter(ApmDataDumper* data_dumper,
141 : SwapQueue<std::vector<std::vector<float>>,
142 : Aec3RenderQueueItemVerifier>* render_transfer_queue,
143 : std::unique_ptr<CascadedBiQuadFilter> render_highpass_filter,
144 : int sample_rate_hz,
145 : int frame_length,
146 : int num_bands);
147 : ~RenderWriter();
148 : bool Insert(AudioBuffer* render);
149 :
150 : private:
151 : ApmDataDumper* data_dumper_;
152 : const int sample_rate_hz_;
153 : const size_t frame_length_;
154 : const int num_bands_;
155 : std::unique_ptr<CascadedBiQuadFilter> render_highpass_filter_;
156 : std::vector<std::vector<float>> render_queue_input_frame_;
157 : SwapQueue<std::vector<std::vector<float>>, Aec3RenderQueueItemVerifier>*
158 : render_transfer_queue_;
159 : RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderWriter);
160 : };
161 :
162 0 : EchoCanceller3::RenderWriter::RenderWriter(
163 : ApmDataDumper* data_dumper,
164 : SwapQueue<std::vector<std::vector<float>>, Aec3RenderQueueItemVerifier>*
165 : render_transfer_queue,
166 : std::unique_ptr<CascadedBiQuadFilter> render_highpass_filter,
167 : int sample_rate_hz,
168 : int frame_length,
169 0 : int num_bands)
170 : : data_dumper_(data_dumper),
171 : sample_rate_hz_(sample_rate_hz),
172 : frame_length_(frame_length),
173 : num_bands_(num_bands),
174 0 : render_highpass_filter_(std::move(render_highpass_filter)),
175 0 : render_queue_input_frame_(num_bands_,
176 0 : std::vector<float>(frame_length_, 0.f)),
177 0 : render_transfer_queue_(render_transfer_queue) {
178 0 : RTC_DCHECK(data_dumper);
179 0 : }
180 :
181 : EchoCanceller3::RenderWriter::~RenderWriter() = default;
182 :
183 0 : bool EchoCanceller3::RenderWriter::Insert(AudioBuffer* input) {
184 0 : RTC_DCHECK_EQ(1, input->num_channels());
185 0 : RTC_DCHECK_EQ(num_bands_, input->num_bands());
186 0 : RTC_DCHECK_EQ(frame_length_, input->num_frames_per_band());
187 0 : data_dumper_->DumpWav("aec3_render_input", frame_length_,
188 0 : &input->split_bands_f(0)[0][0],
189 0 : LowestBandRate(sample_rate_hz_), 1);
190 :
191 0 : CopyAudioBufferIntoFrame(input, num_bands_, frame_length_,
192 0 : &render_queue_input_frame_);
193 :
194 0 : if (render_highpass_filter_) {
195 0 : render_highpass_filter_->Process(render_queue_input_frame_[0]);
196 : }
197 :
198 0 : return render_transfer_queue_->Insert(&render_queue_input_frame_);
199 : }
200 :
201 : int EchoCanceller3::instance_count_ = 0;
202 :
203 0 : EchoCanceller3::EchoCanceller3(int sample_rate_hz, bool use_highpass_filter)
204 : : EchoCanceller3(sample_rate_hz,
205 : use_highpass_filter,
206 0 : std::unique_ptr<BlockProcessor>(
207 0 : BlockProcessor::Create(sample_rate_hz))) {}
208 0 : EchoCanceller3::EchoCanceller3(int sample_rate_hz,
209 : bool use_highpass_filter,
210 0 : std::unique_ptr<BlockProcessor> block_processor)
211 : : data_dumper_(
212 0 : new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
213 : sample_rate_hz_(sample_rate_hz),
214 0 : num_bands_(NumBandsForRate(sample_rate_hz_)),
215 0 : frame_length_(rtc::CheckedDivExact(LowestBandRate(sample_rate_hz_), 100)),
216 0 : output_framer_(num_bands_),
217 0 : capture_blocker_(num_bands_),
218 0 : render_blocker_(num_bands_),
219 : render_transfer_queue_(
220 : kRenderTransferQueueSize,
221 0 : std::vector<std::vector<float>>(
222 0 : num_bands_,
223 0 : std::vector<float>(frame_length_, 0.f)),
224 0 : Aec3RenderQueueItemVerifier(num_bands_, frame_length_)),
225 0 : block_processor_(std::move(block_processor)),
226 0 : render_queue_output_frame_(num_bands_,
227 0 : std::vector<float>(frame_length_, 0.f)),
228 0 : block_(num_bands_, std::vector<float>(kBlockSize, 0.f)),
229 0 : sub_frame_view_(num_bands_) {
230 0 : std::unique_ptr<CascadedBiQuadFilter> render_highpass_filter;
231 0 : if (use_highpass_filter) {
232 : render_highpass_filter.reset(new CascadedBiQuadFilter(
233 0 : sample_rate_hz_ == 8000 ? kHighPassFilterCoefficients_8kHz
234 : : kHighPassFilterCoefficients_16kHz,
235 : sample_rate_hz_ == 8000 ? kNumberOfHighPassBiQuads_8kHz
236 0 : : kNumberOfHighPassBiQuads_16kHz));
237 0 : capture_highpass_filter_.reset(new CascadedBiQuadFilter(
238 0 : sample_rate_hz_ == 8000 ? kHighPassFilterCoefficients_8kHz
239 : : kHighPassFilterCoefficients_16kHz,
240 : sample_rate_hz_ == 8000 ? kNumberOfHighPassBiQuads_8kHz
241 0 : : kNumberOfHighPassBiQuads_16kHz));
242 : }
243 :
244 0 : render_writer_.reset(
245 0 : new RenderWriter(data_dumper_.get(), &render_transfer_queue_,
246 0 : std::move(render_highpass_filter), sample_rate_hz_,
247 0 : frame_length_, num_bands_));
248 :
249 0 : RTC_DCHECK_EQ(num_bands_, std::max(sample_rate_hz_, 16000) / 16000);
250 0 : RTC_DCHECK_GE(kMaxNumBands, num_bands_);
251 0 : }
252 :
253 : EchoCanceller3::~EchoCanceller3() = default;
254 :
255 0 : bool EchoCanceller3::AnalyzeRender(AudioBuffer* render) {
256 0 : RTC_DCHECK_RUNS_SERIALIZED(&render_race_checker_);
257 0 : RTC_DCHECK(render);
258 0 : return render_writer_->Insert(render);
259 : }
260 :
261 0 : void EchoCanceller3::AnalyzeCapture(AudioBuffer* capture) {
262 0 : RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_);
263 0 : RTC_DCHECK(capture);
264 0 : data_dumper_->DumpWav("aec3_capture_analyze_input", frame_length_,
265 0 : capture->channels_f()[0], sample_rate_hz_, 1);
266 :
267 0 : saturated_microphone_signal_ = false;
268 0 : for (size_t k = 0; k < capture->num_channels(); ++k) {
269 0 : saturated_microphone_signal_ |=
270 0 : DetectSaturation(rtc::ArrayView<const float>(capture->channels_f()[k],
271 0 : capture->num_frames()));
272 0 : if (saturated_microphone_signal_) {
273 0 : break;
274 : }
275 : }
276 0 : }
277 :
278 0 : void EchoCanceller3::ProcessCapture(AudioBuffer* capture,
279 : bool known_echo_path_change) {
280 0 : RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_);
281 0 : RTC_DCHECK(capture);
282 0 : RTC_DCHECK_EQ(1u, capture->num_channels());
283 0 : RTC_DCHECK_EQ(num_bands_, capture->num_bands());
284 0 : RTC_DCHECK_EQ(frame_length_, capture->num_frames_per_band());
285 :
286 : rtc::ArrayView<float> capture_lower_band =
287 0 : rtc::ArrayView<float>(&capture->split_bands_f(0)[0][0], frame_length_);
288 :
289 0 : data_dumper_->DumpWav("aec3_capture_input", capture_lower_band,
290 0 : LowestBandRate(sample_rate_hz_), 1);
291 :
292 0 : const bool render_buffer_overrun = EmptyRenderQueue();
293 0 : RTC_DCHECK(!render_buffer_overrun);
294 :
295 0 : if (capture_highpass_filter_) {
296 0 : capture_highpass_filter_->Process(capture_lower_band);
297 : }
298 :
299 0 : ProcessCaptureFrameContent(capture, known_echo_path_change,
300 0 : saturated_microphone_signal_, 0, &capture_blocker_,
301 : &output_framer_, block_processor_.get(), &block_,
302 0 : &sub_frame_view_);
303 :
304 0 : if (sample_rate_hz_ != 8000) {
305 0 : ProcessCaptureFrameContent(
306 0 : capture, known_echo_path_change, saturated_microphone_signal_, 1,
307 : &capture_blocker_, &output_framer_, block_processor_.get(), &block_,
308 0 : &sub_frame_view_);
309 : }
310 :
311 0 : ProcessRemainingCaptureFrameContent(
312 0 : known_echo_path_change, saturated_microphone_signal_, &capture_blocker_,
313 0 : &output_framer_, block_processor_.get(), &block_);
314 :
315 0 : data_dumper_->DumpWav("aec3_capture_output", frame_length_,
316 0 : &capture->split_bands_f(0)[0][0],
317 0 : LowestBandRate(sample_rate_hz_), 1);
318 0 : }
319 :
320 0 : std::string EchoCanceller3::ToString(
321 : const AudioProcessing::Config::EchoCanceller3& config) {
322 0 : std::stringstream ss;
323 : ss << "{"
324 0 : << "enabled: " << (config.enabled ? "true" : "false") << "}";
325 0 : return ss.str();
326 : }
327 :
328 0 : bool EchoCanceller3::Validate(
329 : const AudioProcessing::Config::EchoCanceller3& config) {
330 0 : return true;
331 : }
332 :
333 0 : bool EchoCanceller3::EmptyRenderQueue() {
334 0 : RTC_DCHECK_RUNS_SERIALIZED(&capture_race_checker_);
335 0 : bool render_buffer_overrun = false;
336 : bool frame_to_buffer =
337 0 : render_transfer_queue_.Remove(&render_queue_output_frame_);
338 0 : while (frame_to_buffer) {
339 0 : render_buffer_overrun |= BufferRenderFrameContent(
340 : &render_queue_output_frame_, 0, &render_blocker_,
341 : block_processor_.get(), &block_, &sub_frame_view_);
342 :
343 0 : if (sample_rate_hz_ != 8000) {
344 0 : render_buffer_overrun |= BufferRenderFrameContent(
345 : &render_queue_output_frame_, 1, &render_blocker_,
346 : block_processor_.get(), &block_, &sub_frame_view_);
347 : }
348 :
349 0 : render_buffer_overrun |= BufferRemainingRenderFrameContent(
350 : &render_blocker_, block_processor_.get(), &block_);
351 :
352 : frame_to_buffer =
353 0 : render_transfer_queue_.Remove(&render_queue_output_frame_);
354 : }
355 0 : return render_buffer_overrun;
356 : }
357 :
358 : } // namespace webrtc
|
