Line data Source code
1 : /*
2 : * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 :
11 : #include "webrtc/modules/audio_processing/echo_cancellation_impl.h"
12 :
13 : #include <string.h>
14 :
15 : #include "webrtc/base/checks.h"
16 : #include "webrtc/modules/audio_processing/aec/aec_core.h"
17 : #include "webrtc/modules/audio_processing/aec/echo_cancellation.h"
18 : #include "webrtc/modules/audio_processing/audio_buffer.h"
19 :
20 : namespace webrtc {
21 :
22 : namespace {
23 0 : int16_t MapSetting(EchoCancellation::SuppressionLevel level) {
24 0 : switch (level) {
25 : case EchoCancellation::kLowSuppression:
26 0 : return kAecNlpConservative;
27 : case EchoCancellation::kModerateSuppression:
28 0 : return kAecNlpModerate;
29 : case EchoCancellation::kHighSuppression:
30 0 : return kAecNlpAggressive;
31 : }
32 0 : RTC_NOTREACHED();
33 0 : return -1;
34 : }
35 :
36 0 : AudioProcessing::Error MapError(int err) {
37 0 : switch (err) {
38 : case AEC_UNSUPPORTED_FUNCTION_ERROR:
39 0 : return AudioProcessing::kUnsupportedFunctionError;
40 : case AEC_BAD_PARAMETER_ERROR:
41 0 : return AudioProcessing::kBadParameterError;
42 : case AEC_BAD_PARAMETER_WARNING:
43 0 : return AudioProcessing::kBadStreamParameterWarning;
44 : default:
45 : // AEC_UNSPECIFIED_ERROR
46 : // AEC_UNINITIALIZED_ERROR
47 : // AEC_NULL_POINTER_ERROR
48 0 : return AudioProcessing::kUnspecifiedError;
49 : }
50 : }
51 :
52 : } // namespace
53 :
54 : struct EchoCancellationImpl::StreamProperties {
55 : StreamProperties() = delete;
56 0 : StreamProperties(int sample_rate_hz,
57 : size_t num_reverse_channels,
58 : size_t num_output_channels,
59 : size_t num_proc_channels)
60 0 : : sample_rate_hz(sample_rate_hz),
61 : num_reverse_channels(num_reverse_channels),
62 : num_output_channels(num_output_channels),
63 0 : num_proc_channels(num_proc_channels) {}
64 :
65 : const int sample_rate_hz;
66 : const size_t num_reverse_channels;
67 : const size_t num_output_channels;
68 : const size_t num_proc_channels;
69 : };
70 :
71 : class EchoCancellationImpl::Canceller {
72 : public:
73 0 : Canceller() {
74 0 : state_ = WebRtcAec_Create();
75 0 : RTC_DCHECK(state_);
76 0 : }
77 :
78 0 : ~Canceller() {
79 0 : RTC_CHECK(state_);
80 0 : WebRtcAec_Free(state_);
81 0 : }
82 :
83 0 : void* state() { return state_; }
84 :
85 0 : void Initialize(int sample_rate_hz) {
86 : // TODO(ajm): Drift compensation is disabled in practice. If restored, it
87 : // should be managed internally and not depend on the hardware sample rate.
88 : // For now, just hardcode a 48 kHz value.
89 0 : const int error = WebRtcAec_Init(state_, sample_rate_hz, 48000);
90 0 : RTC_DCHECK_EQ(0, error);
91 0 : }
92 :
93 : private:
94 : void* state_;
95 : };
96 :
97 0 : EchoCancellationImpl::EchoCancellationImpl(rtc::CriticalSection* crit_render,
98 0 : rtc::CriticalSection* crit_capture)
99 : : crit_render_(crit_render),
100 : crit_capture_(crit_capture),
101 : drift_compensation_enabled_(false),
102 : metrics_enabled_(false),
103 : suppression_level_(kModerateSuppression),
104 : stream_drift_samples_(0),
105 : was_stream_drift_set_(false),
106 : stream_has_echo_(false),
107 : delay_logging_enabled_(false),
108 : extended_filter_enabled_(false),
109 0 : delay_agnostic_enabled_(false) {
110 0 : RTC_DCHECK(crit_render);
111 0 : RTC_DCHECK(crit_capture);
112 0 : }
113 :
114 : EchoCancellationImpl::~EchoCancellationImpl() = default;
115 :
116 0 : void EchoCancellationImpl::ProcessRenderAudio(
117 : rtc::ArrayView<const float> packed_render_audio) {
118 0 : rtc::CritScope cs_capture(crit_capture_);
119 0 : if (!enabled_) {
120 0 : return;
121 : }
122 :
123 0 : RTC_DCHECK(stream_properties_);
124 0 : size_t handle_index = 0;
125 0 : size_t buffer_index = 0;
126 : const size_t num_frames_per_band =
127 0 : packed_render_audio.size() / (stream_properties_->num_output_channels *
128 0 : stream_properties_->num_reverse_channels);
129 0 : for (size_t i = 0; i < stream_properties_->num_output_channels; i++) {
130 0 : for (size_t j = 0; j < stream_properties_->num_reverse_channels; j++) {
131 0 : WebRtcAec_BufferFarend(cancellers_[handle_index++]->state(),
132 0 : &packed_render_audio[buffer_index],
133 0 : num_frames_per_band);
134 :
135 0 : buffer_index += num_frames_per_band;
136 : }
137 : }
138 : }
139 :
140 :
141 0 : int EchoCancellationImpl::ProcessCaptureAudio(AudioBuffer* audio,
142 : int stream_delay_ms) {
143 0 : rtc::CritScope cs_capture(crit_capture_);
144 0 : if (!enabled_) {
145 0 : return AudioProcessing::kNoError;
146 : }
147 :
148 0 : if (drift_compensation_enabled_ && !was_stream_drift_set_) {
149 0 : return AudioProcessing::kStreamParameterNotSetError;
150 : }
151 :
152 0 : RTC_DCHECK(stream_properties_);
153 0 : RTC_DCHECK_GE(160, audio->num_frames_per_band());
154 0 : RTC_DCHECK_EQ(audio->num_channels(), stream_properties_->num_proc_channels);
155 :
156 0 : int err = AudioProcessing::kNoError;
157 :
158 : // The ordering convention must be followed to pass to the correct AEC.
159 0 : size_t handle_index = 0;
160 0 : stream_has_echo_ = false;
161 0 : for (size_t i = 0; i < audio->num_channels(); i++) {
162 0 : for (size_t j = 0; j < stream_properties_->num_reverse_channels; j++) {
163 0 : err = WebRtcAec_Process(
164 0 : cancellers_[handle_index]->state(), audio->split_bands_const_f(i),
165 : audio->num_bands(), audio->split_bands_f(i),
166 0 : audio->num_frames_per_band(), stream_delay_ms, stream_drift_samples_);
167 :
168 0 : if (err != AudioProcessing::kNoError) {
169 0 : err = MapError(err);
170 : // TODO(ajm): Figure out how to return warnings properly.
171 0 : if (err != AudioProcessing::kBadStreamParameterWarning) {
172 0 : return err;
173 : }
174 : }
175 :
176 0 : int status = 0;
177 0 : err = WebRtcAec_get_echo_status(cancellers_[handle_index]->state(),
178 0 : &status);
179 0 : if (err != AudioProcessing::kNoError) {
180 0 : return MapError(err);
181 : }
182 :
183 0 : if (status == 1) {
184 0 : stream_has_echo_ = true;
185 : }
186 :
187 0 : handle_index++;
188 : }
189 : }
190 :
191 0 : was_stream_drift_set_ = false;
192 0 : return AudioProcessing::kNoError;
193 : }
194 :
195 0 : int EchoCancellationImpl::Enable(bool enable) {
196 : // Run in a single-threaded manner.
197 0 : rtc::CritScope cs_render(crit_render_);
198 0 : rtc::CritScope cs_capture(crit_capture_);
199 :
200 0 : if (enable && !enabled_) {
201 0 : enabled_ = enable; // Must be set before Initialize() is called.
202 :
203 : // TODO(peah): Simplify once the Enable function has been removed from
204 : // the public APM API.
205 0 : RTC_DCHECK(stream_properties_);
206 0 : Initialize(stream_properties_->sample_rate_hz,
207 0 : stream_properties_->num_reverse_channels,
208 0 : stream_properties_->num_output_channels,
209 0 : stream_properties_->num_proc_channels);
210 : } else {
211 0 : enabled_ = enable;
212 : }
213 0 : return AudioProcessing::kNoError;
214 : }
215 :
216 0 : bool EchoCancellationImpl::is_enabled() const {
217 0 : rtc::CritScope cs(crit_capture_);
218 0 : return enabled_;
219 : }
220 :
221 0 : int EchoCancellationImpl::set_suppression_level(SuppressionLevel level) {
222 : {
223 0 : if (MapSetting(level) == -1) {
224 0 : return AudioProcessing::kBadParameterError;
225 : }
226 0 : rtc::CritScope cs(crit_capture_);
227 0 : suppression_level_ = level;
228 : }
229 0 : return Configure();
230 : }
231 :
232 0 : EchoCancellation::SuppressionLevel EchoCancellationImpl::suppression_level()
233 : const {
234 0 : rtc::CritScope cs(crit_capture_);
235 0 : return suppression_level_;
236 : }
237 :
238 0 : int EchoCancellationImpl::enable_drift_compensation(bool enable) {
239 : {
240 0 : rtc::CritScope cs(crit_capture_);
241 0 : drift_compensation_enabled_ = enable;
242 : }
243 0 : return Configure();
244 : }
245 :
246 0 : bool EchoCancellationImpl::is_drift_compensation_enabled() const {
247 0 : rtc::CritScope cs(crit_capture_);
248 0 : return drift_compensation_enabled_;
249 : }
250 :
251 0 : void EchoCancellationImpl::set_stream_drift_samples(int drift) {
252 0 : rtc::CritScope cs(crit_capture_);
253 0 : was_stream_drift_set_ = true;
254 0 : stream_drift_samples_ = drift;
255 0 : }
256 :
257 0 : int EchoCancellationImpl::stream_drift_samples() const {
258 0 : rtc::CritScope cs(crit_capture_);
259 0 : return stream_drift_samples_;
260 : }
261 :
262 0 : int EchoCancellationImpl::enable_metrics(bool enable) {
263 : {
264 0 : rtc::CritScope cs(crit_capture_);
265 0 : metrics_enabled_ = enable;
266 : }
267 0 : return Configure();
268 : }
269 :
270 0 : bool EchoCancellationImpl::are_metrics_enabled() const {
271 0 : rtc::CritScope cs(crit_capture_);
272 0 : return enabled_ && metrics_enabled_;
273 : }
274 :
275 : // TODO(ajm): we currently just use the metrics from the first AEC. Think more
276 : // aboue the best way to extend this to multi-channel.
277 0 : int EchoCancellationImpl::GetMetrics(Metrics* metrics) {
278 0 : rtc::CritScope cs(crit_capture_);
279 0 : if (metrics == NULL) {
280 0 : return AudioProcessing::kNullPointerError;
281 : }
282 :
283 0 : if (!enabled_ || !metrics_enabled_) {
284 0 : return AudioProcessing::kNotEnabledError;
285 : }
286 :
287 : AecMetrics my_metrics;
288 0 : memset(&my_metrics, 0, sizeof(my_metrics));
289 0 : memset(metrics, 0, sizeof(Metrics));
290 :
291 0 : const int err = WebRtcAec_GetMetrics(cancellers_[0]->state(), &my_metrics);
292 0 : if (err != AudioProcessing::kNoError) {
293 0 : return MapError(err);
294 : }
295 :
296 0 : metrics->residual_echo_return_loss.instant = my_metrics.rerl.instant;
297 0 : metrics->residual_echo_return_loss.average = my_metrics.rerl.average;
298 0 : metrics->residual_echo_return_loss.maximum = my_metrics.rerl.max;
299 0 : metrics->residual_echo_return_loss.minimum = my_metrics.rerl.min;
300 :
301 0 : metrics->echo_return_loss.instant = my_metrics.erl.instant;
302 0 : metrics->echo_return_loss.average = my_metrics.erl.average;
303 0 : metrics->echo_return_loss.maximum = my_metrics.erl.max;
304 0 : metrics->echo_return_loss.minimum = my_metrics.erl.min;
305 :
306 0 : metrics->echo_return_loss_enhancement.instant = my_metrics.erle.instant;
307 0 : metrics->echo_return_loss_enhancement.average = my_metrics.erle.average;
308 0 : metrics->echo_return_loss_enhancement.maximum = my_metrics.erle.max;
309 0 : metrics->echo_return_loss_enhancement.minimum = my_metrics.erle.min;
310 :
311 0 : metrics->a_nlp.instant = my_metrics.aNlp.instant;
312 0 : metrics->a_nlp.average = my_metrics.aNlp.average;
313 0 : metrics->a_nlp.maximum = my_metrics.aNlp.max;
314 0 : metrics->a_nlp.minimum = my_metrics.aNlp.min;
315 :
316 0 : metrics->divergent_filter_fraction = my_metrics.divergent_filter_fraction;
317 0 : return AudioProcessing::kNoError;
318 : }
319 :
320 0 : bool EchoCancellationImpl::stream_has_echo() const {
321 0 : rtc::CritScope cs(crit_capture_);
322 0 : return stream_has_echo_;
323 : }
324 :
325 0 : int EchoCancellationImpl::enable_delay_logging(bool enable) {
326 : {
327 0 : rtc::CritScope cs(crit_capture_);
328 0 : delay_logging_enabled_ = enable;
329 : }
330 0 : return Configure();
331 : }
332 :
333 0 : bool EchoCancellationImpl::is_delay_logging_enabled() const {
334 0 : rtc::CritScope cs(crit_capture_);
335 0 : return enabled_ && delay_logging_enabled_;
336 : }
337 :
338 0 : bool EchoCancellationImpl::is_delay_agnostic_enabled() const {
339 0 : rtc::CritScope cs(crit_capture_);
340 0 : return delay_agnostic_enabled_;
341 : }
342 :
343 0 : std::string EchoCancellationImpl::GetExperimentsDescription() {
344 0 : rtc::CritScope cs(crit_capture_);
345 0 : return refined_adaptive_filter_enabled_ ? "RefinedAdaptiveFilter;" : "";
346 : }
347 :
348 0 : bool EchoCancellationImpl::is_refined_adaptive_filter_enabled() const {
349 0 : rtc::CritScope cs(crit_capture_);
350 0 : return refined_adaptive_filter_enabled_;
351 : }
352 :
353 0 : bool EchoCancellationImpl::is_extended_filter_enabled() const {
354 0 : rtc::CritScope cs(crit_capture_);
355 0 : return extended_filter_enabled_;
356 : }
357 :
358 : // TODO(bjornv): How should we handle the multi-channel case?
359 0 : int EchoCancellationImpl::GetDelayMetrics(int* median, int* std) {
360 0 : rtc::CritScope cs(crit_capture_);
361 0 : float fraction_poor_delays = 0;
362 0 : return GetDelayMetrics(median, std, &fraction_poor_delays);
363 : }
364 :
365 0 : int EchoCancellationImpl::GetDelayMetrics(int* median, int* std,
366 : float* fraction_poor_delays) {
367 0 : rtc::CritScope cs(crit_capture_);
368 0 : if (median == NULL) {
369 0 : return AudioProcessing::kNullPointerError;
370 : }
371 0 : if (std == NULL) {
372 0 : return AudioProcessing::kNullPointerError;
373 : }
374 :
375 0 : if (!enabled_ || !delay_logging_enabled_) {
376 0 : return AudioProcessing::kNotEnabledError;
377 : }
378 :
379 0 : const int err = WebRtcAec_GetDelayMetrics(cancellers_[0]->state(), median,
380 0 : std, fraction_poor_delays);
381 0 : if (err != AudioProcessing::kNoError) {
382 0 : return MapError(err);
383 : }
384 :
385 0 : return AudioProcessing::kNoError;
386 : }
387 :
388 0 : struct AecCore* EchoCancellationImpl::aec_core() const {
389 0 : rtc::CritScope cs(crit_capture_);
390 0 : if (!enabled_) {
391 0 : return NULL;
392 : }
393 0 : return WebRtcAec_aec_core(cancellers_[0]->state());
394 : }
395 :
396 0 : void EchoCancellationImpl::Initialize(int sample_rate_hz,
397 : size_t num_reverse_channels,
398 : size_t num_output_channels,
399 : size_t num_proc_channels) {
400 0 : rtc::CritScope cs_render(crit_render_);
401 0 : rtc::CritScope cs_capture(crit_capture_);
402 :
403 0 : stream_properties_.reset(
404 : new StreamProperties(sample_rate_hz, num_reverse_channels,
405 0 : num_output_channels, num_proc_channels));
406 :
407 0 : if (!enabled_) {
408 0 : return;
409 : }
410 :
411 : const size_t num_cancellers_required =
412 0 : NumCancellersRequired(stream_properties_->num_output_channels,
413 0 : stream_properties_->num_reverse_channels);
414 0 : if (num_cancellers_required > cancellers_.size()) {
415 0 : const size_t cancellers_old_size = cancellers_.size();
416 0 : cancellers_.resize(num_cancellers_required);
417 :
418 0 : for (size_t i = cancellers_old_size; i < cancellers_.size(); ++i) {
419 0 : cancellers_[i].reset(new Canceller());
420 : }
421 : }
422 :
423 0 : for (auto& canceller : cancellers_) {
424 0 : canceller->Initialize(sample_rate_hz);
425 : }
426 :
427 0 : Configure();
428 : }
429 :
430 0 : int EchoCancellationImpl::GetSystemDelayInSamples() const {
431 0 : rtc::CritScope cs(crit_capture_);
432 0 : RTC_DCHECK(enabled_);
433 : // Report the delay for the first AEC component.
434 0 : return WebRtcAec_system_delay(
435 0 : WebRtcAec_aec_core(cancellers_[0]->state()));
436 : }
437 :
438 0 : void EchoCancellationImpl::PackRenderAudioBuffer(
439 : const AudioBuffer* audio,
440 : size_t num_output_channels,
441 : size_t num_channels,
442 : std::vector<float>* packed_buffer) {
443 0 : RTC_DCHECK_GE(160, audio->num_frames_per_band());
444 0 : RTC_DCHECK_EQ(num_channels, audio->num_channels());
445 :
446 0 : packed_buffer->clear();
447 : // The ordering convention must be followed to pass the correct data.
448 0 : for (size_t i = 0; i < num_output_channels; i++) {
449 0 : for (size_t j = 0; j < audio->num_channels(); j++) {
450 : // Buffer the samples in the render queue.
451 0 : packed_buffer->insert(packed_buffer->end(),
452 0 : audio->split_bands_const_f(j)[kBand0To8kHz],
453 0 : (audio->split_bands_const_f(j)[kBand0To8kHz] +
454 0 : audio->num_frames_per_band()));
455 : }
456 : }
457 0 : }
458 :
459 0 : void EchoCancellationImpl::SetExtraOptions(const webrtc::Config& config) {
460 : {
461 0 : rtc::CritScope cs(crit_capture_);
462 0 : extended_filter_enabled_ = config.Get<ExtendedFilter>().enabled;
463 0 : delay_agnostic_enabled_ = config.Get<DelayAgnostic>().enabled;
464 0 : refined_adaptive_filter_enabled_ =
465 0 : config.Get<RefinedAdaptiveFilter>().enabled;
466 : }
467 0 : Configure();
468 0 : }
469 :
470 0 : int EchoCancellationImpl::Configure() {
471 0 : rtc::CritScope cs_render(crit_render_);
472 0 : rtc::CritScope cs_capture(crit_capture_);
473 : AecConfig config;
474 0 : config.metricsMode = metrics_enabled_;
475 0 : config.nlpMode = MapSetting(suppression_level_);
476 0 : config.skewMode = drift_compensation_enabled_;
477 0 : config.delay_logging = delay_logging_enabled_;
478 :
479 0 : int error = AudioProcessing::kNoError;
480 0 : for (auto& canceller : cancellers_) {
481 0 : WebRtcAec_enable_extended_filter(WebRtcAec_aec_core(canceller->state()),
482 0 : extended_filter_enabled_ ? 1 : 0);
483 0 : WebRtcAec_enable_delay_agnostic(WebRtcAec_aec_core(canceller->state()),
484 0 : delay_agnostic_enabled_ ? 1 : 0);
485 0 : WebRtcAec_enable_refined_adaptive_filter(
486 : WebRtcAec_aec_core(canceller->state()),
487 0 : refined_adaptive_filter_enabled_);
488 0 : const int handle_error = WebRtcAec_set_config(canceller->state(), config);
489 0 : if (handle_error != AudioProcessing::kNoError) {
490 0 : error = AudioProcessing::kNoError;
491 : }
492 : }
493 0 : return error;
494 : }
495 :
496 0 : size_t EchoCancellationImpl::NumCancellersRequired(
497 : size_t num_output_channels,
498 : size_t num_reverse_channels) {
499 0 : return num_output_channels * num_reverse_channels;
500 : }
501 :
502 : } // namespace webrtc
|
