Line data Source code
1 : /*
2 : * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 :
11 : /*
12 : * Contains the API functions for the AEC.
13 : */
14 : #include "webrtc/modules/audio_processing/aec/echo_cancellation.h"
15 :
16 : #include <math.h>
17 : #include <stdlib.h>
18 : #include <string.h>
19 :
20 : extern "C" {
21 : #include "webrtc/common_audio/ring_buffer.h"
22 : #include "webrtc/common_audio/signal_processing/include/signal_processing_library.h"
23 : }
24 : #include "webrtc/modules/audio_processing/aec/aec_core.h"
25 : #include "webrtc/modules/audio_processing/aec/aec_resampler.h"
26 : #include "webrtc/modules/audio_processing/logging/apm_data_dumper.h"
27 : #include "webrtc/typedefs.h"
28 :
29 : namespace webrtc {
30 :
31 : Aec::Aec() = default;
32 : Aec::~Aec() = default;
33 :
34 : // Measured delays [ms]
35 : // Device Chrome GTP
36 : // MacBook Air 10
37 : // MacBook Retina 10 100
38 : // MacPro 30?
39 : //
40 : // Win7 Desktop 70 80?
41 : // Win7 T430s 110
42 : // Win8 T420s 70
43 : //
44 : // Daisy 50
45 : // Pixel (w/ preproc?) 240
46 : // Pixel (w/o preproc?) 110 110
47 :
48 : // The extended filter mode gives us the flexibility to ignore the system's
49 : // reported delays. We do this for platforms which we believe provide results
50 : // which are incompatible with the AEC's expectations. Based on measurements
51 : // (some provided above) we set a conservative (i.e. lower than measured)
52 : // fixed delay.
53 : //
54 : // WEBRTC_UNTRUSTED_DELAY will only have an impact when |extended_filter_mode|
55 : // is enabled. See the note along with |DelayCorrection| in
56 : // echo_cancellation_impl.h for more details on the mode.
57 : //
58 : // Justification:
59 : // Chromium/Mac: Here, the true latency is so low (~10-20 ms), that it plays
60 : // havoc with the AEC's buffering. To avoid this, we set a fixed delay of 20 ms
61 : // and then compensate by rewinding by 10 ms (in wideband) through
62 : // kDelayDiffOffsetSamples. This trick does not seem to work for larger rewind
63 : // values, but fortunately this is sufficient.
64 : //
65 : // Chromium/Linux(ChromeOS): The values we get on this platform don't correspond
66 : // well to reality. The variance doesn't match the AEC's buffer changes, and the
67 : // bulk values tend to be too low. However, the range across different hardware
68 : // appears to be too large to choose a single value.
69 : //
70 : // GTP/Linux(ChromeOS): TBD, but for the moment we will trust the values.
71 : #if defined(WEBRTC_CHROMIUM_BUILD) && defined(WEBRTC_MAC)
72 : #define WEBRTC_UNTRUSTED_DELAY
73 : #endif
74 :
75 : #if defined(WEBRTC_UNTRUSTED_DELAY) && defined(WEBRTC_MAC)
76 : static const int kDelayDiffOffsetSamples = -160;
77 : #else
78 : // Not enabled for now.
79 : static const int kDelayDiffOffsetSamples = 0;
80 : #endif
81 :
82 : #if defined(WEBRTC_MAC)
83 : static const int kFixedDelayMs = 20;
84 : #else
85 : static const int kFixedDelayMs = 50;
86 : #endif
87 : #if !defined(WEBRTC_UNTRUSTED_DELAY)
88 : static const int kMinTrustedDelayMs = 20;
89 : #endif
90 : static const int kMaxTrustedDelayMs = 500;
91 :
92 : // Maximum length of resampled signal. Must be an integer multiple of frames
93 : // (ceil(1/(1 + MIN_SKEW)*2) + 1)*FRAME_LEN
94 : // The factor of 2 handles wb, and the + 1 is as a safety margin
95 : // TODO(bjornv): Replace with kResamplerBufferSize
96 : #define MAX_RESAMP_LEN (5 * FRAME_LEN)
97 :
98 : static const int kMaxBufSizeStart = 62; // In partitions
99 : static const int sampMsNb = 8; // samples per ms in nb
100 : static const int initCheck = 42;
101 :
102 : int Aec::instance_count = 0;
103 :
104 : // Estimates delay to set the position of the far-end buffer read pointer
105 : // (controlled by knownDelay)
106 : static void EstBufDelayNormal(Aec* aecInst);
107 : static void EstBufDelayExtended(Aec* aecInst);
108 : static int ProcessNormal(Aec* self,
109 : const float* const* near,
110 : size_t num_bands,
111 : float* const* out,
112 : size_t num_samples,
113 : int16_t reported_delay_ms,
114 : int32_t skew);
115 : static void ProcessExtended(Aec* self,
116 : const float* const* near,
117 : size_t num_bands,
118 : float* const* out,
119 : size_t num_samples,
120 : int16_t reported_delay_ms,
121 : int32_t skew);
122 :
123 0 : void* WebRtcAec_Create() {
124 0 : Aec* aecpc = new Aec();
125 :
126 0 : if (!aecpc) {
127 0 : return NULL;
128 : }
129 0 : aecpc->data_dumper.reset(new ApmDataDumper(aecpc->instance_count));
130 :
131 0 : aecpc->aec = WebRtcAec_CreateAec(aecpc->instance_count);
132 0 : if (!aecpc->aec) {
133 0 : WebRtcAec_Free(aecpc);
134 0 : return NULL;
135 : }
136 0 : aecpc->resampler = WebRtcAec_CreateResampler();
137 0 : if (!aecpc->resampler) {
138 0 : WebRtcAec_Free(aecpc);
139 0 : return NULL;
140 : }
141 : // Create far-end pre-buffer. The buffer size has to be large enough for
142 : // largest possible drift compensation (kResamplerBufferSize) + "almost" an
143 : // FFT buffer (PART_LEN2 - 1).
144 0 : aecpc->far_pre_buf =
145 0 : WebRtc_CreateBuffer(PART_LEN2 + kResamplerBufferSize, sizeof(float));
146 0 : if (!aecpc->far_pre_buf) {
147 0 : WebRtcAec_Free(aecpc);
148 0 : return NULL;
149 : }
150 :
151 0 : aecpc->initFlag = 0;
152 :
153 0 : aecpc->instance_count++;
154 0 : return aecpc;
155 : }
156 :
157 0 : void WebRtcAec_Free(void* aecInst) {
158 0 : Aec* aecpc = reinterpret_cast<Aec*>(aecInst);
159 :
160 0 : if (aecpc == NULL) {
161 0 : return;
162 : }
163 :
164 0 : WebRtc_FreeBuffer(aecpc->far_pre_buf);
165 :
166 0 : WebRtcAec_FreeAec(aecpc->aec);
167 0 : WebRtcAec_FreeResampler(aecpc->resampler);
168 0 : delete aecpc;
169 : }
170 :
171 0 : int32_t WebRtcAec_Init(void* aecInst, int32_t sampFreq, int32_t scSampFreq) {
172 0 : Aec* aecpc = reinterpret_cast<Aec*>(aecInst);
173 0 : aecpc->data_dumper->InitiateNewSetOfRecordings();
174 : AecConfig aecConfig;
175 :
176 0 : if (sampFreq != 8000 && sampFreq != 16000 && sampFreq != 32000 &&
177 : sampFreq != 48000) {
178 0 : return AEC_BAD_PARAMETER_ERROR;
179 : }
180 0 : aecpc->sampFreq = sampFreq;
181 :
182 0 : if (scSampFreq < 1 || scSampFreq > 96000) {
183 0 : return AEC_BAD_PARAMETER_ERROR;
184 : }
185 0 : aecpc->scSampFreq = scSampFreq;
186 :
187 : // Initialize echo canceller core
188 0 : if (WebRtcAec_InitAec(aecpc->aec, aecpc->sampFreq) == -1) {
189 0 : return AEC_UNSPECIFIED_ERROR;
190 : }
191 :
192 0 : if (WebRtcAec_InitResampler(aecpc->resampler, aecpc->scSampFreq) == -1) {
193 0 : return AEC_UNSPECIFIED_ERROR;
194 : }
195 :
196 0 : WebRtc_InitBuffer(aecpc->far_pre_buf);
197 0 : WebRtc_MoveReadPtr(aecpc->far_pre_buf, -PART_LEN); // Start overlap.
198 :
199 0 : aecpc->initFlag = initCheck; // indicates that initialization has been done
200 :
201 0 : if (aecpc->sampFreq == 32000 || aecpc->sampFreq == 48000) {
202 0 : aecpc->splitSampFreq = 16000;
203 : } else {
204 0 : aecpc->splitSampFreq = sampFreq;
205 : }
206 :
207 0 : aecpc->delayCtr = 0;
208 0 : aecpc->sampFactor = (aecpc->scSampFreq * 1.0f) / aecpc->splitSampFreq;
209 : // Sampling frequency multiplier (SWB is processed as 160 frame size).
210 0 : aecpc->rate_factor = aecpc->splitSampFreq / 8000;
211 :
212 0 : aecpc->sum = 0;
213 0 : aecpc->counter = 0;
214 0 : aecpc->checkBuffSize = 1;
215 0 : aecpc->firstVal = 0;
216 :
217 : // We skip the startup_phase completely (setting to 0) if DA-AEC is enabled,
218 : // but not extended_filter mode.
219 0 : aecpc->startup_phase = WebRtcAec_extended_filter_enabled(aecpc->aec) ||
220 0 : !WebRtcAec_delay_agnostic_enabled(aecpc->aec);
221 0 : aecpc->bufSizeStart = 0;
222 0 : aecpc->checkBufSizeCtr = 0;
223 0 : aecpc->msInSndCardBuf = 0;
224 0 : aecpc->filtDelay = -1; // -1 indicates an initialized state.
225 0 : aecpc->timeForDelayChange = 0;
226 0 : aecpc->knownDelay = 0;
227 0 : aecpc->lastDelayDiff = 0;
228 :
229 0 : aecpc->skewFrCtr = 0;
230 0 : aecpc->resample = kAecFalse;
231 0 : aecpc->highSkewCtr = 0;
232 0 : aecpc->skew = 0;
233 :
234 0 : aecpc->farend_started = 0;
235 :
236 : // Default settings.
237 0 : aecConfig.nlpMode = kAecNlpModerate;
238 0 : aecConfig.skewMode = kAecFalse;
239 0 : aecConfig.metricsMode = kAecFalse;
240 0 : aecConfig.delay_logging = kAecFalse;
241 :
242 0 : if (WebRtcAec_set_config(aecpc, aecConfig) == -1) {
243 0 : return AEC_UNSPECIFIED_ERROR;
244 : }
245 :
246 0 : return 0;
247 : }
248 :
249 : // Returns any error that is caused when buffering the
250 : // far-end signal.
251 0 : int32_t WebRtcAec_GetBufferFarendError(void* aecInst,
252 : const float* farend,
253 : size_t nrOfSamples) {
254 0 : Aec* aecpc = reinterpret_cast<Aec*>(aecInst);
255 :
256 0 : if (!farend)
257 0 : return AEC_NULL_POINTER_ERROR;
258 :
259 0 : if (aecpc->initFlag != initCheck)
260 0 : return AEC_UNINITIALIZED_ERROR;
261 :
262 : // number of samples == 160 for SWB input
263 0 : if (nrOfSamples != 80 && nrOfSamples != 160)
264 0 : return AEC_BAD_PARAMETER_ERROR;
265 :
266 0 : return 0;
267 : }
268 :
269 : // only buffer L band for farend
270 0 : int32_t WebRtcAec_BufferFarend(void* aecInst,
271 : const float* farend,
272 : size_t nrOfSamples) {
273 0 : Aec* aecpc = reinterpret_cast<Aec*>(aecInst);
274 0 : size_t newNrOfSamples = nrOfSamples;
275 : float new_farend[MAX_RESAMP_LEN];
276 0 : const float* farend_ptr = farend;
277 :
278 : // Get any error caused by buffering the farend signal.
279 : int32_t error_code =
280 0 : WebRtcAec_GetBufferFarendError(aecInst, farend, nrOfSamples);
281 :
282 0 : if (error_code != 0)
283 0 : return error_code;
284 :
285 0 : if (aecpc->skewMode == kAecTrue && aecpc->resample == kAecTrue) {
286 : // Resample and get a new number of samples
287 0 : WebRtcAec_ResampleLinear(aecpc->resampler, farend, nrOfSamples, aecpc->skew,
288 0 : new_farend, &newNrOfSamples);
289 0 : farend_ptr = new_farend;
290 : }
291 :
292 0 : aecpc->farend_started = 1;
293 0 : WebRtcAec_SetSystemDelay(aecpc->aec, WebRtcAec_system_delay(aecpc->aec) +
294 0 : static_cast<int>(newNrOfSamples));
295 :
296 : // Write the time-domain data to |far_pre_buf|.
297 0 : WebRtc_WriteBuffer(aecpc->far_pre_buf, farend_ptr, newNrOfSamples);
298 :
299 : // TODO(minyue): reduce to |PART_LEN| samples for each buffering.
300 0 : while (WebRtc_available_read(aecpc->far_pre_buf) >= PART_LEN2) {
301 : // We have enough data to pass to the FFT, hence read PART_LEN2 samples.
302 : {
303 0 : float* ptmp = NULL;
304 : float tmp[PART_LEN2];
305 0 : WebRtc_ReadBuffer(aecpc->far_pre_buf,
306 0 : reinterpret_cast<void**>(&ptmp), tmp, PART_LEN2);
307 0 : WebRtcAec_BufferFarendBlock(aecpc->aec, &ptmp[PART_LEN]);
308 : }
309 :
310 : // Rewind |far_pre_buf| PART_LEN samples for overlap before continuing.
311 0 : WebRtc_MoveReadPtr(aecpc->far_pre_buf, -PART_LEN);
312 : }
313 :
314 0 : return 0;
315 : }
316 :
317 0 : int32_t WebRtcAec_Process(void* aecInst,
318 : const float* const* nearend,
319 : size_t num_bands,
320 : float* const* out,
321 : size_t nrOfSamples,
322 : int16_t msInSndCardBuf,
323 : int32_t skew) {
324 0 : Aec* aecpc = reinterpret_cast<Aec*>(aecInst);
325 0 : int32_t retVal = 0;
326 :
327 0 : if (out == NULL) {
328 0 : return AEC_NULL_POINTER_ERROR;
329 : }
330 :
331 0 : if (aecpc->initFlag != initCheck) {
332 0 : return AEC_UNINITIALIZED_ERROR;
333 : }
334 :
335 : // number of samples == 160 for SWB input
336 0 : if (nrOfSamples != 80 && nrOfSamples != 160) {
337 0 : return AEC_BAD_PARAMETER_ERROR;
338 : }
339 :
340 0 : if (msInSndCardBuf < 0) {
341 0 : msInSndCardBuf = 0;
342 0 : retVal = AEC_BAD_PARAMETER_WARNING;
343 0 : } else if (msInSndCardBuf > kMaxTrustedDelayMs) {
344 : // The clamping is now done in ProcessExtended/Normal().
345 0 : retVal = AEC_BAD_PARAMETER_WARNING;
346 : }
347 :
348 : // This returns the value of aec->extended_filter_enabled.
349 0 : if (WebRtcAec_extended_filter_enabled(aecpc->aec)) {
350 0 : ProcessExtended(aecpc, nearend, num_bands, out, nrOfSamples, msInSndCardBuf,
351 0 : skew);
352 : } else {
353 0 : retVal = ProcessNormal(aecpc, nearend, num_bands, out, nrOfSamples,
354 0 : msInSndCardBuf, skew);
355 : }
356 :
357 0 : int far_buf_size_samples = WebRtcAec_system_delay(aecpc->aec);
358 0 : aecpc->data_dumper->DumpRaw("aec_system_delay", 1, &far_buf_size_samples);
359 0 : aecpc->data_dumper->DumpRaw("aec_known_delay", 1, &aecpc->knownDelay);
360 :
361 0 : return retVal;
362 : }
363 :
364 0 : int WebRtcAec_set_config(void* handle, AecConfig config) {
365 0 : Aec* self = reinterpret_cast<Aec*>(handle);
366 0 : if (self->initFlag != initCheck) {
367 0 : return AEC_UNINITIALIZED_ERROR;
368 : }
369 :
370 0 : if (config.skewMode != kAecFalse && config.skewMode != kAecTrue) {
371 0 : return AEC_BAD_PARAMETER_ERROR;
372 : }
373 0 : self->skewMode = config.skewMode;
374 :
375 0 : if (config.nlpMode != kAecNlpConservative &&
376 0 : config.nlpMode != kAecNlpModerate &&
377 0 : config.nlpMode != kAecNlpAggressive) {
378 0 : return AEC_BAD_PARAMETER_ERROR;
379 : }
380 :
381 0 : if (config.metricsMode != kAecFalse && config.metricsMode != kAecTrue) {
382 0 : return AEC_BAD_PARAMETER_ERROR;
383 : }
384 :
385 0 : if (config.delay_logging != kAecFalse && config.delay_logging != kAecTrue) {
386 0 : return AEC_BAD_PARAMETER_ERROR;
387 : }
388 :
389 0 : WebRtcAec_SetConfigCore(self->aec, config.nlpMode, config.metricsMode,
390 0 : config.delay_logging);
391 0 : return 0;
392 : }
393 :
394 0 : int WebRtcAec_get_echo_status(void* handle, int* status) {
395 0 : Aec* self = reinterpret_cast<Aec*>(handle);
396 0 : if (status == NULL) {
397 0 : return AEC_NULL_POINTER_ERROR;
398 : }
399 0 : if (self->initFlag != initCheck) {
400 0 : return AEC_UNINITIALIZED_ERROR;
401 : }
402 :
403 0 : *status = WebRtcAec_echo_state(self->aec);
404 :
405 0 : return 0;
406 : }
407 :
408 0 : int WebRtcAec_GetMetrics(void* handle, AecMetrics* metrics) {
409 0 : const float kUpWeight = 0.7f;
410 : float dtmp;
411 : int stmp;
412 0 : Aec* self = reinterpret_cast<Aec*>(handle);
413 : Stats erl;
414 : Stats erle;
415 : Stats a_nlp;
416 :
417 0 : if (handle == NULL) {
418 0 : return -1;
419 : }
420 0 : if (metrics == NULL) {
421 0 : return AEC_NULL_POINTER_ERROR;
422 : }
423 0 : if (self->initFlag != initCheck) {
424 0 : return AEC_UNINITIALIZED_ERROR;
425 : }
426 :
427 0 : WebRtcAec_GetEchoStats(self->aec, &erl, &erle, &a_nlp,
428 0 : &metrics->divergent_filter_fraction);
429 :
430 : // ERL
431 0 : metrics->erl.instant = static_cast<int>(erl.instant);
432 :
433 0 : if ((erl.himean > kOffsetLevel) && (erl.average > kOffsetLevel)) {
434 : // Use a mix between regular average and upper part average.
435 0 : dtmp = kUpWeight * erl.himean + (1 - kUpWeight) * erl.average;
436 0 : metrics->erl.average = static_cast<int>(dtmp);
437 : } else {
438 0 : metrics->erl.average = kOffsetLevel;
439 : }
440 :
441 0 : metrics->erl.max = static_cast<int>(erl.max);
442 :
443 0 : if (erl.min < (kOffsetLevel * (-1))) {
444 0 : metrics->erl.min = static_cast<int>(erl.min);
445 : } else {
446 0 : metrics->erl.min = kOffsetLevel;
447 : }
448 :
449 : // ERLE
450 0 : metrics->erle.instant = static_cast<int>(erle.instant);
451 :
452 0 : if ((erle.himean > kOffsetLevel) && (erle.average > kOffsetLevel)) {
453 : // Use a mix between regular average and upper part average.
454 0 : dtmp = kUpWeight * erle.himean + (1 - kUpWeight) * erle.average;
455 0 : metrics->erle.average = static_cast<int>(dtmp);
456 : } else {
457 0 : metrics->erle.average = kOffsetLevel;
458 : }
459 :
460 0 : metrics->erle.max = static_cast<int>(erle.max);
461 :
462 0 : if (erle.min < (kOffsetLevel * (-1))) {
463 0 : metrics->erle.min = static_cast<int>(erle.min);
464 : } else {
465 0 : metrics->erle.min = kOffsetLevel;
466 : }
467 :
468 : // RERL
469 0 : if ((metrics->erl.average > kOffsetLevel) &&
470 0 : (metrics->erle.average > kOffsetLevel)) {
471 0 : stmp = metrics->erl.average + metrics->erle.average;
472 : } else {
473 0 : stmp = kOffsetLevel;
474 : }
475 0 : metrics->rerl.average = stmp;
476 :
477 : // No other statistics needed, but returned for completeness.
478 0 : metrics->rerl.instant = stmp;
479 0 : metrics->rerl.max = stmp;
480 0 : metrics->rerl.min = stmp;
481 :
482 : // A_NLP
483 0 : metrics->aNlp.instant = static_cast<int>(a_nlp.instant);
484 :
485 0 : if ((a_nlp.himean > kOffsetLevel) && (a_nlp.average > kOffsetLevel)) {
486 : // Use a mix between regular average and upper part average.
487 0 : dtmp = kUpWeight * a_nlp.himean + (1 - kUpWeight) * a_nlp.average;
488 0 : metrics->aNlp.average = static_cast<int>(dtmp);
489 : } else {
490 0 : metrics->aNlp.average = kOffsetLevel;
491 : }
492 :
493 0 : metrics->aNlp.max = static_cast<int>(a_nlp.max);
494 :
495 0 : if (a_nlp.min < (kOffsetLevel * (-1))) {
496 0 : metrics->aNlp.min = static_cast<int>(a_nlp.min);
497 : } else {
498 0 : metrics->aNlp.min = kOffsetLevel;
499 : }
500 :
501 0 : return 0;
502 : }
503 :
504 0 : int WebRtcAec_GetDelayMetrics(void* handle,
505 : int* median,
506 : int* std,
507 : float* fraction_poor_delays) {
508 0 : Aec* self = reinterpret_cast<Aec*>(handle);
509 0 : if (median == NULL) {
510 0 : return AEC_NULL_POINTER_ERROR;
511 : }
512 0 : if (std == NULL) {
513 0 : return AEC_NULL_POINTER_ERROR;
514 : }
515 0 : if (self->initFlag != initCheck) {
516 0 : return AEC_UNINITIALIZED_ERROR;
517 : }
518 0 : if (WebRtcAec_GetDelayMetricsCore(self->aec, median, std,
519 : fraction_poor_delays) == -1) {
520 : // Logging disabled.
521 0 : return AEC_UNSUPPORTED_FUNCTION_ERROR;
522 : }
523 :
524 0 : return 0;
525 : }
526 :
527 0 : AecCore* WebRtcAec_aec_core(void* handle) {
528 0 : if (!handle) {
529 0 : return NULL;
530 : }
531 0 : return reinterpret_cast<Aec*>(handle)->aec;
532 : }
533 :
534 0 : static int ProcessNormal(Aec* aecpc,
535 : const float* const* nearend,
536 : size_t num_bands,
537 : float* const* out,
538 : size_t nrOfSamples,
539 : int16_t msInSndCardBuf,
540 : int32_t skew) {
541 0 : int retVal = 0;
542 : size_t i;
543 : size_t nBlocks10ms;
544 : // Limit resampling to doubling/halving of signal
545 0 : const float minSkewEst = -0.5f;
546 0 : const float maxSkewEst = 1.0f;
547 :
548 0 : msInSndCardBuf =
549 : msInSndCardBuf > kMaxTrustedDelayMs ? kMaxTrustedDelayMs : msInSndCardBuf;
550 : // TODO(andrew): we need to investigate if this +10 is really wanted.
551 0 : msInSndCardBuf += 10;
552 0 : aecpc->msInSndCardBuf = msInSndCardBuf;
553 :
554 0 : if (aecpc->skewMode == kAecTrue) {
555 0 : if (aecpc->skewFrCtr < 25) {
556 0 : aecpc->skewFrCtr++;
557 : } else {
558 0 : retVal = WebRtcAec_GetSkew(aecpc->resampler, skew, &aecpc->skew);
559 0 : if (retVal == -1) {
560 0 : aecpc->skew = 0;
561 0 : retVal = AEC_BAD_PARAMETER_WARNING;
562 : }
563 :
564 0 : aecpc->skew /= aecpc->sampFactor * nrOfSamples;
565 :
566 0 : if (aecpc->skew < 1.0e-3 && aecpc->skew > -1.0e-3) {
567 0 : aecpc->resample = kAecFalse;
568 : } else {
569 0 : aecpc->resample = kAecTrue;
570 : }
571 :
572 0 : if (aecpc->skew < minSkewEst) {
573 0 : aecpc->skew = minSkewEst;
574 0 : } else if (aecpc->skew > maxSkewEst) {
575 0 : aecpc->skew = maxSkewEst;
576 : }
577 :
578 0 : aecpc->data_dumper->DumpRaw("aec_skew", 1, &aecpc->skew);
579 : }
580 : }
581 :
582 0 : nBlocks10ms = nrOfSamples / (FRAME_LEN * aecpc->rate_factor);
583 :
584 0 : if (aecpc->startup_phase) {
585 0 : for (i = 0; i < num_bands; ++i) {
586 : // Only needed if they don't already point to the same place.
587 0 : if (nearend[i] != out[i]) {
588 0 : memcpy(out[i], nearend[i], sizeof(nearend[i][0]) * nrOfSamples);
589 : }
590 : }
591 :
592 : // The AEC is in the start up mode
593 : // AEC is disabled until the system delay is OK
594 :
595 : // Mechanism to ensure that the system delay is reasonably stable.
596 0 : if (aecpc->checkBuffSize) {
597 0 : aecpc->checkBufSizeCtr++;
598 : // Before we fill up the far-end buffer we require the system delay
599 : // to be stable (+/-8 ms) compared to the first value. This
600 : // comparison is made during the following 6 consecutive 10 ms
601 : // blocks. If it seems to be stable then we start to fill up the
602 : // far-end buffer.
603 0 : if (aecpc->counter == 0) {
604 0 : aecpc->firstVal = aecpc->msInSndCardBuf;
605 0 : aecpc->sum = 0;
606 : }
607 :
608 0 : if (abs(aecpc->firstVal - aecpc->msInSndCardBuf) <
609 0 : WEBRTC_SPL_MAX(0.2 * aecpc->msInSndCardBuf, sampMsNb)) {
610 0 : aecpc->sum += aecpc->msInSndCardBuf;
611 0 : aecpc->counter++;
612 : } else {
613 0 : aecpc->counter = 0;
614 : }
615 :
616 0 : if (aecpc->counter * nBlocks10ms >= 6) {
617 : // The far-end buffer size is determined in partitions of
618 : // PART_LEN samples. Use 75% of the average value of the system
619 : // delay as buffer size to start with.
620 0 : aecpc->bufSizeStart =
621 0 : WEBRTC_SPL_MIN((3 * aecpc->sum * aecpc->rate_factor * 8) /
622 : (4 * aecpc->counter * PART_LEN),
623 : kMaxBufSizeStart);
624 : // Buffer size has now been determined.
625 0 : aecpc->checkBuffSize = 0;
626 : }
627 :
628 0 : if (aecpc->checkBufSizeCtr * nBlocks10ms > 50) {
629 : // For really bad systems, don't disable the echo canceller for
630 : // more than 0.5 sec.
631 0 : aecpc->bufSizeStart = WEBRTC_SPL_MIN(
632 : (aecpc->msInSndCardBuf * aecpc->rate_factor * 3) / 40,
633 : kMaxBufSizeStart);
634 0 : aecpc->checkBuffSize = 0;
635 : }
636 : }
637 :
638 : // If |checkBuffSize| changed in the if-statement above.
639 0 : if (!aecpc->checkBuffSize) {
640 : // The system delay is now reasonably stable (or has been unstable
641 : // for too long). When the far-end buffer is filled with
642 : // approximately the same amount of data as reported by the system
643 : // we end the startup phase.
644 : int overhead_elements =
645 0 : WebRtcAec_system_delay(aecpc->aec) / PART_LEN - aecpc->bufSizeStart;
646 0 : if (overhead_elements == 0) {
647 : // Enable the AEC
648 0 : aecpc->startup_phase = 0;
649 0 : } else if (overhead_elements > 0) {
650 : // TODO(bjornv): Do we need a check on how much we actually
651 : // moved the read pointer? It should always be possible to move
652 : // the pointer |overhead_elements| since we have only added data
653 : // to the buffer and no delay compensation nor AEC processing
654 : // has been done.
655 0 : WebRtcAec_AdjustFarendBufferSizeAndSystemDelay(aecpc->aec,
656 0 : overhead_elements);
657 :
658 : // Enable the AEC
659 0 : aecpc->startup_phase = 0;
660 : }
661 : }
662 : } else {
663 : // AEC is enabled.
664 0 : EstBufDelayNormal(aecpc);
665 :
666 : // Call the AEC.
667 : // TODO(bjornv): Re-structure such that we don't have to pass
668 : // |aecpc->knownDelay| as input. Change name to something like
669 : // |system_buffer_diff|.
670 0 : WebRtcAec_ProcessFrames(aecpc->aec, nearend, num_bands, nrOfSamples,
671 0 : aecpc->knownDelay, out);
672 : }
673 :
674 0 : return retVal;
675 : }
676 :
677 0 : static void ProcessExtended(Aec* self,
678 : const float* const* near,
679 : size_t num_bands,
680 : float* const* out,
681 : size_t num_samples,
682 : int16_t reported_delay_ms,
683 : int32_t skew) {
684 : size_t i;
685 0 : const int delay_diff_offset = kDelayDiffOffsetSamples;
686 0 : RTC_DCHECK(num_samples == 80 || num_samples == 160);
687 : #if defined(WEBRTC_UNTRUSTED_DELAY)
688 : reported_delay_ms = kFixedDelayMs;
689 : #else
690 : // This is the usual mode where we trust the reported system delay values.
691 : // Due to the longer filter, we no longer add 10 ms to the reported delay
692 : // to reduce chance of non-causality. Instead we apply a minimum here to avoid
693 : // issues with the read pointer jumping around needlessly.
694 0 : reported_delay_ms = reported_delay_ms < kMinTrustedDelayMs
695 : ? kMinTrustedDelayMs
696 : : reported_delay_ms;
697 : // If the reported delay appears to be bogus, we attempt to recover by using
698 : // the measured fixed delay values. We use >= here because higher layers
699 : // may already clamp to this maximum value, and we would otherwise not
700 : // detect it here.
701 0 : reported_delay_ms = reported_delay_ms >= kMaxTrustedDelayMs
702 : ? kFixedDelayMs
703 0 : : reported_delay_ms;
704 : #endif
705 0 : self->msInSndCardBuf = reported_delay_ms;
706 :
707 0 : if (!self->farend_started) {
708 0 : for (i = 0; i < num_bands; ++i) {
709 : // Only needed if they don't already point to the same place.
710 0 : if (near[i] != out[i]) {
711 0 : memcpy(out[i], near[i], sizeof(near[i][0]) * num_samples);
712 : }
713 : }
714 0 : return;
715 : }
716 0 : if (self->startup_phase) {
717 : // In the extended mode, there isn't a startup "phase", just a special
718 : // action on the first frame. In the trusted delay case, we'll take the
719 : // current reported delay, unless it's less then our conservative
720 : // measurement.
721 : int startup_size_ms =
722 0 : reported_delay_ms < kFixedDelayMs ? kFixedDelayMs : reported_delay_ms;
723 : #if defined(WEBRTC_ANDROID)
724 : int target_delay = startup_size_ms * self->rate_factor * 8;
725 : #else
726 : // To avoid putting the AEC in a non-causal state we're being slightly
727 : // conservative and scale by 2. On Android we use a fixed delay and
728 : // therefore there is no need to scale the target_delay.
729 0 : int target_delay = startup_size_ms * self->rate_factor * 8 / 2;
730 : #endif
731 : int overhead_elements =
732 0 : (WebRtcAec_system_delay(self->aec) - target_delay) / PART_LEN;
733 0 : WebRtcAec_AdjustFarendBufferSizeAndSystemDelay(self->aec,
734 0 : overhead_elements);
735 0 : self->startup_phase = 0;
736 : }
737 :
738 0 : EstBufDelayExtended(self);
739 :
740 : {
741 : // |delay_diff_offset| gives us the option to manually rewind the delay on
742 : // very low delay platforms which can't be expressed purely through
743 : // |reported_delay_ms|.
744 : const int adjusted_known_delay =
745 0 : WEBRTC_SPL_MAX(0, self->knownDelay + delay_diff_offset);
746 :
747 0 : WebRtcAec_ProcessFrames(self->aec, near, num_bands, num_samples,
748 0 : adjusted_known_delay, out);
749 : }
750 : }
751 :
752 0 : static void EstBufDelayNormal(Aec* aecpc) {
753 0 : int nSampSndCard = aecpc->msInSndCardBuf * sampMsNb * aecpc->rate_factor;
754 0 : int current_delay = nSampSndCard - WebRtcAec_system_delay(aecpc->aec);
755 0 : int delay_difference = 0;
756 :
757 : // Before we proceed with the delay estimate filtering we:
758 : // 1) Compensate for the frame that will be read.
759 : // 2) Compensate for drift resampling.
760 : // 3) Compensate for non-causality if needed, since the estimated delay can't
761 : // be negative.
762 :
763 : // 1) Compensating for the frame(s) that will be read/processed.
764 0 : current_delay += FRAME_LEN * aecpc->rate_factor;
765 :
766 : // 2) Account for resampling frame delay.
767 0 : if (aecpc->skewMode == kAecTrue && aecpc->resample == kAecTrue) {
768 0 : current_delay -= kResamplingDelay;
769 : }
770 :
771 : // 3) Compensate for non-causality, if needed, by flushing one block.
772 0 : if (current_delay < PART_LEN) {
773 0 : current_delay +=
774 0 : WebRtcAec_AdjustFarendBufferSizeAndSystemDelay(aecpc->aec, 1) *
775 : PART_LEN;
776 : }
777 :
778 : // We use -1 to signal an initialized state in the "extended" implementation;
779 : // compensate for that.
780 0 : aecpc->filtDelay = aecpc->filtDelay < 0 ? 0 : aecpc->filtDelay;
781 0 : aecpc->filtDelay =
782 0 : WEBRTC_SPL_MAX(0, static_cast<int16_t>(0.8 *
783 : aecpc->filtDelay +
784 : 0.2 * current_delay));
785 :
786 0 : delay_difference = aecpc->filtDelay - aecpc->knownDelay;
787 0 : if (delay_difference > 224) {
788 0 : if (aecpc->lastDelayDiff < 96) {
789 0 : aecpc->timeForDelayChange = 0;
790 : } else {
791 0 : aecpc->timeForDelayChange++;
792 : }
793 0 : } else if (delay_difference < 96 && aecpc->knownDelay > 0) {
794 0 : if (aecpc->lastDelayDiff > 224) {
795 0 : aecpc->timeForDelayChange = 0;
796 : } else {
797 0 : aecpc->timeForDelayChange++;
798 : }
799 : } else {
800 0 : aecpc->timeForDelayChange = 0;
801 : }
802 0 : aecpc->lastDelayDiff = delay_difference;
803 :
804 0 : if (aecpc->timeForDelayChange > 25) {
805 0 : aecpc->knownDelay = WEBRTC_SPL_MAX((int)aecpc->filtDelay - 160, 0);
806 : }
807 0 : }
808 :
809 0 : static void EstBufDelayExtended(Aec* self) {
810 0 : int reported_delay = self->msInSndCardBuf * sampMsNb * self->rate_factor;
811 0 : int current_delay = reported_delay - WebRtcAec_system_delay(self->aec);
812 0 : int delay_difference = 0;
813 :
814 : // Before we proceed with the delay estimate filtering we:
815 : // 1) Compensate for the frame that will be read.
816 : // 2) Compensate for drift resampling.
817 : // 3) Compensate for non-causality if needed, since the estimated delay can't
818 : // be negative.
819 :
820 : // 1) Compensating for the frame(s) that will be read/processed.
821 0 : current_delay += FRAME_LEN * self->rate_factor;
822 :
823 : // 2) Account for resampling frame delay.
824 0 : if (self->skewMode == kAecTrue && self->resample == kAecTrue) {
825 0 : current_delay -= kResamplingDelay;
826 : }
827 :
828 : // 3) Compensate for non-causality, if needed, by flushing two blocks.
829 0 : if (current_delay < PART_LEN) {
830 0 : current_delay +=
831 0 : WebRtcAec_AdjustFarendBufferSizeAndSystemDelay(self->aec, 2) * PART_LEN;
832 : }
833 :
834 0 : if (self->filtDelay == -1) {
835 0 : self->filtDelay = WEBRTC_SPL_MAX(0, 0.5 * current_delay);
836 : } else {
837 0 : self->filtDelay = WEBRTC_SPL_MAX(
838 : 0, static_cast<int16_t>(0.95 * self->filtDelay + 0.05 * current_delay));
839 : }
840 :
841 0 : delay_difference = self->filtDelay - self->knownDelay;
842 0 : if (delay_difference > 384) {
843 0 : if (self->lastDelayDiff < 128) {
844 0 : self->timeForDelayChange = 0;
845 : } else {
846 0 : self->timeForDelayChange++;
847 : }
848 0 : } else if (delay_difference < 128 && self->knownDelay > 0) {
849 0 : if (self->lastDelayDiff > 384) {
850 0 : self->timeForDelayChange = 0;
851 : } else {
852 0 : self->timeForDelayChange++;
853 : }
854 : } else {
855 0 : self->timeForDelayChange = 0;
856 : }
857 0 : self->lastDelayDiff = delay_difference;
858 :
859 0 : if (self->timeForDelayChange > 25) {
860 0 : self->knownDelay = WEBRTC_SPL_MAX((int)self->filtDelay - 256, 0);
861 : }
862 0 : }
863 : } // namespace webrtc
|
