Line data Source code
1 : /*
2 : * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 :
11 : #include "webrtc/modules/audio_coding/neteq/neteq_impl.h"
12 :
13 : #include <assert.h>
14 : #include <memory.h> // memset
15 :
16 : #include <algorithm>
17 : #include <utility>
18 : #include <vector>
19 :
20 : #include "webrtc/base/checks.h"
21 : #include "webrtc/base/logging.h"
22 : #include "webrtc/base/safe_conversions.h"
23 : #include "webrtc/base/sanitizer.h"
24 : #include "webrtc/base/trace_event.h"
25 : #include "webrtc/common_audio/signal_processing/include/signal_processing_library.h"
26 : #include "webrtc/modules/audio_coding/codecs/audio_decoder.h"
27 : #include "webrtc/modules/audio_coding/neteq/accelerate.h"
28 : #include "webrtc/modules/audio_coding/neteq/background_noise.h"
29 : #include "webrtc/modules/audio_coding/neteq/buffer_level_filter.h"
30 : #include "webrtc/modules/audio_coding/neteq/comfort_noise.h"
31 : #include "webrtc/modules/audio_coding/neteq/decision_logic.h"
32 : #include "webrtc/modules/audio_coding/neteq/decoder_database.h"
33 : #include "webrtc/modules/audio_coding/neteq/defines.h"
34 : #include "webrtc/modules/audio_coding/neteq/delay_manager.h"
35 : #include "webrtc/modules/audio_coding/neteq/delay_peak_detector.h"
36 : #include "webrtc/modules/audio_coding/neteq/dtmf_buffer.h"
37 : #include "webrtc/modules/audio_coding/neteq/dtmf_tone_generator.h"
38 : #include "webrtc/modules/audio_coding/neteq/expand.h"
39 : #include "webrtc/modules/audio_coding/neteq/merge.h"
40 : #include "webrtc/modules/audio_coding/neteq/nack_tracker.h"
41 : #include "webrtc/modules/audio_coding/neteq/normal.h"
42 : #include "webrtc/modules/audio_coding/neteq/packet_buffer.h"
43 : #include "webrtc/modules/audio_coding/neteq/packet.h"
44 : #include "webrtc/modules/audio_coding/neteq/red_payload_splitter.h"
45 : #include "webrtc/modules/audio_coding/neteq/post_decode_vad.h"
46 : #include "webrtc/modules/audio_coding/neteq/preemptive_expand.h"
47 : #include "webrtc/modules/audio_coding/neteq/sync_buffer.h"
48 : #include "webrtc/modules/audio_coding/neteq/tick_timer.h"
49 : #include "webrtc/modules/audio_coding/neteq/timestamp_scaler.h"
50 : #include "webrtc/modules/include/module_common_types.h"
51 :
52 : namespace webrtc {
53 :
54 0 : NetEqImpl::Dependencies::Dependencies(
55 : const NetEq::Config& config,
56 0 : const rtc::scoped_refptr<AudioDecoderFactory>& decoder_factory)
57 0 : : tick_timer(new TickTimer),
58 0 : buffer_level_filter(new BufferLevelFilter),
59 0 : decoder_database(new DecoderDatabase(decoder_factory)),
60 0 : delay_peak_detector(new DelayPeakDetector(tick_timer.get())),
61 0 : delay_manager(new DelayManager(config.max_packets_in_buffer,
62 0 : delay_peak_detector.get(),
63 0 : tick_timer.get())),
64 0 : dtmf_buffer(new DtmfBuffer(config.sample_rate_hz)),
65 0 : dtmf_tone_generator(new DtmfToneGenerator),
66 : packet_buffer(
67 0 : new PacketBuffer(config.max_packets_in_buffer, tick_timer.get())),
68 0 : red_payload_splitter(new RedPayloadSplitter),
69 0 : timestamp_scaler(new TimestampScaler(*decoder_database)),
70 0 : accelerate_factory(new AccelerateFactory),
71 0 : expand_factory(new ExpandFactory),
72 0 : preemptive_expand_factory(new PreemptiveExpandFactory) {}
73 :
74 : NetEqImpl::Dependencies::~Dependencies() = default;
75 :
76 0 : NetEqImpl::NetEqImpl(const NetEq::Config& config,
77 : Dependencies&& deps,
78 0 : bool create_components)
79 0 : : tick_timer_(std::move(deps.tick_timer)),
80 0 : buffer_level_filter_(std::move(deps.buffer_level_filter)),
81 0 : decoder_database_(std::move(deps.decoder_database)),
82 0 : delay_manager_(std::move(deps.delay_manager)),
83 0 : delay_peak_detector_(std::move(deps.delay_peak_detector)),
84 0 : dtmf_buffer_(std::move(deps.dtmf_buffer)),
85 0 : dtmf_tone_generator_(std::move(deps.dtmf_tone_generator)),
86 0 : packet_buffer_(std::move(deps.packet_buffer)),
87 0 : red_payload_splitter_(std::move(deps.red_payload_splitter)),
88 0 : timestamp_scaler_(std::move(deps.timestamp_scaler)),
89 0 : vad_(new PostDecodeVad()),
90 0 : expand_factory_(std::move(deps.expand_factory)),
91 0 : accelerate_factory_(std::move(deps.accelerate_factory)),
92 0 : preemptive_expand_factory_(std::move(deps.preemptive_expand_factory)),
93 : last_mode_(kModeNormal),
94 : decoded_buffer_length_(kMaxFrameSize),
95 0 : decoded_buffer_(new int16_t[decoded_buffer_length_]),
96 : playout_timestamp_(0),
97 : new_codec_(false),
98 : timestamp_(0),
99 : reset_decoder_(false),
100 : ssrc_(0),
101 : first_packet_(true),
102 : error_code_(0),
103 : decoder_error_code_(0),
104 0 : background_noise_mode_(config.background_noise_mode),
105 0 : playout_mode_(config.playout_mode),
106 0 : enable_fast_accelerate_(config.enable_fast_accelerate),
107 : nack_enabled_(false),
108 0 : enable_muted_state_(config.enable_muted_state) {
109 0 : LOG(LS_INFO) << "NetEq config: " << config.ToString();
110 0 : int fs = config.sample_rate_hz;
111 0 : if (fs != 8000 && fs != 16000 && fs != 32000 && fs != 48000) {
112 0 : LOG(LS_ERROR) << "Sample rate " << fs << " Hz not supported. " <<
113 0 : "Changing to 8000 Hz.";
114 0 : fs = 8000;
115 : }
116 0 : delay_manager_->SetMaximumDelay(config.max_delay_ms);
117 0 : fs_hz_ = fs;
118 0 : fs_mult_ = fs / 8000;
119 0 : last_output_sample_rate_hz_ = fs;
120 0 : output_size_samples_ = static_cast<size_t>(kOutputSizeMs * 8 * fs_mult_);
121 0 : decoder_frame_length_ = 3 * output_size_samples_;
122 0 : WebRtcSpl_Init();
123 0 : if (create_components) {
124 0 : SetSampleRateAndChannels(fs, 1); // Default is 1 channel.
125 : }
126 0 : RTC_DCHECK(!vad_->enabled());
127 0 : if (config.enable_post_decode_vad) {
128 0 : vad_->Enable();
129 : }
130 0 : }
131 :
132 : NetEqImpl::~NetEqImpl() = default;
133 :
134 0 : int NetEqImpl::InsertPacket(const WebRtcRTPHeader& rtp_header,
135 : rtc::ArrayView<const uint8_t> payload,
136 : uint32_t receive_timestamp) {
137 0 : rtc::MsanCheckInitialized(payload);
138 0 : TRACE_EVENT0("webrtc", "NetEqImpl::InsertPacket");
139 0 : rtc::CritScope lock(&crit_sect_);
140 : int error =
141 0 : InsertPacketInternal(rtp_header, payload, receive_timestamp);
142 0 : if (error != 0) {
143 0 : error_code_ = error;
144 0 : return kFail;
145 : }
146 0 : return kOK;
147 : }
148 :
149 : namespace {
150 0 : void SetAudioFrameActivityAndType(bool vad_enabled,
151 : NetEqImpl::OutputType type,
152 : AudioFrame::VADActivity last_vad_activity,
153 : AudioFrame* audio_frame) {
154 0 : switch (type) {
155 : case NetEqImpl::OutputType::kNormalSpeech: {
156 0 : audio_frame->speech_type_ = AudioFrame::kNormalSpeech;
157 0 : audio_frame->vad_activity_ = AudioFrame::kVadActive;
158 0 : break;
159 : }
160 : case NetEqImpl::OutputType::kVadPassive: {
161 : // This should only be reached if the VAD is enabled.
162 0 : RTC_DCHECK(vad_enabled);
163 0 : audio_frame->speech_type_ = AudioFrame::kNormalSpeech;
164 0 : audio_frame->vad_activity_ = AudioFrame::kVadPassive;
165 0 : break;
166 : }
167 : case NetEqImpl::OutputType::kCNG: {
168 0 : audio_frame->speech_type_ = AudioFrame::kCNG;
169 0 : audio_frame->vad_activity_ = AudioFrame::kVadPassive;
170 0 : break;
171 : }
172 : case NetEqImpl::OutputType::kPLC: {
173 0 : audio_frame->speech_type_ = AudioFrame::kPLC;
174 0 : audio_frame->vad_activity_ = last_vad_activity;
175 0 : break;
176 : }
177 : case NetEqImpl::OutputType::kPLCCNG: {
178 0 : audio_frame->speech_type_ = AudioFrame::kPLCCNG;
179 0 : audio_frame->vad_activity_ = AudioFrame::kVadPassive;
180 0 : break;
181 : }
182 : default:
183 0 : RTC_NOTREACHED();
184 : }
185 0 : if (!vad_enabled) {
186 : // Always set kVadUnknown when receive VAD is inactive.
187 0 : audio_frame->vad_activity_ = AudioFrame::kVadUnknown;
188 : }
189 0 : }
190 : } // namespace
191 :
192 0 : int NetEqImpl::GetAudio(AudioFrame* audio_frame, bool* muted) {
193 0 : TRACE_EVENT0("webrtc", "NetEqImpl::GetAudio");
194 0 : rtc::CritScope lock(&crit_sect_);
195 0 : int error = GetAudioInternal(audio_frame, muted);
196 0 : if (error != 0) {
197 0 : error_code_ = error;
198 0 : return kFail;
199 : }
200 0 : RTC_DCHECK_EQ(
201 : audio_frame->sample_rate_hz_,
202 0 : rtc::checked_cast<int>(audio_frame->samples_per_channel_ * 100));
203 0 : SetAudioFrameActivityAndType(vad_->enabled(), LastOutputType(),
204 0 : last_vad_activity_, audio_frame);
205 0 : last_vad_activity_ = audio_frame->vad_activity_;
206 0 : last_output_sample_rate_hz_ = audio_frame->sample_rate_hz_;
207 0 : RTC_DCHECK(last_output_sample_rate_hz_ == 8000 ||
208 : last_output_sample_rate_hz_ == 16000 ||
209 : last_output_sample_rate_hz_ == 32000 ||
210 : last_output_sample_rate_hz_ == 48000)
211 0 : << "Unexpected sample rate " << last_output_sample_rate_hz_;
212 0 : return kOK;
213 : }
214 :
215 0 : int NetEqImpl::RegisterPayloadType(NetEqDecoder codec,
216 : const std::string& name,
217 : uint8_t rtp_payload_type) {
218 0 : rtc::CritScope lock(&crit_sect_);
219 0 : LOG(LS_VERBOSE) << "RegisterPayloadType "
220 : << static_cast<int>(rtp_payload_type) << " "
221 0 : << static_cast<int>(codec);
222 0 : int ret = decoder_database_->RegisterPayload(rtp_payload_type, codec, name);
223 0 : if (ret != DecoderDatabase::kOK) {
224 0 : switch (ret) {
225 : case DecoderDatabase::kInvalidRtpPayloadType:
226 0 : error_code_ = kInvalidRtpPayloadType;
227 0 : break;
228 : case DecoderDatabase::kCodecNotSupported:
229 0 : error_code_ = kCodecNotSupported;
230 0 : break;
231 : case DecoderDatabase::kDecoderExists:
232 0 : error_code_ = kDecoderExists;
233 0 : break;
234 : default:
235 0 : error_code_ = kOtherError;
236 : }
237 0 : return kFail;
238 : }
239 0 : return kOK;
240 : }
241 :
242 0 : int NetEqImpl::RegisterExternalDecoder(AudioDecoder* decoder,
243 : NetEqDecoder codec,
244 : const std::string& codec_name,
245 : uint8_t rtp_payload_type) {
246 0 : rtc::CritScope lock(&crit_sect_);
247 0 : LOG(LS_VERBOSE) << "RegisterExternalDecoder "
248 : << static_cast<int>(rtp_payload_type) << " "
249 0 : << static_cast<int>(codec);
250 0 : if (!decoder) {
251 0 : LOG(LS_ERROR) << "Cannot register external decoder with NULL pointer";
252 0 : assert(false);
253 : return kFail;
254 : }
255 0 : int ret = decoder_database_->InsertExternal(rtp_payload_type, codec,
256 0 : codec_name, decoder);
257 0 : if (ret != DecoderDatabase::kOK) {
258 0 : switch (ret) {
259 : case DecoderDatabase::kInvalidRtpPayloadType:
260 0 : error_code_ = kInvalidRtpPayloadType;
261 0 : break;
262 : case DecoderDatabase::kCodecNotSupported:
263 0 : error_code_ = kCodecNotSupported;
264 0 : break;
265 : case DecoderDatabase::kDecoderExists:
266 0 : error_code_ = kDecoderExists;
267 0 : break;
268 : case DecoderDatabase::kInvalidSampleRate:
269 0 : error_code_ = kInvalidSampleRate;
270 0 : break;
271 : case DecoderDatabase::kInvalidPointer:
272 0 : error_code_ = kInvalidPointer;
273 0 : break;
274 : default:
275 0 : error_code_ = kOtherError;
276 : }
277 0 : return kFail;
278 : }
279 0 : return kOK;
280 : }
281 :
282 0 : bool NetEqImpl::RegisterPayloadType(int rtp_payload_type,
283 : const SdpAudioFormat& audio_format) {
284 0 : LOG(LS_VERBOSE) << "NetEqImpl::RegisterPayloadType: payload type "
285 0 : << rtp_payload_type << ", codec " << audio_format;
286 0 : rtc::CritScope lock(&crit_sect_);
287 0 : switch (decoder_database_->RegisterPayload(rtp_payload_type, audio_format)) {
288 : case DecoderDatabase::kOK:
289 0 : return true;
290 : case DecoderDatabase::kInvalidRtpPayloadType:
291 0 : error_code_ = kInvalidRtpPayloadType;
292 0 : return false;
293 : case DecoderDatabase::kCodecNotSupported:
294 0 : error_code_ = kCodecNotSupported;
295 0 : return false;
296 : case DecoderDatabase::kDecoderExists:
297 0 : error_code_ = kDecoderExists;
298 0 : return false;
299 : case DecoderDatabase::kInvalidSampleRate:
300 0 : error_code_ = kInvalidSampleRate;
301 0 : return false;
302 : case DecoderDatabase::kInvalidPointer:
303 0 : error_code_ = kInvalidPointer;
304 0 : return false;
305 : default:
306 0 : error_code_ = kOtherError;
307 0 : return false;
308 : }
309 : }
310 :
311 0 : int NetEqImpl::RemovePayloadType(uint8_t rtp_payload_type) {
312 0 : rtc::CritScope lock(&crit_sect_);
313 0 : int ret = decoder_database_->Remove(rtp_payload_type);
314 0 : if (ret == DecoderDatabase::kOK) {
315 0 : packet_buffer_->DiscardPacketsWithPayloadType(rtp_payload_type);
316 0 : return kOK;
317 0 : } else if (ret == DecoderDatabase::kDecoderNotFound) {
318 0 : error_code_ = kDecoderNotFound;
319 : } else {
320 0 : error_code_ = kOtherError;
321 : }
322 0 : return kFail;
323 : }
324 :
325 0 : void NetEqImpl::RemoveAllPayloadTypes() {
326 0 : rtc::CritScope lock(&crit_sect_);
327 0 : decoder_database_->RemoveAll();
328 0 : }
329 :
330 0 : bool NetEqImpl::SetMinimumDelay(int delay_ms) {
331 0 : rtc::CritScope lock(&crit_sect_);
332 0 : if (delay_ms >= 0 && delay_ms < 10000) {
333 0 : assert(delay_manager_.get());
334 0 : return delay_manager_->SetMinimumDelay(delay_ms);
335 : }
336 0 : return false;
337 : }
338 :
339 0 : bool NetEqImpl::SetMaximumDelay(int delay_ms) {
340 0 : rtc::CritScope lock(&crit_sect_);
341 0 : if (delay_ms >= 0 && delay_ms < 10000) {
342 0 : assert(delay_manager_.get());
343 0 : return delay_manager_->SetMaximumDelay(delay_ms);
344 : }
345 0 : return false;
346 : }
347 :
348 0 : int NetEqImpl::LeastRequiredDelayMs() const {
349 0 : rtc::CritScope lock(&crit_sect_);
350 0 : assert(delay_manager_.get());
351 0 : return delay_manager_->least_required_delay_ms();
352 : }
353 :
354 0 : int NetEqImpl::SetTargetDelay() {
355 0 : return kNotImplemented;
356 : }
357 :
358 0 : int NetEqImpl::TargetDelay() {
359 0 : return kNotImplemented;
360 : }
361 :
362 0 : int NetEqImpl::CurrentDelayMs() const {
363 0 : rtc::CritScope lock(&crit_sect_);
364 0 : if (fs_hz_ == 0)
365 0 : return 0;
366 : // Sum up the samples in the packet buffer with the future length of the sync
367 : // buffer, and divide the sum by the sample rate.
368 : const size_t delay_samples =
369 0 : packet_buffer_->NumSamplesInBuffer(decoder_frame_length_) +
370 0 : sync_buffer_->FutureLength();
371 : // The division below will truncate.
372 : const int delay_ms =
373 0 : static_cast<int>(delay_samples) / rtc::CheckedDivExact(fs_hz_, 1000);
374 0 : return delay_ms;
375 : }
376 :
377 0 : int NetEqImpl::FilteredCurrentDelayMs() const {
378 0 : rtc::CritScope lock(&crit_sect_);
379 : // Calculate the filtered packet buffer level in samples. The value from
380 : // |buffer_level_filter_| is in number of packets, represented in Q8.
381 : const size_t packet_buffer_samples =
382 0 : (buffer_level_filter_->filtered_current_level() *
383 0 : decoder_frame_length_) >>
384 0 : 8;
385 : // Sum up the filtered packet buffer level with the future length of the sync
386 : // buffer, and divide the sum by the sample rate.
387 : const size_t delay_samples =
388 0 : packet_buffer_samples + sync_buffer_->FutureLength();
389 : // The division below will truncate. The return value is in ms.
390 0 : return static_cast<int>(delay_samples) / rtc::CheckedDivExact(fs_hz_, 1000);
391 : }
392 :
393 : // Deprecated.
394 : // TODO(henrik.lundin) Delete.
395 0 : void NetEqImpl::SetPlayoutMode(NetEqPlayoutMode mode) {
396 0 : rtc::CritScope lock(&crit_sect_);
397 0 : if (mode != playout_mode_) {
398 0 : playout_mode_ = mode;
399 0 : CreateDecisionLogic();
400 : }
401 0 : }
402 :
403 : // Deprecated.
404 : // TODO(henrik.lundin) Delete.
405 0 : NetEqPlayoutMode NetEqImpl::PlayoutMode() const {
406 0 : rtc::CritScope lock(&crit_sect_);
407 0 : return playout_mode_;
408 : }
409 :
410 0 : int NetEqImpl::NetworkStatistics(NetEqNetworkStatistics* stats) {
411 0 : rtc::CritScope lock(&crit_sect_);
412 0 : assert(decoder_database_.get());
413 : const size_t total_samples_in_buffers =
414 0 : packet_buffer_->NumSamplesInBuffer(decoder_frame_length_) +
415 0 : sync_buffer_->FutureLength();
416 0 : assert(delay_manager_.get());
417 0 : assert(decision_logic_.get());
418 0 : stats_.GetNetworkStatistics(fs_hz_, total_samples_in_buffers,
419 0 : decoder_frame_length_, *delay_manager_.get(),
420 0 : *decision_logic_.get(), stats);
421 0 : return 0;
422 : }
423 :
424 0 : void NetEqImpl::GetRtcpStatistics(RtcpStatistics* stats) {
425 0 : rtc::CritScope lock(&crit_sect_);
426 0 : if (stats) {
427 0 : rtcp_.GetStatistics(false, stats);
428 : }
429 0 : }
430 :
431 0 : void NetEqImpl::GetRtcpStatisticsNoReset(RtcpStatistics* stats) {
432 0 : rtc::CritScope lock(&crit_sect_);
433 0 : if (stats) {
434 0 : rtcp_.GetStatistics(true, stats);
435 : }
436 0 : }
437 :
438 0 : void NetEqImpl::EnableVad() {
439 0 : rtc::CritScope lock(&crit_sect_);
440 0 : assert(vad_.get());
441 0 : vad_->Enable();
442 0 : }
443 :
444 0 : void NetEqImpl::DisableVad() {
445 0 : rtc::CritScope lock(&crit_sect_);
446 0 : assert(vad_.get());
447 0 : vad_->Disable();
448 0 : }
449 :
450 0 : rtc::Optional<uint32_t> NetEqImpl::GetPlayoutTimestamp() const {
451 0 : rtc::CritScope lock(&crit_sect_);
452 0 : if (first_packet_ || last_mode_ == kModeRfc3389Cng ||
453 0 : last_mode_ == kModeCodecInternalCng) {
454 : // We don't have a valid RTP timestamp until we have decoded our first
455 : // RTP packet. Also, the RTP timestamp is not accurate while playing CNG,
456 : // which is indicated by returning an empty value.
457 0 : return rtc::Optional<uint32_t>();
458 : }
459 : return rtc::Optional<uint32_t>(
460 0 : timestamp_scaler_->ToExternal(playout_timestamp_));
461 : }
462 :
463 0 : int NetEqImpl::last_output_sample_rate_hz() const {
464 0 : rtc::CritScope lock(&crit_sect_);
465 0 : return last_output_sample_rate_hz_;
466 : }
467 :
468 0 : rtc::Optional<CodecInst> NetEqImpl::GetDecoder(int payload_type) const {
469 0 : rtc::CritScope lock(&crit_sect_);
470 : const DecoderDatabase::DecoderInfo* di =
471 0 : decoder_database_->GetDecoderInfo(payload_type);
472 0 : if (!di) {
473 0 : return rtc::Optional<CodecInst>();
474 : }
475 :
476 : // Create a CodecInst with some fields set. The remaining fields are zeroed,
477 : // but we tell MSan to consider them uninitialized.
478 0 : CodecInst ci = {0};
479 0 : rtc::MsanMarkUninitialized(rtc::MakeArrayView(&ci, 1));
480 0 : ci.pltype = payload_type;
481 0 : std::strncpy(ci.plname, di->get_name().c_str(), sizeof(ci.plname));
482 0 : ci.plname[sizeof(ci.plname) - 1] = '\0';
483 0 : ci.plfreq = di->IsRed() ? 8000 : di->SampleRateHz();
484 0 : AudioDecoder* const decoder = di->GetDecoder();
485 0 : ci.channels = decoder ? decoder->Channels() : 1;
486 0 : return rtc::Optional<CodecInst>(ci);
487 : }
488 :
489 0 : rtc::Optional<SdpAudioFormat> NetEqImpl::GetDecoderFormat(
490 : int payload_type) const {
491 0 : rtc::CritScope lock(&crit_sect_);
492 : const DecoderDatabase::DecoderInfo* const di =
493 0 : decoder_database_->GetDecoderInfo(payload_type);
494 0 : if (!di) {
495 0 : return rtc::Optional<SdpAudioFormat>(); // Payload type not registered.
496 : }
497 0 : return rtc::Optional<SdpAudioFormat>(di->GetFormat());
498 : }
499 :
500 0 : int NetEqImpl::SetTargetNumberOfChannels() {
501 0 : return kNotImplemented;
502 : }
503 :
504 0 : int NetEqImpl::SetTargetSampleRate() {
505 0 : return kNotImplemented;
506 : }
507 :
508 0 : int NetEqImpl::LastError() const {
509 0 : rtc::CritScope lock(&crit_sect_);
510 0 : return error_code_;
511 : }
512 :
513 0 : int NetEqImpl::LastDecoderError() {
514 0 : rtc::CritScope lock(&crit_sect_);
515 0 : return decoder_error_code_;
516 : }
517 :
518 0 : void NetEqImpl::FlushBuffers() {
519 0 : rtc::CritScope lock(&crit_sect_);
520 0 : LOG(LS_VERBOSE) << "FlushBuffers";
521 0 : packet_buffer_->Flush();
522 0 : assert(sync_buffer_.get());
523 0 : assert(expand_.get());
524 0 : sync_buffer_->Flush();
525 0 : sync_buffer_->set_next_index(sync_buffer_->next_index() -
526 0 : expand_->overlap_length());
527 : // Set to wait for new codec.
528 0 : first_packet_ = true;
529 0 : }
530 :
531 0 : void NetEqImpl::PacketBufferStatistics(int* current_num_packets,
532 : int* max_num_packets) const {
533 0 : rtc::CritScope lock(&crit_sect_);
534 0 : packet_buffer_->BufferStat(current_num_packets, max_num_packets);
535 0 : }
536 :
537 0 : void NetEqImpl::EnableNack(size_t max_nack_list_size) {
538 0 : rtc::CritScope lock(&crit_sect_);
539 0 : if (!nack_enabled_) {
540 0 : const int kNackThresholdPackets = 2;
541 0 : nack_.reset(NackTracker::Create(kNackThresholdPackets));
542 0 : nack_enabled_ = true;
543 0 : nack_->UpdateSampleRate(fs_hz_);
544 : }
545 0 : nack_->SetMaxNackListSize(max_nack_list_size);
546 0 : }
547 :
548 0 : void NetEqImpl::DisableNack() {
549 0 : rtc::CritScope lock(&crit_sect_);
550 0 : nack_.reset();
551 0 : nack_enabled_ = false;
552 0 : }
553 :
554 0 : std::vector<uint16_t> NetEqImpl::GetNackList(int64_t round_trip_time_ms) const {
555 0 : rtc::CritScope lock(&crit_sect_);
556 0 : if (!nack_enabled_) {
557 0 : return std::vector<uint16_t>();
558 : }
559 0 : RTC_DCHECK(nack_.get());
560 0 : return nack_->GetNackList(round_trip_time_ms);
561 : }
562 :
563 0 : const SyncBuffer* NetEqImpl::sync_buffer_for_test() const {
564 0 : rtc::CritScope lock(&crit_sect_);
565 0 : return sync_buffer_.get();
566 : }
567 :
568 0 : Operations NetEqImpl::last_operation_for_test() const {
569 0 : rtc::CritScope lock(&crit_sect_);
570 0 : return last_operation_;
571 : }
572 :
573 : // Methods below this line are private.
574 :
575 0 : int NetEqImpl::InsertPacketInternal(const WebRtcRTPHeader& rtp_header,
576 : rtc::ArrayView<const uint8_t> payload,
577 : uint32_t receive_timestamp) {
578 0 : if (payload.empty()) {
579 0 : LOG_F(LS_ERROR) << "payload is empty";
580 0 : return kInvalidPointer;
581 : }
582 :
583 0 : PacketList packet_list;
584 : // Insert packet in a packet list.
585 0 : packet_list.push_back([&rtp_header, &payload] {
586 : // Convert to Packet.
587 0 : Packet packet;
588 0 : packet.payload_type = rtp_header.header.payloadType;
589 0 : packet.sequence_number = rtp_header.header.sequenceNumber;
590 0 : packet.timestamp = rtp_header.header.timestamp;
591 0 : packet.payload.SetData(payload.data(), payload.size());
592 : // Waiting time will be set upon inserting the packet in the buffer.
593 0 : RTC_DCHECK(!packet.waiting_time);
594 0 : return packet;
595 0 : }());
596 :
597 0 : bool update_sample_rate_and_channels = false;
598 : // Reinitialize NetEq if it's needed (changed SSRC or first call).
599 0 : if ((rtp_header.header.ssrc != ssrc_) || first_packet_) {
600 : // Note: |first_packet_| will be cleared further down in this method, once
601 : // the packet has been successfully inserted into the packet buffer.
602 :
603 0 : rtcp_.Init(rtp_header.header.sequenceNumber);
604 :
605 : // Flush the packet buffer and DTMF buffer.
606 0 : packet_buffer_->Flush();
607 0 : dtmf_buffer_->Flush();
608 :
609 : // Store new SSRC.
610 0 : ssrc_ = rtp_header.header.ssrc;
611 :
612 : // Update audio buffer timestamp.
613 0 : sync_buffer_->IncreaseEndTimestamp(rtp_header.header.timestamp -
614 0 : timestamp_);
615 :
616 : // Update codecs.
617 0 : timestamp_ = rtp_header.header.timestamp;
618 :
619 : // Reset timestamp scaling.
620 0 : timestamp_scaler_->Reset();
621 :
622 : // Trigger an update of sampling rate and the number of channels.
623 0 : update_sample_rate_and_channels = true;
624 : }
625 :
626 : // Update RTCP statistics, only for regular packets.
627 0 : rtcp_.Update(rtp_header.header, receive_timestamp);
628 :
629 0 : if (nack_enabled_) {
630 0 : RTC_DCHECK(nack_);
631 0 : if (update_sample_rate_and_channels) {
632 0 : nack_->Reset();
633 : }
634 0 : nack_->UpdateLastReceivedPacket(rtp_header.header.sequenceNumber,
635 0 : rtp_header.header.timestamp);
636 : }
637 :
638 : // Check for RED payload type, and separate payloads into several packets.
639 0 : if (decoder_database_->IsRed(rtp_header.header.payloadType)) {
640 0 : if (!red_payload_splitter_->SplitRed(&packet_list)) {
641 0 : return kRedundancySplitError;
642 : }
643 : // Only accept a few RED payloads of the same type as the main data,
644 : // DTMF events and CNG.
645 0 : red_payload_splitter_->CheckRedPayloads(&packet_list, *decoder_database_);
646 : }
647 :
648 : // Check payload types.
649 0 : if (decoder_database_->CheckPayloadTypes(packet_list) ==
650 : DecoderDatabase::kDecoderNotFound) {
651 0 : return kUnknownRtpPayloadType;
652 : }
653 :
654 0 : RTC_DCHECK(!packet_list.empty());
655 : // Store these for later use, since the first packet may very well disappear
656 : // before we need these values.
657 0 : const uint32_t main_timestamp = packet_list.front().timestamp;
658 0 : const uint8_t main_payload_type = packet_list.front().payload_type;
659 0 : const uint16_t main_sequence_number = packet_list.front().sequence_number;
660 :
661 : // Scale timestamp to internal domain (only for some codecs).
662 0 : timestamp_scaler_->ToInternal(&packet_list);
663 :
664 : // Process DTMF payloads. Cycle through the list of packets, and pick out any
665 : // DTMF payloads found.
666 0 : PacketList::iterator it = packet_list.begin();
667 0 : while (it != packet_list.end()) {
668 0 : const Packet& current_packet = (*it);
669 0 : RTC_DCHECK(!current_packet.payload.empty());
670 0 : if (decoder_database_->IsDtmf(current_packet.payload_type)) {
671 0 : DtmfEvent event;
672 0 : int ret = DtmfBuffer::ParseEvent(current_packet.timestamp,
673 : current_packet.payload.data(),
674 0 : current_packet.payload.size(), &event);
675 0 : if (ret != DtmfBuffer::kOK) {
676 0 : return kDtmfParsingError;
677 : }
678 0 : if (dtmf_buffer_->InsertEvent(event) != DtmfBuffer::kOK) {
679 0 : return kDtmfInsertError;
680 : }
681 0 : it = packet_list.erase(it);
682 : } else {
683 0 : ++it;
684 : }
685 : }
686 :
687 : // Update bandwidth estimate, if the packet is not comfort noise.
688 0 : if (!packet_list.empty() &&
689 0 : !decoder_database_->IsComfortNoise(main_payload_type)) {
690 : // The list can be empty here if we got nothing but DTMF payloads.
691 0 : AudioDecoder* decoder = decoder_database_->GetDecoder(main_payload_type);
692 0 : RTC_DCHECK(decoder); // Should always get a valid object, since we have
693 : // already checked that the payload types are known.
694 0 : decoder->IncomingPacket(packet_list.front().payload.data(),
695 0 : packet_list.front().payload.size(),
696 0 : packet_list.front().sequence_number,
697 0 : packet_list.front().timestamp,
698 0 : receive_timestamp);
699 : }
700 :
701 0 : PacketList parsed_packet_list;
702 0 : while (!packet_list.empty()) {
703 0 : Packet& packet = packet_list.front();
704 : const DecoderDatabase::DecoderInfo* info =
705 0 : decoder_database_->GetDecoderInfo(packet.payload_type);
706 0 : if (!info) {
707 0 : LOG(LS_WARNING) << "SplitAudio unknown payload type";
708 0 : return kUnknownRtpPayloadType;
709 : }
710 :
711 0 : if (info->IsComfortNoise()) {
712 : // Carry comfort noise packets along.
713 0 : parsed_packet_list.splice(parsed_packet_list.end(), packet_list,
714 0 : packet_list.begin());
715 : } else {
716 0 : const auto sequence_number = packet.sequence_number;
717 0 : const auto payload_type = packet.payload_type;
718 0 : const Packet::Priority original_priority = packet.priority;
719 0 : auto packet_from_result = [&] (AudioDecoder::ParseResult& result) {
720 0 : Packet new_packet;
721 0 : new_packet.sequence_number = sequence_number;
722 0 : new_packet.payload_type = payload_type;
723 0 : new_packet.timestamp = result.timestamp;
724 0 : new_packet.priority.codec_level = result.priority;
725 0 : new_packet.priority.red_level = original_priority.red_level;
726 0 : new_packet.frame = std::move(result.frame);
727 0 : return new_packet;
728 0 : };
729 :
730 : std::vector<AudioDecoder::ParseResult> results =
731 0 : info->GetDecoder()->ParsePayload(std::move(packet.payload),
732 0 : packet.timestamp);
733 0 : if (results.empty()) {
734 0 : packet_list.pop_front();
735 : } else {
736 0 : bool first = true;
737 0 : for (auto& result : results) {
738 0 : RTC_DCHECK(result.frame);
739 0 : RTC_DCHECK_GE(result.priority, 0);
740 0 : if (first) {
741 : // Re-use the node and move it to parsed_packet_list.
742 0 : packet_list.front() = packet_from_result(result);
743 0 : parsed_packet_list.splice(parsed_packet_list.end(), packet_list,
744 0 : packet_list.begin());
745 0 : first = false;
746 : } else {
747 0 : parsed_packet_list.push_back(packet_from_result(result));
748 : }
749 : }
750 : }
751 : }
752 : }
753 :
754 : // Insert packets in buffer.
755 : const size_t buffer_length_before_insert =
756 0 : packet_buffer_->NumPacketsInBuffer();
757 0 : const int ret = packet_buffer_->InsertPacketList(
758 0 : &parsed_packet_list, *decoder_database_, ¤t_rtp_payload_type_,
759 0 : ¤t_cng_rtp_payload_type_);
760 0 : if (ret == PacketBuffer::kFlushed) {
761 : // Reset DSP timestamp etc. if packet buffer flushed.
762 0 : new_codec_ = true;
763 0 : update_sample_rate_and_channels = true;
764 0 : } else if (ret != PacketBuffer::kOK) {
765 0 : return kOtherError;
766 : }
767 :
768 0 : if (first_packet_) {
769 0 : first_packet_ = false;
770 : // Update the codec on the next GetAudio call.
771 0 : new_codec_ = true;
772 : }
773 :
774 0 : if (current_rtp_payload_type_) {
775 0 : RTC_DCHECK(decoder_database_->GetDecoderInfo(*current_rtp_payload_type_))
776 0 : << "Payload type " << static_cast<int>(*current_rtp_payload_type_)
777 0 : << " is unknown where it shouldn't be";
778 : }
779 :
780 0 : if (update_sample_rate_and_channels && !packet_buffer_->Empty()) {
781 : // We do not use |current_rtp_payload_type_| to |set payload_type|, but
782 : // get the next RTP header from |packet_buffer_| to obtain the payload type.
783 : // The reason for it is the following corner case. If NetEq receives a
784 : // CNG packet with a sample rate different than the current CNG then it
785 : // flushes its buffer, assuming send codec must have been changed. However,
786 : // payload type of the hypothetically new send codec is not known.
787 0 : const Packet* next_packet = packet_buffer_->PeekNextPacket();
788 0 : RTC_DCHECK(next_packet);
789 0 : const int payload_type = next_packet->payload_type;
790 0 : size_t channels = 1;
791 0 : if (!decoder_database_->IsComfortNoise(payload_type)) {
792 0 : AudioDecoder* decoder = decoder_database_->GetDecoder(payload_type);
793 0 : assert(decoder); // Payloads are already checked to be valid.
794 0 : channels = decoder->Channels();
795 : }
796 : const DecoderDatabase::DecoderInfo* decoder_info =
797 0 : decoder_database_->GetDecoderInfo(payload_type);
798 0 : assert(decoder_info);
799 0 : if (decoder_info->SampleRateHz() != fs_hz_ ||
800 0 : channels != algorithm_buffer_->Channels()) {
801 0 : SetSampleRateAndChannels(decoder_info->SampleRateHz(),
802 0 : channels);
803 : }
804 0 : if (nack_enabled_) {
805 0 : RTC_DCHECK(nack_);
806 : // Update the sample rate even if the rate is not new, because of Reset().
807 0 : nack_->UpdateSampleRate(fs_hz_);
808 : }
809 : }
810 :
811 : // TODO(hlundin): Move this code to DelayManager class.
812 : const DecoderDatabase::DecoderInfo* dec_info =
813 0 : decoder_database_->GetDecoderInfo(main_payload_type);
814 0 : assert(dec_info); // Already checked that the payload type is known.
815 0 : delay_manager_->LastDecodedWasCngOrDtmf(dec_info->IsComfortNoise() ||
816 0 : dec_info->IsDtmf());
817 0 : if (delay_manager_->last_pack_cng_or_dtmf() == 0) {
818 : // Calculate the total speech length carried in each packet.
819 : const size_t buffer_length_after_insert =
820 0 : packet_buffer_->NumPacketsInBuffer();
821 :
822 0 : if (buffer_length_after_insert > buffer_length_before_insert) {
823 : const size_t packet_length_samples =
824 0 : (buffer_length_after_insert - buffer_length_before_insert) *
825 0 : decoder_frame_length_;
826 0 : if (packet_length_samples != decision_logic_->packet_length_samples()) {
827 0 : decision_logic_->set_packet_length_samples(packet_length_samples);
828 0 : delay_manager_->SetPacketAudioLength(
829 0 : rtc::checked_cast<int>((1000 * packet_length_samples) / fs_hz_));
830 : }
831 : }
832 :
833 : // Update statistics.
834 0 : if ((int32_t)(main_timestamp - timestamp_) >= 0 && !new_codec_) {
835 : // Only update statistics if incoming packet is not older than last played
836 : // out packet, and if new codec flag is not set.
837 0 : delay_manager_->Update(main_sequence_number, main_timestamp, fs_hz_);
838 : }
839 0 : } else if (delay_manager_->last_pack_cng_or_dtmf() == -1) {
840 : // This is first "normal" packet after CNG or DTMF.
841 : // Reset packet time counter and measure time until next packet,
842 : // but don't update statistics.
843 0 : delay_manager_->set_last_pack_cng_or_dtmf(0);
844 0 : delay_manager_->ResetPacketIatCount();
845 : }
846 0 : return 0;
847 : }
848 :
849 0 : int NetEqImpl::GetAudioInternal(AudioFrame* audio_frame, bool* muted) {
850 0 : PacketList packet_list;
851 0 : DtmfEvent dtmf_event;
852 : Operations operation;
853 : bool play_dtmf;
854 0 : *muted = false;
855 0 : tick_timer_->Increment();
856 0 : stats_.IncreaseCounter(output_size_samples_, fs_hz_);
857 :
858 : // Check for muted state.
859 0 : if (enable_muted_state_ && expand_->Muted() && packet_buffer_->Empty()) {
860 0 : RTC_DCHECK_EQ(last_mode_, kModeExpand);
861 0 : playout_timestamp_ += static_cast<uint32_t>(output_size_samples_);
862 0 : audio_frame->sample_rate_hz_ = fs_hz_;
863 0 : audio_frame->samples_per_channel_ = output_size_samples_;
864 0 : audio_frame->timestamp_ =
865 0 : first_packet_
866 0 : ? 0
867 0 : : timestamp_scaler_->ToExternal(playout_timestamp_) -
868 0 : static_cast<uint32_t>(audio_frame->samples_per_channel_);
869 0 : audio_frame->num_channels_ = sync_buffer_->Channels();
870 0 : stats_.ExpandedNoiseSamples(output_size_samples_);
871 0 : *muted = true;
872 0 : return 0;
873 : }
874 :
875 : int return_value = GetDecision(&operation, &packet_list, &dtmf_event,
876 0 : &play_dtmf);
877 0 : if (return_value != 0) {
878 0 : last_mode_ = kModeError;
879 0 : return return_value;
880 : }
881 :
882 : AudioDecoder::SpeechType speech_type;
883 0 : int length = 0;
884 : int decode_return_value = Decode(&packet_list, &operation,
885 0 : &length, &speech_type);
886 :
887 0 : assert(vad_.get());
888 : bool sid_frame_available =
889 0 : (operation == kRfc3389Cng && !packet_list.empty());
890 0 : vad_->Update(decoded_buffer_.get(), static_cast<size_t>(length), speech_type,
891 0 : sid_frame_available, fs_hz_);
892 :
893 0 : if (sid_frame_available || speech_type == AudioDecoder::kComfortNoise) {
894 : // Start a new stopwatch since we are decoding a new CNG packet.
895 0 : generated_noise_stopwatch_ = tick_timer_->GetNewStopwatch();
896 : }
897 :
898 0 : algorithm_buffer_->Clear();
899 0 : switch (operation) {
900 : case kNormal: {
901 0 : DoNormal(decoded_buffer_.get(), length, speech_type, play_dtmf);
902 0 : break;
903 : }
904 : case kMerge: {
905 0 : DoMerge(decoded_buffer_.get(), length, speech_type, play_dtmf);
906 0 : break;
907 : }
908 : case kExpand: {
909 0 : return_value = DoExpand(play_dtmf);
910 0 : break;
911 : }
912 : case kAccelerate:
913 : case kFastAccelerate: {
914 : const bool fast_accelerate =
915 0 : enable_fast_accelerate_ && (operation == kFastAccelerate);
916 0 : return_value = DoAccelerate(decoded_buffer_.get(), length, speech_type,
917 0 : play_dtmf, fast_accelerate);
918 0 : break;
919 : }
920 : case kPreemptiveExpand: {
921 0 : return_value = DoPreemptiveExpand(decoded_buffer_.get(), length,
922 0 : speech_type, play_dtmf);
923 0 : break;
924 : }
925 : case kRfc3389Cng:
926 : case kRfc3389CngNoPacket: {
927 0 : return_value = DoRfc3389Cng(&packet_list, play_dtmf);
928 0 : break;
929 : }
930 : case kCodecInternalCng: {
931 : // This handles the case when there is no transmission and the decoder
932 : // should produce internal comfort noise.
933 : // TODO(hlundin): Write test for codec-internal CNG.
934 0 : DoCodecInternalCng(decoded_buffer_.get(), length);
935 0 : break;
936 : }
937 : case kDtmf: {
938 : // TODO(hlundin): Write test for this.
939 0 : return_value = DoDtmf(dtmf_event, &play_dtmf);
940 0 : break;
941 : }
942 : case kAlternativePlc: {
943 : // TODO(hlundin): Write test for this.
944 0 : DoAlternativePlc(false);
945 0 : break;
946 : }
947 : case kAlternativePlcIncreaseTimestamp: {
948 : // TODO(hlundin): Write test for this.
949 0 : DoAlternativePlc(true);
950 0 : break;
951 : }
952 : case kAudioRepetitionIncreaseTimestamp: {
953 : // TODO(hlundin): Write test for this.
954 0 : sync_buffer_->IncreaseEndTimestamp(
955 0 : static_cast<uint32_t>(output_size_samples_));
956 : // Skipping break on purpose. Execution should move on into the
957 : // next case.
958 : FALLTHROUGH();
959 : }
960 : case kAudioRepetition: {
961 : // TODO(hlundin): Write test for this.
962 : // Copy last |output_size_samples_| from |sync_buffer_| to
963 : // |algorithm_buffer|.
964 0 : algorithm_buffer_->PushBackFromIndex(
965 0 : *sync_buffer_, sync_buffer_->Size() - output_size_samples_);
966 0 : expand_->Reset();
967 0 : break;
968 : }
969 : case kUndefined: {
970 0 : LOG(LS_ERROR) << "Invalid operation kUndefined.";
971 0 : assert(false); // This should not happen.
972 : last_mode_ = kModeError;
973 : return kInvalidOperation;
974 : }
975 : } // End of switch.
976 0 : last_operation_ = operation;
977 0 : if (return_value < 0) {
978 0 : return return_value;
979 : }
980 :
981 0 : if (last_mode_ != kModeRfc3389Cng) {
982 0 : comfort_noise_->Reset();
983 : }
984 :
985 : // Copy from |algorithm_buffer| to |sync_buffer_|.
986 0 : sync_buffer_->PushBack(*algorithm_buffer_);
987 :
988 : // Extract data from |sync_buffer_| to |output|.
989 0 : size_t num_output_samples_per_channel = output_size_samples_;
990 0 : size_t num_output_samples = output_size_samples_ * sync_buffer_->Channels();
991 0 : if (num_output_samples > AudioFrame::kMaxDataSizeSamples) {
992 0 : LOG(LS_WARNING) << "Output array is too short. "
993 0 : << AudioFrame::kMaxDataSizeSamples << " < "
994 0 : << output_size_samples_ << " * "
995 0 : << sync_buffer_->Channels();
996 0 : num_output_samples = AudioFrame::kMaxDataSizeSamples;
997 0 : num_output_samples_per_channel =
998 0 : AudioFrame::kMaxDataSizeSamples / sync_buffer_->Channels();
999 : }
1000 0 : sync_buffer_->GetNextAudioInterleaved(num_output_samples_per_channel,
1001 0 : audio_frame);
1002 0 : audio_frame->sample_rate_hz_ = fs_hz_;
1003 0 : if (sync_buffer_->FutureLength() < expand_->overlap_length()) {
1004 : // The sync buffer should always contain |overlap_length| samples, but now
1005 : // too many samples have been extracted. Reinstall the |overlap_length|
1006 : // lookahead by moving the index.
1007 : const size_t missing_lookahead_samples =
1008 0 : expand_->overlap_length() - sync_buffer_->FutureLength();
1009 0 : RTC_DCHECK_GE(sync_buffer_->next_index(), missing_lookahead_samples);
1010 0 : sync_buffer_->set_next_index(sync_buffer_->next_index() -
1011 0 : missing_lookahead_samples);
1012 : }
1013 0 : if (audio_frame->samples_per_channel_ != output_size_samples_) {
1014 0 : LOG(LS_ERROR) << "audio_frame->samples_per_channel_ ("
1015 0 : << audio_frame->samples_per_channel_
1016 0 : << ") != output_size_samples_ (" << output_size_samples_
1017 0 : << ")";
1018 : // TODO(minyue): treatment of under-run, filling zeros
1019 0 : memset(audio_frame->data_, 0, num_output_samples * sizeof(int16_t));
1020 0 : return kSampleUnderrun;
1021 : }
1022 :
1023 : // Should always have overlap samples left in the |sync_buffer_|.
1024 0 : RTC_DCHECK_GE(sync_buffer_->FutureLength(), expand_->overlap_length());
1025 :
1026 0 : if (play_dtmf) {
1027 : return_value =
1028 0 : DtmfOverdub(dtmf_event, sync_buffer_->Channels(), audio_frame->data_);
1029 : }
1030 :
1031 : // Update the background noise parameters if last operation wrote data
1032 : // straight from the decoder to the |sync_buffer_|. That is, none of the
1033 : // operations that modify the signal can be followed by a parameter update.
1034 0 : if ((last_mode_ == kModeNormal) ||
1035 0 : (last_mode_ == kModeAccelerateFail) ||
1036 0 : (last_mode_ == kModePreemptiveExpandFail) ||
1037 0 : (last_mode_ == kModeRfc3389Cng) ||
1038 0 : (last_mode_ == kModeCodecInternalCng)) {
1039 0 : background_noise_->Update(*sync_buffer_, *vad_.get());
1040 : }
1041 :
1042 0 : if (operation == kDtmf) {
1043 : // DTMF data was written the end of |sync_buffer_|.
1044 : // Update index to end of DTMF data in |sync_buffer_|.
1045 0 : sync_buffer_->set_dtmf_index(sync_buffer_->Size());
1046 : }
1047 :
1048 0 : if (last_mode_ != kModeExpand) {
1049 : // If last operation was not expand, calculate the |playout_timestamp_| from
1050 : // the |sync_buffer_|. However, do not update the |playout_timestamp_| if it
1051 : // would be moved "backwards".
1052 0 : uint32_t temp_timestamp = sync_buffer_->end_timestamp() -
1053 0 : static_cast<uint32_t>(sync_buffer_->FutureLength());
1054 0 : if (static_cast<int32_t>(temp_timestamp - playout_timestamp_) > 0) {
1055 0 : playout_timestamp_ = temp_timestamp;
1056 : }
1057 : } else {
1058 : // Use dead reckoning to estimate the |playout_timestamp_|.
1059 0 : playout_timestamp_ += static_cast<uint32_t>(output_size_samples_);
1060 : }
1061 : // Set the timestamp in the audio frame to zero before the first packet has
1062 : // been inserted. Otherwise, subtract the frame size in samples to get the
1063 : // timestamp of the first sample in the frame (playout_timestamp_ is the
1064 : // last + 1).
1065 0 : audio_frame->timestamp_ =
1066 0 : first_packet_
1067 0 : ? 0
1068 0 : : timestamp_scaler_->ToExternal(playout_timestamp_) -
1069 0 : static_cast<uint32_t>(audio_frame->samples_per_channel_);
1070 :
1071 0 : if (!(last_mode_ == kModeRfc3389Cng ||
1072 0 : last_mode_ == kModeCodecInternalCng ||
1073 0 : last_mode_ == kModeExpand)) {
1074 0 : generated_noise_stopwatch_.reset();
1075 : }
1076 :
1077 0 : if (decode_return_value) return decode_return_value;
1078 0 : return return_value;
1079 : }
1080 :
1081 0 : int NetEqImpl::GetDecision(Operations* operation,
1082 : PacketList* packet_list,
1083 : DtmfEvent* dtmf_event,
1084 : bool* play_dtmf) {
1085 : // Initialize output variables.
1086 0 : *play_dtmf = false;
1087 0 : *operation = kUndefined;
1088 :
1089 0 : assert(sync_buffer_.get());
1090 0 : uint32_t end_timestamp = sync_buffer_->end_timestamp();
1091 0 : if (!new_codec_) {
1092 0 : const uint32_t five_seconds_samples = 5 * fs_hz_;
1093 0 : packet_buffer_->DiscardOldPackets(end_timestamp, five_seconds_samples);
1094 : }
1095 0 : const Packet* packet = packet_buffer_->PeekNextPacket();
1096 :
1097 0 : RTC_DCHECK(!generated_noise_stopwatch_ ||
1098 0 : generated_noise_stopwatch_->ElapsedTicks() >= 1);
1099 : uint64_t generated_noise_samples =
1100 : generated_noise_stopwatch_
1101 0 : ? (generated_noise_stopwatch_->ElapsedTicks() - 1) *
1102 0 : output_size_samples_ +
1103 0 : decision_logic_->noise_fast_forward()
1104 0 : : 0;
1105 :
1106 0 : if (decision_logic_->CngRfc3389On() || last_mode_ == kModeRfc3389Cng) {
1107 : // Because of timestamp peculiarities, we have to "manually" disallow using
1108 : // a CNG packet with the same timestamp as the one that was last played.
1109 : // This can happen when using redundancy and will cause the timing to shift.
1110 0 : while (packet && decoder_database_->IsComfortNoise(packet->payload_type) &&
1111 0 : (end_timestamp >= packet->timestamp ||
1112 0 : end_timestamp + generated_noise_samples > packet->timestamp)) {
1113 : // Don't use this packet, discard it.
1114 0 : if (packet_buffer_->DiscardNextPacket() != PacketBuffer::kOK) {
1115 0 : assert(false); // Must be ok by design.
1116 : }
1117 : // Check buffer again.
1118 0 : if (!new_codec_) {
1119 0 : packet_buffer_->DiscardOldPackets(end_timestamp, 5 * fs_hz_);
1120 : }
1121 0 : packet = packet_buffer_->PeekNextPacket();
1122 : }
1123 : }
1124 :
1125 0 : assert(expand_.get());
1126 0 : const int samples_left = static_cast<int>(sync_buffer_->FutureLength() -
1127 0 : expand_->overlap_length());
1128 0 : if (last_mode_ == kModeAccelerateSuccess ||
1129 0 : last_mode_ == kModeAccelerateLowEnergy ||
1130 0 : last_mode_ == kModePreemptiveExpandSuccess ||
1131 0 : last_mode_ == kModePreemptiveExpandLowEnergy) {
1132 : // Subtract (samples_left + output_size_samples_) from sampleMemory.
1133 0 : decision_logic_->AddSampleMemory(
1134 0 : -(samples_left + rtc::checked_cast<int>(output_size_samples_)));
1135 : }
1136 :
1137 : // Check if it is time to play a DTMF event.
1138 0 : if (dtmf_buffer_->GetEvent(
1139 : static_cast<uint32_t>(
1140 : end_timestamp + generated_noise_samples),
1141 0 : dtmf_event)) {
1142 0 : *play_dtmf = true;
1143 : }
1144 :
1145 : // Get instruction.
1146 0 : assert(sync_buffer_.get());
1147 0 : assert(expand_.get());
1148 0 : generated_noise_samples =
1149 : generated_noise_stopwatch_
1150 0 : ? generated_noise_stopwatch_->ElapsedTicks() * output_size_samples_ +
1151 0 : decision_logic_->noise_fast_forward()
1152 0 : : 0;
1153 0 : *operation = decision_logic_->GetDecision(
1154 0 : *sync_buffer_, *expand_, decoder_frame_length_, packet, last_mode_,
1155 0 : *play_dtmf, generated_noise_samples, &reset_decoder_);
1156 :
1157 : // Check if we already have enough samples in the |sync_buffer_|. If so,
1158 : // change decision to normal, unless the decision was merge, accelerate, or
1159 : // preemptive expand.
1160 0 : if (samples_left >= rtc::checked_cast<int>(output_size_samples_) &&
1161 0 : *operation != kMerge &&
1162 0 : *operation != kAccelerate &&
1163 0 : *operation != kFastAccelerate &&
1164 0 : *operation != kPreemptiveExpand) {
1165 0 : *operation = kNormal;
1166 0 : return 0;
1167 : }
1168 :
1169 0 : decision_logic_->ExpandDecision(*operation);
1170 :
1171 : // Check conditions for reset.
1172 0 : if (new_codec_ || *operation == kUndefined) {
1173 : // The only valid reason to get kUndefined is that new_codec_ is set.
1174 0 : assert(new_codec_);
1175 0 : if (*play_dtmf && !packet) {
1176 0 : timestamp_ = dtmf_event->timestamp;
1177 : } else {
1178 0 : if (!packet) {
1179 0 : LOG(LS_ERROR) << "Packet missing where it shouldn't.";
1180 0 : return -1;
1181 : }
1182 0 : timestamp_ = packet->timestamp;
1183 0 : if (*operation == kRfc3389CngNoPacket &&
1184 0 : decoder_database_->IsComfortNoise(packet->payload_type)) {
1185 : // Change decision to CNG packet, since we do have a CNG packet, but it
1186 : // was considered too early to use. Now, use it anyway.
1187 0 : *operation = kRfc3389Cng;
1188 0 : } else if (*operation != kRfc3389Cng) {
1189 0 : *operation = kNormal;
1190 : }
1191 : }
1192 : // Adjust |sync_buffer_| timestamp before setting |end_timestamp| to the
1193 : // new value.
1194 0 : sync_buffer_->IncreaseEndTimestamp(timestamp_ - end_timestamp);
1195 0 : end_timestamp = timestamp_;
1196 0 : new_codec_ = false;
1197 0 : decision_logic_->SoftReset();
1198 0 : buffer_level_filter_->Reset();
1199 0 : delay_manager_->Reset();
1200 0 : stats_.ResetMcu();
1201 : }
1202 :
1203 0 : size_t required_samples = output_size_samples_;
1204 0 : const size_t samples_10_ms = static_cast<size_t>(80 * fs_mult_);
1205 0 : const size_t samples_20_ms = 2 * samples_10_ms;
1206 0 : const size_t samples_30_ms = 3 * samples_10_ms;
1207 :
1208 0 : switch (*operation) {
1209 : case kExpand: {
1210 0 : timestamp_ = end_timestamp;
1211 0 : return 0;
1212 : }
1213 : case kRfc3389CngNoPacket:
1214 : case kCodecInternalCng: {
1215 0 : return 0;
1216 : }
1217 : case kDtmf: {
1218 : // TODO(hlundin): Write test for this.
1219 : // Update timestamp.
1220 0 : timestamp_ = end_timestamp;
1221 : const uint64_t generated_noise_samples =
1222 : generated_noise_stopwatch_
1223 0 : ? generated_noise_stopwatch_->ElapsedTicks() *
1224 0 : output_size_samples_ +
1225 0 : decision_logic_->noise_fast_forward()
1226 0 : : 0;
1227 0 : if (generated_noise_samples > 0 && last_mode_ != kModeDtmf) {
1228 : // Make a jump in timestamp due to the recently played comfort noise.
1229 : uint32_t timestamp_jump =
1230 0 : static_cast<uint32_t>(generated_noise_samples);
1231 0 : sync_buffer_->IncreaseEndTimestamp(timestamp_jump);
1232 0 : timestamp_ += timestamp_jump;
1233 : }
1234 0 : return 0;
1235 : }
1236 : case kAccelerate:
1237 : case kFastAccelerate: {
1238 : // In order to do an accelerate we need at least 30 ms of audio data.
1239 0 : if (samples_left >= static_cast<int>(samples_30_ms)) {
1240 : // Already have enough data, so we do not need to extract any more.
1241 0 : decision_logic_->set_sample_memory(samples_left);
1242 0 : decision_logic_->set_prev_time_scale(true);
1243 0 : return 0;
1244 0 : } else if (samples_left >= static_cast<int>(samples_10_ms) &&
1245 0 : decoder_frame_length_ >= samples_30_ms) {
1246 : // Avoid decoding more data as it might overflow the playout buffer.
1247 0 : *operation = kNormal;
1248 0 : return 0;
1249 0 : } else if (samples_left < static_cast<int>(samples_20_ms) &&
1250 0 : decoder_frame_length_ < samples_30_ms) {
1251 : // Build up decoded data by decoding at least 20 ms of audio data. Do
1252 : // not perform accelerate yet, but wait until we only need to do one
1253 : // decoding.
1254 0 : required_samples = 2 * output_size_samples_;
1255 0 : *operation = kNormal;
1256 : }
1257 : // If none of the above is true, we have one of two possible situations:
1258 : // (1) 20 ms <= samples_left < 30 ms and decoder_frame_length_ < 30 ms; or
1259 : // (2) samples_left < 10 ms and decoder_frame_length_ >= 30 ms.
1260 : // In either case, we move on with the accelerate decision, and decode one
1261 : // frame now.
1262 0 : break;
1263 : }
1264 : case kPreemptiveExpand: {
1265 : // In order to do a preemptive expand we need at least 30 ms of decoded
1266 : // audio data.
1267 0 : if ((samples_left >= static_cast<int>(samples_30_ms)) ||
1268 0 : (samples_left >= static_cast<int>(samples_10_ms) &&
1269 0 : decoder_frame_length_ >= samples_30_ms)) {
1270 : // Already have enough data, so we do not need to extract any more.
1271 : // Or, avoid decoding more data as it might overflow the playout buffer.
1272 : // Still try preemptive expand, though.
1273 0 : decision_logic_->set_sample_memory(samples_left);
1274 0 : decision_logic_->set_prev_time_scale(true);
1275 0 : return 0;
1276 : }
1277 0 : if (samples_left < static_cast<int>(samples_20_ms) &&
1278 0 : decoder_frame_length_ < samples_30_ms) {
1279 : // Build up decoded data by decoding at least 20 ms of audio data.
1280 : // Still try to perform preemptive expand.
1281 0 : required_samples = 2 * output_size_samples_;
1282 : }
1283 : // Move on with the preemptive expand decision.
1284 0 : break;
1285 : }
1286 : case kMerge: {
1287 0 : required_samples =
1288 0 : std::max(merge_->RequiredFutureSamples(), required_samples);
1289 0 : break;
1290 : }
1291 : default: {
1292 : // Do nothing.
1293 : }
1294 : }
1295 :
1296 : // Get packets from buffer.
1297 0 : int extracted_samples = 0;
1298 0 : if (packet && *operation != kAlternativePlc &&
1299 0 : *operation != kAlternativePlcIncreaseTimestamp &&
1300 0 : *operation != kAudioRepetition &&
1301 0 : *operation != kAudioRepetitionIncreaseTimestamp) {
1302 0 : sync_buffer_->IncreaseEndTimestamp(packet->timestamp - end_timestamp);
1303 0 : if (decision_logic_->CngOff()) {
1304 : // Adjustment of timestamp only corresponds to an actual packet loss
1305 : // if comfort noise is not played. If comfort noise was just played,
1306 : // this adjustment of timestamp is only done to get back in sync with the
1307 : // stream timestamp; no loss to report.
1308 0 : stats_.LostSamples(packet->timestamp - end_timestamp);
1309 : }
1310 :
1311 0 : if (*operation != kRfc3389Cng) {
1312 : // We are about to decode and use a non-CNG packet.
1313 0 : decision_logic_->SetCngOff();
1314 : }
1315 :
1316 0 : extracted_samples = ExtractPackets(required_samples, packet_list);
1317 0 : if (extracted_samples < 0) {
1318 0 : return kPacketBufferCorruption;
1319 : }
1320 : }
1321 :
1322 0 : if (*operation == kAccelerate || *operation == kFastAccelerate ||
1323 0 : *operation == kPreemptiveExpand) {
1324 0 : decision_logic_->set_sample_memory(samples_left + extracted_samples);
1325 0 : decision_logic_->set_prev_time_scale(true);
1326 : }
1327 :
1328 0 : if (*operation == kAccelerate || *operation == kFastAccelerate) {
1329 : // Check that we have enough data (30ms) to do accelerate.
1330 0 : if (extracted_samples + samples_left < static_cast<int>(samples_30_ms)) {
1331 : // TODO(hlundin): Write test for this.
1332 : // Not enough, do normal operation instead.
1333 0 : *operation = kNormal;
1334 : }
1335 : }
1336 :
1337 0 : timestamp_ = end_timestamp;
1338 0 : return 0;
1339 : }
1340 :
1341 0 : int NetEqImpl::Decode(PacketList* packet_list, Operations* operation,
1342 : int* decoded_length,
1343 : AudioDecoder::SpeechType* speech_type) {
1344 0 : *speech_type = AudioDecoder::kSpeech;
1345 :
1346 : // When packet_list is empty, we may be in kCodecInternalCng mode, and for
1347 : // that we use current active decoder.
1348 0 : AudioDecoder* decoder = decoder_database_->GetActiveDecoder();
1349 :
1350 0 : if (!packet_list->empty()) {
1351 0 : const Packet& packet = packet_list->front();
1352 0 : uint8_t payload_type = packet.payload_type;
1353 0 : if (!decoder_database_->IsComfortNoise(payload_type)) {
1354 0 : decoder = decoder_database_->GetDecoder(payload_type);
1355 0 : assert(decoder);
1356 0 : if (!decoder) {
1357 0 : LOG(LS_WARNING) << "Unknown payload type "
1358 0 : << static_cast<int>(payload_type);
1359 0 : packet_list->clear();
1360 0 : return kDecoderNotFound;
1361 : }
1362 : bool decoder_changed;
1363 0 : decoder_database_->SetActiveDecoder(payload_type, &decoder_changed);
1364 0 : if (decoder_changed) {
1365 : // We have a new decoder. Re-init some values.
1366 : const DecoderDatabase::DecoderInfo* decoder_info = decoder_database_
1367 0 : ->GetDecoderInfo(payload_type);
1368 0 : assert(decoder_info);
1369 0 : if (!decoder_info) {
1370 0 : LOG(LS_WARNING) << "Unknown payload type "
1371 0 : << static_cast<int>(payload_type);
1372 0 : packet_list->clear();
1373 0 : return kDecoderNotFound;
1374 : }
1375 : // If sampling rate or number of channels has changed, we need to make
1376 : // a reset.
1377 0 : if (decoder_info->SampleRateHz() != fs_hz_ ||
1378 0 : decoder->Channels() != algorithm_buffer_->Channels()) {
1379 : // TODO(tlegrand): Add unittest to cover this event.
1380 0 : SetSampleRateAndChannels(decoder_info->SampleRateHz(),
1381 0 : decoder->Channels());
1382 : }
1383 0 : sync_buffer_->set_end_timestamp(timestamp_);
1384 0 : playout_timestamp_ = timestamp_;
1385 : }
1386 : }
1387 : }
1388 :
1389 0 : if (reset_decoder_) {
1390 : // TODO(hlundin): Write test for this.
1391 0 : if (decoder)
1392 0 : decoder->Reset();
1393 :
1394 : // Reset comfort noise decoder.
1395 0 : ComfortNoiseDecoder* cng_decoder = decoder_database_->GetActiveCngDecoder();
1396 0 : if (cng_decoder)
1397 0 : cng_decoder->Reset();
1398 :
1399 0 : reset_decoder_ = false;
1400 : }
1401 :
1402 0 : *decoded_length = 0;
1403 : // Update codec-internal PLC state.
1404 0 : if ((*operation == kMerge) && decoder && decoder->HasDecodePlc()) {
1405 0 : decoder->DecodePlc(1, &decoded_buffer_[*decoded_length]);
1406 : }
1407 :
1408 : int return_value;
1409 0 : if (*operation == kCodecInternalCng) {
1410 0 : RTC_DCHECK(packet_list->empty());
1411 0 : return_value = DecodeCng(decoder, decoded_length, speech_type);
1412 : } else {
1413 : return_value = DecodeLoop(packet_list, *operation, decoder,
1414 0 : decoded_length, speech_type);
1415 : }
1416 :
1417 0 : if (*decoded_length < 0) {
1418 : // Error returned from the decoder.
1419 0 : *decoded_length = 0;
1420 0 : sync_buffer_->IncreaseEndTimestamp(
1421 0 : static_cast<uint32_t>(decoder_frame_length_));
1422 0 : int error_code = 0;
1423 0 : if (decoder)
1424 0 : error_code = decoder->ErrorCode();
1425 0 : if (error_code != 0) {
1426 : // Got some error code from the decoder.
1427 0 : decoder_error_code_ = error_code;
1428 0 : return_value = kDecoderErrorCode;
1429 0 : LOG(LS_WARNING) << "Decoder returned error code: " << error_code;
1430 : } else {
1431 : // Decoder does not implement error codes. Return generic error.
1432 0 : return_value = kOtherDecoderError;
1433 0 : LOG(LS_WARNING) << "Decoder error (no error code)";
1434 : }
1435 0 : *operation = kExpand; // Do expansion to get data instead.
1436 : }
1437 0 : if (*speech_type != AudioDecoder::kComfortNoise) {
1438 : // Don't increment timestamp if codec returned CNG speech type
1439 : // since in this case, the we will increment the CNGplayedTS counter.
1440 : // Increase with number of samples per channel.
1441 0 : assert(*decoded_length == 0 ||
1442 : (decoder && decoder->Channels() == sync_buffer_->Channels()));
1443 0 : sync_buffer_->IncreaseEndTimestamp(
1444 0 : *decoded_length / static_cast<int>(sync_buffer_->Channels()));
1445 : }
1446 0 : return return_value;
1447 : }
1448 :
1449 0 : int NetEqImpl::DecodeCng(AudioDecoder* decoder, int* decoded_length,
1450 : AudioDecoder::SpeechType* speech_type) {
1451 0 : if (!decoder) {
1452 : // This happens when active decoder is not defined.
1453 0 : *decoded_length = -1;
1454 0 : return 0;
1455 : }
1456 :
1457 0 : while (*decoded_length < rtc::checked_cast<int>(output_size_samples_)) {
1458 0 : const int length = decoder->Decode(
1459 : nullptr, 0, fs_hz_,
1460 0 : (decoded_buffer_length_ - *decoded_length) * sizeof(int16_t),
1461 0 : &decoded_buffer_[*decoded_length], speech_type);
1462 0 : if (length > 0) {
1463 0 : *decoded_length += length;
1464 : } else {
1465 : // Error.
1466 0 : LOG(LS_WARNING) << "Failed to decode CNG";
1467 0 : *decoded_length = -1;
1468 0 : break;
1469 : }
1470 0 : if (*decoded_length > static_cast<int>(decoded_buffer_length_)) {
1471 : // Guard against overflow.
1472 0 : LOG(LS_WARNING) << "Decoded too much CNG.";
1473 0 : return kDecodedTooMuch;
1474 : }
1475 : }
1476 0 : return 0;
1477 : }
1478 :
1479 0 : int NetEqImpl::DecodeLoop(PacketList* packet_list, const Operations& operation,
1480 : AudioDecoder* decoder, int* decoded_length,
1481 : AudioDecoder::SpeechType* speech_type) {
1482 : // Do decoding.
1483 0 : while (
1484 0 : !packet_list->empty() &&
1485 0 : !decoder_database_->IsComfortNoise(packet_list->front().payload_type)) {
1486 0 : assert(decoder); // At this point, we must have a decoder object.
1487 : // The number of channels in the |sync_buffer_| should be the same as the
1488 : // number decoder channels.
1489 0 : assert(sync_buffer_->Channels() == decoder->Channels());
1490 0 : assert(decoded_buffer_length_ >= kMaxFrameSize * decoder->Channels());
1491 0 : assert(operation == kNormal || operation == kAccelerate ||
1492 : operation == kFastAccelerate || operation == kMerge ||
1493 : operation == kPreemptiveExpand);
1494 :
1495 0 : auto opt_result = packet_list->front().frame->Decode(
1496 0 : rtc::ArrayView<int16_t>(&decoded_buffer_[*decoded_length],
1497 0 : decoded_buffer_length_ - *decoded_length));
1498 0 : packet_list->pop_front();
1499 0 : if (opt_result) {
1500 0 : const auto& result = *opt_result;
1501 0 : *speech_type = result.speech_type;
1502 0 : if (result.num_decoded_samples > 0) {
1503 0 : *decoded_length += rtc::checked_cast<int>(result.num_decoded_samples);
1504 : // Update |decoder_frame_length_| with number of samples per channel.
1505 0 : decoder_frame_length_ =
1506 0 : result.num_decoded_samples / decoder->Channels();
1507 : }
1508 : } else {
1509 : // Error.
1510 : // TODO(ossu): What to put here?
1511 0 : LOG(LS_WARNING) << "Decode error";
1512 0 : *decoded_length = -1;
1513 0 : packet_list->clear();
1514 0 : break;
1515 : }
1516 0 : if (*decoded_length > rtc::checked_cast<int>(decoded_buffer_length_)) {
1517 : // Guard against overflow.
1518 0 : LOG(LS_WARNING) << "Decoded too much.";
1519 0 : packet_list->clear();
1520 0 : return kDecodedTooMuch;
1521 : }
1522 : } // End of decode loop.
1523 :
1524 : // If the list is not empty at this point, either a decoding error terminated
1525 : // the while-loop, or list must hold exactly one CNG packet.
1526 0 : assert(
1527 : packet_list->empty() || *decoded_length < 0 ||
1528 : (packet_list->size() == 1 &&
1529 : decoder_database_->IsComfortNoise(packet_list->front().payload_type)));
1530 0 : return 0;
1531 : }
1532 :
1533 0 : void NetEqImpl::DoNormal(const int16_t* decoded_buffer, size_t decoded_length,
1534 : AudioDecoder::SpeechType speech_type, bool play_dtmf) {
1535 0 : assert(normal_.get());
1536 0 : assert(mute_factor_array_.get());
1537 0 : normal_->Process(decoded_buffer, decoded_length, last_mode_,
1538 0 : mute_factor_array_.get(), algorithm_buffer_.get());
1539 0 : if (decoded_length != 0) {
1540 0 : last_mode_ = kModeNormal;
1541 : }
1542 :
1543 : // If last packet was decoded as an inband CNG, set mode to CNG instead.
1544 0 : if ((speech_type == AudioDecoder::kComfortNoise)
1545 0 : || ((last_mode_ == kModeCodecInternalCng)
1546 0 : && (decoded_length == 0))) {
1547 : // TODO(hlundin): Remove second part of || statement above.
1548 0 : last_mode_ = kModeCodecInternalCng;
1549 : }
1550 :
1551 0 : if (!play_dtmf) {
1552 0 : dtmf_tone_generator_->Reset();
1553 : }
1554 0 : }
1555 :
1556 0 : void NetEqImpl::DoMerge(int16_t* decoded_buffer, size_t decoded_length,
1557 : AudioDecoder::SpeechType speech_type, bool play_dtmf) {
1558 0 : assert(mute_factor_array_.get());
1559 0 : assert(merge_.get());
1560 0 : size_t new_length = merge_->Process(decoded_buffer, decoded_length,
1561 : mute_factor_array_.get(),
1562 0 : algorithm_buffer_.get());
1563 : size_t expand_length_correction = new_length -
1564 0 : decoded_length / algorithm_buffer_->Channels();
1565 :
1566 : // Update in-call and post-call statistics.
1567 0 : if (expand_->MuteFactor(0) == 0) {
1568 : // Expand generates only noise.
1569 0 : stats_.ExpandedNoiseSamples(expand_length_correction);
1570 : } else {
1571 : // Expansion generates more than only noise.
1572 0 : stats_.ExpandedVoiceSamples(expand_length_correction);
1573 : }
1574 :
1575 0 : last_mode_ = kModeMerge;
1576 : // If last packet was decoded as an inband CNG, set mode to CNG instead.
1577 0 : if (speech_type == AudioDecoder::kComfortNoise) {
1578 0 : last_mode_ = kModeCodecInternalCng;
1579 : }
1580 0 : expand_->Reset();
1581 0 : if (!play_dtmf) {
1582 0 : dtmf_tone_generator_->Reset();
1583 : }
1584 0 : }
1585 :
1586 0 : int NetEqImpl::DoExpand(bool play_dtmf) {
1587 0 : while ((sync_buffer_->FutureLength() - expand_->overlap_length()) <
1588 0 : output_size_samples_) {
1589 0 : algorithm_buffer_->Clear();
1590 0 : int return_value = expand_->Process(algorithm_buffer_.get());
1591 0 : size_t length = algorithm_buffer_->Size();
1592 :
1593 : // Update in-call and post-call statistics.
1594 0 : if (expand_->MuteFactor(0) == 0) {
1595 : // Expand operation generates only noise.
1596 0 : stats_.ExpandedNoiseSamples(length);
1597 : } else {
1598 : // Expand operation generates more than only noise.
1599 0 : stats_.ExpandedVoiceSamples(length);
1600 : }
1601 :
1602 0 : last_mode_ = kModeExpand;
1603 :
1604 0 : if (return_value < 0) {
1605 0 : return return_value;
1606 : }
1607 :
1608 0 : sync_buffer_->PushBack(*algorithm_buffer_);
1609 0 : algorithm_buffer_->Clear();
1610 : }
1611 0 : if (!play_dtmf) {
1612 0 : dtmf_tone_generator_->Reset();
1613 : }
1614 :
1615 0 : if (!generated_noise_stopwatch_) {
1616 : // Start a new stopwatch since we may be covering for a lost CNG packet.
1617 0 : generated_noise_stopwatch_ = tick_timer_->GetNewStopwatch();
1618 : }
1619 :
1620 0 : return 0;
1621 : }
1622 :
1623 0 : int NetEqImpl::DoAccelerate(int16_t* decoded_buffer,
1624 : size_t decoded_length,
1625 : AudioDecoder::SpeechType speech_type,
1626 : bool play_dtmf,
1627 : bool fast_accelerate) {
1628 : const size_t required_samples =
1629 0 : static_cast<size_t>(240 * fs_mult_); // Must have 30 ms.
1630 0 : size_t borrowed_samples_per_channel = 0;
1631 0 : size_t num_channels = algorithm_buffer_->Channels();
1632 0 : size_t decoded_length_per_channel = decoded_length / num_channels;
1633 0 : if (decoded_length_per_channel < required_samples) {
1634 : // Must move data from the |sync_buffer_| in order to get 30 ms.
1635 0 : borrowed_samples_per_channel = static_cast<int>(required_samples -
1636 : decoded_length_per_channel);
1637 0 : memmove(&decoded_buffer[borrowed_samples_per_channel * num_channels],
1638 : decoded_buffer,
1639 0 : sizeof(int16_t) * decoded_length);
1640 0 : sync_buffer_->ReadInterleavedFromEnd(borrowed_samples_per_channel,
1641 0 : decoded_buffer);
1642 0 : decoded_length = required_samples * num_channels;
1643 : }
1644 :
1645 : size_t samples_removed;
1646 : Accelerate::ReturnCodes return_code =
1647 0 : accelerate_->Process(decoded_buffer, decoded_length, fast_accelerate,
1648 0 : algorithm_buffer_.get(), &samples_removed);
1649 0 : stats_.AcceleratedSamples(samples_removed);
1650 0 : switch (return_code) {
1651 : case Accelerate::kSuccess:
1652 0 : last_mode_ = kModeAccelerateSuccess;
1653 0 : break;
1654 : case Accelerate::kSuccessLowEnergy:
1655 0 : last_mode_ = kModeAccelerateLowEnergy;
1656 0 : break;
1657 : case Accelerate::kNoStretch:
1658 0 : last_mode_ = kModeAccelerateFail;
1659 0 : break;
1660 : case Accelerate::kError:
1661 : // TODO(hlundin): Map to kModeError instead?
1662 0 : last_mode_ = kModeAccelerateFail;
1663 0 : return kAccelerateError;
1664 : }
1665 :
1666 0 : if (borrowed_samples_per_channel > 0) {
1667 : // Copy borrowed samples back to the |sync_buffer_|.
1668 0 : size_t length = algorithm_buffer_->Size();
1669 0 : if (length < borrowed_samples_per_channel) {
1670 : // This destroys the beginning of the buffer, but will not cause any
1671 : // problems.
1672 0 : sync_buffer_->ReplaceAtIndex(*algorithm_buffer_,
1673 0 : sync_buffer_->Size() -
1674 0 : borrowed_samples_per_channel);
1675 0 : sync_buffer_->PushFrontZeros(borrowed_samples_per_channel - length);
1676 0 : algorithm_buffer_->PopFront(length);
1677 0 : assert(algorithm_buffer_->Empty());
1678 : } else {
1679 0 : sync_buffer_->ReplaceAtIndex(*algorithm_buffer_,
1680 : borrowed_samples_per_channel,
1681 0 : sync_buffer_->Size() -
1682 0 : borrowed_samples_per_channel);
1683 0 : algorithm_buffer_->PopFront(borrowed_samples_per_channel);
1684 : }
1685 : }
1686 :
1687 : // If last packet was decoded as an inband CNG, set mode to CNG instead.
1688 0 : if (speech_type == AudioDecoder::kComfortNoise) {
1689 0 : last_mode_ = kModeCodecInternalCng;
1690 : }
1691 0 : if (!play_dtmf) {
1692 0 : dtmf_tone_generator_->Reset();
1693 : }
1694 0 : expand_->Reset();
1695 0 : return 0;
1696 : }
1697 :
1698 0 : int NetEqImpl::DoPreemptiveExpand(int16_t* decoded_buffer,
1699 : size_t decoded_length,
1700 : AudioDecoder::SpeechType speech_type,
1701 : bool play_dtmf) {
1702 : const size_t required_samples =
1703 0 : static_cast<size_t>(240 * fs_mult_); // Must have 30 ms.
1704 0 : size_t num_channels = algorithm_buffer_->Channels();
1705 0 : size_t borrowed_samples_per_channel = 0;
1706 0 : size_t old_borrowed_samples_per_channel = 0;
1707 0 : size_t decoded_length_per_channel = decoded_length / num_channels;
1708 0 : if (decoded_length_per_channel < required_samples) {
1709 : // Must move data from the |sync_buffer_| in order to get 30 ms.
1710 0 : borrowed_samples_per_channel =
1711 : required_samples - decoded_length_per_channel;
1712 : // Calculate how many of these were already played out.
1713 0 : old_borrowed_samples_per_channel =
1714 0 : (borrowed_samples_per_channel > sync_buffer_->FutureLength()) ?
1715 0 : (borrowed_samples_per_channel - sync_buffer_->FutureLength()) : 0;
1716 0 : memmove(&decoded_buffer[borrowed_samples_per_channel * num_channels],
1717 : decoded_buffer,
1718 0 : sizeof(int16_t) * decoded_length);
1719 0 : sync_buffer_->ReadInterleavedFromEnd(borrowed_samples_per_channel,
1720 0 : decoded_buffer);
1721 0 : decoded_length = required_samples * num_channels;
1722 : }
1723 :
1724 : size_t samples_added;
1725 0 : PreemptiveExpand::ReturnCodes return_code = preemptive_expand_->Process(
1726 : decoded_buffer, decoded_length,
1727 : old_borrowed_samples_per_channel,
1728 0 : algorithm_buffer_.get(), &samples_added);
1729 0 : stats_.PreemptiveExpandedSamples(samples_added);
1730 0 : switch (return_code) {
1731 : case PreemptiveExpand::kSuccess:
1732 0 : last_mode_ = kModePreemptiveExpandSuccess;
1733 0 : break;
1734 : case PreemptiveExpand::kSuccessLowEnergy:
1735 0 : last_mode_ = kModePreemptiveExpandLowEnergy;
1736 0 : break;
1737 : case PreemptiveExpand::kNoStretch:
1738 0 : last_mode_ = kModePreemptiveExpandFail;
1739 0 : break;
1740 : case PreemptiveExpand::kError:
1741 : // TODO(hlundin): Map to kModeError instead?
1742 0 : last_mode_ = kModePreemptiveExpandFail;
1743 0 : return kPreemptiveExpandError;
1744 : }
1745 :
1746 0 : if (borrowed_samples_per_channel > 0) {
1747 : // Copy borrowed samples back to the |sync_buffer_|.
1748 0 : sync_buffer_->ReplaceAtIndex(
1749 0 : *algorithm_buffer_, borrowed_samples_per_channel,
1750 0 : sync_buffer_->Size() - borrowed_samples_per_channel);
1751 0 : algorithm_buffer_->PopFront(borrowed_samples_per_channel);
1752 : }
1753 :
1754 : // If last packet was decoded as an inband CNG, set mode to CNG instead.
1755 0 : if (speech_type == AudioDecoder::kComfortNoise) {
1756 0 : last_mode_ = kModeCodecInternalCng;
1757 : }
1758 0 : if (!play_dtmf) {
1759 0 : dtmf_tone_generator_->Reset();
1760 : }
1761 0 : expand_->Reset();
1762 0 : return 0;
1763 : }
1764 :
1765 0 : int NetEqImpl::DoRfc3389Cng(PacketList* packet_list, bool play_dtmf) {
1766 0 : if (!packet_list->empty()) {
1767 : // Must have exactly one SID frame at this point.
1768 0 : assert(packet_list->size() == 1);
1769 0 : const Packet& packet = packet_list->front();
1770 0 : if (!decoder_database_->IsComfortNoise(packet.payload_type)) {
1771 0 : LOG(LS_ERROR) << "Trying to decode non-CNG payload as CNG.";
1772 0 : return kOtherError;
1773 : }
1774 0 : if (comfort_noise_->UpdateParameters(packet) ==
1775 : ComfortNoise::kInternalError) {
1776 0 : algorithm_buffer_->Zeros(output_size_samples_);
1777 0 : return -comfort_noise_->internal_error_code();
1778 : }
1779 : }
1780 0 : int cn_return = comfort_noise_->Generate(output_size_samples_,
1781 0 : algorithm_buffer_.get());
1782 0 : expand_->Reset();
1783 0 : last_mode_ = kModeRfc3389Cng;
1784 0 : if (!play_dtmf) {
1785 0 : dtmf_tone_generator_->Reset();
1786 : }
1787 0 : if (cn_return == ComfortNoise::kInternalError) {
1788 0 : decoder_error_code_ = comfort_noise_->internal_error_code();
1789 0 : return kComfortNoiseErrorCode;
1790 0 : } else if (cn_return == ComfortNoise::kUnknownPayloadType) {
1791 0 : return kUnknownRtpPayloadType;
1792 : }
1793 0 : return 0;
1794 : }
1795 :
1796 0 : void NetEqImpl::DoCodecInternalCng(const int16_t* decoded_buffer,
1797 : size_t decoded_length) {
1798 0 : RTC_DCHECK(normal_.get());
1799 0 : RTC_DCHECK(mute_factor_array_.get());
1800 0 : normal_->Process(decoded_buffer, decoded_length, last_mode_,
1801 0 : mute_factor_array_.get(), algorithm_buffer_.get());
1802 0 : last_mode_ = kModeCodecInternalCng;
1803 0 : expand_->Reset();
1804 0 : }
1805 :
1806 0 : int NetEqImpl::DoDtmf(const DtmfEvent& dtmf_event, bool* play_dtmf) {
1807 : // This block of the code and the block further down, handling |dtmf_switch|
1808 : // are commented out. Otherwise playing out-of-band DTMF would fail in VoE
1809 : // test, DtmfTest.ManualSuccessfullySendsOutOfBandTelephoneEvents. This is
1810 : // equivalent to |dtmf_switch| always be false.
1811 : //
1812 : // See http://webrtc-codereview.appspot.com/1195004/ for discussion
1813 : // On this issue. This change might cause some glitches at the point of
1814 : // switch from audio to DTMF. Issue 1545 is filed to track this.
1815 : //
1816 : // bool dtmf_switch = false;
1817 : // if ((last_mode_ != kModeDtmf) && dtmf_tone_generator_->initialized()) {
1818 : // // Special case; see below.
1819 : // // We must catch this before calling Generate, since |initialized| is
1820 : // // modified in that call.
1821 : // dtmf_switch = true;
1822 : // }
1823 :
1824 0 : int dtmf_return_value = 0;
1825 0 : if (!dtmf_tone_generator_->initialized()) {
1826 : // Initialize if not already done.
1827 0 : dtmf_return_value = dtmf_tone_generator_->Init(fs_hz_, dtmf_event.event_no,
1828 0 : dtmf_event.volume);
1829 : }
1830 :
1831 0 : if (dtmf_return_value == 0) {
1832 : // Generate DTMF signal.
1833 0 : dtmf_return_value = dtmf_tone_generator_->Generate(output_size_samples_,
1834 0 : algorithm_buffer_.get());
1835 : }
1836 :
1837 0 : if (dtmf_return_value < 0) {
1838 0 : algorithm_buffer_->Zeros(output_size_samples_);
1839 0 : return dtmf_return_value;
1840 : }
1841 :
1842 : // if (dtmf_switch) {
1843 : // // This is the special case where the previous operation was DTMF
1844 : // // overdub, but the current instruction is "regular" DTMF. We must make
1845 : // // sure that the DTMF does not have any discontinuities. The first DTMF
1846 : // // sample that we generate now must be played out immediately, therefore
1847 : // // it must be copied to the speech buffer.
1848 : // // TODO(hlundin): This code seems incorrect. (Legacy.) Write test and
1849 : // // verify correct operation.
1850 : // assert(false);
1851 : // // Must generate enough data to replace all of the |sync_buffer_|
1852 : // // "future".
1853 : // int required_length = sync_buffer_->FutureLength();
1854 : // assert(dtmf_tone_generator_->initialized());
1855 : // dtmf_return_value = dtmf_tone_generator_->Generate(required_length,
1856 : // algorithm_buffer_);
1857 : // assert((size_t) required_length == algorithm_buffer_->Size());
1858 : // if (dtmf_return_value < 0) {
1859 : // algorithm_buffer_->Zeros(output_size_samples_);
1860 : // return dtmf_return_value;
1861 : // }
1862 : //
1863 : // // Overwrite the "future" part of the speech buffer with the new DTMF
1864 : // // data.
1865 : // // TODO(hlundin): It seems that this overwriting has gone lost.
1866 : // // Not adapted for multi-channel yet.
1867 : // assert(algorithm_buffer_->Channels() == 1);
1868 : // if (algorithm_buffer_->Channels() != 1) {
1869 : // LOG(LS_WARNING) << "DTMF not supported for more than one channel";
1870 : // return kStereoNotSupported;
1871 : // }
1872 : // // Shuffle the remaining data to the beginning of algorithm buffer.
1873 : // algorithm_buffer_->PopFront(sync_buffer_->FutureLength());
1874 : // }
1875 :
1876 0 : sync_buffer_->IncreaseEndTimestamp(
1877 0 : static_cast<uint32_t>(output_size_samples_));
1878 0 : expand_->Reset();
1879 0 : last_mode_ = kModeDtmf;
1880 :
1881 : // Set to false because the DTMF is already in the algorithm buffer.
1882 0 : *play_dtmf = false;
1883 0 : return 0;
1884 : }
1885 :
1886 0 : void NetEqImpl::DoAlternativePlc(bool increase_timestamp) {
1887 0 : AudioDecoder* decoder = decoder_database_->GetActiveDecoder();
1888 : size_t length;
1889 0 : if (decoder && decoder->HasDecodePlc()) {
1890 : // Use the decoder's packet-loss concealment.
1891 : // TODO(hlundin): Will probably need a longer buffer for multi-channel.
1892 : int16_t decoded_buffer[kMaxFrameSize];
1893 0 : length = decoder->DecodePlc(1, decoded_buffer);
1894 0 : if (length > 0)
1895 0 : algorithm_buffer_->PushBackInterleaved(decoded_buffer, length);
1896 : } else {
1897 : // Do simple zero-stuffing.
1898 0 : length = output_size_samples_;
1899 0 : algorithm_buffer_->Zeros(length);
1900 : // By not advancing the timestamp, NetEq inserts samples.
1901 0 : stats_.AddZeros(length);
1902 : }
1903 0 : if (increase_timestamp) {
1904 0 : sync_buffer_->IncreaseEndTimestamp(static_cast<uint32_t>(length));
1905 : }
1906 0 : expand_->Reset();
1907 0 : }
1908 :
1909 0 : int NetEqImpl::DtmfOverdub(const DtmfEvent& dtmf_event, size_t num_channels,
1910 : int16_t* output) const {
1911 0 : size_t out_index = 0;
1912 0 : size_t overdub_length = output_size_samples_; // Default value.
1913 :
1914 0 : if (sync_buffer_->dtmf_index() > sync_buffer_->next_index()) {
1915 : // Special operation for transition from "DTMF only" to "DTMF overdub".
1916 0 : out_index = std::min(
1917 0 : sync_buffer_->dtmf_index() - sync_buffer_->next_index(),
1918 0 : output_size_samples_);
1919 0 : overdub_length = output_size_samples_ - out_index;
1920 : }
1921 :
1922 0 : AudioMultiVector dtmf_output(num_channels);
1923 0 : int dtmf_return_value = 0;
1924 0 : if (!dtmf_tone_generator_->initialized()) {
1925 0 : dtmf_return_value = dtmf_tone_generator_->Init(fs_hz_, dtmf_event.event_no,
1926 0 : dtmf_event.volume);
1927 : }
1928 0 : if (dtmf_return_value == 0) {
1929 0 : dtmf_return_value = dtmf_tone_generator_->Generate(overdub_length,
1930 0 : &dtmf_output);
1931 0 : assert(overdub_length == dtmf_output.Size());
1932 : }
1933 0 : dtmf_output.ReadInterleaved(overdub_length, &output[out_index]);
1934 0 : return dtmf_return_value < 0 ? dtmf_return_value : 0;
1935 : }
1936 :
1937 0 : int NetEqImpl::ExtractPackets(size_t required_samples,
1938 : PacketList* packet_list) {
1939 0 : bool first_packet = true;
1940 0 : uint8_t prev_payload_type = 0;
1941 0 : uint32_t prev_timestamp = 0;
1942 0 : uint16_t prev_sequence_number = 0;
1943 0 : bool next_packet_available = false;
1944 :
1945 0 : const Packet* next_packet = packet_buffer_->PeekNextPacket();
1946 0 : RTC_DCHECK(next_packet);
1947 0 : if (!next_packet) {
1948 0 : LOG(LS_ERROR) << "Packet buffer unexpectedly empty.";
1949 0 : return -1;
1950 : }
1951 0 : uint32_t first_timestamp = next_packet->timestamp;
1952 0 : size_t extracted_samples = 0;
1953 :
1954 : // Packet extraction loop.
1955 0 : do {
1956 0 : timestamp_ = next_packet->timestamp;
1957 0 : rtc::Optional<Packet> packet = packet_buffer_->GetNextPacket();
1958 : // |next_packet| may be invalid after the |packet_buffer_| operation.
1959 0 : next_packet = nullptr;
1960 0 : if (!packet) {
1961 0 : LOG(LS_ERROR) << "Should always be able to extract a packet here";
1962 0 : assert(false); // Should always be able to extract a packet here.
1963 : return -1;
1964 : }
1965 0 : stats_.StoreWaitingTime(packet->waiting_time->ElapsedMs());
1966 0 : RTC_DCHECK(!packet->empty());
1967 :
1968 0 : if (first_packet) {
1969 0 : first_packet = false;
1970 0 : if (nack_enabled_) {
1971 0 : RTC_DCHECK(nack_);
1972 : // TODO(henrik.lundin): Should we update this for all decoded packets?
1973 0 : nack_->UpdateLastDecodedPacket(packet->sequence_number,
1974 0 : packet->timestamp);
1975 : }
1976 0 : prev_sequence_number = packet->sequence_number;
1977 0 : prev_timestamp = packet->timestamp;
1978 0 : prev_payload_type = packet->payload_type;
1979 : }
1980 :
1981 : const bool has_cng_packet =
1982 0 : decoder_database_->IsComfortNoise(packet->payload_type);
1983 : // Store number of extracted samples.
1984 0 : size_t packet_duration = 0;
1985 0 : if (packet->frame) {
1986 0 : packet_duration = packet->frame->Duration();
1987 : // TODO(ossu): Is this the correct way to track Opus FEC packets?
1988 0 : if (packet->priority.codec_level > 0) {
1989 0 : stats_.SecondaryDecodedSamples(rtc::checked_cast<int>(packet_duration));
1990 : }
1991 0 : } else if (!has_cng_packet) {
1992 0 : LOG(LS_WARNING) << "Unknown payload type "
1993 0 : << static_cast<int>(packet->payload_type);
1994 0 : RTC_NOTREACHED();
1995 : }
1996 :
1997 0 : if (packet_duration == 0) {
1998 : // Decoder did not return a packet duration. Assume that the packet
1999 : // contains the same number of samples as the previous one.
2000 0 : packet_duration = decoder_frame_length_;
2001 : }
2002 0 : extracted_samples = packet->timestamp - first_timestamp + packet_duration;
2003 :
2004 0 : packet_list->push_back(std::move(*packet)); // Store packet in list.
2005 0 : packet = rtc::Optional<Packet>(); // Ensure it's never used after the move.
2006 :
2007 : // Check what packet is available next.
2008 0 : next_packet = packet_buffer_->PeekNextPacket();
2009 0 : next_packet_available = false;
2010 0 : if (next_packet && prev_payload_type == next_packet->payload_type &&
2011 0 : !has_cng_packet) {
2012 0 : int16_t seq_no_diff = next_packet->sequence_number - prev_sequence_number;
2013 0 : size_t ts_diff = next_packet->timestamp - prev_timestamp;
2014 0 : if (seq_no_diff == 1 ||
2015 0 : (seq_no_diff == 0 && ts_diff == decoder_frame_length_)) {
2016 : // The next sequence number is available, or the next part of a packet
2017 : // that was split into pieces upon insertion.
2018 0 : next_packet_available = true;
2019 : }
2020 0 : prev_sequence_number = next_packet->sequence_number;
2021 : }
2022 0 : } while (extracted_samples < required_samples && next_packet_available);
2023 :
2024 0 : if (extracted_samples > 0) {
2025 : // Delete old packets only when we are going to decode something. Otherwise,
2026 : // we could end up in the situation where we never decode anything, since
2027 : // all incoming packets are considered too old but the buffer will also
2028 : // never be flooded and flushed.
2029 0 : packet_buffer_->DiscardAllOldPackets(timestamp_);
2030 : }
2031 :
2032 0 : return rtc::checked_cast<int>(extracted_samples);
2033 : }
2034 :
2035 0 : void NetEqImpl::UpdatePlcComponents(int fs_hz, size_t channels) {
2036 : // Delete objects and create new ones.
2037 0 : expand_.reset(expand_factory_->Create(background_noise_.get(),
2038 : sync_buffer_.get(), &random_vector_,
2039 0 : &stats_, fs_hz, channels));
2040 0 : merge_.reset(new Merge(fs_hz, channels, expand_.get(), sync_buffer_.get()));
2041 0 : }
2042 :
2043 0 : void NetEqImpl::SetSampleRateAndChannels(int fs_hz, size_t channels) {
2044 0 : LOG(LS_VERBOSE) << "SetSampleRateAndChannels " << fs_hz << " " << channels;
2045 : // TODO(hlundin): Change to an enumerator and skip assert.
2046 0 : assert(fs_hz == 8000 || fs_hz == 16000 || fs_hz == 32000 || fs_hz == 48000);
2047 0 : assert(channels > 0);
2048 :
2049 0 : fs_hz_ = fs_hz;
2050 0 : fs_mult_ = fs_hz / 8000;
2051 0 : output_size_samples_ = static_cast<size_t>(kOutputSizeMs * 8 * fs_mult_);
2052 0 : decoder_frame_length_ = 3 * output_size_samples_; // Initialize to 30ms.
2053 :
2054 0 : last_mode_ = kModeNormal;
2055 :
2056 : // Create a new array of mute factors and set all to 1.
2057 0 : mute_factor_array_.reset(new int16_t[channels]);
2058 0 : for (size_t i = 0; i < channels; ++i) {
2059 0 : mute_factor_array_[i] = 16384; // 1.0 in Q14.
2060 : }
2061 :
2062 0 : ComfortNoiseDecoder* cng_decoder = decoder_database_->GetActiveCngDecoder();
2063 0 : if (cng_decoder)
2064 0 : cng_decoder->Reset();
2065 :
2066 : // Reinit post-decode VAD with new sample rate.
2067 0 : assert(vad_.get()); // Cannot be NULL here.
2068 0 : vad_->Init();
2069 :
2070 : // Delete algorithm buffer and create a new one.
2071 0 : algorithm_buffer_.reset(new AudioMultiVector(channels));
2072 :
2073 : // Delete sync buffer and create a new one.
2074 0 : sync_buffer_.reset(new SyncBuffer(channels, kSyncBufferSize * fs_mult_));
2075 :
2076 : // Delete BackgroundNoise object and create a new one.
2077 0 : background_noise_.reset(new BackgroundNoise(channels));
2078 0 : background_noise_->set_mode(background_noise_mode_);
2079 :
2080 : // Reset random vector.
2081 0 : random_vector_.Reset();
2082 :
2083 0 : UpdatePlcComponents(fs_hz, channels);
2084 :
2085 : // Move index so that we create a small set of future samples (all 0).
2086 0 : sync_buffer_->set_next_index(sync_buffer_->next_index() -
2087 0 : expand_->overlap_length());
2088 :
2089 0 : normal_.reset(new Normal(fs_hz, decoder_database_.get(), *background_noise_,
2090 0 : expand_.get()));
2091 0 : accelerate_.reset(
2092 0 : accelerate_factory_->Create(fs_hz, channels, *background_noise_));
2093 0 : preemptive_expand_.reset(preemptive_expand_factory_->Create(
2094 0 : fs_hz, channels, *background_noise_, expand_->overlap_length()));
2095 :
2096 : // Delete ComfortNoise object and create a new one.
2097 0 : comfort_noise_.reset(new ComfortNoise(fs_hz, decoder_database_.get(),
2098 0 : sync_buffer_.get()));
2099 :
2100 : // Verify that |decoded_buffer_| is long enough.
2101 0 : if (decoded_buffer_length_ < kMaxFrameSize * channels) {
2102 : // Reallocate to larger size.
2103 0 : decoded_buffer_length_ = kMaxFrameSize * channels;
2104 0 : decoded_buffer_.reset(new int16_t[decoded_buffer_length_]);
2105 : }
2106 :
2107 : // Create DecisionLogic if it is not created yet, then communicate new sample
2108 : // rate and output size to DecisionLogic object.
2109 0 : if (!decision_logic_.get()) {
2110 0 : CreateDecisionLogic();
2111 : }
2112 0 : decision_logic_->SetSampleRate(fs_hz_, output_size_samples_);
2113 0 : }
2114 :
2115 0 : NetEqImpl::OutputType NetEqImpl::LastOutputType() {
2116 0 : assert(vad_.get());
2117 0 : assert(expand_.get());
2118 0 : if (last_mode_ == kModeCodecInternalCng || last_mode_ == kModeRfc3389Cng) {
2119 0 : return OutputType::kCNG;
2120 0 : } else if (last_mode_ == kModeExpand && expand_->MuteFactor(0) == 0) {
2121 : // Expand mode has faded down to background noise only (very long expand).
2122 0 : return OutputType::kPLCCNG;
2123 0 : } else if (last_mode_ == kModeExpand) {
2124 0 : return OutputType::kPLC;
2125 0 : } else if (vad_->running() && !vad_->active_speech()) {
2126 0 : return OutputType::kVadPassive;
2127 : } else {
2128 0 : return OutputType::kNormalSpeech;
2129 : }
2130 : }
2131 :
2132 0 : void NetEqImpl::CreateDecisionLogic() {
2133 0 : decision_logic_.reset(DecisionLogic::Create(
2134 : fs_hz_, output_size_samples_, playout_mode_, decoder_database_.get(),
2135 0 : *packet_buffer_.get(), delay_manager_.get(), buffer_level_filter_.get(),
2136 0 : tick_timer_.get()));
2137 0 : }
2138 : } // namespace webrtc
|
