Line data Source code
1 : /*
2 : * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 :
11 : #include "webrtc/modules/audio_coding/acm2/acm_receiver.h"
12 :
13 : #include <stdlib.h> // malloc
14 :
15 : #include <algorithm> // sort
16 : #include <vector>
17 :
18 : #include "webrtc/base/checks.h"
19 : #include "webrtc/base/format_macros.h"
20 : #include "webrtc/base/logging.h"
21 : #include "webrtc/base/safe_conversions.h"
22 : #include "webrtc/common_audio/signal_processing/include/signal_processing_library.h"
23 : #include "webrtc/common_types.h"
24 : #include "webrtc/modules/audio_coding/codecs/audio_decoder.h"
25 : #include "webrtc/modules/audio_coding/acm2/acm_resampler.h"
26 : #include "webrtc/modules/audio_coding/acm2/call_statistics.h"
27 : #include "webrtc/modules/audio_coding/neteq/include/neteq.h"
28 : #include "webrtc/system_wrappers/include/clock.h"
29 : #include "webrtc/system_wrappers/include/trace.h"
30 :
31 : namespace webrtc {
32 :
33 : namespace acm2 {
34 :
35 0 : AcmReceiver::AcmReceiver(const AudioCodingModule::Config& config)
36 0 : : last_audio_buffer_(new int16_t[AudioFrame::kMaxDataSizeSamples]),
37 0 : neteq_(NetEq::Create(config.neteq_config, config.decoder_factory)),
38 0 : clock_(config.clock),
39 0 : resampled_last_output_frame_(true) {
40 0 : assert(clock_);
41 0 : memset(last_audio_buffer_.get(), 0, AudioFrame::kMaxDataSizeSamples);
42 0 : }
43 :
44 0 : AcmReceiver::~AcmReceiver() {
45 0 : delete neteq_;
46 0 : }
47 :
48 0 : int AcmReceiver::SetMinimumDelay(int delay_ms) {
49 0 : if (neteq_->SetMinimumDelay(delay_ms))
50 0 : return 0;
51 0 : LOG(LERROR) << "AcmReceiver::SetExtraDelay " << delay_ms;
52 0 : return -1;
53 : }
54 :
55 0 : int AcmReceiver::SetMaximumDelay(int delay_ms) {
56 0 : if (neteq_->SetMaximumDelay(delay_ms))
57 0 : return 0;
58 0 : LOG(LERROR) << "AcmReceiver::SetExtraDelay " << delay_ms;
59 0 : return -1;
60 : }
61 :
62 0 : int AcmReceiver::LeastRequiredDelayMs() const {
63 0 : return neteq_->LeastRequiredDelayMs();
64 : }
65 :
66 0 : rtc::Optional<int> AcmReceiver::last_packet_sample_rate_hz() const {
67 0 : rtc::CritScope lock(&crit_sect_);
68 0 : return last_packet_sample_rate_hz_;
69 : }
70 :
71 0 : int AcmReceiver::last_output_sample_rate_hz() const {
72 0 : return neteq_->last_output_sample_rate_hz();
73 : }
74 :
75 0 : int AcmReceiver::InsertPacket(const WebRtcRTPHeader& rtp_header,
76 : rtc::ArrayView<const uint8_t> incoming_payload) {
77 0 : uint32_t receive_timestamp = 0;
78 0 : const RTPHeader* header = &rtp_header.header; // Just a shorthand.
79 :
80 : {
81 0 : rtc::CritScope lock(&crit_sect_);
82 :
83 : const rtc::Optional<CodecInst> ci =
84 0 : RtpHeaderToDecoder(*header, incoming_payload[0]);
85 0 : if (!ci) {
86 0 : LOG_F(LS_ERROR) << "Payload-type "
87 0 : << static_cast<int>(header->payloadType)
88 0 : << " is not registered.";
89 0 : return -1;
90 : }
91 0 : receive_timestamp = NowInTimestamp(ci->plfreq);
92 :
93 0 : if (STR_CASE_CMP(ci->plname, "cn") == 0) {
94 0 : if (last_audio_decoder_ && last_audio_decoder_->channels > 1) {
95 : // This is a CNG and the audio codec is not mono, so skip pushing in
96 : // packets into NetEq.
97 0 : return 0;
98 : }
99 : } else {
100 0 : last_audio_decoder_ = ci;
101 0 : last_audio_format_ = neteq_->GetDecoderFormat(ci->pltype);
102 0 : RTC_DCHECK(last_audio_format_);
103 0 : last_packet_sample_rate_hz_ = rtc::Optional<int>(ci->plfreq);
104 : }
105 : } // |crit_sect_| is released.
106 :
107 0 : if (neteq_->InsertPacket(rtp_header, incoming_payload, receive_timestamp) <
108 : 0) {
109 0 : LOG(LERROR) << "AcmReceiver::InsertPacket "
110 0 : << static_cast<int>(header->payloadType)
111 0 : << " Failed to insert packet";
112 0 : return -1;
113 : }
114 0 : return 0;
115 : }
116 :
117 0 : int AcmReceiver::GetAudio(int desired_freq_hz,
118 : AudioFrame* audio_frame,
119 : bool* muted) {
120 0 : RTC_DCHECK(muted);
121 : // Accessing members, take the lock.
122 0 : rtc::CritScope lock(&crit_sect_);
123 :
124 0 : if (neteq_->GetAudio(audio_frame, muted) != NetEq::kOK) {
125 0 : LOG(LERROR) << "AcmReceiver::GetAudio - NetEq Failed.";
126 0 : return -1;
127 : }
128 :
129 0 : const int current_sample_rate_hz = neteq_->last_output_sample_rate_hz();
130 :
131 : // Update if resampling is required.
132 : const bool need_resampling =
133 0 : (desired_freq_hz != -1) && (current_sample_rate_hz != desired_freq_hz);
134 :
135 0 : if (need_resampling && !resampled_last_output_frame_) {
136 : // Prime the resampler with the last frame.
137 : int16_t temp_output[AudioFrame::kMaxDataSizeSamples];
138 0 : int samples_per_channel_int = resampler_.Resample10Msec(
139 0 : last_audio_buffer_.get(), current_sample_rate_hz, desired_freq_hz,
140 : audio_frame->num_channels_, AudioFrame::kMaxDataSizeSamples,
141 0 : temp_output);
142 0 : if (samples_per_channel_int < 0) {
143 0 : LOG(LERROR) << "AcmReceiver::GetAudio - "
144 0 : "Resampling last_audio_buffer_ failed.";
145 0 : return -1;
146 : }
147 : }
148 :
149 : // TODO(henrik.lundin) Glitches in the output may appear if the output rate
150 : // from NetEq changes. See WebRTC issue 3923.
151 0 : if (need_resampling) {
152 0 : int samples_per_channel_int = resampler_.Resample10Msec(
153 : audio_frame->data_, current_sample_rate_hz, desired_freq_hz,
154 : audio_frame->num_channels_, AudioFrame::kMaxDataSizeSamples,
155 0 : audio_frame->data_);
156 0 : if (samples_per_channel_int < 0) {
157 0 : LOG(LERROR) << "AcmReceiver::GetAudio - Resampling audio_buffer_ failed.";
158 0 : return -1;
159 : }
160 0 : audio_frame->samples_per_channel_ =
161 0 : static_cast<size_t>(samples_per_channel_int);
162 0 : audio_frame->sample_rate_hz_ = desired_freq_hz;
163 0 : RTC_DCHECK_EQ(
164 : audio_frame->sample_rate_hz_,
165 0 : rtc::checked_cast<int>(audio_frame->samples_per_channel_ * 100));
166 0 : resampled_last_output_frame_ = true;
167 : } else {
168 0 : resampled_last_output_frame_ = false;
169 : // We might end up here ONLY if codec is changed.
170 : }
171 :
172 : // Store current audio in |last_audio_buffer_| for next time.
173 0 : memcpy(last_audio_buffer_.get(), audio_frame->data_,
174 0 : sizeof(int16_t) * audio_frame->samples_per_channel_ *
175 0 : audio_frame->num_channels_);
176 :
177 0 : call_stats_.DecodedByNetEq(audio_frame->speech_type_, *muted);
178 0 : return 0;
179 : }
180 :
181 0 : int32_t AcmReceiver::AddCodec(int acm_codec_id,
182 : uint8_t payload_type,
183 : size_t channels,
184 : int /*sample_rate_hz*/,
185 : AudioDecoder* audio_decoder,
186 : const std::string& name) {
187 : // TODO(kwiberg): This function has been ignoring the |sample_rate_hz|
188 : // argument for a long time. Arguably, it should simply be removed.
189 :
190 0 : const auto neteq_decoder = [acm_codec_id, channels]() -> NetEqDecoder {
191 0 : if (acm_codec_id == -1)
192 0 : return NetEqDecoder::kDecoderArbitrary; // External decoder.
193 : const rtc::Optional<RentACodec::CodecId> cid =
194 0 : RentACodec::CodecIdFromIndex(acm_codec_id);
195 0 : RTC_DCHECK(cid) << "Invalid codec index: " << acm_codec_id;
196 : const rtc::Optional<NetEqDecoder> ned =
197 0 : RentACodec::NetEqDecoderFromCodecId(*cid, channels);
198 0 : RTC_DCHECK(ned) << "Invalid codec ID: " << static_cast<int>(*cid);
199 0 : return *ned;
200 0 : }();
201 : const rtc::Optional<SdpAudioFormat> new_format =
202 0 : RentACodec::NetEqDecoderToSdpAudioFormat(neteq_decoder);
203 :
204 0 : rtc::CritScope lock(&crit_sect_);
205 :
206 0 : const auto old_format = neteq_->GetDecoderFormat(payload_type);
207 0 : if (old_format && new_format && *old_format == *new_format) {
208 : // Re-registering the same codec. Do nothing and return.
209 0 : return 0;
210 : }
211 :
212 0 : if (neteq_->RemovePayloadType(payload_type) != NetEq::kOK &&
213 0 : neteq_->LastError() != NetEq::kDecoderNotFound) {
214 0 : LOG(LERROR) << "Cannot remove payload " << static_cast<int>(payload_type);
215 0 : return -1;
216 : }
217 :
218 : int ret_val;
219 0 : if (!audio_decoder) {
220 0 : ret_val = neteq_->RegisterPayloadType(neteq_decoder, name, payload_type);
221 : } else {
222 0 : ret_val = neteq_->RegisterExternalDecoder(
223 0 : audio_decoder, neteq_decoder, name, payload_type);
224 : }
225 0 : if (ret_val != NetEq::kOK) {
226 0 : LOG(LERROR) << "AcmReceiver::AddCodec " << acm_codec_id
227 : << static_cast<int>(payload_type)
228 0 : << " channels: " << channels;
229 0 : return -1;
230 : }
231 0 : return 0;
232 : }
233 :
234 0 : bool AcmReceiver::AddCodec(int rtp_payload_type,
235 : const SdpAudioFormat& audio_format) {
236 0 : const auto old_format = neteq_->GetDecoderFormat(rtp_payload_type);
237 0 : if (old_format && *old_format == audio_format) {
238 : // Re-registering the same codec. Do nothing and return.
239 0 : return true;
240 : }
241 :
242 0 : if (neteq_->RemovePayloadType(rtp_payload_type) != NetEq::kOK &&
243 0 : neteq_->LastError() != NetEq::kDecoderNotFound) {
244 0 : LOG(LERROR) << "AcmReceiver::AddCodec: Could not remove existing decoder"
245 0 : " for payload type "
246 0 : << rtp_payload_type;
247 0 : return false;
248 : }
249 :
250 : const bool success =
251 0 : neteq_->RegisterPayloadType(rtp_payload_type, audio_format);
252 0 : if (!success) {
253 0 : LOG(LERROR) << "AcmReceiver::AddCodec failed for payload type "
254 0 : << rtp_payload_type << ", decoder format " << audio_format;
255 : }
256 0 : return success;
257 : }
258 :
259 0 : void AcmReceiver::FlushBuffers() {
260 0 : neteq_->FlushBuffers();
261 0 : }
262 :
263 0 : void AcmReceiver::RemoveAllCodecs() {
264 0 : rtc::CritScope lock(&crit_sect_);
265 0 : neteq_->RemoveAllPayloadTypes();
266 0 : last_audio_decoder_ = rtc::Optional<CodecInst>();
267 0 : last_audio_format_ = rtc::Optional<SdpAudioFormat>();
268 0 : last_packet_sample_rate_hz_ = rtc::Optional<int>();
269 0 : }
270 :
271 0 : int AcmReceiver::RemoveCodec(uint8_t payload_type) {
272 0 : rtc::CritScope lock(&crit_sect_);
273 0 : if (neteq_->RemovePayloadType(payload_type) != NetEq::kOK &&
274 0 : neteq_->LastError() != NetEq::kDecoderNotFound) {
275 0 : LOG(LERROR) << "AcmReceiver::RemoveCodec" << static_cast<int>(payload_type);
276 0 : return -1;
277 : }
278 0 : if (last_audio_decoder_ && payload_type == last_audio_decoder_->pltype) {
279 0 : last_audio_decoder_ = rtc::Optional<CodecInst>();
280 0 : last_audio_format_ = rtc::Optional<SdpAudioFormat>();
281 0 : last_packet_sample_rate_hz_ = rtc::Optional<int>();
282 : }
283 0 : return 0;
284 : }
285 :
286 0 : rtc::Optional<uint32_t> AcmReceiver::GetPlayoutTimestamp() {
287 0 : return neteq_->GetPlayoutTimestamp();
288 : }
289 :
290 0 : int AcmReceiver::FilteredCurrentDelayMs() const {
291 0 : return neteq_->FilteredCurrentDelayMs();
292 : }
293 :
294 0 : int AcmReceiver::LastAudioCodec(CodecInst* codec) const {
295 0 : rtc::CritScope lock(&crit_sect_);
296 0 : if (!last_audio_decoder_) {
297 0 : return -1;
298 : }
299 0 : *codec = *last_audio_decoder_;
300 0 : return 0;
301 : }
302 :
303 0 : rtc::Optional<SdpAudioFormat> AcmReceiver::LastAudioFormat() const {
304 0 : rtc::CritScope lock(&crit_sect_);
305 0 : return last_audio_format_;
306 : }
307 :
308 0 : void AcmReceiver::GetNetworkStatistics(NetworkStatistics* acm_stat) {
309 : NetEqNetworkStatistics neteq_stat;
310 : // NetEq function always returns zero, so we don't check the return value.
311 0 : neteq_->NetworkStatistics(&neteq_stat);
312 :
313 0 : acm_stat->currentBufferSize = neteq_stat.current_buffer_size_ms;
314 0 : acm_stat->preferredBufferSize = neteq_stat.preferred_buffer_size_ms;
315 0 : acm_stat->jitterPeaksFound = neteq_stat.jitter_peaks_found ? true : false;
316 0 : acm_stat->currentPacketLossRate = neteq_stat.packet_loss_rate;
317 0 : acm_stat->currentDiscardRate = neteq_stat.packet_discard_rate;
318 0 : acm_stat->currentExpandRate = neteq_stat.expand_rate;
319 0 : acm_stat->currentSpeechExpandRate = neteq_stat.speech_expand_rate;
320 0 : acm_stat->currentPreemptiveRate = neteq_stat.preemptive_rate;
321 0 : acm_stat->currentAccelerateRate = neteq_stat.accelerate_rate;
322 0 : acm_stat->currentSecondaryDecodedRate = neteq_stat.secondary_decoded_rate;
323 0 : acm_stat->clockDriftPPM = neteq_stat.clockdrift_ppm;
324 0 : acm_stat->addedSamples = neteq_stat.added_zero_samples;
325 0 : acm_stat->meanWaitingTimeMs = neteq_stat.mean_waiting_time_ms;
326 0 : acm_stat->medianWaitingTimeMs = neteq_stat.median_waiting_time_ms;
327 0 : acm_stat->minWaitingTimeMs = neteq_stat.min_waiting_time_ms;
328 0 : acm_stat->maxWaitingTimeMs = neteq_stat.max_waiting_time_ms;
329 0 : }
330 :
331 0 : int AcmReceiver::DecoderByPayloadType(uint8_t payload_type,
332 : CodecInst* codec) const {
333 0 : rtc::CritScope lock(&crit_sect_);
334 0 : const rtc::Optional<CodecInst> ci = neteq_->GetDecoder(payload_type);
335 0 : if (ci) {
336 0 : *codec = *ci;
337 0 : return 0;
338 : } else {
339 0 : LOG(LERROR) << "AcmReceiver::DecoderByPayloadType "
340 0 : << static_cast<int>(payload_type);
341 0 : return -1;
342 : }
343 : }
344 :
345 0 : int AcmReceiver::EnableNack(size_t max_nack_list_size) {
346 0 : neteq_->EnableNack(max_nack_list_size);
347 0 : return 0;
348 : }
349 :
350 0 : void AcmReceiver::DisableNack() {
351 0 : neteq_->DisableNack();
352 0 : }
353 :
354 0 : std::vector<uint16_t> AcmReceiver::GetNackList(
355 : int64_t round_trip_time_ms) const {
356 0 : return neteq_->GetNackList(round_trip_time_ms);
357 : }
358 :
359 0 : void AcmReceiver::ResetInitialDelay() {
360 0 : neteq_->SetMinimumDelay(0);
361 : // TODO(turajs): Should NetEq Buffer be flushed?
362 0 : }
363 :
364 0 : const rtc::Optional<CodecInst> AcmReceiver::RtpHeaderToDecoder(
365 : const RTPHeader& rtp_header,
366 : uint8_t first_payload_byte) const {
367 : const rtc::Optional<CodecInst> ci =
368 0 : neteq_->GetDecoder(rtp_header.payloadType);
369 0 : if (ci && STR_CASE_CMP(ci->plname, "red") == 0) {
370 : // This is a RED packet. Get the payload of the audio codec.
371 0 : return neteq_->GetDecoder(first_payload_byte & 0x7f);
372 : } else {
373 0 : return ci;
374 : }
375 : }
376 :
377 0 : uint32_t AcmReceiver::NowInTimestamp(int decoder_sampling_rate) const {
378 : // Down-cast the time to (32-6)-bit since we only care about
379 : // the least significant bits. (32-6) bits cover 2^(32-6) = 67108864 ms.
380 : // We masked 6 most significant bits of 32-bit so there is no overflow in
381 : // the conversion from milliseconds to timestamp.
382 : const uint32_t now_in_ms = static_cast<uint32_t>(
383 0 : clock_->TimeInMilliseconds() & 0x03ffffff);
384 : return static_cast<uint32_t>(
385 0 : (decoder_sampling_rate / 1000) * now_in_ms);
386 : }
387 :
388 0 : void AcmReceiver::GetDecodingCallStatistics(
389 : AudioDecodingCallStats* stats) const {
390 0 : rtc::CritScope lock(&crit_sect_);
391 0 : *stats = call_stats_.GetDecodingStatistics();
392 0 : }
393 :
394 : } // namespace acm2
395 :
396 : } // namespace webrtc
|
