Line data Source code
1 : /*
2 : * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 :
11 : #include "webrtc/audio/audio_send_stream.h"
12 :
13 : #include <string>
14 :
15 : #include "webrtc/audio/audio_state.h"
16 : #include "webrtc/audio/conversion.h"
17 : #include "webrtc/audio/scoped_voe_interface.h"
18 : #include "webrtc/base/checks.h"
19 : #include "webrtc/base/event.h"
20 : #include "webrtc/base/logging.h"
21 : #include "webrtc/base/task_queue.h"
22 : #include "webrtc/modules/congestion_controller/include/congestion_controller.h"
23 : #include "webrtc/modules/pacing/paced_sender.h"
24 : #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h"
25 : #include "webrtc/voice_engine/channel_proxy.h"
26 : #include "webrtc/voice_engine/include/voe_audio_processing.h"
27 : #include "webrtc/voice_engine/include/voe_codec.h"
28 : #include "webrtc/voice_engine/include/voe_rtp_rtcp.h"
29 : #include "webrtc/voice_engine/include/voe_volume_control.h"
30 : #include "webrtc/voice_engine/voice_engine_impl.h"
31 :
32 : namespace webrtc {
33 :
34 : namespace {
35 :
36 : constexpr char kOpusCodecName[] = "opus";
37 :
38 0 : bool IsCodec(const webrtc::CodecInst& codec, const char* ref_name) {
39 0 : return (_stricmp(codec.plname, ref_name) == 0);
40 : }
41 : } // namespace
42 :
43 : namespace internal {
44 0 : AudioSendStream::AudioSendStream(
45 : const webrtc::AudioSendStream::Config& config,
46 : const rtc::scoped_refptr<webrtc::AudioState>& audio_state,
47 : rtc::TaskQueue* worker_queue,
48 : PacketRouter* packet_router,
49 : CongestionController* congestion_controller,
50 : BitrateAllocator* bitrate_allocator,
51 : RtcEventLog* event_log,
52 0 : RtcpRttStats* rtcp_rtt_stats)
53 : : worker_queue_(worker_queue),
54 : config_(config),
55 : audio_state_(audio_state),
56 : bitrate_allocator_(bitrate_allocator),
57 0 : congestion_controller_(congestion_controller) {
58 0 : LOG(LS_INFO) << "AudioSendStream: " << config_.ToString();
59 0 : RTC_DCHECK_NE(config_.voe_channel_id, -1);
60 0 : RTC_DCHECK(audio_state_.get());
61 0 : RTC_DCHECK(congestion_controller);
62 :
63 0 : VoiceEngineImpl* voe_impl = static_cast<VoiceEngineImpl*>(voice_engine());
64 0 : channel_proxy_ = voe_impl->GetChannelProxy(config_.voe_channel_id);
65 0 : channel_proxy_->SetRtcEventLog(event_log);
66 0 : channel_proxy_->SetRtcpRttStats(rtcp_rtt_stats);
67 0 : channel_proxy_->RegisterSenderCongestionControlObjects(
68 0 : congestion_controller->pacer(),
69 0 : congestion_controller->GetTransportFeedbackObserver(), packet_router);
70 0 : channel_proxy_->SetRTCPStatus(true);
71 0 : channel_proxy_->SetLocalSSRC(config.rtp.ssrc);
72 0 : channel_proxy_->SetRTCP_CNAME(config.rtp.c_name);
73 : // TODO(solenberg): Config NACK history window (which is a packet count),
74 : // using the actual packet size for the configured codec.
75 0 : channel_proxy_->SetNACKStatus(config_.rtp.nack.rtp_history_ms != 0,
76 0 : config_.rtp.nack.rtp_history_ms / 20);
77 :
78 0 : channel_proxy_->RegisterExternalTransport(config.send_transport);
79 :
80 0 : for (const auto& extension : config.rtp.extensions) {
81 0 : if (extension.uri == RtpExtension::kAudioLevelUri) {
82 0 : channel_proxy_->SetSendAudioLevelIndicationStatus(true, extension.id);
83 0 : } else if (extension.uri == RtpExtension::kTransportSequenceNumberUri) {
84 0 : channel_proxy_->EnableSendTransportSequenceNumber(extension.id);
85 : } else {
86 0 : RTC_NOTREACHED() << "Registering unsupported RTP extension.";
87 : }
88 : }
89 0 : if (!SetupSendCodec()) {
90 0 : LOG(LS_ERROR) << "Failed to set up send codec state.";
91 : }
92 0 : }
93 :
94 0 : AudioSendStream::~AudioSendStream() {
95 0 : RTC_DCHECK(thread_checker_.CalledOnValidThread());
96 0 : LOG(LS_INFO) << "~AudioSendStream: " << config_.ToString();
97 0 : channel_proxy_->DeRegisterExternalTransport();
98 0 : channel_proxy_->ResetCongestionControlObjects();
99 0 : channel_proxy_->SetRtcEventLog(nullptr);
100 0 : channel_proxy_->SetRtcpRttStats(nullptr);
101 0 : }
102 :
103 0 : void AudioSendStream::Start() {
104 0 : RTC_DCHECK(thread_checker_.CalledOnValidThread());
105 0 : if (config_.min_bitrate_bps != -1 && config_.max_bitrate_bps != -1) {
106 0 : RTC_DCHECK_GE(config_.max_bitrate_bps, config_.min_bitrate_bps);
107 0 : rtc::Event thread_sync_event(false /* manual_reset */, false);
108 0 : worker_queue_->PostTask([this, &thread_sync_event] {
109 0 : bitrate_allocator_->AddObserver(this, config_.min_bitrate_bps,
110 0 : config_.max_bitrate_bps, 0, true);
111 0 : thread_sync_event.Set();
112 0 : });
113 0 : thread_sync_event.Wait(rtc::Event::kForever);
114 : }
115 :
116 0 : ScopedVoEInterface<VoEBase> base(voice_engine());
117 0 : int error = base->StartSend(config_.voe_channel_id);
118 0 : if (error != 0) {
119 0 : LOG(LS_ERROR) << "AudioSendStream::Start failed with error: " << error;
120 : }
121 0 : }
122 :
123 0 : void AudioSendStream::Stop() {
124 0 : RTC_DCHECK(thread_checker_.CalledOnValidThread());
125 0 : rtc::Event thread_sync_event(false /* manual_reset */, false);
126 0 : worker_queue_->PostTask([this, &thread_sync_event] {
127 0 : bitrate_allocator_->RemoveObserver(this);
128 0 : thread_sync_event.Set();
129 0 : });
130 0 : thread_sync_event.Wait(rtc::Event::kForever);
131 :
132 0 : ScopedVoEInterface<VoEBase> base(voice_engine());
133 0 : int error = base->StopSend(config_.voe_channel_id);
134 0 : if (error != 0) {
135 0 : LOG(LS_ERROR) << "AudioSendStream::Stop failed with error: " << error;
136 : }
137 0 : }
138 :
139 0 : bool AudioSendStream::SendTelephoneEvent(int payload_type,
140 : int payload_frequency, int event,
141 : int duration_ms) {
142 0 : RTC_DCHECK(thread_checker_.CalledOnValidThread());
143 0 : return channel_proxy_->SetSendTelephoneEventPayloadType(payload_type,
144 0 : payload_frequency) &&
145 0 : channel_proxy_->SendTelephoneEventOutband(event, duration_ms);
146 : }
147 :
148 0 : void AudioSendStream::SetMuted(bool muted) {
149 0 : RTC_DCHECK(thread_checker_.CalledOnValidThread());
150 0 : channel_proxy_->SetInputMute(muted);
151 0 : }
152 :
153 0 : webrtc::AudioSendStream::Stats AudioSendStream::GetStats() const {
154 0 : RTC_DCHECK(thread_checker_.CalledOnValidThread());
155 0 : webrtc::AudioSendStream::Stats stats;
156 0 : stats.local_ssrc = config_.rtp.ssrc;
157 0 : ScopedVoEInterface<VoEAudioProcessing> processing(voice_engine());
158 0 : ScopedVoEInterface<VoECodec> codec(voice_engine());
159 0 : ScopedVoEInterface<VoEVolumeControl> volume(voice_engine());
160 :
161 0 : webrtc::CallStatistics call_stats = channel_proxy_->GetRTCPStatistics();
162 0 : stats.bytes_sent = call_stats.bytesSent;
163 0 : stats.packets_sent = call_stats.packetsSent;
164 : // RTT isn't known until a RTCP report is received. Until then, VoiceEngine
165 : // returns 0 to indicate an error value.
166 0 : if (call_stats.rttMs > 0) {
167 0 : stats.rtt_ms = call_stats.rttMs;
168 : }
169 : // TODO(solenberg): [was ajm]: Re-enable this metric once we have a reliable
170 : // implementation.
171 0 : stats.aec_quality_min = -1;
172 :
173 0 : webrtc::CodecInst codec_inst = {0};
174 0 : if (codec->GetSendCodec(config_.voe_channel_id, codec_inst) != -1) {
175 0 : RTC_DCHECK_NE(codec_inst.pltype, -1);
176 0 : stats.codec_name = codec_inst.plname;
177 0 : stats.codec_payload_type = rtc::Optional<int>(codec_inst.pltype);
178 :
179 : // Get data from the last remote RTCP report.
180 0 : for (const auto& block : channel_proxy_->GetRemoteRTCPReportBlocks()) {
181 : // Lookup report for send ssrc only.
182 0 : if (block.source_SSRC == stats.local_ssrc) {
183 0 : stats.packets_lost = block.cumulative_num_packets_lost;
184 0 : stats.fraction_lost = Q8ToFloat(block.fraction_lost);
185 0 : stats.ext_seqnum = block.extended_highest_sequence_number;
186 : // Convert samples to milliseconds.
187 0 : if (codec_inst.plfreq / 1000 > 0) {
188 0 : stats.jitter_ms =
189 0 : block.interarrival_jitter / (codec_inst.plfreq / 1000);
190 : }
191 0 : break;
192 : }
193 : }
194 : }
195 :
196 : // Local speech level.
197 : {
198 0 : unsigned int level = 0;
199 0 : int error = volume->GetSpeechInputLevelFullRange(level);
200 0 : RTC_DCHECK_EQ(0, error);
201 0 : stats.audio_level = static_cast<int32_t>(level);
202 : }
203 :
204 0 : ScopedVoEInterface<VoEBase> base(voice_engine());
205 0 : RTC_DCHECK(base->audio_processing());
206 0 : auto audio_processing_stats = base->audio_processing()->GetStatistics();
207 0 : stats.echo_delay_median_ms = audio_processing_stats.delay_median;
208 0 : stats.echo_delay_std_ms = audio_processing_stats.delay_standard_deviation;
209 0 : stats.echo_return_loss = audio_processing_stats.echo_return_loss.instant();
210 0 : stats.echo_return_loss_enhancement =
211 0 : audio_processing_stats.echo_return_loss_enhancement.instant();
212 0 : stats.residual_echo_likelihood =
213 0 : audio_processing_stats.residual_echo_likelihood;
214 0 : stats.residual_echo_likelihood_recent_max =
215 0 : audio_processing_stats.residual_echo_likelihood_recent_max;
216 :
217 : internal::AudioState* audio_state =
218 0 : static_cast<internal::AudioState*>(audio_state_.get());
219 0 : stats.typing_noise_detected = audio_state->typing_noise_detected();
220 :
221 0 : return stats;
222 : }
223 :
224 0 : void AudioSendStream::SignalNetworkState(NetworkState state) {
225 0 : RTC_DCHECK(thread_checker_.CalledOnValidThread());
226 0 : }
227 :
228 0 : bool AudioSendStream::DeliverRtcp(const uint8_t* packet, size_t length) {
229 : // TODO(solenberg): Tests call this function on a network thread, libjingle
230 : // calls on the worker thread. We should move towards always using a network
231 : // thread. Then this check can be enabled.
232 : // RTC_DCHECK(!thread_checker_.CalledOnValidThread());
233 0 : return channel_proxy_->ReceivedRTCPPacket(packet, length);
234 : }
235 :
236 0 : uint32_t AudioSendStream::OnBitrateUpdated(uint32_t bitrate_bps,
237 : uint8_t fraction_loss,
238 : int64_t rtt,
239 : int64_t probing_interval_ms) {
240 0 : RTC_DCHECK_GE(bitrate_bps,
241 0 : static_cast<uint32_t>(config_.min_bitrate_bps));
242 : // The bitrate allocator might allocate an higher than max configured bitrate
243 : // if there is room, to allow for, as example, extra FEC. Ignore that for now.
244 0 : const uint32_t max_bitrate_bps = config_.max_bitrate_bps;
245 0 : if (bitrate_bps > max_bitrate_bps)
246 0 : bitrate_bps = max_bitrate_bps;
247 :
248 0 : channel_proxy_->SetBitrate(bitrate_bps, probing_interval_ms);
249 :
250 : // The amount of audio protection is not exposed by the encoder, hence
251 : // always returning 0.
252 0 : return 0;
253 : }
254 :
255 0 : const webrtc::AudioSendStream::Config& AudioSendStream::config() const {
256 0 : RTC_DCHECK(thread_checker_.CalledOnValidThread());
257 0 : return config_;
258 : }
259 :
260 0 : void AudioSendStream::SetTransportOverhead(int transport_overhead_per_packet) {
261 0 : RTC_DCHECK(thread_checker_.CalledOnValidThread());
262 0 : congestion_controller_->SetTransportOverhead(transport_overhead_per_packet);
263 0 : channel_proxy_->SetTransportOverhead(transport_overhead_per_packet);
264 0 : }
265 :
266 0 : VoiceEngine* AudioSendStream::voice_engine() const {
267 : internal::AudioState* audio_state =
268 0 : static_cast<internal::AudioState*>(audio_state_.get());
269 0 : VoiceEngine* voice_engine = audio_state->voice_engine();
270 0 : RTC_DCHECK(voice_engine);
271 0 : return voice_engine;
272 : }
273 :
274 : // Apply current codec settings to a single voe::Channel used for sending.
275 0 : bool AudioSendStream::SetupSendCodec() {
276 0 : ScopedVoEInterface<VoEBase> base(voice_engine());
277 0 : ScopedVoEInterface<VoECodec> codec(voice_engine());
278 :
279 0 : const int channel = config_.voe_channel_id;
280 :
281 : // Disable VAD and FEC unless we know the other side wants them.
282 0 : codec->SetVADStatus(channel, false);
283 0 : codec->SetFECStatus(channel, false);
284 :
285 : // We disable audio network adaptor here. This will on one hand make sure that
286 : // audio network adaptor is disabled by default, and on the other allow audio
287 : // network adaptor to be reconfigured, since SetReceiverFrameLengthRange can
288 : // be only called when audio network adaptor is disabled.
289 0 : channel_proxy_->DisableAudioNetworkAdaptor();
290 :
291 0 : const auto& send_codec_spec = config_.send_codec_spec;
292 :
293 : // We set the codec first, since the below extra configuration is only applied
294 : // to the "current" codec.
295 :
296 : // If codec is already configured, we do not it again.
297 : // TODO(minyue): check if this check is really needed, or can we move it into
298 : // |codec->SetSendCodec|.
299 0 : webrtc::CodecInst current_codec = {0};
300 0 : if (codec->GetSendCodec(channel, current_codec) != 0 ||
301 0 : (send_codec_spec.codec_inst != current_codec)) {
302 0 : if (codec->SetSendCodec(channel, send_codec_spec.codec_inst) == -1) {
303 0 : LOG(LS_WARNING) << "SetSendCodec() failed: " << base->LastError();
304 0 : return false;
305 : }
306 : }
307 :
308 : // Codec internal FEC. Treat any failure as fatal internal error.
309 0 : if (send_codec_spec.enable_codec_fec) {
310 0 : if (codec->SetFECStatus(channel, true) != 0) {
311 0 : LOG(LS_WARNING) << "SetFECStatus() failed: " << base->LastError();
312 0 : return false;
313 : }
314 : }
315 :
316 : // DTX and maxplaybackrate are only set if current codec is Opus.
317 0 : if (IsCodec(send_codec_spec.codec_inst, kOpusCodecName)) {
318 0 : if (codec->SetOpusDtx(channel, send_codec_spec.enable_opus_dtx) != 0) {
319 0 : LOG(LS_WARNING) << "SetOpusDtx() failed: " << base->LastError();
320 0 : return false;
321 : }
322 :
323 : // If opus_max_playback_rate <= 0, the default maximum playback rate
324 : // (48 kHz) will be used.
325 0 : if (send_codec_spec.opus_max_playback_rate > 0) {
326 0 : if (codec->SetOpusMaxPlaybackRate(
327 0 : channel, send_codec_spec.opus_max_playback_rate) != 0) {
328 0 : LOG(LS_WARNING) << "SetOpusMaxPlaybackRate() failed: "
329 0 : << base->LastError();
330 0 : return false;
331 : }
332 : }
333 :
334 0 : if (config_.audio_network_adaptor_config) {
335 : // Audio network adaptor is only allowed for Opus currently.
336 : // |SetReceiverFrameLengthRange| needs to be called before
337 : // |EnableAudioNetworkAdaptor|.
338 0 : channel_proxy_->SetReceiverFrameLengthRange(send_codec_spec.min_ptime_ms,
339 0 : send_codec_spec.max_ptime_ms);
340 0 : channel_proxy_->EnableAudioNetworkAdaptor(
341 0 : *config_.audio_network_adaptor_config);
342 0 : LOG(LS_INFO) << "Audio network adaptor enabled on SSRC "
343 0 : << config_.rtp.ssrc;
344 : }
345 : }
346 :
347 : // Set the CN payloadtype and the VAD status.
348 0 : if (send_codec_spec.cng_payload_type != -1) {
349 : // The CN payload type for 8000 Hz clockrate is fixed at 13.
350 0 : if (send_codec_spec.cng_plfreq != 8000) {
351 : webrtc::PayloadFrequencies cn_freq;
352 0 : switch (send_codec_spec.cng_plfreq) {
353 : case 16000:
354 0 : cn_freq = webrtc::kFreq16000Hz;
355 0 : break;
356 : case 32000:
357 0 : cn_freq = webrtc::kFreq32000Hz;
358 0 : break;
359 : default:
360 0 : RTC_NOTREACHED();
361 0 : return false;
362 : }
363 0 : if (codec->SetSendCNPayloadType(channel, send_codec_spec.cng_payload_type,
364 0 : cn_freq) != 0) {
365 0 : LOG(LS_WARNING) << "SetSendCNPayloadType() failed: "
366 0 : << base->LastError();
367 : // TODO(ajm): This failure condition will be removed from VoE.
368 : // Restore the return here when we update to a new enough webrtc.
369 : //
370 : // Not returning false because the SetSendCNPayloadType will fail if
371 : // the channel is already sending.
372 : // This can happen if the remote description is applied twice, for
373 : // example in the case of ROAP on top of JSEP, where both side will
374 : // send the offer.
375 : }
376 : }
377 :
378 : // Only turn on VAD if we have a CN payload type that matches the
379 : // clockrate for the codec we are going to use.
380 0 : if (send_codec_spec.cng_plfreq == send_codec_spec.codec_inst.plfreq &&
381 0 : send_codec_spec.codec_inst.channels == 1) {
382 : // TODO(minyue): If CN frequency == 48000 Hz is allowed, consider the
383 : // interaction between VAD and Opus FEC.
384 0 : if (codec->SetVADStatus(channel, true) != 0) {
385 0 : LOG(LS_WARNING) << "SetVADStatus() failed: " << base->LastError();
386 0 : return false;
387 : }
388 : }
389 : }
390 0 : return true;
391 : }
392 :
393 : } // namespace internal
394 : } // namespace webrtc
|
