Line data Source code
1 : /*
2 : * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 :
11 : #include "webrtc/video/video_receive_stream.h"
12 :
13 : #include <stdlib.h>
14 :
15 : #include <set>
16 : #include <string>
17 : #include <utility>
18 :
19 : #include "webrtc/base/checks.h"
20 : #include "webrtc/base/logging.h"
21 : #include "webrtc/base/optional.h"
22 : #include "webrtc/common_video/h264/profile_level_id.h"
23 : #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
24 : #include "webrtc/modules/congestion_controller/include/congestion_controller.h"
25 : #include "webrtc/modules/utility/include/process_thread.h"
26 : #include "webrtc/modules/video_coding/frame_object.h"
27 : #include "webrtc/modules/video_coding/include/video_coding.h"
28 : #include "webrtc/modules/video_coding/jitter_estimator.h"
29 : #include "webrtc/modules/video_coding/timing.h"
30 : #include "webrtc/modules/video_coding/utility/ivf_file_writer.h"
31 : #include "webrtc/system_wrappers/include/clock.h"
32 : #include "webrtc/system_wrappers/include/field_trial.h"
33 : #include "webrtc/video/call_stats.h"
34 : #include "webrtc/video/receive_statistics_proxy.h"
35 : #include "webrtc/video_receive_stream.h"
36 : #include "webrtc/voice_engine/include/voe_video_sync.h"
37 :
38 : namespace webrtc {
39 :
40 0 : static bool UseSendSideBwe(const VideoReceiveStream::Config& config) {
41 0 : if (!config.rtp.transport_cc)
42 0 : return false;
43 0 : for (const auto& extension : config.rtp.extensions) {
44 0 : if (extension.uri == RtpExtension::kTransportSequenceNumberUri)
45 0 : return true;
46 : }
47 0 : return false;
48 : }
49 :
50 0 : std::string VideoReceiveStream::Decoder::ToString() const {
51 0 : std::stringstream ss;
52 0 : ss << "{decoder: " << (decoder ? "(VideoDecoder)" : "nullptr");
53 0 : ss << ", payload_type: " << payload_type;
54 0 : ss << ", payload_name: " << payload_name;
55 0 : ss << ", codec_params: {";
56 0 : for (const auto& it : codec_params)
57 0 : ss << it.first << ": " << it.second;
58 0 : ss << '}';
59 0 : ss << '}';
60 :
61 0 : return ss.str();
62 : }
63 :
64 0 : std::string VideoReceiveStream::Config::ToString() const {
65 0 : std::stringstream ss;
66 0 : ss << "{decoders: [";
67 0 : for (size_t i = 0; i < decoders.size(); ++i) {
68 0 : ss << decoders[i].ToString();
69 0 : if (i != decoders.size() - 1)
70 0 : ss << ", ";
71 : }
72 0 : ss << ']';
73 0 : ss << ", rtp: " << rtp.ToString();
74 0 : ss << ", renderer: " << (renderer ? "(renderer)" : "nullptr");
75 0 : ss << ", render_delay_ms: " << render_delay_ms;
76 0 : if (!sync_group.empty())
77 0 : ss << ", sync_group: " << sync_group;
78 : ss << ", pre_decode_callback: "
79 0 : << (pre_decode_callback ? "(EncodedFrameObserver)" : "nullptr");
80 : ss << ", pre_render_callback: "
81 0 : << (pre_render_callback ? "(I420FrameCallback)" : "nullptr");
82 0 : ss << ", target_delay_ms: " << target_delay_ms;
83 0 : ss << '}';
84 :
85 0 : return ss.str();
86 : }
87 :
88 0 : std::string VideoReceiveStream::Config::Rtp::ToString() const {
89 0 : std::stringstream ss;
90 0 : ss << "{remote_ssrc: " << remote_ssrc;
91 0 : ss << ", local_ssrc: " << local_ssrc;
92 : ss << ", rtcp_mode: "
93 0 : << (rtcp_mode == RtcpMode::kCompound ? "RtcpMode::kCompound"
94 0 : : "RtcpMode::kReducedSize");
95 0 : ss << ", rtcp_xr: ";
96 : ss << "{receiver_reference_time_report: "
97 0 : << (rtcp_xr.receiver_reference_time_report ? "on" : "off");
98 0 : ss << '}';
99 0 : ss << ", remb: " << (remb ? "on" : "off");
100 0 : ss << ", transport_cc: " << (transport_cc ? "on" : "off");
101 0 : ss << ", nack: {rtp_history_ms: " << nack.rtp_history_ms << '}';
102 0 : ss << ", ulpfec: " << ulpfec.ToString();
103 0 : ss << ", rtx: {";
104 0 : for (auto& kv : rtx) {
105 0 : ss << kv.first << " -> ";
106 0 : ss << "{ssrc: " << kv.second.ssrc;
107 0 : ss << ", payload_type: " << kv.second.payload_type;
108 0 : ss << '}';
109 : }
110 0 : ss << '}';
111 0 : ss << ", extensions: [";
112 0 : for (size_t i = 0; i < extensions.size(); ++i) {
113 0 : ss << extensions[i].ToString();
114 0 : if (i != extensions.size() - 1)
115 0 : ss << ", ";
116 : }
117 0 : ss << ']';
118 0 : ss << '}';
119 0 : return ss.str();
120 : }
121 :
122 0 : std::string VideoReceiveStream::Stats::ToString(int64_t time_ms) const {
123 0 : std::stringstream ss;
124 0 : ss << "VideoReceiveStream stats: " << time_ms << ", {ssrc: " << ssrc << ", ";
125 0 : ss << "total_bps: " << total_bitrate_bps << ", ";
126 0 : ss << "width: " << width << ", ";
127 0 : ss << "height: " << height << ", ";
128 0 : ss << "key: " << frame_counts.key_frames << ", ";
129 0 : ss << "delta: " << frame_counts.delta_frames << ", ";
130 0 : ss << "network_fps: " << network_frame_rate << ", ";
131 0 : ss << "decode_fps: " << decode_frame_rate << ", ";
132 0 : ss << "render_fps: " << render_frame_rate << ", ";
133 0 : ss << "decode_ms: " << decode_ms << ", ";
134 0 : ss << "max_decode_ms: " << max_decode_ms << ", ";
135 0 : ss << "cur_delay_ms: " << current_delay_ms << ", ";
136 0 : ss << "targ_delay_ms: " << target_delay_ms << ", ";
137 0 : ss << "jb_delay_ms: " << jitter_buffer_ms << ", ";
138 0 : ss << "min_playout_delay_ms: " << min_playout_delay_ms << ", ";
139 0 : ss << "discarded: " << discarded_packets << ", ";
140 0 : ss << "sync_offset_ms: " << sync_offset_ms << ", ";
141 0 : ss << "cum_loss: " << rtcp_stats.cumulative_lost << ", ";
142 0 : ss << "max_ext_seq: " << rtcp_stats.extended_max_sequence_number << ", ";
143 0 : ss << "nack: " << rtcp_packet_type_counts.nack_packets << ", ";
144 0 : ss << "fir: " << rtcp_packet_type_counts.fir_packets << ", ";
145 0 : ss << "pli: " << rtcp_packet_type_counts.pli_packets;
146 0 : ss << '}';
147 0 : return ss.str();
148 : }
149 :
150 : namespace {
151 0 : VideoCodec CreateDecoderVideoCodec(const VideoReceiveStream::Decoder& decoder) {
152 0 : VideoCodec codec;
153 0 : memset(&codec, 0, sizeof(codec));
154 :
155 0 : codec.plType = decoder.payload_type;
156 0 : strncpy(codec.plName, decoder.payload_name.c_str(), sizeof(codec.plName));
157 0 : if (decoder.payload_name == "VP8") {
158 0 : codec.codecType = kVideoCodecVP8;
159 0 : } else if (decoder.payload_name == "VP9") {
160 0 : codec.codecType = kVideoCodecVP9;
161 0 : } else if (decoder.payload_name == "H264") {
162 0 : codec.codecType = kVideoCodecH264;
163 : } else {
164 0 : codec.codecType = kVideoCodecGeneric;
165 : }
166 :
167 0 : if (codec.codecType == kVideoCodecVP8) {
168 0 : *(codec.VP8()) = VideoEncoder::GetDefaultVp8Settings();
169 0 : } else if (codec.codecType == kVideoCodecVP9) {
170 0 : *(codec.VP9()) = VideoEncoder::GetDefaultVp9Settings();
171 0 : } else if (codec.codecType == kVideoCodecH264) {
172 0 : *(codec.H264()) = VideoEncoder::GetDefaultH264Settings();
173 0 : codec.H264()->profile =
174 0 : H264::ParseSdpProfileLevelId(decoder.codec_params)->profile;
175 : }
176 :
177 0 : codec.width = 320;
178 0 : codec.height = 180;
179 0 : const int kDefaultStartBitrate = 300;
180 0 : codec.startBitrate = codec.minBitrate = codec.maxBitrate =
181 : kDefaultStartBitrate;
182 :
183 0 : return codec;
184 : }
185 : } // namespace
186 :
187 : namespace internal {
188 :
189 0 : VideoReceiveStream::VideoReceiveStream(
190 : int num_cpu_cores,
191 : CongestionController* congestion_controller,
192 : PacketRouter* packet_router,
193 : VideoReceiveStream::Config config,
194 : webrtc::VoiceEngine* voice_engine,
195 : ProcessThread* process_thread,
196 : CallStats* call_stats,
197 0 : VieRemb* remb)
198 : : transport_adapter_(config.rtcp_send_transport),
199 0 : config_(std::move(config)),
200 : num_cpu_cores_(num_cpu_cores),
201 : process_thread_(process_thread),
202 0 : clock_(Clock::GetRealTimeClock()),
203 : decode_thread_(DecodeThreadFunction, this, "DecodingThread"),
204 : congestion_controller_(congestion_controller),
205 : call_stats_(call_stats),
206 0 : timing_(new VCMTiming(clock_)),
207 0 : video_receiver_(clock_, nullptr, this, timing_.get(), this, this),
208 0 : stats_proxy_(&config_, clock_),
209 : rtp_stream_receiver_(
210 : &video_receiver_,
211 0 : congestion_controller_->GetRemoteBitrateEstimator(
212 0 : UseSendSideBwe(config_)),
213 : &transport_adapter_,
214 0 : call_stats_->rtcp_rtt_stats(),
215 0 : congestion_controller_->pacer(),
216 : packet_router,
217 : remb,
218 : &config_,
219 : &stats_proxy_,
220 0 : process_thread_,
221 0 : congestion_controller_->GetRetransmissionRateLimiter(),
222 : this, // NackSender
223 : this, // KeyFrameRequestSender
224 : this, // OnCompleteFrameCallback
225 : timing_.get()),
226 : rtp_stream_sync_(&video_receiver_, &rtp_stream_receiver_),
227 : jitter_buffer_experiment_(
228 0 : field_trial::FindFullName("WebRTC-NewVideoJitterBuffer") ==
229 0 : "Enabled") {
230 0 : LOG(LS_INFO) << "VideoReceiveStream: " << config_.ToString();
231 :
232 0 : RTC_DCHECK(process_thread_);
233 0 : RTC_DCHECK(congestion_controller_);
234 0 : RTC_DCHECK(call_stats_);
235 :
236 0 : RTC_DCHECK(!config_.decoders.empty());
237 0 : std::set<int> decoder_payload_types;
238 0 : for (const Decoder& decoder : config_.decoders) {
239 0 : RTC_CHECK(decoder.decoder);
240 0 : RTC_CHECK(decoder_payload_types.find(decoder.payload_type) ==
241 : decoder_payload_types.end())
242 0 : << "Duplicate payload type (" << decoder.payload_type
243 0 : << ") for different decoders.";
244 0 : decoder_payload_types.insert(decoder.payload_type);
245 : }
246 :
247 0 : video_receiver_.SetRenderDelay(config.render_delay_ms);
248 :
249 0 : if (jitter_buffer_experiment_) {
250 0 : jitter_estimator_.reset(new VCMJitterEstimator(clock_));
251 0 : frame_buffer_.reset(new video_coding::FrameBuffer(
252 0 : clock_, jitter_estimator_.get(), timing_.get()));
253 : }
254 :
255 0 : process_thread_->RegisterModule(&video_receiver_);
256 0 : process_thread_->RegisterModule(&rtp_stream_sync_);
257 0 : }
258 :
259 0 : VideoReceiveStream::~VideoReceiveStream() {
260 0 : LOG(LS_INFO) << "~VideoReceiveStream: " << config_.ToString();
261 0 : Stop();
262 :
263 0 : process_thread_->DeRegisterModule(&rtp_stream_sync_);
264 0 : process_thread_->DeRegisterModule(&video_receiver_);
265 :
266 0 : congestion_controller_->GetRemoteBitrateEstimator(UseSendSideBwe(config_))
267 0 : ->RemoveStream(rtp_stream_receiver_.GetRemoteSsrc());
268 0 : }
269 :
270 0 : void VideoReceiveStream::SignalNetworkState(NetworkState state) {
271 0 : rtp_stream_receiver_.SignalNetworkState(state);
272 0 : }
273 :
274 :
275 0 : bool VideoReceiveStream::DeliverRtcp(const uint8_t* packet, size_t length) {
276 0 : return rtp_stream_receiver_.DeliverRtcp(packet, length);
277 : }
278 :
279 0 : bool VideoReceiveStream::DeliverRtp(const uint8_t* packet,
280 : size_t length,
281 : const PacketTime& packet_time) {
282 0 : return rtp_stream_receiver_.DeliverRtp(packet, length, packet_time);
283 : }
284 :
285 0 : bool VideoReceiveStream::OnRecoveredPacket(const uint8_t* packet,
286 : size_t length) {
287 0 : return rtp_stream_receiver_.OnRecoveredPacket(packet, length);
288 : }
289 :
290 0 : void VideoReceiveStream::Start() {
291 0 : if (decode_thread_.IsRunning())
292 0 : return;
293 0 : video_receiver_.Reset();
294 0 : if (jitter_buffer_experiment_) {
295 0 : frame_buffer_->Start();
296 0 : call_stats_->RegisterStatsObserver(&rtp_stream_receiver_);
297 :
298 0 : if (rtp_stream_receiver_.IsRetransmissionsEnabled() &&
299 0 : rtp_stream_receiver_.IsUlpfecEnabled()) {
300 0 : frame_buffer_->SetProtectionMode(kProtectionNackFEC);
301 : }
302 : }
303 0 : transport_adapter_.Enable();
304 0 : rtc::VideoSinkInterface<VideoFrame>* renderer = nullptr;
305 0 : if (config_.renderer) {
306 0 : if (config_.disable_prerenderer_smoothing) {
307 0 : renderer = this;
308 : } else {
309 0 : incoming_video_stream_.reset(
310 0 : new IncomingVideoStream(config_.render_delay_ms, this));
311 0 : renderer = incoming_video_stream_.get();
312 : }
313 : }
314 0 : RTC_DCHECK(renderer != nullptr);
315 :
316 0 : for (const Decoder& decoder : config_.decoders) {
317 0 : video_receiver_.RegisterExternalDecoder(decoder.decoder,
318 0 : decoder.payload_type);
319 : // TODO(johan): make Decoder.codec_params accessible for RtpStreamReceiver
320 : // which holds H264SpsPpsTracker
321 0 : VideoCodec codec = CreateDecoderVideoCodec(decoder);
322 0 : RTC_CHECK(rtp_stream_receiver_.AddReceiveCodec(codec));
323 0 : RTC_CHECK_EQ(VCM_OK, video_receiver_.RegisterReceiveCodec(
324 0 : &codec, num_cpu_cores_, false));
325 : }
326 :
327 0 : video_stream_decoder_.reset(new VideoStreamDecoder(
328 : &video_receiver_, &rtp_stream_receiver_, &rtp_stream_receiver_,
329 0 : rtp_stream_receiver_.IsRetransmissionsEnabled(),
330 0 : rtp_stream_receiver_.IsUlpfecEnabled(), &stats_proxy_, renderer,
331 0 : config_.pre_render_callback));
332 : // Register the channel to receive stats updates.
333 0 : call_stats_->RegisterStatsObserver(video_stream_decoder_.get());
334 : // Start the decode thread
335 0 : decode_thread_.Start();
336 0 : decode_thread_.SetPriority(rtc::kHighestPriority);
337 0 : rtp_stream_receiver_.StartReceive();
338 : }
339 :
340 0 : void VideoReceiveStream::Stop() {
341 0 : rtp_stream_receiver_.StopReceive();
342 : // TriggerDecoderShutdown will release any waiting decoder thread and make it
343 : // stop immediately, instead of waiting for a timeout. Needs to be called
344 : // before joining the decoder thread thread.
345 0 : video_receiver_.TriggerDecoderShutdown();
346 :
347 0 : if (jitter_buffer_experiment_) {
348 0 : frame_buffer_->Stop();
349 0 : call_stats_->DeregisterStatsObserver(&rtp_stream_receiver_);
350 : }
351 :
352 0 : if (decode_thread_.IsRunning()) {
353 0 : decode_thread_.Stop();
354 : // Deregister external decoders so they are no longer running during
355 : // destruction. This effectively stops the VCM since the decoder thread is
356 : // stopped, the VCM is deregistered and no asynchronous decoder threads are
357 : // running.
358 0 : for (const Decoder& decoder : config_.decoders)
359 0 : video_receiver_.RegisterExternalDecoder(nullptr, decoder.payload_type);
360 : }
361 :
362 0 : call_stats_->DeregisterStatsObserver(video_stream_decoder_.get());
363 0 : video_stream_decoder_.reset();
364 0 : incoming_video_stream_.reset();
365 0 : transport_adapter_.Disable();
366 0 : }
367 :
368 0 : void VideoReceiveStream::SetSyncChannel(VoiceEngine* voice_engine,
369 : int audio_channel_id) {
370 0 : if (voice_engine && audio_channel_id != -1) {
371 0 : VoEVideoSync* voe_sync_interface = VoEVideoSync::GetInterface(voice_engine);
372 0 : rtp_stream_sync_.ConfigureSync(audio_channel_id, voe_sync_interface);
373 0 : voe_sync_interface->Release();
374 : } else {
375 0 : rtp_stream_sync_.ConfigureSync(-1, nullptr);
376 : }
377 0 : }
378 :
379 0 : VideoReceiveStream::Stats VideoReceiveStream::GetStats() const {
380 0 : return stats_proxy_.GetStats();
381 : }
382 :
383 : // TODO(tommi): This method grabs a lock 6 times.
384 0 : void VideoReceiveStream::OnFrame(const VideoFrame& video_frame) {
385 : // TODO(tommi): OnDecodedFrame grabs a lock, incidentally the same lock
386 : // that OnSyncOffsetUpdated() and OnRenderedFrame() below grab.
387 0 : stats_proxy_.OnDecodedFrame();
388 :
389 : int64_t sync_offset_ms;
390 : double estimated_freq_khz;
391 : // TODO(tommi): GetStreamSyncOffsetInMs grabs three locks. One inside the
392 : // function itself, another in GetChannel() and a third in
393 : // GetPlayoutTimestamp. Seems excessive. Anyhow, I'm assuming the function
394 : // succeeds most of the time, which leads to grabbing a fourth lock.
395 0 : if (rtp_stream_sync_.GetStreamSyncOffsetInMs(video_frame, &sync_offset_ms,
396 : &estimated_freq_khz)) {
397 : // TODO(tommi): OnSyncOffsetUpdated grabs a lock.
398 0 : stats_proxy_.OnSyncOffsetUpdated(sync_offset_ms, estimated_freq_khz);
399 : }
400 :
401 : // config_.renderer must never be null if we're getting this callback.
402 0 : config_.renderer->OnFrame(video_frame);
403 :
404 : // TODO(tommi): OnRenderFrame grabs a lock too.
405 0 : stats_proxy_.OnRenderedFrame(video_frame);
406 0 : }
407 :
408 0 : void VideoReceiveStream::OnCompleteFrame(
409 : std::unique_ptr<video_coding::FrameObject> frame) {
410 0 : int last_continuous_pid = frame_buffer_->InsertFrame(std::move(frame));
411 0 : if (last_continuous_pid != -1)
412 0 : rtp_stream_receiver_.FrameContinuous(last_continuous_pid);
413 0 : }
414 :
415 : // TODO(asapersson): Consider moving callback from video_encoder.h or
416 : // creating a different callback.
417 0 : EncodedImageCallback::Result VideoReceiveStream::OnEncodedImage(
418 : const EncodedImage& encoded_image,
419 : const CodecSpecificInfo* codec_specific_info,
420 : const RTPFragmentationHeader* fragmentation) {
421 0 : stats_proxy_.OnPreDecode(encoded_image, codec_specific_info);
422 0 : if (config_.pre_decode_callback) {
423 0 : config_.pre_decode_callback->EncodedFrameCallback(
424 0 : EncodedFrame(encoded_image._buffer, encoded_image._length,
425 0 : encoded_image._frameType));
426 : }
427 : {
428 0 : rtc::CritScope lock(&ivf_writer_lock_);
429 0 : if (ivf_writer_.get()) {
430 0 : RTC_DCHECK(codec_specific_info);
431 0 : bool ok = ivf_writer_->WriteFrame(encoded_image,
432 0 : codec_specific_info->codecType);
433 0 : RTC_DCHECK(ok);
434 : }
435 : }
436 :
437 0 : return Result(Result::OK, encoded_image._timeStamp);
438 : }
439 :
440 0 : bool VideoReceiveStream::DecodeThreadFunction(void* ptr) {
441 0 : static_cast<VideoReceiveStream*>(ptr)->Decode();
442 0 : return true;
443 : }
444 :
445 0 : void VideoReceiveStream::Decode() {
446 : static const int kMaxDecodeWaitTimeMs = 50;
447 0 : if (jitter_buffer_experiment_) {
448 : static const int kMaxWaitForFrameMs = 3000;
449 0 : std::unique_ptr<video_coding::FrameObject> frame;
450 : video_coding::FrameBuffer::ReturnReason res =
451 0 : frame_buffer_->NextFrame(kMaxWaitForFrameMs, &frame);
452 :
453 0 : if (res == video_coding::FrameBuffer::ReturnReason::kStopped)
454 0 : return;
455 :
456 0 : if (frame) {
457 0 : if (video_receiver_.Decode(frame.get()) == VCM_OK)
458 0 : rtp_stream_receiver_.FrameDecoded(frame->picture_id);
459 : } else {
460 0 : LOG(LS_WARNING) << "No decodable frame in " << kMaxWaitForFrameMs
461 0 : << " ms, requesting keyframe.";
462 0 : RequestKeyFrame();
463 : }
464 : } else {
465 0 : video_receiver_.Decode(kMaxDecodeWaitTimeMs);
466 : }
467 : }
468 :
469 0 : void VideoReceiveStream::SendNack(
470 : const std::vector<uint16_t>& sequence_numbers) {
471 0 : rtp_stream_receiver_.RequestPacketRetransmit(sequence_numbers);
472 0 : }
473 :
474 0 : void VideoReceiveStream::EnableEncodedFrameRecording(rtc::PlatformFile file,
475 : size_t byte_limit) {
476 : {
477 0 : rtc::CritScope lock(&ivf_writer_lock_);
478 0 : if (file == rtc::kInvalidPlatformFileValue) {
479 0 : ivf_writer_.reset();
480 : } else {
481 0 : ivf_writer_ = IvfFileWriter::Wrap(rtc::File(file), byte_limit);
482 : }
483 : }
484 :
485 0 : if (file != rtc::kInvalidPlatformFileValue) {
486 : // Make a keyframe appear as early as possible in the logs, to give actually
487 : // decodable output.
488 0 : RequestKeyFrame();
489 : }
490 0 : }
491 :
492 0 : void VideoReceiveStream::RequestKeyFrame() {
493 0 : rtp_stream_receiver_.RequestKeyFrame();
494 0 : }
495 :
496 : bool
497 0 : VideoReceiveStream::GetRemoteRTCPSenderInfo(RTCPSenderInfo* sender_info) const {
498 0 : return -1 != rtp_stream_receiver_.rtp_rtcp()->RemoteRTCPStat(sender_info);
499 : }
500 :
501 : } // namespace internal
502 : } // namespace webrtc
|
