Line data Source code
1 : /*
2 : * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 :
11 : #include "webrtc/base/checks.h"
12 : #include "webrtc/base/logging.h"
13 : #include "webrtc/base/trace_event.h"
14 : #include "webrtc/common_types.h"
15 : #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
16 : #include "webrtc/modules/video_coding/include/video_codec_interface.h"
17 : #include "webrtc/modules/video_coding/encoded_frame.h"
18 : #include "webrtc/modules/video_coding/jitter_buffer.h"
19 : #include "webrtc/modules/video_coding/packet.h"
20 : #include "webrtc/modules/video_coding/video_coding_impl.h"
21 : #include "webrtc/system_wrappers/include/clock.h"
22 :
23 : namespace webrtc {
24 : namespace vcm {
25 :
26 0 : VideoReceiver::VideoReceiver(Clock* clock,
27 : EventFactory* event_factory,
28 : EncodedImageCallback* pre_decode_image_callback,
29 : VCMTiming* timing,
30 : NackSender* nack_sender,
31 0 : KeyFrameRequestSender* keyframe_request_sender)
32 : : clock_(clock),
33 : _receiveState(kReceiveStateInitial),
34 : _timing(timing),
35 : _receiver(_timing,
36 0 : clock_,
37 : event_factory,
38 : nack_sender,
39 : keyframe_request_sender),
40 0 : _decodedFrameCallback(_timing, clock_),
41 : _frameTypeCallback(nullptr),
42 : _receiveStatsCallback(nullptr),
43 : _decoderTimingCallback(nullptr),
44 : _packetRequestCallback(nullptr),
45 : _receiveStateCallback(nullptr),
46 : _decoder(nullptr),
47 : _frameFromFile(),
48 : _scheduleKeyRequest(false),
49 : drop_frames_until_keyframe_(false),
50 : max_nack_list_size_(0),
51 : _codecDataBase(nullptr),
52 : pre_decode_image_callback_(pre_decode_image_callback),
53 0 : _receiveStatsTimer(1000, clock_),
54 0 : _retransmissionTimer(10, clock_),
55 0 : _keyRequestTimer(500, clock_) {}
56 :
57 0 : VideoReceiver::~VideoReceiver() {}
58 :
59 0 : void VideoReceiver::Process() {
60 : // Receive-side statistics
61 0 : if (_receiveStatsTimer.TimeUntilProcess() == 0) {
62 0 : _receiveStatsTimer.Processed();
63 0 : rtc::CritScope cs(&process_crit_);
64 0 : if (_receiveStatsCallback != nullptr) {
65 : uint32_t bitRate;
66 : uint32_t frameRate;
67 0 : _receiver.ReceiveStatistics(&bitRate, &frameRate);
68 0 : _receiveStatsCallback->OnReceiveRatesUpdated(bitRate, frameRate);
69 : }
70 :
71 0 : if (_decoderTimingCallback != nullptr) {
72 : int decode_ms;
73 : int max_decode_ms;
74 : int current_delay_ms;
75 : int target_delay_ms;
76 : int jitter_buffer_ms;
77 : int min_playout_delay_ms;
78 : int render_delay_ms;
79 0 : if (_timing->GetTimings(&decode_ms, &max_decode_ms, ¤t_delay_ms,
80 : &target_delay_ms, &jitter_buffer_ms,
81 : &min_playout_delay_ms, &render_delay_ms)) {
82 0 : _decoderTimingCallback->OnDecoderTiming(
83 : decode_ms, max_decode_ms, current_delay_ms, target_delay_ms,
84 0 : jitter_buffer_ms, min_playout_delay_ms, render_delay_ms);
85 : }
86 : }
87 : }
88 :
89 : // Key frame requests
90 0 : if (_keyRequestTimer.TimeUntilProcess() == 0) {
91 0 : _keyRequestTimer.Processed();
92 0 : bool request_key_frame = false;
93 : {
94 0 : rtc::CritScope cs(&process_crit_);
95 0 : request_key_frame = _scheduleKeyRequest && _frameTypeCallback != nullptr;
96 : }
97 0 : if (request_key_frame)
98 0 : RequestKeyFrame();
99 : }
100 :
101 : // Packet retransmission requests
102 : // TODO(holmer): Add API for changing Process interval and make sure it's
103 : // disabled when NACK is off.
104 0 : if (_retransmissionTimer.TimeUntilProcess() == 0) {
105 0 : _retransmissionTimer.Processed();
106 0 : bool callback_registered = false;
107 : uint16_t length;
108 : {
109 0 : rtc::CritScope cs(&process_crit_);
110 0 : length = max_nack_list_size_;
111 0 : callback_registered = _packetRequestCallback != nullptr;
112 : }
113 0 : if (callback_registered && length > 0) {
114 : // Collect sequence numbers from the default receiver.
115 0 : bool request_key_frame = false;
116 0 : std::vector<uint16_t> nackList = _receiver.NackList(&request_key_frame);
117 0 : int32_t ret = VCM_OK;
118 0 : if (request_key_frame) {
119 0 : ret = RequestKeyFrame();
120 : }
121 0 : if (ret == VCM_OK && !nackList.empty()) {
122 0 : rtc::CritScope cs(&process_crit_);
123 0 : if (_packetRequestCallback != nullptr) {
124 0 : _packetRequestCallback->ResendPackets(&nackList[0], nackList.size());
125 : }
126 : }
127 : }
128 : }
129 0 : }
130 :
131 0 : void VideoReceiver::SetReceiveState(VideoReceiveState state) {
132 0 : if (state == _receiveState) {
133 0 : return;
134 : }
135 0 : if (state == kReceiveStatePreemptiveNACK &&
136 0 : (_receiveState == kReceiveStateWaitingKey ||
137 0 : _receiveState == kReceiveStateDecodingWithErrors)) {
138 : // invalid state transition - this lets us try to set it on NACK
139 : // without worrying about the current state
140 0 : return;
141 : }
142 0 : _receiveState = state;
143 :
144 0 : rtc::CritScope cs(&process_crit_);
145 0 : if (_receiveStateCallback != NULL) {
146 0 : _receiveStateCallback->ReceiveStateChange(_receiveState);
147 : }
148 : }
149 :
150 0 : int64_t VideoReceiver::TimeUntilNextProcess() {
151 0 : int64_t timeUntilNextProcess = _receiveStatsTimer.TimeUntilProcess();
152 0 : if (_receiver.NackMode() != kNoNack) {
153 : // We need a Process call more often if we are relying on
154 : // retransmissions
155 0 : timeUntilNextProcess =
156 0 : VCM_MIN(timeUntilNextProcess, _retransmissionTimer.TimeUntilProcess());
157 : }
158 0 : timeUntilNextProcess =
159 0 : VCM_MIN(timeUntilNextProcess, _keyRequestTimer.TimeUntilProcess());
160 :
161 0 : return timeUntilNextProcess;
162 : }
163 :
164 0 : int32_t VideoReceiver::SetReceiveChannelParameters(int64_t rtt) {
165 0 : rtc::CritScope cs(&receive_crit_);
166 0 : _receiver.UpdateRtt(rtt);
167 0 : return 0;
168 : }
169 :
170 : // Enable or disable a video protection method.
171 : // Note: This API should be deprecated, as it does not offer a distinction
172 : // between the protection method and decoding with or without errors. If such a
173 : // behavior is desired, use the following API: SetReceiverRobustnessMode.
174 0 : int32_t VideoReceiver::SetVideoProtection(VCMVideoProtection videoProtection,
175 : bool enable) {
176 : // By default, do not decode with errors.
177 0 : _receiver.SetDecodeErrorMode(kNoErrors);
178 0 : switch (videoProtection) {
179 : case kProtectionNack: {
180 0 : RTC_DCHECK(enable);
181 0 : _receiver.SetNackMode(kNack, -1, -1);
182 0 : break;
183 : }
184 :
185 : case kProtectionNackFEC: {
186 0 : rtc::CritScope cs(&receive_crit_);
187 0 : RTC_DCHECK(enable);
188 0 : _receiver.SetNackMode(kNack,
189 : media_optimization::kLowRttNackMs,
190 0 : media_optimization::kMaxRttDelayThreshold);
191 0 : _receiver.SetDecodeErrorMode(kNoErrors);
192 0 : break;
193 : }
194 : case kProtectionFEC:
195 : case kProtectionNone:
196 : // No receiver-side protection.
197 0 : RTC_DCHECK(enable);
198 0 : _receiver.SetNackMode(kNoNack, -1, -1);
199 0 : _receiver.SetDecodeErrorMode(kWithErrors);
200 0 : break;
201 : }
202 0 : return VCM_OK;
203 : }
204 :
205 : // Register a receive callback. Will be called whenever there is a new frame
206 : // ready for rendering.
207 0 : int32_t VideoReceiver::RegisterReceiveCallback(
208 : VCMReceiveCallback* receiveCallback) {
209 0 : rtc::CritScope cs(&receive_crit_);
210 0 : _decodedFrameCallback.SetUserReceiveCallback(receiveCallback);
211 0 : return VCM_OK;
212 : }
213 :
214 0 : int32_t VideoReceiver::RegisterReceiveStatisticsCallback(
215 : VCMReceiveStatisticsCallback* receiveStats) {
216 0 : rtc::CritScope cs(&process_crit_);
217 0 : _receiver.RegisterStatsCallback(receiveStats);
218 0 : _receiveStatsCallback = receiveStats;
219 0 : return VCM_OK;
220 : }
221 :
222 0 : int32_t VideoReceiver::RegisterDecoderTimingCallback(
223 : VCMDecoderTimingCallback* decoderTiming) {
224 0 : rtc::CritScope cs(&process_crit_);
225 0 : _decoderTimingCallback = decoderTiming;
226 0 : return VCM_OK;
227 : }
228 :
229 : // Register an externally defined decoder object.
230 0 : void VideoReceiver::RegisterExternalDecoder(VideoDecoder* externalDecoder,
231 : uint8_t payloadType) {
232 0 : rtc::CritScope cs(&receive_crit_);
233 0 : if (externalDecoder == nullptr) {
234 : // Make sure the VCM updates the decoder next time it decodes.
235 0 : _decoder = nullptr;
236 0 : RTC_CHECK(_codecDataBase.DeregisterExternalDecoder(payloadType));
237 0 : return;
238 : }
239 0 : _codecDataBase.RegisterExternalDecoder(externalDecoder, payloadType);
240 : }
241 :
242 : // Register a frame type request callback.
243 0 : int32_t VideoReceiver::RegisterFrameTypeCallback(
244 : VCMFrameTypeCallback* frameTypeCallback) {
245 0 : rtc::CritScope cs(&process_crit_);
246 0 : _frameTypeCallback = frameTypeCallback;
247 0 : return VCM_OK;
248 : }
249 :
250 0 : int32_t VideoReceiver::RegisterPacketRequestCallback(
251 : VCMPacketRequestCallback* callback) {
252 0 : rtc::CritScope cs(&process_crit_);
253 0 : _packetRequestCallback = callback;
254 0 : return VCM_OK;
255 : }
256 :
257 0 : int32_t VideoReceiver::RegisterReceiveStateCallback(
258 : VCMReceiveStateCallback* callback) {
259 0 : rtc::CritScope cs(&process_crit_);
260 0 : _receiveStateCallback = callback;
261 0 : return VCM_OK;
262 : }
263 :
264 0 : void VideoReceiver::TriggerDecoderShutdown() {
265 0 : _receiver.TriggerDecoderShutdown();
266 0 : }
267 :
268 0 : void VideoReceiver::Reset() {
269 0 : _receiver.Reset();
270 0 : }
271 :
272 : // Decode next frame, blocking.
273 : // Should be called as often as possible to get the most out of the decoder.
274 0 : int32_t VideoReceiver::Decode(uint16_t maxWaitTimeMs) {
275 0 : bool prefer_late_decoding = false;
276 : {
277 0 : rtc::CritScope cs(&receive_crit_);
278 0 : prefer_late_decoding = _codecDataBase.PrefersLateDecoding();
279 : }
280 :
281 : VCMEncodedFrame* frame =
282 0 : _receiver.FrameForDecoding(maxWaitTimeMs, prefer_late_decoding);
283 :
284 0 : if (!frame)
285 0 : return VCM_FRAME_NOT_READY;
286 :
287 : {
288 0 : rtc::CritScope cs(&process_crit_);
289 0 : if (drop_frames_until_keyframe_) {
290 : // Still getting delta frames, schedule another keyframe request as if
291 : // decode failed.
292 0 : if (frame->FrameType() != kVideoFrameKey) {
293 0 : LOG(LS_INFO) << "Dropping delta frame for receiver " << (void*) this;
294 0 : _scheduleKeyRequest = true;
295 0 : _receiver.ReleaseFrame(frame);
296 0 : return VCM_FRAME_NOT_READY;
297 : }
298 0 : drop_frames_until_keyframe_ = false;
299 : }
300 : }
301 :
302 0 : if (pre_decode_image_callback_) {
303 0 : EncodedImage encoded_image(frame->EncodedImage());
304 0 : int qp = -1;
305 0 : if (qp_parser_.GetQp(*frame, &qp)) {
306 0 : encoded_image.qp_ = qp;
307 : }
308 0 : pre_decode_image_callback_->OnEncodedImage(encoded_image,
309 0 : frame->CodecSpecific(), nullptr);
310 : }
311 :
312 0 : rtc::CritScope cs(&receive_crit_);
313 : // If this frame was too late, we should adjust the delay accordingly
314 0 : _timing->UpdateCurrentDelay(frame->RenderTimeMs(),
315 0 : clock_->TimeInMilliseconds());
316 :
317 0 : if (first_frame_received_()) {
318 0 : LOG(LS_INFO) << "Received first "
319 0 : << (frame->Complete() ? "complete" : "incomplete")
320 0 : << " decodable video frame";
321 : }
322 :
323 0 : const int32_t ret = Decode(*frame);
324 0 : _receiver.ReleaseFrame(frame);
325 0 : return ret;
326 : }
327 :
328 : // Used for the WebRTC-NewVideoJitterBuffer experiment.
329 : // TODO(philipel): Clean up among the Decode functions as we replace
330 : // VCMEncodedFrame with FrameObject.
331 0 : int32_t VideoReceiver::Decode(const webrtc::VCMEncodedFrame* frame) {
332 0 : rtc::CritScope lock(&receive_crit_);
333 0 : if (pre_decode_image_callback_) {
334 0 : EncodedImage encoded_image(frame->EncodedImage());
335 0 : int qp = -1;
336 0 : if (qp_parser_.GetQp(*frame, &qp)) {
337 0 : encoded_image.qp_ = qp;
338 : }
339 0 : pre_decode_image_callback_->OnEncodedImage(encoded_image,
340 0 : frame->CodecSpecific(), nullptr);
341 : }
342 0 : return Decode(*frame);
343 : }
344 :
345 0 : int32_t VideoReceiver::RequestSliceLossIndication(
346 : const uint64_t pictureID) const {
347 0 : TRACE_EVENT1("webrtc", "RequestSLI", "picture_id", pictureID);
348 0 : rtc::CritScope cs(&process_crit_);
349 0 : if (_frameTypeCallback != nullptr) {
350 : const int32_t ret =
351 0 : _frameTypeCallback->SliceLossIndicationRequest(pictureID);
352 0 : if (ret < 0) {
353 0 : return ret;
354 : }
355 : } else {
356 0 : return VCM_MISSING_CALLBACK;
357 : }
358 0 : return VCM_OK;
359 : }
360 :
361 0 : int32_t VideoReceiver::RequestKeyFrame() {
362 0 : TRACE_EVENT0("webrtc", "RequestKeyFrame");
363 0 : rtc::CritScope cs(&process_crit_);
364 0 : if (_frameTypeCallback != nullptr) {
365 0 : const int32_t ret = _frameTypeCallback->RequestKeyFrame();
366 0 : if (ret < 0) {
367 0 : return ret;
368 : }
369 0 : _scheduleKeyRequest = false;
370 : } else {
371 0 : return VCM_MISSING_CALLBACK;
372 : }
373 0 : return VCM_OK;
374 : }
375 :
376 : // Must be called from inside the receive side critical section.
377 0 : int32_t VideoReceiver::Decode(const VCMEncodedFrame& frame) {
378 0 : TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", frame.TimeStamp(), "Decode",
379 : "type", frame.FrameType());
380 : // Change decoder if payload type has changed
381 0 : _decoder = _codecDataBase.GetDecoder(frame, &_decodedFrameCallback);
382 0 : if (_decoder == nullptr) {
383 0 : return VCM_NO_CODEC_REGISTERED;
384 : }
385 : // Decode a frame
386 0 : int32_t ret = _decoder->Decode(frame, clock_->TimeInMilliseconds());
387 :
388 : // Check for failed decoding, run frame type request callback if needed.
389 0 : bool request_key_frame = false;
390 0 : if (ret < 0) {
391 0 : if (ret == VCM_ERROR_REQUEST_SLI) {
392 0 : return RequestSliceLossIndication(
393 0 : _decodedFrameCallback.LastReceivedPictureID() + 1);
394 : } else {
395 0 : request_key_frame = true;
396 : }
397 0 : } else if (ret == VCM_REQUEST_SLI) {
398 0 : ret = RequestSliceLossIndication(
399 0 : _decodedFrameCallback.LastReceivedPictureID() + 1);
400 : }
401 0 : if (!frame.Complete() || frame.MissingFrame()) {
402 0 : request_key_frame = true;
403 0 : ret = VCM_OK;
404 : }
405 0 : if (request_key_frame) {
406 0 : rtc::CritScope cs(&process_crit_);
407 0 : _scheduleKeyRequest = true;
408 : }
409 0 : TRACE_EVENT_ASYNC_END0("webrtc", "Video", frame.TimeStamp());
410 0 : return ret;
411 : }
412 :
413 : // Register possible receive codecs, can be called multiple times
414 0 : int32_t VideoReceiver::RegisterReceiveCodec(const VideoCodec* receiveCodec,
415 : int32_t numberOfCores,
416 : bool requireKeyFrame) {
417 0 : rtc::CritScope cs(&receive_crit_);
418 0 : if (receiveCodec == nullptr) {
419 0 : return VCM_PARAMETER_ERROR;
420 : }
421 0 : if (!_codecDataBase.RegisterReceiveCodec(receiveCodec, numberOfCores,
422 : requireKeyFrame)) {
423 0 : return -1;
424 : }
425 0 : return 0;
426 : }
427 :
428 : // Get current received codec
429 0 : int32_t VideoReceiver::ReceiveCodec(VideoCodec* currentReceiveCodec) const {
430 0 : rtc::CritScope cs(&receive_crit_);
431 0 : if (currentReceiveCodec == nullptr) {
432 0 : return VCM_PARAMETER_ERROR;
433 : }
434 0 : return _codecDataBase.ReceiveCodec(currentReceiveCodec) ? 0 : -1;
435 : }
436 :
437 : // Get current received codec
438 0 : VideoCodecType VideoReceiver::ReceiveCodec() const {
439 0 : rtc::CritScope cs(&receive_crit_);
440 0 : return _codecDataBase.ReceiveCodec();
441 : }
442 :
443 : // Incoming packet from network parsed and ready for decode, non blocking.
444 0 : int32_t VideoReceiver::IncomingPacket(const uint8_t* incomingPayload,
445 : size_t payloadLength,
446 : const WebRtcRTPHeader& rtpInfo) {
447 0 : if (rtpInfo.frameType == kVideoFrameKey) {
448 0 : TRACE_EVENT1("webrtc", "VCM::PacketKeyFrame", "seqnum",
449 : rtpInfo.header.sequenceNumber);
450 : }
451 0 : if (incomingPayload == nullptr) {
452 : // The jitter buffer doesn't handle non-zero payload lengths for packets
453 : // without payload.
454 : // TODO(holmer): We should fix this in the jitter buffer.
455 0 : payloadLength = 0;
456 : }
457 0 : const VCMPacket packet(incomingPayload, payloadLength, rtpInfo);
458 0 : int32_t ret = _receiver.InsertPacket(packet);
459 :
460 : // TODO(holmer): Investigate if this somehow should use the key frame
461 : // request scheduling to throttle the requests.
462 0 : if (ret == VCM_FLUSH_INDICATOR) {
463 : {
464 0 : rtc::CritScope cs(&process_crit_);
465 0 : drop_frames_until_keyframe_ = true;
466 : }
467 0 : RequestKeyFrame();
468 0 : SetReceiveState(kReceiveStateWaitingKey);
469 0 : } else if (ret < 0) {
470 0 : return ret;
471 : }
472 0 : return VCM_OK;
473 : }
474 :
475 : // Minimum playout delay (used for lip-sync). This is the minimum delay required
476 : // to sync with audio. Not included in VideoCodingModule::Delay()
477 : // Defaults to 0 ms.
478 0 : int32_t VideoReceiver::SetMinimumPlayoutDelay(uint32_t minPlayoutDelayMs) {
479 0 : _timing->set_min_playout_delay(minPlayoutDelayMs);
480 0 : return VCM_OK;
481 : }
482 :
483 : // The estimated delay caused by rendering, defaults to
484 : // kDefaultRenderDelayMs = 10 ms
485 0 : int32_t VideoReceiver::SetRenderDelay(uint32_t timeMS) {
486 0 : _timing->set_render_delay(timeMS);
487 0 : return VCM_OK;
488 : }
489 :
490 : // Current video delay
491 0 : int32_t VideoReceiver::Delay() const {
492 0 : return _timing->TargetVideoDelay();
493 : }
494 :
495 0 : uint32_t VideoReceiver::DiscardedPackets() const {
496 0 : return _receiver.DiscardedPackets();
497 : }
498 :
499 0 : int VideoReceiver::SetReceiverRobustnessMode(
500 : ReceiverRobustness robustnessMode,
501 : VCMDecodeErrorMode decode_error_mode) {
502 0 : rtc::CritScope cs(&receive_crit_);
503 0 : switch (robustnessMode) {
504 : case VideoCodingModule::kNone:
505 0 : _receiver.SetNackMode(kNoNack, -1, -1);
506 0 : break;
507 : case VideoCodingModule::kHardNack:
508 : // Always wait for retransmissions (except when decoding with errors).
509 0 : _receiver.SetNackMode(kNack, -1, -1);
510 0 : break;
511 : case VideoCodingModule::kSoftNack:
512 : #if 1
513 0 : assert(false); // TODO(hlundin): Not completed.
514 : return VCM_NOT_IMPLEMENTED;
515 : #else
516 : // Enable hybrid NACK/FEC. Always wait for retransmissions and don't add
517 : // extra delay when RTT is above kLowRttNackMs.
518 : _receiver.SetNackMode(kNack, media_optimization::kLowRttNackMs, -1);
519 : break;
520 : #endif
521 : case VideoCodingModule::kReferenceSelection:
522 : #if 1
523 0 : assert(false); // TODO(hlundin): Not completed.
524 : return VCM_NOT_IMPLEMENTED;
525 : #else
526 : if (decode_error_mode == kNoErrors) {
527 : return VCM_PARAMETER_ERROR;
528 : }
529 : _receiver.SetNackMode(kNoNack, -1, -1);
530 : break;
531 : #endif
532 : }
533 0 : _receiver.SetDecodeErrorMode(decode_error_mode);
534 0 : return VCM_OK;
535 : }
536 :
537 0 : void VideoReceiver::SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode) {
538 0 : rtc::CritScope cs(&receive_crit_);
539 0 : _receiver.SetDecodeErrorMode(decode_error_mode);
540 0 : }
541 :
542 0 : void VideoReceiver::SetNackSettings(size_t max_nack_list_size,
543 : int max_packet_age_to_nack,
544 : int max_incomplete_time_ms) {
545 0 : if (max_nack_list_size != 0) {
546 0 : rtc::CritScope cs(&process_crit_);
547 0 : max_nack_list_size_ = max_nack_list_size;
548 : }
549 0 : _receiver.SetNackSettings(max_nack_list_size, max_packet_age_to_nack,
550 0 : max_incomplete_time_ms);
551 0 : }
552 :
553 0 : int VideoReceiver::SetMinReceiverDelay(int desired_delay_ms) {
554 0 : return _receiver.SetMinReceiverDelay(desired_delay_ms);
555 : }
556 :
557 : } // namespace vcm
558 : } // namespace webrtc
|
