Line data Source code
1 : /*
2 : * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 :
11 : #include "webrtc/modules/video_coding/receiver.h"
12 :
13 : #include <assert.h>
14 :
15 : #include <cstdlib>
16 : #include <utility>
17 : #include <vector>
18 :
19 : #include "webrtc/base/logging.h"
20 : #include "webrtc/base/trace_event.h"
21 : #include "webrtc/modules/video_coding/encoded_frame.h"
22 : #include "webrtc/modules/video_coding/internal_defines.h"
23 : #include "webrtc/modules/video_coding/media_opt_util.h"
24 : #include "webrtc/system_wrappers/include/clock.h"
25 :
26 : namespace webrtc {
27 :
28 : enum { kMaxReceiverDelayMs = 10000 };
29 :
30 0 : VCMReceiver::VCMReceiver(VCMTiming* timing,
31 : Clock* clock,
32 0 : EventFactory* event_factory)
33 : : VCMReceiver::VCMReceiver(timing,
34 : clock,
35 : event_factory,
36 : nullptr, // NackSender
37 0 : nullptr) // KeyframeRequestSender
38 0 : {}
39 :
40 0 : VCMReceiver::VCMReceiver(VCMTiming* timing,
41 : Clock* clock,
42 : EventFactory* event_factory,
43 : NackSender* nack_sender,
44 0 : KeyFrameRequestSender* keyframe_request_sender)
45 : : VCMReceiver(
46 : timing,
47 : clock,
48 0 : std::unique_ptr<EventWrapper>(event_factory
49 0 : ? event_factory->CreateEvent()
50 : : EventWrapper::Create()),
51 0 : std::unique_ptr<EventWrapper>(event_factory
52 0 : ? event_factory->CreateEvent()
53 : : EventWrapper::Create()),
54 : nack_sender,
55 0 : keyframe_request_sender) {}
56 :
57 0 : VCMReceiver::VCMReceiver(VCMTiming* timing,
58 : Clock* clock,
59 : std::unique_ptr<EventWrapper> receiver_event,
60 0 : std::unique_ptr<EventWrapper> jitter_buffer_event)
61 : : VCMReceiver::VCMReceiver(timing,
62 : clock,
63 0 : std::move(receiver_event),
64 0 : std::move(jitter_buffer_event),
65 : nullptr, // NackSender
66 0 : nullptr) // KeyframeRequestSender
67 0 : {}
68 :
69 0 : VCMReceiver::VCMReceiver(VCMTiming* timing,
70 : Clock* clock,
71 : std::unique_ptr<EventWrapper> receiver_event,
72 : std::unique_ptr<EventWrapper> jitter_buffer_event,
73 : NackSender* nack_sender,
74 0 : KeyFrameRequestSender* keyframe_request_sender)
75 0 : : crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
76 : clock_(clock),
77 0 : jitter_buffer_(clock_,
78 0 : std::move(jitter_buffer_event),
79 : nack_sender,
80 : keyframe_request_sender),
81 : timing_(timing),
82 0 : render_wait_event_(std::move(receiver_event)),
83 : receiveState_(kReceiveStateInitial),
84 0 : max_video_delay_ms_(kMaxVideoDelayMs) {
85 0 : Reset();
86 0 : }
87 :
88 0 : VCMReceiver::~VCMReceiver() {
89 0 : render_wait_event_->Set();
90 0 : delete crit_sect_;
91 0 : }
92 :
93 0 : void VCMReceiver::Reset() {
94 0 : CriticalSectionScoped cs(crit_sect_);
95 0 : if (!jitter_buffer_.Running()) {
96 0 : jitter_buffer_.Start();
97 : } else {
98 0 : jitter_buffer_.Flush();
99 : }
100 0 : receiveState_ = kReceiveStateInitial;
101 0 : }
102 :
103 0 : void VCMReceiver::UpdateRtt(int64_t rtt) {
104 0 : jitter_buffer_.UpdateRtt(rtt);
105 0 : }
106 :
107 0 : int32_t VCMReceiver::InsertPacket(const VCMPacket& packet) {
108 : // Insert the packet into the jitter buffer. The packet can either be empty or
109 : // contain media at this point.
110 0 : bool retransmitted = false;
111 : const VCMFrameBufferEnum ret =
112 0 : jitter_buffer_.InsertPacket(packet, &retransmitted);
113 0 : if (ret == kOldPacket) {
114 0 : return VCM_OK;
115 0 : } else if (ret == kFlushIndicator) {
116 0 : return VCM_FLUSH_INDICATOR;
117 0 : } else if (ret < 0) {
118 0 : return VCM_JITTER_BUFFER_ERROR;
119 : }
120 0 : if (ret == kCompleteSession && !retransmitted) {
121 : // We don't want to include timestamps which have suffered from
122 : // retransmission here, since we compensate with extra retransmission
123 : // delay within the jitter estimate.
124 0 : timing_->IncomingTimestamp(packet.timestamp, clock_->TimeInMilliseconds());
125 : }
126 0 : return VCM_OK;
127 : }
128 :
129 0 : void VCMReceiver::TriggerDecoderShutdown() {
130 0 : jitter_buffer_.Stop();
131 0 : render_wait_event_->Set();
132 0 : }
133 :
134 0 : VCMEncodedFrame* VCMReceiver::FrameForDecoding(uint16_t max_wait_time_ms,
135 : bool prefer_late_decoding) {
136 0 : const int64_t start_time_ms = clock_->TimeInMilliseconds();
137 0 : uint32_t frame_timestamp = 0;
138 0 : int min_playout_delay_ms = -1;
139 0 : int max_playout_delay_ms = -1;
140 0 : int64_t render_time_ms = 0;
141 : // Exhaust wait time to get a complete frame for decoding.
142 : VCMEncodedFrame* found_frame =
143 0 : jitter_buffer_.NextCompleteFrame(max_wait_time_ms);
144 :
145 0 : if (found_frame) {
146 0 : frame_timestamp = found_frame->TimeStamp();
147 0 : min_playout_delay_ms = found_frame->EncodedImage().playout_delay_.min_ms;
148 0 : max_playout_delay_ms = found_frame->EncodedImage().playout_delay_.max_ms;
149 : } else {
150 0 : if (!jitter_buffer_.NextMaybeIncompleteTimestamp(&frame_timestamp))
151 0 : return nullptr;
152 : }
153 :
154 0 : if (min_playout_delay_ms >= 0)
155 0 : timing_->set_min_playout_delay(min_playout_delay_ms);
156 :
157 0 : if (max_playout_delay_ms >= 0)
158 0 : timing_->set_max_playout_delay(max_playout_delay_ms);
159 :
160 : // We have a frame - Set timing and render timestamp.
161 0 : timing_->SetJitterDelay(jitter_buffer_.EstimatedJitterMs());
162 0 : const int64_t now_ms = clock_->TimeInMilliseconds();
163 0 : timing_->UpdateCurrentDelay(frame_timestamp);
164 0 : render_time_ms = timing_->RenderTimeMs(frame_timestamp, now_ms);
165 : // Check render timing.
166 0 : bool timing_error = false;
167 : // Assume that render timing errors are due to changes in the video stream.
168 0 : if (render_time_ms < 0) {
169 0 : timing_error = true;
170 0 : } else if (std::abs(render_time_ms - now_ms) > max_video_delay_ms_) {
171 0 : int frame_delay = std::abs(render_time_ms - now_ms);
172 0 : LOG(LS_WARNING) << "A frame about to be decoded is out of the configured "
173 0 : << "delay bounds (" << frame_delay << " > "
174 : << max_video_delay_ms_
175 0 : << "). Resetting the video jitter buffer.";
176 0 : timing_error = true;
177 0 : } else if (static_cast<int>(timing_->TargetVideoDelay()) >
178 0 : max_video_delay_ms_) {
179 0 : LOG(LS_WARNING) << "The video target delay has grown larger than "
180 0 : << max_video_delay_ms_ << " ms. Resetting jitter buffer.";
181 0 : timing_error = true;
182 : }
183 :
184 0 : if (timing_error) {
185 : // Timing error => reset timing and flush the jitter buffer.
186 0 : jitter_buffer_.Flush();
187 0 : timing_->Reset();
188 0 : return NULL;
189 : }
190 :
191 0 : if (prefer_late_decoding) {
192 : // Decode frame as close as possible to the render timestamp.
193 : const int32_t available_wait_time =
194 0 : max_wait_time_ms -
195 0 : static_cast<int32_t>(clock_->TimeInMilliseconds() - start_time_ms);
196 : uint16_t new_max_wait_time =
197 0 : static_cast<uint16_t>(VCM_MAX(available_wait_time, 0));
198 : uint32_t wait_time_ms =
199 0 : timing_->MaxWaitingTime(render_time_ms, clock_->TimeInMilliseconds());
200 0 : if (new_max_wait_time < wait_time_ms) {
201 : // We're not allowed to wait until the frame is supposed to be rendered,
202 : // waiting as long as we're allowed to avoid busy looping, and then return
203 : // NULL. Next call to this function might return the frame.
204 0 : render_wait_event_->Wait(new_max_wait_time);
205 0 : return NULL;
206 : }
207 : // Wait until it's time to render.
208 0 : render_wait_event_->Wait(wait_time_ms);
209 : }
210 :
211 : // Extract the frame from the jitter buffer and set the render time.
212 0 : VCMEncodedFrame* frame = jitter_buffer_.ExtractAndSetDecode(frame_timestamp);
213 0 : if (frame == NULL) {
214 0 : return NULL;
215 : }
216 0 : frame->SetRenderTime(render_time_ms);
217 0 : TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", frame->TimeStamp(), "SetRenderTS",
218 : "render_time", frame->RenderTimeMs());
219 0 : UpdateReceiveState(*frame);
220 0 : if (!frame->Complete()) {
221 : // Update stats for incomplete frames.
222 0 : bool retransmitted = false;
223 : const int64_t last_packet_time_ms =
224 0 : jitter_buffer_.LastPacketTime(frame, &retransmitted);
225 0 : if (last_packet_time_ms >= 0 && !retransmitted) {
226 : // We don't want to include timestamps which have suffered from
227 : // retransmission here, since we compensate with extra retransmission
228 : // delay within the jitter estimate.
229 0 : timing_->IncomingTimestamp(frame_timestamp, last_packet_time_ms);
230 : }
231 : }
232 0 : return frame;
233 : }
234 :
235 0 : void VCMReceiver::ReleaseFrame(VCMEncodedFrame* frame) {
236 0 : jitter_buffer_.ReleaseFrame(frame);
237 0 : }
238 :
239 0 : void VCMReceiver::ReceiveStatistics(uint32_t* bitrate, uint32_t* framerate) {
240 0 : assert(bitrate);
241 0 : assert(framerate);
242 0 : jitter_buffer_.IncomingRateStatistics(framerate, bitrate);
243 0 : }
244 :
245 0 : uint32_t VCMReceiver::DiscardedPackets() const {
246 0 : return jitter_buffer_.num_discarded_packets();
247 : }
248 :
249 0 : void VCMReceiver::SetNackMode(VCMNackMode nackMode,
250 : int64_t low_rtt_nack_threshold_ms,
251 : int64_t high_rtt_nack_threshold_ms) {
252 0 : CriticalSectionScoped cs(crit_sect_);
253 : // Default to always having NACK enabled in hybrid mode.
254 0 : jitter_buffer_.SetNackMode(nackMode, low_rtt_nack_threshold_ms,
255 0 : high_rtt_nack_threshold_ms);
256 0 : }
257 :
258 0 : void VCMReceiver::SetNackSettings(size_t max_nack_list_size,
259 : int max_packet_age_to_nack,
260 : int max_incomplete_time_ms) {
261 0 : jitter_buffer_.SetNackSettings(max_nack_list_size, max_packet_age_to_nack,
262 0 : max_incomplete_time_ms);
263 0 : }
264 :
265 0 : VCMNackMode VCMReceiver::NackMode() const {
266 0 : CriticalSectionScoped cs(crit_sect_);
267 0 : return jitter_buffer_.nack_mode();
268 : }
269 :
270 0 : std::vector<uint16_t> VCMReceiver::NackList(bool* request_key_frame) {
271 0 : return jitter_buffer_.GetNackList(request_key_frame);
272 : }
273 :
274 0 : VideoReceiveState VCMReceiver::ReceiveState() const {
275 0 : CriticalSectionScoped cs(crit_sect_);
276 0 : return receiveState_;
277 : }
278 :
279 0 : void VCMReceiver::SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode) {
280 0 : jitter_buffer_.SetDecodeErrorMode(decode_error_mode);
281 0 : }
282 :
283 0 : VCMDecodeErrorMode VCMReceiver::DecodeErrorMode() const {
284 0 : return jitter_buffer_.decode_error_mode();
285 : }
286 :
287 0 : int VCMReceiver::SetMinReceiverDelay(int desired_delay_ms) {
288 0 : CriticalSectionScoped cs(crit_sect_);
289 0 : if (desired_delay_ms < 0 || desired_delay_ms > kMaxReceiverDelayMs) {
290 0 : return -1;
291 : }
292 0 : max_video_delay_ms_ = desired_delay_ms + kMaxVideoDelayMs;
293 : // Initializing timing to the desired delay.
294 0 : timing_->set_min_playout_delay(desired_delay_ms);
295 0 : return 0;
296 : }
297 :
298 0 : void VCMReceiver::UpdateReceiveState(const VCMEncodedFrame& frame) {
299 0 : if (frame.Complete() && frame.FrameType() == kVideoFrameKey) {
300 0 : receiveState_ = kReceiveStateNormal;
301 0 : return;
302 : }
303 0 : if (frame.MissingFrame() || !frame.Complete()) {
304 : // State is corrupted
305 0 : receiveState_ = kReceiveStateWaitingKey;
306 : }
307 : // state continues
308 : }
309 :
310 0 : void VCMReceiver::RegisterStatsCallback(
311 : VCMReceiveStatisticsCallback* callback) {
312 0 : jitter_buffer_.RegisterStatsCallback(callback);
313 0 : }
314 :
315 : } // namespace webrtc
|
