Line data Source code
1 : /*
2 : * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 :
11 : #include "webrtc/modules/video_coding/frame_buffer2.h"
12 :
13 : #include <algorithm>
14 : #include <cstring>
15 : #include <queue>
16 :
17 : #include "webrtc/base/checks.h"
18 : #include "webrtc/base/logging.h"
19 : #include "webrtc/modules/video_coding/jitter_estimator.h"
20 : #include "webrtc/modules/video_coding/timing.h"
21 : #include "webrtc/system_wrappers/include/clock.h"
22 : #include "webrtc/system_wrappers/include/metrics.h"
23 :
24 : namespace webrtc {
25 : namespace video_coding {
26 :
27 : namespace {
28 : // Max number of frames the buffer will hold.
29 : constexpr int kMaxFramesBuffered = 600;
30 :
31 : // Max number of decoded frame info that will be saved.
32 : constexpr int kMaxFramesHistory = 50;
33 : } // namespace
34 :
35 0 : FrameBuffer::FrameBuffer(Clock* clock,
36 : VCMJitterEstimator* jitter_estimator,
37 0 : VCMTiming* timing)
38 : : clock_(clock),
39 : new_countinuous_frame_event_(false, false),
40 : jitter_estimator_(jitter_estimator),
41 : timing_(timing),
42 0 : inter_frame_delay_(clock_->TimeInMilliseconds()),
43 0 : last_decoded_frame_it_(frames_.end()),
44 0 : last_continuous_frame_it_(frames_.end()),
45 : num_frames_history_(0),
46 : num_frames_buffered_(0),
47 : stopped_(false),
48 0 : protection_mode_(kProtectionNack) {}
49 :
50 0 : FrameBuffer::~FrameBuffer() {
51 0 : UpdateHistograms();
52 0 : }
53 :
54 0 : FrameBuffer::ReturnReason FrameBuffer::NextFrame(
55 : int64_t max_wait_time_ms,
56 : std::unique_ptr<FrameObject>* frame_out) {
57 0 : int64_t latest_return_time = clock_->TimeInMilliseconds() + max_wait_time_ms;
58 0 : int64_t wait_ms = max_wait_time_ms;
59 0 : FrameMap::iterator next_frame_it;
60 :
61 0 : do {
62 0 : int64_t now_ms = clock_->TimeInMilliseconds();
63 : {
64 0 : rtc::CritScope lock(&crit_);
65 0 : new_countinuous_frame_event_.Reset();
66 0 : if (stopped_)
67 0 : return kStopped;
68 :
69 0 : wait_ms = max_wait_time_ms;
70 :
71 : // Need to hold |crit_| in order to use |frames_|, therefore we
72 : // set it here in the loop instead of outside the loop in order to not
73 : // acquire the lock unnecesserily.
74 0 : next_frame_it = frames_.end();
75 :
76 : // |frame_it| points to the first frame after the
77 : // |last_decoded_frame_it_|.
78 0 : auto frame_it = frames_.end();
79 0 : if (last_decoded_frame_it_ == frames_.end()) {
80 0 : frame_it = frames_.begin();
81 : } else {
82 0 : frame_it = last_decoded_frame_it_;
83 0 : ++frame_it;
84 : }
85 :
86 : // |continuous_end_it| points to the first frame after the
87 : // |last_continuous_frame_it_|.
88 0 : auto continuous_end_it = last_continuous_frame_it_;
89 0 : if (continuous_end_it != frames_.end())
90 0 : ++continuous_end_it;
91 :
92 0 : for (; frame_it != continuous_end_it && frame_it != frames_.end();
93 : ++frame_it) {
94 0 : if (!frame_it->second.continuous ||
95 0 : frame_it->second.num_missing_decodable > 0) {
96 0 : continue;
97 : }
98 :
99 0 : FrameObject* frame = frame_it->second.frame.get();
100 0 : next_frame_it = frame_it;
101 0 : if (frame->RenderTime() == -1)
102 0 : frame->SetRenderTime(timing_->RenderTimeMs(frame->timestamp, now_ms));
103 0 : wait_ms = timing_->MaxWaitingTime(frame->RenderTime(), now_ms);
104 :
105 : // This will cause the frame buffer to prefer high framerate rather
106 : // than high resolution in the case of the decoder not decoding fast
107 : // enough and the stream has multiple spatial and temporal layers.
108 0 : if (wait_ms == 0)
109 0 : continue;
110 :
111 0 : break;
112 : }
113 : } // rtc::Critscope lock(&crit_);
114 :
115 0 : wait_ms = std::min<int64_t>(wait_ms, latest_return_time - now_ms);
116 0 : wait_ms = std::max<int64_t>(wait_ms, 0);
117 0 : } while (new_countinuous_frame_event_.Wait(wait_ms));
118 :
119 0 : rtc::CritScope lock(&crit_);
120 0 : if (next_frame_it != frames_.end()) {
121 0 : std::unique_ptr<FrameObject> frame = std::move(next_frame_it->second.frame);
122 0 : int64_t received_time = frame->ReceivedTime();
123 0 : uint32_t timestamp = frame->timestamp;
124 :
125 : int64_t frame_delay;
126 0 : if (inter_frame_delay_.CalculateDelay(timestamp, &frame_delay,
127 : received_time)) {
128 0 : jitter_estimator_->UpdateEstimate(frame_delay, frame->size());
129 : }
130 0 : float rtt_mult = protection_mode_ == kProtectionNackFEC ? 0.0 : 1.0;
131 0 : timing_->SetJitterDelay(jitter_estimator_->GetJitterEstimate(rtt_mult));
132 0 : timing_->UpdateCurrentDelay(frame->RenderTime(),
133 0 : clock_->TimeInMilliseconds());
134 :
135 0 : UpdateJitterDelay();
136 :
137 0 : PropagateDecodability(next_frame_it->second);
138 0 : AdvanceLastDecodedFrame(next_frame_it);
139 0 : *frame_out = std::move(frame);
140 0 : return kFrameFound;
141 : } else {
142 0 : return kTimeout;
143 : }
144 : }
145 :
146 0 : void FrameBuffer::SetProtectionMode(VCMVideoProtection mode) {
147 0 : rtc::CritScope lock(&crit_);
148 0 : protection_mode_ = mode;
149 0 : }
150 :
151 0 : void FrameBuffer::Start() {
152 0 : rtc::CritScope lock(&crit_);
153 0 : stopped_ = false;
154 0 : }
155 :
156 0 : void FrameBuffer::Stop() {
157 0 : rtc::CritScope lock(&crit_);
158 0 : stopped_ = true;
159 0 : new_countinuous_frame_event_.Set();
160 0 : }
161 :
162 0 : int FrameBuffer::InsertFrame(std::unique_ptr<FrameObject> frame) {
163 0 : rtc::CritScope lock(&crit_);
164 0 : RTC_DCHECK(frame);
165 :
166 0 : ++num_total_frames_;
167 0 : if (frame->num_references == 0)
168 0 : ++num_key_frames_;
169 :
170 0 : FrameKey key(frame->picture_id, frame->spatial_layer);
171 : int last_continuous_picture_id =
172 0 : last_continuous_frame_it_ == frames_.end()
173 0 : ? -1
174 0 : : last_continuous_frame_it_->first.picture_id;
175 :
176 0 : if (num_frames_buffered_ >= kMaxFramesBuffered) {
177 0 : LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) (" << key.picture_id
178 0 : << ":" << static_cast<int>(key.spatial_layer)
179 : << ") could not be inserted due to the frame "
180 0 : << "buffer being full, dropping frame.";
181 0 : return last_continuous_picture_id;
182 : }
183 :
184 0 : if (frame->inter_layer_predicted && frame->spatial_layer == 0) {
185 0 : LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) (" << key.picture_id
186 0 : << ":" << static_cast<int>(key.spatial_layer)
187 0 : << ") is marked as inter layer predicted, dropping frame.";
188 0 : return last_continuous_picture_id;
189 : }
190 :
191 0 : if (last_decoded_frame_it_ != frames_.end() &&
192 0 : key < last_decoded_frame_it_->first) {
193 0 : LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) (" << key.picture_id
194 0 : << ":" << static_cast<int>(key.spatial_layer)
195 0 : << ") inserted after frame ("
196 0 : << last_decoded_frame_it_->first.picture_id << ":"
197 : << static_cast<int>(
198 0 : last_decoded_frame_it_->first.spatial_layer)
199 0 : << ") was handed off for decoding, dropping frame.";
200 0 : return last_continuous_picture_id;
201 : }
202 :
203 : // Test if inserting this frame would cause the order of the frames to become
204 : // ambiguous (covering more than half the interval of 2^16). This can happen
205 : // when the picture id make large jumps mid stream.
206 0 : if (!frames_.empty() &&
207 0 : key < frames_.begin()->first &&
208 0 : frames_.rbegin()->first < key) {
209 0 : LOG(LS_WARNING) << "A jump in picture id was detected, clearing buffer.";
210 0 : ClearFramesAndHistory();
211 0 : last_continuous_picture_id = -1;
212 : }
213 :
214 0 : auto info = frames_.insert(std::make_pair(key, FrameInfo())).first;
215 :
216 0 : if (info->second.frame) {
217 0 : LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) (" << key.picture_id
218 0 : << ":" << static_cast<int>(key.spatial_layer)
219 0 : << ") already inserted, dropping frame.";
220 0 : return last_continuous_picture_id;
221 : }
222 :
223 0 : if (!UpdateFrameInfoWithIncomingFrame(*frame, info))
224 0 : return last_continuous_picture_id;
225 :
226 0 : info->second.frame = std::move(frame);
227 0 : ++num_frames_buffered_;
228 :
229 0 : if (info->second.num_missing_continuous == 0) {
230 0 : info->second.continuous = true;
231 0 : PropagateContinuity(info);
232 0 : last_continuous_picture_id = last_continuous_frame_it_->first.picture_id;
233 :
234 : // Since we now have new continuous frames there might be a better frame
235 : // to return from NextFrame. Signal that thread so that it again can choose
236 : // which frame to return.
237 0 : new_countinuous_frame_event_.Set();
238 : }
239 :
240 0 : return last_continuous_picture_id;
241 : }
242 :
243 0 : void FrameBuffer::PropagateContinuity(FrameMap::iterator start) {
244 0 : RTC_DCHECK(start->second.continuous);
245 0 : if (last_continuous_frame_it_ == frames_.end())
246 0 : last_continuous_frame_it_ = start;
247 :
248 0 : std::queue<FrameMap::iterator> continuous_frames;
249 0 : continuous_frames.push(start);
250 :
251 : // A simple BFS to traverse continuous frames.
252 0 : while (!continuous_frames.empty()) {
253 0 : auto frame = continuous_frames.front();
254 0 : continuous_frames.pop();
255 :
256 0 : if (last_continuous_frame_it_->first < frame->first)
257 0 : last_continuous_frame_it_ = frame;
258 :
259 : // Loop through all dependent frames, and if that frame no longer has
260 : // any unfulfilled dependencies then that frame is continuous as well.
261 0 : for (size_t d = 0; d < frame->second.num_dependent_frames; ++d) {
262 0 : auto frame_ref = frames_.find(frame->second.dependent_frames[d]);
263 0 : --frame_ref->second.num_missing_continuous;
264 :
265 0 : if (frame_ref->second.num_missing_continuous == 0) {
266 0 : frame_ref->second.continuous = true;
267 0 : continuous_frames.push(frame_ref);
268 : }
269 : }
270 : }
271 0 : }
272 :
273 0 : void FrameBuffer::PropagateDecodability(const FrameInfo& info) {
274 0 : for (size_t d = 0; d < info.num_dependent_frames; ++d) {
275 0 : auto ref_info = frames_.find(info.dependent_frames[d]);
276 0 : RTC_DCHECK(ref_info != frames_.end());
277 0 : RTC_DCHECK_GT(ref_info->second.num_missing_decodable, 0U);
278 0 : --ref_info->second.num_missing_decodable;
279 : }
280 0 : }
281 :
282 0 : void FrameBuffer::AdvanceLastDecodedFrame(FrameMap::iterator decoded) {
283 0 : if (last_decoded_frame_it_ == frames_.end()) {
284 0 : last_decoded_frame_it_ = frames_.begin();
285 : } else {
286 0 : RTC_DCHECK(last_decoded_frame_it_->first < decoded->first);
287 0 : ++last_decoded_frame_it_;
288 : }
289 0 : --num_frames_buffered_;
290 0 : ++num_frames_history_;
291 :
292 : // First, delete non-decoded frames from the history.
293 0 : while (last_decoded_frame_it_ != decoded) {
294 0 : if (last_decoded_frame_it_->second.frame)
295 0 : --num_frames_buffered_;
296 0 : last_decoded_frame_it_ = frames_.erase(last_decoded_frame_it_);
297 : }
298 :
299 : // Then remove old history if we have too much history saved.
300 0 : if (num_frames_history_ > kMaxFramesHistory) {
301 0 : frames_.erase(frames_.begin());
302 0 : --num_frames_history_;
303 : }
304 0 : }
305 :
306 0 : bool FrameBuffer::UpdateFrameInfoWithIncomingFrame(const FrameObject& frame,
307 : FrameMap::iterator info) {
308 0 : FrameKey key(frame.picture_id, frame.spatial_layer);
309 0 : info->second.num_missing_continuous = frame.num_references;
310 0 : info->second.num_missing_decodable = frame.num_references;
311 :
312 0 : RTC_DCHECK(last_decoded_frame_it_ == frames_.end() ||
313 0 : last_decoded_frame_it_->first < info->first);
314 :
315 : // Check how many dependencies that have already been fulfilled.
316 0 : for (size_t i = 0; i < frame.num_references; ++i) {
317 0 : FrameKey ref_key(frame.references[i], frame.spatial_layer);
318 0 : auto ref_info = frames_.find(ref_key);
319 :
320 : // Does |frame| depend on a frame earlier than the last decoded frame?
321 0 : if (last_decoded_frame_it_ != frames_.end() &&
322 0 : ref_key <= last_decoded_frame_it_->first) {
323 0 : if (ref_info == frames_.end()) {
324 0 : LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) ("
325 0 : << key.picture_id << ":"
326 0 : << static_cast<int>(key.spatial_layer)
327 : << " depends on a non-decoded frame more previous than "
328 0 : << "the last decoded frame, dropping frame.";
329 0 : return false;
330 : }
331 :
332 0 : --info->second.num_missing_continuous;
333 0 : --info->second.num_missing_decodable;
334 : } else {
335 0 : if (ref_info == frames_.end())
336 0 : ref_info = frames_.insert(std::make_pair(ref_key, FrameInfo())).first;
337 :
338 0 : if (ref_info->second.continuous)
339 0 : --info->second.num_missing_continuous;
340 :
341 : // Add backwards reference so |frame| can be updated when new
342 : // frames are inserted or decoded.
343 0 : ref_info->second.dependent_frames[ref_info->second.num_dependent_frames] =
344 0 : key;
345 0 : ++ref_info->second.num_dependent_frames;
346 : }
347 0 : RTC_DCHECK_LE(ref_info->second.num_missing_continuous,
348 0 : ref_info->second.num_missing_decodable);
349 : }
350 :
351 : // Check if we have the lower spatial layer frame.
352 0 : if (frame.inter_layer_predicted) {
353 0 : ++info->second.num_missing_continuous;
354 0 : ++info->second.num_missing_decodable;
355 :
356 0 : FrameKey ref_key(frame.picture_id, frame.spatial_layer - 1);
357 : // Gets or create the FrameInfo for the referenced frame.
358 0 : auto ref_info = frames_.insert(std::make_pair(ref_key, FrameInfo())).first;
359 0 : if (ref_info->second.continuous)
360 0 : --info->second.num_missing_continuous;
361 :
362 0 : if (ref_info == last_decoded_frame_it_) {
363 0 : --info->second.num_missing_decodable;
364 : } else {
365 0 : ref_info->second.dependent_frames[ref_info->second.num_dependent_frames] =
366 0 : key;
367 0 : ++ref_info->second.num_dependent_frames;
368 : }
369 0 : RTC_DCHECK_LE(ref_info->second.num_missing_continuous,
370 0 : ref_info->second.num_missing_decodable);
371 : }
372 :
373 0 : RTC_DCHECK_LE(info->second.num_missing_continuous,
374 0 : info->second.num_missing_decodable);
375 :
376 0 : return true;
377 : }
378 :
379 0 : void FrameBuffer::UpdateJitterDelay() {
380 : int unused;
381 : int delay;
382 0 : timing_->GetTimings(&unused, &unused, &unused, &unused, &delay, &unused,
383 0 : &unused);
384 :
385 0 : accumulated_delay_ += delay;
386 0 : ++accumulated_delay_samples_;
387 0 : }
388 :
389 0 : void FrameBuffer::UpdateHistograms() const {
390 0 : rtc::CritScope lock(&crit_);
391 0 : if (num_total_frames_ > 0) {
392 0 : int key_frames_permille = (static_cast<float>(num_key_frames_) * 1000.0f /
393 0 : static_cast<float>(num_total_frames_) +
394 0 : 0.5f);
395 0 : RTC_HISTOGRAM_COUNTS_1000("WebRTC.Video.KeyFramesReceivedInPermille",
396 : key_frames_permille);
397 : }
398 :
399 0 : if (accumulated_delay_samples_ > 0) {
400 0 : RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.JitterBufferDelayInMs",
401 : accumulated_delay_ / accumulated_delay_samples_);
402 : }
403 0 : }
404 :
405 0 : void FrameBuffer::ClearFramesAndHistory() {
406 0 : frames_.clear();
407 0 : last_decoded_frame_it_ = frames_.end();
408 0 : last_continuous_frame_it_ = frames_.end();
409 0 : num_frames_history_ = 0;
410 0 : num_frames_buffered_ = 0;
411 0 : }
412 :
413 : } // namespace video_coding
414 : } // namespace webrtc
|
