Line data Source code
1 : /*
2 : * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 :
11 : #include "webrtc/modules/video_coding/rtp_frame_reference_finder.h"
12 :
13 : #include <algorithm>
14 : #include <limits>
15 :
16 : #include "webrtc/base/checks.h"
17 : #include "webrtc/base/logging.h"
18 : #include "webrtc/modules/video_coding/frame_object.h"
19 : #include "webrtc/modules/video_coding/packet_buffer.h"
20 :
21 : namespace webrtc {
22 : namespace video_coding {
23 :
24 0 : RtpFrameReferenceFinder::RtpFrameReferenceFinder(
25 0 : OnCompleteFrameCallback* frame_callback)
26 : : last_picture_id_(-1),
27 : last_unwrap_(-1),
28 : current_ss_idx_(0),
29 : cleared_to_seq_num_(-1),
30 0 : frame_callback_(frame_callback) {}
31 :
32 0 : void RtpFrameReferenceFinder::ManageFrame(
33 : std::unique_ptr<RtpFrameObject> frame) {
34 0 : rtc::CritScope lock(&crit_);
35 :
36 : // If we have cleared past this frame, drop it.
37 0 : if (cleared_to_seq_num_ != -1 &&
38 0 : AheadOf<uint16_t>(cleared_to_seq_num_, frame->first_seq_num())) {
39 0 : return;
40 : }
41 :
42 0 : switch (frame->codec_type()) {
43 : case kVideoCodecFlexfec:
44 : case kVideoCodecULPFEC:
45 : case kVideoCodecRED:
46 0 : RTC_NOTREACHED();
47 0 : break;
48 : case kVideoCodecVP8:
49 0 : ManageFrameVp8(std::move(frame));
50 0 : break;
51 : case kVideoCodecVP9:
52 0 : ManageFrameVp9(std::move(frame));
53 0 : break;
54 : // Since the EndToEndTests use kVicdeoCodecUnknow we treat it the same as
55 : // kVideoCodecGeneric.
56 : // TODO(philipel): Take a look at the EndToEndTests and see if maybe they
57 : // should be changed to use kVideoCodecGeneric instead.
58 : case kVideoCodecUnknown:
59 : case kVideoCodecH264:
60 : case kVideoCodecI420:
61 : case kVideoCodecGeneric:
62 0 : ManageFrameGeneric(std::move(frame), kNoPictureId);
63 0 : break;
64 : }
65 : }
66 :
67 0 : void RtpFrameReferenceFinder::PaddingReceived(uint16_t seq_num) {
68 0 : rtc::CritScope lock(&crit_);
69 : auto clean_padding_to =
70 0 : stashed_padding_.lower_bound(seq_num - kMaxPaddingAge);
71 0 : stashed_padding_.erase(stashed_padding_.begin(), clean_padding_to);
72 0 : stashed_padding_.insert(seq_num);
73 0 : UpdateLastPictureIdWithPadding(seq_num);
74 0 : RetryStashedFrames();
75 0 : }
76 :
77 0 : void RtpFrameReferenceFinder::ClearTo(uint16_t seq_num) {
78 0 : rtc::CritScope lock(&crit_);
79 0 : cleared_to_seq_num_ = seq_num;
80 :
81 0 : auto it = stashed_frames_.begin();
82 0 : while (it != stashed_frames_.end()) {
83 0 : if (AheadOf<uint16_t>(cleared_to_seq_num_, (*it)->first_seq_num())) {
84 0 : it = stashed_frames_.erase(it);
85 : } else {
86 0 : ++it;
87 : }
88 : }
89 0 : }
90 :
91 0 : void RtpFrameReferenceFinder::UpdateLastPictureIdWithPadding(uint16_t seq_num) {
92 0 : auto gop_seq_num_it = last_seq_num_gop_.upper_bound(seq_num);
93 :
94 : // If this padding packet "belongs" to a group of pictures that we don't track
95 : // anymore, do nothing.
96 0 : if (gop_seq_num_it == last_seq_num_gop_.begin())
97 0 : return;
98 0 : --gop_seq_num_it;
99 :
100 : // Calculate the next contiuous sequence number and search for it in
101 : // the padding packets we have stashed.
102 0 : uint16_t next_seq_num_with_padding = gop_seq_num_it->second.second + 1;
103 : auto padding_seq_num_it =
104 0 : stashed_padding_.lower_bound(next_seq_num_with_padding);
105 :
106 : // While there still are padding packets and those padding packets are
107 : // continuous, then advance the "last-picture-id-with-padding" and remove
108 : // the stashed padding packet.
109 0 : while (padding_seq_num_it != stashed_padding_.end() &&
110 0 : *padding_seq_num_it == next_seq_num_with_padding) {
111 0 : gop_seq_num_it->second.second = next_seq_num_with_padding;
112 0 : ++next_seq_num_with_padding;
113 0 : padding_seq_num_it = stashed_padding_.erase(padding_seq_num_it);
114 : }
115 : }
116 :
117 0 : void RtpFrameReferenceFinder::RetryStashedFrames() {
118 0 : size_t num_stashed_frames = stashed_frames_.size();
119 :
120 : // Clean up stashed frames if there are too many.
121 0 : while (stashed_frames_.size() > kMaxStashedFrames)
122 0 : stashed_frames_.pop_front();
123 :
124 : // Since frames are stashed if there is not enough data to determine their
125 : // frame references we should at most check |stashed_frames_.size()| in
126 : // order to not pop and push frames in and endless loop.
127 : // NOTE! This function may be called recursively, hence the
128 : // "!stashed_frames_.empty()" condition.
129 0 : for (size_t i = 0; i < num_stashed_frames && !stashed_frames_.empty(); ++i) {
130 0 : std::unique_ptr<RtpFrameObject> frame = std::move(stashed_frames_.front());
131 0 : stashed_frames_.pop_front();
132 0 : ManageFrame(std::move(frame));
133 : }
134 0 : }
135 :
136 0 : void RtpFrameReferenceFinder::ManageFrameGeneric(
137 : std::unique_ptr<RtpFrameObject> frame,
138 : int picture_id) {
139 : // If |picture_id| is specified then we use that to set the frame references,
140 : // otherwise we use sequence number.
141 0 : if (picture_id != kNoPictureId) {
142 0 : if (last_unwrap_ == -1)
143 0 : last_unwrap_ = picture_id;
144 :
145 0 : frame->picture_id = UnwrapPictureId(picture_id % kPicIdLength);
146 0 : frame->num_references = frame->frame_type() == kVideoFrameKey ? 0 : 1;
147 0 : frame->references[0] = frame->picture_id - 1;
148 0 : frame_callback_->OnCompleteFrame(std::move(frame));
149 0 : return;
150 : }
151 :
152 0 : if (frame->frame_type() == kVideoFrameKey) {
153 0 : last_seq_num_gop_.insert(std::make_pair(
154 0 : frame->last_seq_num(),
155 0 : std::make_pair(frame->last_seq_num(), frame->last_seq_num())));
156 : }
157 :
158 : // We have received a frame but not yet a keyframe, stash this frame.
159 0 : if (last_seq_num_gop_.empty()) {
160 0 : stashed_frames_.push_back(std::move(frame));
161 0 : return;
162 : }
163 :
164 : // Clean up info for old keyframes but make sure to keep info
165 : // for the last keyframe.
166 0 : auto clean_to = last_seq_num_gop_.lower_bound(frame->last_seq_num() - 100);
167 0 : if (clean_to != last_seq_num_gop_.end())
168 0 : last_seq_num_gop_.erase(last_seq_num_gop_.begin(), clean_to);
169 :
170 : // Find the last sequence number of the last frame for the keyframe
171 : // that this frame indirectly references.
172 0 : auto seq_num_it = last_seq_num_gop_.upper_bound(frame->last_seq_num());
173 0 : if (seq_num_it == last_seq_num_gop_.begin()) {
174 0 : LOG(LS_WARNING) << "Generic frame with packet range ["
175 0 : << frame->first_seq_num() << ", " << frame->last_seq_num()
176 0 : << "] has no Gop, dropping frame.";
177 0 : return;
178 : }
179 0 : seq_num_it--;
180 :
181 : // Make sure the packet sequence numbers are continuous, otherwise stash
182 : // this frame.
183 0 : uint16_t last_picture_id_gop = seq_num_it->second.first;
184 0 : uint16_t last_picture_id_with_padding_gop = seq_num_it->second.second;
185 0 : if (frame->frame_type() == kVideoFrameDelta) {
186 0 : uint16_t prev_seq_num = frame->first_seq_num() - 1;
187 0 : if (prev_seq_num != last_picture_id_with_padding_gop) {
188 0 : stashed_frames_.push_back(std::move(frame));
189 0 : return;
190 : }
191 : }
192 :
193 0 : RTC_DCHECK(AheadOrAt(frame->last_seq_num(), seq_num_it->first));
194 :
195 : // Since keyframes can cause reordering we can't simply assign the
196 : // picture id according to some incrementing counter.
197 0 : frame->picture_id = frame->last_seq_num();
198 0 : frame->num_references = frame->frame_type() == kVideoFrameDelta;
199 0 : frame->references[0] = last_picture_id_gop;
200 0 : if (AheadOf(frame->picture_id, last_picture_id_gop)) {
201 0 : seq_num_it->second.first = frame->picture_id;
202 0 : seq_num_it->second.second = frame->picture_id;
203 : }
204 :
205 0 : last_picture_id_ = frame->picture_id;
206 0 : UpdateLastPictureIdWithPadding(frame->picture_id);
207 0 : frame_callback_->OnCompleteFrame(std::move(frame));
208 0 : RetryStashedFrames();
209 : }
210 :
211 0 : void RtpFrameReferenceFinder::ManageFrameVp8(
212 : std::unique_ptr<RtpFrameObject> frame) {
213 0 : rtc::Optional<RTPVideoTypeHeader> rtp_codec_header = frame->GetCodecHeader();
214 0 : if (!rtp_codec_header)
215 0 : return;
216 :
217 0 : const RTPVideoHeaderVP8& codec_header = rtp_codec_header->VP8;
218 :
219 0 : if (codec_header.pictureId == kNoPictureId ||
220 0 : codec_header.temporalIdx == kNoTemporalIdx ||
221 0 : codec_header.tl0PicIdx == kNoTl0PicIdx) {
222 0 : ManageFrameGeneric(std::move(frame), codec_header.pictureId);
223 0 : return;
224 : }
225 :
226 0 : frame->picture_id = codec_header.pictureId % kPicIdLength;
227 :
228 0 : if (last_unwrap_ == -1)
229 0 : last_unwrap_ = codec_header.pictureId;
230 :
231 0 : if (last_picture_id_ == -1)
232 0 : last_picture_id_ = frame->picture_id;
233 :
234 : // Find if there has been a gap in fully received frames and save the picture
235 : // id of those frames in |not_yet_received_frames_|.
236 0 : if (AheadOf<uint16_t, kPicIdLength>(frame->picture_id, last_picture_id_)) {
237 0 : last_picture_id_ = Add<kPicIdLength>(last_picture_id_, 1);
238 0 : while (last_picture_id_ != frame->picture_id) {
239 0 : not_yet_received_frames_.insert(last_picture_id_);
240 0 : last_picture_id_ = Add<kPicIdLength>(last_picture_id_, 1);
241 : }
242 : }
243 :
244 : // Clean up info for base layers that are too old.
245 0 : uint8_t old_tl0_pic_idx = codec_header.tl0PicIdx - kMaxLayerInfo;
246 0 : auto clean_layer_info_to = layer_info_.lower_bound(old_tl0_pic_idx);
247 0 : layer_info_.erase(layer_info_.begin(), clean_layer_info_to);
248 :
249 : // Clean up info about not yet received frames that are too old.
250 : uint16_t old_picture_id =
251 0 : Subtract<kPicIdLength>(frame->picture_id, kMaxNotYetReceivedFrames);
252 0 : auto clean_frames_to = not_yet_received_frames_.lower_bound(old_picture_id);
253 : not_yet_received_frames_.erase(not_yet_received_frames_.begin(),
254 0 : clean_frames_to);
255 :
256 0 : if (frame->frame_type() == kVideoFrameKey) {
257 0 : frame->num_references = 0;
258 0 : layer_info_[codec_header.tl0PicIdx].fill(-1);
259 0 : CompletedFrameVp8(std::move(frame));
260 0 : return;
261 : }
262 :
263 0 : auto layer_info_it = layer_info_.find(codec_header.temporalIdx == 0
264 0 : ? codec_header.tl0PicIdx - 1
265 0 : : codec_header.tl0PicIdx);
266 :
267 : // If we don't have the base layer frame yet, stash this frame.
268 0 : if (layer_info_it == layer_info_.end()) {
269 0 : stashed_frames_.push_back(std::move(frame));
270 0 : return;
271 : }
272 :
273 : // A non keyframe base layer frame has been received, copy the layer info
274 : // from the previous base layer frame and set a reference to the previous
275 : // base layer frame.
276 0 : if (codec_header.temporalIdx == 0) {
277 0 : layer_info_it =
278 : layer_info_
279 0 : .insert(make_pair(codec_header.tl0PicIdx, layer_info_it->second))
280 : .first;
281 0 : frame->num_references = 1;
282 0 : frame->references[0] = layer_info_it->second[0];
283 0 : CompletedFrameVp8(std::move(frame));
284 0 : return;
285 : }
286 :
287 : // Layer sync frame, this frame only references its base layer frame.
288 0 : if (codec_header.layerSync) {
289 0 : frame->num_references = 1;
290 0 : frame->references[0] = layer_info_it->second[0];
291 :
292 0 : CompletedFrameVp8(std::move(frame));
293 0 : return;
294 : }
295 :
296 : // Find all references for this frame.
297 0 : frame->num_references = 0;
298 0 : for (uint8_t layer = 0; layer <= codec_header.temporalIdx; ++layer) {
299 : // If we have not yet received a previous frame on this temporal layer,
300 : // stash this frame.
301 0 : if (layer_info_it->second[layer] == -1) {
302 0 : stashed_frames_.push_back(std::move(frame));
303 0 : return;
304 : }
305 :
306 : // If the last frame on this layer is ahead of this frame it means that
307 : // a layer sync frame has been received after this frame for the same
308 : // base layer frame, drop this frame.
309 0 : if (AheadOf<uint16_t, kPicIdLength>(layer_info_it->second[layer],
310 0 : frame->picture_id)) {
311 0 : return;
312 : }
313 :
314 : // If we have not yet received a frame between this frame and the referenced
315 : // frame then we have to wait for that frame to be completed first.
316 : auto not_received_frame_it =
317 0 : not_yet_received_frames_.upper_bound(layer_info_it->second[layer]);
318 0 : if (not_received_frame_it != not_yet_received_frames_.end() &&
319 0 : AheadOf<uint16_t, kPicIdLength>(frame->picture_id,
320 0 : *not_received_frame_it)) {
321 0 : stashed_frames_.push_back(std::move(frame));
322 0 : return;
323 : }
324 :
325 0 : RTC_DCHECK((AheadOf<uint16_t, kPicIdLength>(frame->picture_id,
326 0 : layer_info_it->second[layer])));
327 0 : ++frame->num_references;
328 0 : frame->references[layer] = layer_info_it->second[layer];
329 : }
330 :
331 0 : CompletedFrameVp8(std::move(frame));
332 : }
333 :
334 0 : void RtpFrameReferenceFinder::CompletedFrameVp8(
335 : std::unique_ptr<RtpFrameObject> frame) {
336 0 : rtc::Optional<RTPVideoTypeHeader> rtp_codec_header = frame->GetCodecHeader();
337 0 : if (!rtp_codec_header)
338 0 : return;
339 :
340 0 : const RTPVideoHeaderVP8& codec_header = rtp_codec_header->VP8;
341 :
342 0 : uint8_t tl0_pic_idx = codec_header.tl0PicIdx;
343 0 : uint8_t temporal_index = codec_header.temporalIdx;
344 0 : auto layer_info_it = layer_info_.find(tl0_pic_idx);
345 :
346 : // Update this layer info and newer.
347 0 : while (layer_info_it != layer_info_.end()) {
348 0 : if (layer_info_it->second[temporal_index] != -1 &&
349 0 : AheadOf<uint16_t, kPicIdLength>(layer_info_it->second[temporal_index],
350 0 : frame->picture_id)) {
351 : // The frame was not newer, then no subsequent layer info have to be
352 : // update.
353 0 : break;
354 : }
355 :
356 0 : layer_info_it->second[codec_header.temporalIdx] = frame->picture_id;
357 0 : ++tl0_pic_idx;
358 0 : layer_info_it = layer_info_.find(tl0_pic_idx);
359 : }
360 0 : not_yet_received_frames_.erase(frame->picture_id);
361 :
362 0 : for (size_t i = 0; i < frame->num_references; ++i)
363 0 : frame->references[i] = UnwrapPictureId(frame->references[i]);
364 0 : frame->picture_id = UnwrapPictureId(frame->picture_id);
365 :
366 0 : frame_callback_->OnCompleteFrame(std::move(frame));
367 0 : RetryStashedFrames();
368 : }
369 :
370 0 : void RtpFrameReferenceFinder::ManageFrameVp9(
371 : std::unique_ptr<RtpFrameObject> frame) {
372 0 : rtc::Optional<RTPVideoTypeHeader> rtp_codec_header = frame->GetCodecHeader();
373 0 : if (!rtp_codec_header)
374 0 : return;
375 :
376 0 : const RTPVideoHeaderVP9& codec_header = rtp_codec_header->VP9;
377 :
378 0 : bool old_frame = Vp9PidTl0Fix(*frame, &rtp_codec_header->VP9.picture_id,
379 0 : &rtp_codec_header->VP9.tl0_pic_idx);
380 0 : if (old_frame)
381 0 : return;
382 :
383 0 : if (codec_header.picture_id == kNoPictureId ||
384 0 : codec_header.temporal_idx == kNoTemporalIdx) {
385 0 : ManageFrameGeneric(std::move(frame), codec_header.picture_id);
386 0 : return;
387 : }
388 :
389 0 : frame->spatial_layer = codec_header.spatial_idx;
390 0 : frame->inter_layer_predicted = codec_header.inter_layer_predicted;
391 0 : frame->picture_id = codec_header.picture_id % kPicIdLength;
392 :
393 0 : if (last_unwrap_ == -1)
394 0 : last_unwrap_ = codec_header.picture_id;
395 :
396 0 : if (last_picture_id_ == -1)
397 0 : last_picture_id_ = frame->picture_id;
398 :
399 0 : if (codec_header.flexible_mode) {
400 0 : frame->num_references = codec_header.num_ref_pics;
401 0 : for (size_t i = 0; i < frame->num_references; ++i) {
402 0 : frame->references[i] =
403 0 : Subtract<1 << 16>(frame->picture_id, codec_header.pid_diff[i]);
404 : }
405 :
406 0 : CompletedFrameVp9(std::move(frame));
407 0 : return;
408 : }
409 :
410 0 : if (codec_header.ss_data_available) {
411 : // Scalability structures can only be sent with tl0 frames.
412 0 : if (codec_header.temporal_idx != 0) {
413 0 : LOG(LS_WARNING) << "Received scalability structure on a non base layer"
414 0 : " frame. Scalability structure ignored.";
415 : } else {
416 0 : current_ss_idx_ = Add<kMaxGofSaved>(current_ss_idx_, 1);
417 0 : scalability_structures_[current_ss_idx_] = codec_header.gof;
418 0 : scalability_structures_[current_ss_idx_].pid_start = frame->picture_id;
419 :
420 0 : GofInfo info(&scalability_structures_[current_ss_idx_],
421 0 : frame->picture_id);
422 0 : gof_info_.insert(std::make_pair(codec_header.tl0_pic_idx, info));
423 : }
424 : }
425 :
426 : // Clean up info for base layers that are too old.
427 0 : uint8_t old_tl0_pic_idx = codec_header.tl0_pic_idx - kMaxGofSaved;
428 0 : auto clean_gof_info_to = gof_info_.lower_bound(old_tl0_pic_idx);
429 0 : gof_info_.erase(gof_info_.begin(), clean_gof_info_to);
430 :
431 0 : if (frame->frame_type() == kVideoFrameKey) {
432 : // When using GOF all keyframes must include the scalability structure.
433 0 : if (!codec_header.ss_data_available)
434 0 : LOG(LS_WARNING) << "Received keyframe without scalability structure";
435 :
436 0 : frame->num_references = 0;
437 0 : GofInfo info = gof_info_.find(codec_header.tl0_pic_idx)->second;
438 0 : FrameReceivedVp9(frame->picture_id, &info);
439 0 : CompletedFrameVp9(std::move(frame));
440 0 : return;
441 : }
442 :
443 : auto gof_info_it = gof_info_.find(
444 0 : (codec_header.temporal_idx == 0 && !codec_header.ss_data_available)
445 0 : ? codec_header.tl0_pic_idx - 1
446 0 : : codec_header.tl0_pic_idx);
447 :
448 : // Gof info for this frame is not available yet, stash this frame.
449 0 : if (gof_info_it == gof_info_.end()) {
450 0 : stashed_frames_.push_back(std::move(frame));
451 0 : return;
452 : }
453 :
454 0 : GofInfo* info = &gof_info_it->second;
455 0 : FrameReceivedVp9(frame->picture_id, info);
456 :
457 : // Make sure we don't miss any frame that could potentially have the
458 : // up switch flag set.
459 0 : if (MissingRequiredFrameVp9(frame->picture_id, *info)) {
460 0 : stashed_frames_.push_back(std::move(frame));
461 0 : return;
462 : }
463 :
464 0 : if (codec_header.temporal_up_switch) {
465 : auto pid_tidx =
466 0 : std::make_pair(frame->picture_id, codec_header.temporal_idx);
467 0 : up_switch_.insert(pid_tidx);
468 : }
469 :
470 : // If this is a base layer frame that contains a scalability structure
471 : // then gof info has already been inserted earlier, so we only want to
472 : // insert if we haven't done so already.
473 0 : if (codec_header.temporal_idx == 0 && !codec_header.ss_data_available) {
474 0 : GofInfo new_info(info->gof, frame->picture_id);
475 0 : gof_info_.insert(std::make_pair(codec_header.tl0_pic_idx, new_info));
476 : }
477 :
478 : // Clean out old info about up switch frames.
479 0 : uint16_t old_picture_id = Subtract<kPicIdLength>(frame->picture_id, 50);
480 0 : auto up_switch_erase_to = up_switch_.lower_bound(old_picture_id);
481 0 : up_switch_.erase(up_switch_.begin(), up_switch_erase_to);
482 :
483 0 : size_t diff = ForwardDiff<uint16_t, kPicIdLength>(info->gof->pid_start,
484 0 : frame->picture_id);
485 0 : size_t gof_idx = diff % info->gof->num_frames_in_gof;
486 :
487 : // Populate references according to the scalability structure.
488 0 : frame->num_references = info->gof->num_ref_pics[gof_idx];
489 0 : for (size_t i = 0; i < frame->num_references; ++i) {
490 0 : frame->references[i] = Subtract<kPicIdLength>(
491 0 : frame->picture_id, info->gof->pid_diff[gof_idx][i]);
492 :
493 : // If this is a reference to a frame earlier than the last up switch point,
494 : // then ignore this reference.
495 0 : if (UpSwitchInIntervalVp9(frame->picture_id, codec_header.temporal_idx,
496 0 : frame->references[i])) {
497 0 : --frame->num_references;
498 : }
499 : }
500 :
501 0 : CompletedFrameVp9(std::move(frame));
502 : }
503 :
504 0 : bool RtpFrameReferenceFinder::MissingRequiredFrameVp9(uint16_t picture_id,
505 : const GofInfo& info) {
506 : size_t diff =
507 0 : ForwardDiff<uint16_t, kPicIdLength>(info.gof->pid_start, picture_id);
508 0 : size_t gof_idx = diff % info.gof->num_frames_in_gof;
509 0 : size_t temporal_idx = info.gof->temporal_idx[gof_idx];
510 :
511 : // For every reference this frame has, check if there is a frame missing in
512 : // the interval (|ref_pid|, |picture_id|) in any of the lower temporal
513 : // layers. If so, we are missing a required frame.
514 0 : uint8_t num_references = info.gof->num_ref_pics[gof_idx];
515 0 : for (size_t i = 0; i < num_references; ++i) {
516 : uint16_t ref_pid =
517 0 : Subtract<kPicIdLength>(picture_id, info.gof->pid_diff[gof_idx][i]);
518 0 : for (size_t l = 0; l < temporal_idx; ++l) {
519 0 : auto missing_frame_it = missing_frames_for_layer_[l].lower_bound(ref_pid);
520 0 : if (missing_frame_it != missing_frames_for_layer_[l].end() &&
521 0 : AheadOf<uint16_t, kPicIdLength>(picture_id, *missing_frame_it)) {
522 0 : return true;
523 : }
524 : }
525 : }
526 0 : return false;
527 : }
528 :
529 0 : void RtpFrameReferenceFinder::FrameReceivedVp9(uint16_t picture_id,
530 : GofInfo* info) {
531 0 : int last_picture_id = info->last_picture_id;
532 :
533 : // If there is a gap, find which temporal layer the missing frames
534 : // belong to and add the frame as missing for that temporal layer.
535 : // Otherwise, remove this frame from the set of missing frames.
536 0 : if (AheadOf<uint16_t, kPicIdLength>(picture_id, last_picture_id)) {
537 0 : size_t diff = ForwardDiff<uint16_t, kPicIdLength>(info->gof->pid_start,
538 0 : last_picture_id);
539 0 : size_t gof_idx = diff % info->gof->num_frames_in_gof;
540 :
541 0 : last_picture_id = Add<kPicIdLength>(last_picture_id, 1);
542 0 : while (last_picture_id != picture_id) {
543 0 : ++gof_idx;
544 0 : RTC_DCHECK_NE(0ul, gof_idx % info->gof->num_frames_in_gof);
545 0 : size_t temporal_idx = info->gof->temporal_idx[gof_idx];
546 0 : missing_frames_for_layer_[temporal_idx].insert(last_picture_id);
547 0 : last_picture_id = Add<kPicIdLength>(last_picture_id, 1);
548 : }
549 0 : info->last_picture_id = last_picture_id;
550 : } else {
551 : size_t diff =
552 0 : ForwardDiff<uint16_t, kPicIdLength>(info->gof->pid_start, picture_id);
553 0 : size_t gof_idx = diff % info->gof->num_frames_in_gof;
554 0 : size_t temporal_idx = info->gof->temporal_idx[gof_idx];
555 0 : missing_frames_for_layer_[temporal_idx].erase(picture_id);
556 : }
557 0 : }
558 :
559 0 : bool RtpFrameReferenceFinder::UpSwitchInIntervalVp9(uint16_t picture_id,
560 : uint8_t temporal_idx,
561 : uint16_t pid_ref) {
562 0 : for (auto up_switch_it = up_switch_.upper_bound(pid_ref);
563 0 : up_switch_it != up_switch_.end() &&
564 0 : AheadOf<uint16_t, kPicIdLength>(picture_id, up_switch_it->first);
565 : ++up_switch_it) {
566 0 : if (up_switch_it->second < temporal_idx)
567 0 : return true;
568 : }
569 :
570 0 : return false;
571 : }
572 :
573 0 : void RtpFrameReferenceFinder::CompletedFrameVp9(
574 : std::unique_ptr<RtpFrameObject> frame) {
575 0 : for (size_t i = 0; i < frame->num_references; ++i)
576 0 : frame->references[i] = UnwrapPictureId(frame->references[i]);
577 0 : frame->picture_id = UnwrapPictureId(frame->picture_id);
578 :
579 0 : frame_callback_->OnCompleteFrame(std::move(frame));
580 0 : RetryStashedFrames();
581 0 : }
582 :
583 0 : uint16_t RtpFrameReferenceFinder::UnwrapPictureId(uint16_t picture_id) {
584 0 : RTC_DCHECK_NE(-1, last_unwrap_);
585 :
586 0 : uint16_t unwrap_truncated = last_unwrap_ % kPicIdLength;
587 0 : uint16_t diff = MinDiff<uint16_t, kPicIdLength>(unwrap_truncated, picture_id);
588 :
589 0 : if (AheadOf<uint16_t, kPicIdLength>(picture_id, unwrap_truncated))
590 0 : last_unwrap_ = Add<1 << 16>(last_unwrap_, diff);
591 : else
592 0 : last_unwrap_ = Subtract<1 << 16>(last_unwrap_, diff);
593 :
594 0 : return last_unwrap_;
595 : }
596 :
597 0 : bool RtpFrameReferenceFinder::Vp9PidTl0Fix(const RtpFrameObject& frame,
598 : int16_t* picture_id,
599 : int16_t* tl0_pic_idx) {
600 0 : const int kTl0PicIdLength = 256;
601 0 : const uint8_t kMaxPidDiff = 128;
602 :
603 : // We are currently receiving VP9 without PID, nothing to fix.
604 0 : if (*picture_id == kNoPictureId)
605 0 : return false;
606 :
607 : // If |vp9_fix_jump_timestamp_| != -1 then a jump has occurred recently.
608 0 : if (vp9_fix_jump_timestamp_ != -1) {
609 : // If this frame has a timestamp older than |vp9_fix_jump_timestamp_| then
610 : // this frame is old (more previous than the frame where we detected the
611 : // jump) and should be dropped.
612 0 : if (AheadOf<uint32_t>(vp9_fix_jump_timestamp_, frame.timestamp))
613 0 : return true;
614 :
615 : // After 60 seconds, reset |vp9_fix_jump_timestamp_| in order to not
616 : // discard old frames when the timestamp wraps.
617 : int diff_ms =
618 0 : ForwardDiff<uint32_t>(vp9_fix_jump_timestamp_, frame.timestamp) / 90;
619 0 : if (diff_ms > 60 * 1000)
620 0 : vp9_fix_jump_timestamp_ = -1;
621 : }
622 :
623 : // Update |vp9_fix_last_timestamp_| with the most recent timestamp.
624 0 : if (vp9_fix_last_timestamp_ == -1)
625 0 : vp9_fix_last_timestamp_ = frame.timestamp;
626 0 : if (AheadOf<uint32_t>(frame.timestamp, vp9_fix_last_timestamp_))
627 0 : vp9_fix_last_timestamp_ = frame.timestamp;
628 :
629 0 : uint16_t fixed_pid = Add<kPicIdLength>(*picture_id, vp9_fix_pid_offset_);
630 0 : if (vp9_fix_last_picture_id_ == -1)
631 0 : vp9_fix_last_picture_id_ = *picture_id;
632 :
633 0 : int16_t fixed_tl0 = kNoTl0PicIdx;
634 0 : if (*tl0_pic_idx != kNoTl0PicIdx) {
635 0 : fixed_tl0 = Add<kTl0PicIdLength>(*tl0_pic_idx, vp9_fix_tl0_pic_idx_offset_);
636 : // Update |vp9_fix_last_tl0_pic_idx_| with the most recent tl0 pic index.
637 0 : if (vp9_fix_last_tl0_pic_idx_ == -1)
638 0 : vp9_fix_last_tl0_pic_idx_ = *tl0_pic_idx;
639 0 : if (AheadOf<uint8_t>(fixed_tl0, vp9_fix_last_tl0_pic_idx_))
640 0 : vp9_fix_last_tl0_pic_idx_ = fixed_tl0;
641 : }
642 :
643 0 : bool has_jumped = DetectVp9PicIdJump(fixed_pid, fixed_tl0, frame.timestamp);
644 0 : if (!has_jumped)
645 0 : has_jumped = DetectVp9Tl0PicIdxJump(fixed_tl0, frame.timestamp);
646 :
647 0 : if (has_jumped) {
648 : // First we calculate the offset to get to the previous picture id, and then
649 : // we add kMaxPid to avoid accidently referencing any previous
650 : // frames that was inserted into the FrameBuffer.
651 0 : vp9_fix_pid_offset_ = ForwardDiff<uint16_t, kPicIdLength>(
652 0 : *picture_id, vp9_fix_last_picture_id_);
653 0 : vp9_fix_pid_offset_ += kMaxPidDiff;
654 :
655 0 : fixed_pid = Add<kPicIdLength>(*picture_id, vp9_fix_pid_offset_);
656 0 : vp9_fix_last_picture_id_ = fixed_pid;
657 0 : vp9_fix_jump_timestamp_ = frame.timestamp;
658 0 : gof_info_.clear();
659 :
660 0 : vp9_fix_tl0_pic_idx_offset_ =
661 0 : ForwardDiff<uint8_t>(*tl0_pic_idx, vp9_fix_last_tl0_pic_idx_);
662 0 : vp9_fix_tl0_pic_idx_offset_ += kMaxGofSaved;
663 0 : fixed_tl0 = Add<kTl0PicIdLength>(*tl0_pic_idx, vp9_fix_tl0_pic_idx_offset_);
664 0 : vp9_fix_last_tl0_pic_idx_ = fixed_tl0;
665 : }
666 :
667 : // Update |vp9_fix_last_picture_id_| with the most recent picture id.
668 0 : if (AheadOf<uint16_t, kPicIdLength>(fixed_pid, vp9_fix_last_picture_id_))
669 0 : vp9_fix_last_picture_id_ = fixed_pid;
670 :
671 0 : *picture_id = fixed_pid;
672 0 : *tl0_pic_idx = fixed_tl0;
673 :
674 0 : return false;
675 : }
676 :
677 0 : bool RtpFrameReferenceFinder::DetectVp9PicIdJump(int fixed_pid,
678 : int fixed_tl0,
679 : uint32_t timestamp) const {
680 : // Test if there has been a jump backwards in the picture id.
681 0 : if (AheadOrAt<uint32_t>(timestamp, vp9_fix_last_timestamp_) &&
682 0 : AheadOf<uint16_t, kPicIdLength>(vp9_fix_last_picture_id_, fixed_pid)) {
683 0 : return true;
684 : }
685 :
686 : // Test if we have jumped forward too much. The reason we have to do this
687 : // is because the FrameBuffer holds history of old frames and inserting
688 : // frames with a much advanced picture id can result in the frame buffer
689 : // holding more than half of the interval of picture ids.
690 0 : if (AheadOrAt<uint32_t>(timestamp, vp9_fix_last_timestamp_) &&
691 0 : ForwardDiff<uint16_t, kPicIdLength>(vp9_fix_last_picture_id_, fixed_pid) >
692 : 128) {
693 0 : return true;
694 : }
695 :
696 : // Special case where the picture id jump forward but not by much and the
697 : // tl0 jumps to the id of an already saved gof for that id. In order to
698 : // detect this we check if the picture id span over the length of the GOF.
699 0 : if (fixed_tl0 != kNoTl0PicIdx) {
700 0 : auto info_it = gof_info_.find(fixed_tl0);
701 0 : if (info_it != gof_info_.end()) {
702 : int last_pid_gof_idx_0 =
703 0 : Subtract<kPicIdLength>(info_it->second.last_picture_id,
704 0 : info_it->second.last_picture_id %
705 0 : info_it->second.gof->num_frames_in_gof);
706 0 : int pif_gof_end = Add<kPicIdLength>(
707 0 : last_pid_gof_idx_0, info_it->second.gof->num_frames_in_gof);
708 0 : if (AheadOf<uint16_t, kPicIdLength>(fixed_pid, pif_gof_end))
709 0 : return true;
710 : }
711 : }
712 :
713 0 : return false;
714 : }
715 :
716 0 : bool RtpFrameReferenceFinder::DetectVp9Tl0PicIdxJump(int fixed_tl0,
717 : uint32_t timestamp) const {
718 0 : if (fixed_tl0 != kNoTl0PicIdx) {
719 : // Test if there has been a jump backwards in tl0 pic index.
720 0 : if (AheadOrAt<uint32_t>(timestamp, vp9_fix_last_timestamp_) &&
721 0 : AheadOf<uint8_t>(vp9_fix_last_tl0_pic_idx_, fixed_tl0)) {
722 0 : return true;
723 : }
724 :
725 : // Test if there has been a jump forward. If the jump forward results
726 : // in the tl0 pic index for this frame to be considered smaller than the
727 : // smallest item in |gof_info_| then we have jumped forward far enough to
728 : // wrap.
729 0 : if (!gof_info_.empty() &&
730 0 : AheadOf<uint8_t>(gof_info_.begin()->first, fixed_tl0)) {
731 0 : return true;
732 : }
733 : }
734 0 : return false;
735 : }
736 :
737 : } // namespace video_coding
738 : } // namespace webrtc
|
