Line data Source code
1 : /*
2 : * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 :
11 : #ifndef WEBRTC_MODULES_VIDEO_CODING_RTP_FRAME_REFERENCE_FINDER_H_
12 : #define WEBRTC_MODULES_VIDEO_CODING_RTP_FRAME_REFERENCE_FINDER_H_
13 :
14 : #include <array>
15 : #include <map>
16 : #include <memory>
17 : #include <deque>
18 : #include <set>
19 : #include <utility>
20 :
21 : #include "webrtc/base/criticalsection.h"
22 : #include "webrtc/base/thread_annotations.h"
23 : #include "webrtc/modules/include/module_common_types.h"
24 : #include "webrtc/modules/video_coding/sequence_number_util.h"
25 :
26 : namespace webrtc {
27 : namespace video_coding {
28 :
29 : class FrameObject;
30 : class RtpFrameObject;
31 :
32 : // A complete frame is a frame which has received all its packets and all its
33 : // references are known.
34 0 : class OnCompleteFrameCallback {
35 : public:
36 0 : virtual ~OnCompleteFrameCallback() {}
37 : virtual void OnCompleteFrame(std::unique_ptr<FrameObject> frame) = 0;
38 : };
39 :
40 0 : class RtpFrameReferenceFinder {
41 : public:
42 : explicit RtpFrameReferenceFinder(OnCompleteFrameCallback* frame_callback);
43 :
44 : // Manage this frame until:
45 : // - We have all information needed to determine its references, after
46 : // which |frame_callback_| is called with the completed frame, or
47 : // - We have too many stashed frames (determined by |kMaxStashedFrames)
48 : // so we drop this frame, or
49 : // - It gets cleared by ClearTo, which also means we drop it.
50 : void ManageFrame(std::unique_ptr<RtpFrameObject> frame);
51 :
52 : // Notifies that padding has been received, which the reference finder
53 : // might need to calculate the references of a frame.
54 : void PaddingReceived(uint16_t seq_num);
55 :
56 : // Clear all stashed frames that include packets older than |seq_num|.
57 : void ClearTo(uint16_t seq_num);
58 :
59 : private:
60 : static const uint16_t kPicIdLength = 1 << 15;
61 : static const uint8_t kMaxTemporalLayers = 5;
62 : static const int kMaxLayerInfo = 50;
63 : static const int kMaxStashedFrames = 50;
64 : static const int kMaxNotYetReceivedFrames = 100;
65 : static const int kMaxGofSaved = 50;
66 : static const int kMaxPaddingAge = 100;
67 :
68 :
69 : struct GofInfo {
70 0 : GofInfo(GofInfoVP9* gof, uint16_t last_picture_id)
71 0 : : gof(gof), last_picture_id(last_picture_id) {}
72 : GofInfoVP9* gof;
73 : uint16_t last_picture_id;
74 : };
75 :
76 : rtc::CriticalSection crit_;
77 :
78 : // Find the relevant group of pictures and update its "last-picture-id-with
79 : // padding" sequence number.
80 : void UpdateLastPictureIdWithPadding(uint16_t seq_num)
81 : EXCLUSIVE_LOCKS_REQUIRED(crit_);
82 :
83 : // Retry finding references for all frames that previously didn't have
84 : // all information needed.
85 : void RetryStashedFrames() EXCLUSIVE_LOCKS_REQUIRED(crit_);
86 :
87 : // Find references for generic frames. If |picture_id| is unspecified
88 : // then packet sequence numbers will be used to determine the references
89 : // of the frames.
90 : void ManageFrameGeneric(std::unique_ptr<RtpFrameObject> frame,
91 : int picture_id) EXCLUSIVE_LOCKS_REQUIRED(crit_);
92 :
93 : // Find references for Vp8 frames
94 : void ManageFrameVp8(std::unique_ptr<RtpFrameObject> frame)
95 : EXCLUSIVE_LOCKS_REQUIRED(crit_);
96 :
97 : // Updates all necessary state used to determine frame references
98 : // for Vp8 and then calls the |frame_callback| callback with the
99 : // completed frame.
100 : void CompletedFrameVp8(std::unique_ptr<RtpFrameObject> frame)
101 : EXCLUSIVE_LOCKS_REQUIRED(crit_);
102 :
103 : // Find references for Vp9 frames
104 : void ManageFrameVp9(std::unique_ptr<RtpFrameObject> frame)
105 : EXCLUSIVE_LOCKS_REQUIRED(crit_);
106 :
107 : // Unwrap the picture id and the frame references and then call the
108 : // |frame_callback| callback with the completed frame.
109 : void CompletedFrameVp9(std::unique_ptr<RtpFrameObject> frame)
110 : EXCLUSIVE_LOCKS_REQUIRED(crit_);
111 :
112 : // Check if we are missing a frame necessary to determine the references
113 : // for this frame.
114 : bool MissingRequiredFrameVp9(uint16_t picture_id, const GofInfo& info)
115 : EXCLUSIVE_LOCKS_REQUIRED(crit_);
116 :
117 : // Updates which frames that have been received. If there is a gap,
118 : // missing frames will be added to |missing_frames_for_layer_| or
119 : // if this is an already missing frame then it will be removed.
120 : void FrameReceivedVp9(uint16_t picture_id, GofInfo* info)
121 : EXCLUSIVE_LOCKS_REQUIRED(crit_);
122 :
123 : // Check if there is a frame with the up-switch flag set in the interval
124 : // (|pid_ref|, |picture_id|) with temporal layer smaller than |temporal_idx|.
125 : bool UpSwitchInIntervalVp9(uint16_t picture_id,
126 : uint8_t temporal_idx,
127 : uint16_t pid_ref) EXCLUSIVE_LOCKS_REQUIRED(crit_);
128 :
129 : // All picture ids are unwrapped to 16 bits.
130 : uint16_t UnwrapPictureId(uint16_t picture_id) EXCLUSIVE_LOCKS_REQUIRED(crit_);
131 :
132 : // Returns true if the frame is old and should be dropped.
133 : // TODO(philipel): Remove when VP9 PID/TL0 does not jump mid-stream (should be
134 : // around M59).
135 : bool Vp9PidTl0Fix(const RtpFrameObject& frame,
136 : int16_t* picture_id,
137 : int16_t* tl0_pic_idx) EXCLUSIVE_LOCKS_REQUIRED(crit_);
138 :
139 : // TODO(philipel): Remove when VP9 PID/TL0 does not jump mid-stream (should be
140 : // around M59).
141 : bool DetectVp9PicIdJump(int fixed_pid,
142 : int fixed_tl0,
143 : uint32_t timestamp) const
144 : EXCLUSIVE_LOCKS_REQUIRED(crit_);
145 :
146 : // TODO(philipel): Remove when VP9 PID/TL0 does not jump mid-stream (should be
147 : // around M59).
148 : bool DetectVp9Tl0PicIdxJump(int fixed_tl0, uint32_t timestamp) const
149 : EXCLUSIVE_LOCKS_REQUIRED(crit_);
150 :
151 : // For every group of pictures, hold two sequence numbers. The first being
152 : // the sequence number of the last packet of the last completed frame, and
153 : // the second being the sequence number of the last packet of the last
154 : // completed frame advanced by any potential continuous packets of padding.
155 : std::map<uint16_t,
156 : std::pair<uint16_t, uint16_t>,
157 : DescendingSeqNumComp<uint16_t>>
158 : last_seq_num_gop_ GUARDED_BY(crit_);
159 :
160 : // Save the last picture id in order to detect when there is a gap in frames
161 : // that have not yet been fully received.
162 : int last_picture_id_ GUARDED_BY(crit_);
163 :
164 : // Padding packets that have been received but that are not yet continuous
165 : // with any group of pictures.
166 : std::set<uint16_t, DescendingSeqNumComp<uint16_t>> stashed_padding_
167 : GUARDED_BY(crit_);
168 :
169 : // The last unwrapped picture id. Used to unwrap the picture id from a length
170 : // of |kPicIdLength| to 16 bits.
171 : int last_unwrap_ GUARDED_BY(crit_);
172 :
173 : // Frames earlier than the last received frame that have not yet been
174 : // fully received.
175 : std::set<uint16_t, DescendingSeqNumComp<uint16_t, kPicIdLength>>
176 : not_yet_received_frames_ GUARDED_BY(crit_);
177 :
178 : // Frames that have been fully received but didn't have all the information
179 : // needed to determine their references.
180 : std::deque<std::unique_ptr<RtpFrameObject>> stashed_frames_ GUARDED_BY(crit_);
181 :
182 : // Holds the information about the last completed frame for a given temporal
183 : // layer given a Tl0 picture index.
184 : std::map<uint8_t,
185 : std::array<int16_t, kMaxTemporalLayers>,
186 : DescendingSeqNumComp<uint8_t>>
187 : layer_info_ GUARDED_BY(crit_);
188 :
189 : // Where the current scalability structure is in the
190 : // |scalability_structures_| array.
191 : uint8_t current_ss_idx_;
192 :
193 : // Holds received scalability structures.
194 : std::array<GofInfoVP9, kMaxGofSaved> scalability_structures_
195 : GUARDED_BY(crit_);
196 :
197 : // Holds the the Gof information for a given TL0 picture index.
198 : std::map<uint8_t, GofInfo, DescendingSeqNumComp<uint8_t>> gof_info_
199 : GUARDED_BY(crit_);
200 :
201 : // Keep track of which picture id and which temporal layer that had the
202 : // up switch flag set.
203 : std::map<uint16_t, uint8_t, DescendingSeqNumComp<uint16_t, kPicIdLength>>
204 : up_switch_ GUARDED_BY(crit_);
205 :
206 : // For every temporal layer, keep a set of which frames that are missing.
207 : std::array<std::set<uint16_t, DescendingSeqNumComp<uint16_t, kPicIdLength>>,
208 : kMaxTemporalLayers>
209 : missing_frames_for_layer_ GUARDED_BY(crit_);
210 :
211 : // How far frames have been cleared by sequence number. A frame will be
212 : // cleared if it contains a packet with a sequence number older than
213 : // |cleared_to_seq_num_|.
214 : int cleared_to_seq_num_ GUARDED_BY(crit_);
215 :
216 : OnCompleteFrameCallback* frame_callback_;
217 :
218 : // Vp9PidFix variables
219 : // TODO(philipel): Remove when VP9 PID does not jump mid-stream.
220 : int vp9_fix_last_timestamp_ = -1;
221 : int vp9_fix_jump_timestamp_ = -1;
222 : int vp9_fix_last_picture_id_ = -1;
223 : int vp9_fix_pid_offset_ = 0;
224 : int vp9_fix_last_tl0_pic_idx_ = -1;
225 : int vp9_fix_tl0_pic_idx_offset_ = 0;
226 : };
227 :
228 : } // namespace video_coding
229 : } // namespace webrtc
230 :
231 : #endif // WEBRTC_MODULES_VIDEO_CODING_RTP_FRAME_REFERENCE_FINDER_H_
|
