Line data Source code
1 : /*
2 : * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 :
11 : #ifndef WEBRTC_MODULES_VIDEO_CODING_JITTER_BUFFER_H_
12 : #define WEBRTC_MODULES_VIDEO_CODING_JITTER_BUFFER_H_
13 :
14 : #include <list>
15 : #include <map>
16 : #include <memory>
17 : #include <set>
18 : #include <vector>
19 :
20 : #include "webrtc/base/constructormagic.h"
21 : #include "webrtc/base/thread_annotations.h"
22 : #include "webrtc/modules/include/module_common_types.h"
23 : #include "webrtc/modules/utility/include/process_thread.h"
24 : #include "webrtc/modules/video_coding/include/video_coding.h"
25 : #include "webrtc/modules/video_coding/include/video_coding_defines.h"
26 : #include "webrtc/modules/video_coding/decoding_state.h"
27 : #include "webrtc/modules/video_coding/inter_frame_delay.h"
28 : #include "webrtc/modules/video_coding/jitter_buffer_common.h"
29 : #include "webrtc/modules/video_coding/jitter_estimator.h"
30 : #include "webrtc/modules/video_coding/nack_module.h"
31 : #include "webrtc/system_wrappers/include/critical_section_wrapper.h"
32 : #include "webrtc/typedefs.h"
33 :
34 : namespace webrtc {
35 :
36 : enum VCMNackMode { kNack, kNoNack };
37 :
38 : // forward declarations
39 : class Clock;
40 : class EventFactory;
41 : class EventWrapper;
42 : class VCMFrameBuffer;
43 : class VCMPacket;
44 : class VCMEncodedFrame;
45 :
46 : typedef std::list<VCMFrameBuffer*> UnorderedFrameList;
47 :
48 : struct VCMJitterSample {
49 0 : VCMJitterSample() : timestamp(0), frame_size(0), latest_packet_time(-1) {}
50 : uint32_t timestamp;
51 : uint32_t frame_size;
52 : int64_t latest_packet_time;
53 : };
54 :
55 : class TimestampLessThan {
56 : public:
57 0 : bool operator()(uint32_t timestamp1, uint32_t timestamp2) const {
58 0 : return IsNewerTimestamp(timestamp2, timestamp1);
59 : }
60 : };
61 :
62 0 : class FrameList
63 : : public std::map<uint32_t, VCMFrameBuffer*, TimestampLessThan> {
64 : public:
65 : void InsertFrame(VCMFrameBuffer* frame);
66 : VCMFrameBuffer* FindFrame(uint16_t seq_num, uint32_t timestamp);
67 : VCMFrameBuffer* PopFrame(uint32_t timestamp);
68 : VCMFrameBuffer* Front() const;
69 : VCMFrameBuffer* Back() const;
70 : int RecycleFramesUntilKeyFrame(FrameList::iterator* key_frame_it,
71 : UnorderedFrameList* free_frames);
72 : void CleanUpOldOrEmptyFrames(VCMDecodingState* decoding_state,
73 : UnorderedFrameList* free_frames);
74 : void Reset(UnorderedFrameList* free_frames);
75 : };
76 :
77 : class Vp9SsMap {
78 : public:
79 : typedef std::map<uint32_t, GofInfoVP9, TimestampLessThan> SsMap;
80 : bool Insert(const VCMPacket& packet);
81 : void Reset();
82 :
83 : // Removes SS data that are older than |timestamp|.
84 : // The |timestamp| should be an old timestamp, i.e. packets with older
85 : // timestamps should no longer be inserted.
86 : void RemoveOld(uint32_t timestamp);
87 :
88 : bool UpdatePacket(VCMPacket* packet);
89 : void UpdateFrames(FrameList* frames);
90 :
91 : // Public for testing.
92 : // Returns an iterator to the corresponding SS data for the input |timestamp|.
93 : bool Find(uint32_t timestamp, SsMap::iterator* it);
94 :
95 : private:
96 : // These two functions are called by RemoveOld.
97 : // Checks if it is time to do a clean up (done each kSsCleanupIntervalSec).
98 : bool TimeForCleanup(uint32_t timestamp) const;
99 :
100 : // Advances the oldest SS data to handle timestamp wrap in cases where SS data
101 : // are received very seldom (e.g. only once in beginning, second when
102 : // IsNewerTimestamp is not true).
103 : void AdvanceFront(uint32_t timestamp);
104 :
105 : SsMap ss_map_;
106 : };
107 :
108 : class VCMJitterBuffer {
109 : public:
110 : VCMJitterBuffer(Clock* clock,
111 : std::unique_ptr<EventWrapper> event,
112 : NackSender* nack_sender = nullptr,
113 : KeyFrameRequestSender* keyframe_request_sender = nullptr);
114 :
115 : ~VCMJitterBuffer();
116 :
117 : // Initializes and starts jitter buffer.
118 : void Start();
119 :
120 : // Signals all internal events and stops the jitter buffer.
121 : void Stop();
122 :
123 : // Returns true if the jitter buffer is running.
124 : bool Running() const;
125 :
126 : // Empty the jitter buffer of all its data.
127 : void Flush();
128 :
129 : // Get the number of received frames, by type, since the jitter buffer
130 : // was started.
131 : FrameCounts FrameStatistics() const;
132 :
133 : // Gets number of packets received.
134 : int num_packets() const;
135 :
136 : // Gets number of duplicated packets received.
137 : int num_duplicated_packets() const;
138 :
139 : // Gets number of packets discarded by the jitter buffer.
140 : int num_discarded_packets() const;
141 :
142 : // Statistics, Calculate frame and bit rates.
143 : void IncomingRateStatistics(unsigned int* framerate, unsigned int* bitrate);
144 :
145 : // Wait |max_wait_time_ms| for a complete frame to arrive.
146 : // If found, a pointer to the frame is returned. Returns nullptr otherwise.
147 : VCMEncodedFrame* NextCompleteFrame(uint32_t max_wait_time_ms);
148 :
149 : // Locates a frame for decoding (even an incomplete) without delay.
150 : // The function returns true once such a frame is found, its corresponding
151 : // timestamp is returned. Otherwise, returns false.
152 : bool NextMaybeIncompleteTimestamp(uint32_t* timestamp);
153 :
154 : // Extract frame corresponding to input timestamp.
155 : // Frame will be set to a decoding state.
156 : VCMEncodedFrame* ExtractAndSetDecode(uint32_t timestamp);
157 :
158 : // Releases a frame returned from the jitter buffer, should be called when
159 : // done with decoding.
160 : void ReleaseFrame(VCMEncodedFrame* frame);
161 :
162 : // Returns the time in ms when the latest packet was inserted into the frame.
163 : // Retransmitted is set to true if any of the packets belonging to the frame
164 : // has been retransmitted.
165 : int64_t LastPacketTime(const VCMEncodedFrame* frame,
166 : bool* retransmitted) const;
167 :
168 : // Inserts a packet into a frame returned from GetFrame().
169 : // If the return value is <= 0, |frame| is invalidated and the pointer must
170 : // be dropped after this function returns.
171 : VCMFrameBufferEnum InsertPacket(const VCMPacket& packet, bool* retransmitted);
172 :
173 : // Returns the estimated jitter in milliseconds.
174 : uint32_t EstimatedJitterMs();
175 :
176 : // Updates the round-trip time estimate.
177 : void UpdateRtt(int64_t rtt_ms);
178 :
179 : // Set the NACK mode. |high_rtt_nack_threshold_ms| is an RTT threshold in ms
180 : // above which NACK will be disabled if the NACK mode is |kNack|, -1 meaning
181 : // that NACK is always enabled in the |kNack| mode.
182 : // |low_rtt_nack_threshold_ms| is an RTT threshold in ms below which we expect
183 : // to rely on NACK only, and therefore are using larger buffers to have time
184 : // to wait for retransmissions.
185 : void SetNackMode(VCMNackMode mode,
186 : int64_t low_rtt_nack_threshold_ms,
187 : int64_t high_rtt_nack_threshold_ms);
188 :
189 : void SetNackSettings(size_t max_nack_list_size,
190 : int max_packet_age_to_nack,
191 : int max_incomplete_time_ms);
192 :
193 : // Returns the current NACK mode.
194 : VCMNackMode nack_mode() const;
195 :
196 : // Returns a list of the sequence numbers currently missing.
197 : std::vector<uint16_t> GetNackList(bool* request_key_frame);
198 :
199 : // Set decode error mode - Should not be changed in the middle of the
200 : // session. Changes will not influence frames already in the buffer.
201 : void SetDecodeErrorMode(VCMDecodeErrorMode error_mode);
202 0 : VCMDecodeErrorMode decode_error_mode() const { return decode_error_mode_; }
203 :
204 : void RegisterStatsCallback(VCMReceiveStatisticsCallback* callback);
205 :
206 : private:
207 : class SequenceNumberLessThan {
208 : public:
209 0 : bool operator()(const uint16_t& sequence_number1,
210 : const uint16_t& sequence_number2) const {
211 0 : return IsNewerSequenceNumber(sequence_number2, sequence_number1);
212 : }
213 : };
214 : typedef std::set<uint16_t, SequenceNumberLessThan> SequenceNumberSet;
215 :
216 : // Gets the frame assigned to the timestamp of the packet. May recycle
217 : // existing frames if no free frames are available. Returns an error code if
218 : // failing, or kNoError on success. |frame_list| contains which list the
219 : // packet was in, or NULL if it was not in a FrameList (a new frame).
220 : VCMFrameBufferEnum GetFrame(const VCMPacket& packet,
221 : VCMFrameBuffer** frame,
222 : FrameList** frame_list)
223 : EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
224 :
225 : // Returns true if |frame| is continuous in |decoding_state|, not taking
226 : // decodable frames into account.
227 : bool IsContinuousInState(const VCMFrameBuffer& frame,
228 : const VCMDecodingState& decoding_state) const
229 : EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
230 : // Returns true if |frame| is continuous in the |last_decoded_state_|, taking
231 : // all decodable frames into account.
232 : bool IsContinuous(const VCMFrameBuffer& frame) const
233 : EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
234 : // Looks for frames in |incomplete_frames_| which are continuous in the
235 : // provided |decoded_state|. Starts the search from the timestamp of
236 : // |decoded_state|.
237 : void FindAndInsertContinuousFramesWithState(
238 : const VCMDecodingState& decoded_state)
239 : EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
240 : // Looks for frames in |incomplete_frames_| which are continuous in
241 : // |last_decoded_state_| taking all decodable frames into account. Starts
242 : // the search from |new_frame|.
243 : void FindAndInsertContinuousFrames(const VCMFrameBuffer& new_frame)
244 : EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
245 : VCMFrameBuffer* NextFrame() const EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
246 : // Returns true if the NACK list was updated to cover sequence numbers up to
247 : // |sequence_number|. If false a key frame is needed to get into a state where
248 : // we can continue decoding.
249 : bool UpdateNackList(uint16_t sequence_number)
250 : EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
251 : bool TooLargeNackList() const;
252 : // Returns true if the NACK list was reduced without problem. If false a key
253 : // frame is needed to get into a state where we can continue decoding.
254 : bool HandleTooLargeNackList() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
255 : bool MissingTooOldPacket(uint16_t latest_sequence_number) const
256 : EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
257 : // Returns true if the too old packets was successfully removed from the NACK
258 : // list. If false, a key frame is needed to get into a state where we can
259 : // continue decoding.
260 : bool HandleTooOldPackets(uint16_t latest_sequence_number)
261 : EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
262 : // Drops all packets in the NACK list up until |last_decoded_sequence_number|.
263 : void DropPacketsFromNackList(uint16_t last_decoded_sequence_number);
264 :
265 : // Gets an empty frame, creating a new frame if necessary (i.e. increases
266 : // jitter buffer size).
267 : VCMFrameBuffer* GetEmptyFrame() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
268 :
269 : // Attempts to increase the size of the jitter buffer. Returns true on
270 : // success, false otherwise.
271 : bool TryToIncreaseJitterBufferSize() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
272 :
273 : // Recycles oldest frames until a key frame is found. Used if jitter buffer is
274 : // completely full. Returns true if a key frame was found.
275 : bool RecycleFramesUntilKeyFrame() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
276 :
277 : // Updates the frame statistics.
278 : // Counts only complete frames, so decodable incomplete frames will not be
279 : // counted.
280 : void CountFrame(const VCMFrameBuffer& frame)
281 : EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
282 :
283 : // Update rolling average of packets per frame.
284 : void UpdateAveragePacketsPerFrame(int current_number_packets_);
285 :
286 : // Cleans the frame list in the JB from old/empty frames.
287 : // Should only be called prior to actual use.
288 : void CleanUpOldOrEmptyFrames() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
289 :
290 : // Returns true if |packet| is likely to have been retransmitted.
291 : bool IsPacketRetransmitted(const VCMPacket& packet) const;
292 :
293 : // The following three functions update the jitter estimate with the
294 : // payload size, receive time and RTP timestamp of a frame.
295 : void UpdateJitterEstimate(const VCMJitterSample& sample,
296 : bool incomplete_frame);
297 : void UpdateJitterEstimate(const VCMFrameBuffer& frame, bool incomplete_frame);
298 : void UpdateJitterEstimate(int64_t latest_packet_time_ms,
299 : uint32_t timestamp,
300 : unsigned int frame_size,
301 : bool incomplete_frame);
302 :
303 : // Returns true if we should wait for retransmissions, false otherwise.
304 : bool WaitForRetransmissions();
305 :
306 : int NonContinuousOrIncompleteDuration() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
307 :
308 : uint16_t EstimatedLowSequenceNumber(const VCMFrameBuffer& frame) const;
309 :
310 : void UpdateHistograms() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
311 :
312 : // Reset frame buffer and return it to free_frames_.
313 : void RecycleFrameBuffer(VCMFrameBuffer* frame)
314 : EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
315 :
316 : Clock* clock_;
317 : // If we are running (have started) or not.
318 : bool running_;
319 : CriticalSectionWrapper* crit_sect_;
320 : // Event to signal when we have a frame ready for decoder.
321 : std::unique_ptr<EventWrapper> frame_event_;
322 : // Number of allocated frames.
323 : int max_number_of_frames_;
324 : UnorderedFrameList free_frames_ GUARDED_BY(crit_sect_);
325 : FrameList decodable_frames_ GUARDED_BY(crit_sect_);
326 : FrameList incomplete_frames_ GUARDED_BY(crit_sect_);
327 : VCMDecodingState last_decoded_state_ GUARDED_BY(crit_sect_);
328 : bool first_packet_since_reset_;
329 :
330 : // Statistics.
331 : VCMReceiveStatisticsCallback* stats_callback_ GUARDED_BY(crit_sect_);
332 : // Frame counts for each type (key, delta, ...)
333 : FrameCounts receive_statistics_;
334 : // Latest calculated frame rates of incoming stream.
335 : unsigned int incoming_frame_rate_;
336 : unsigned int incoming_frame_count_;
337 : int64_t time_last_incoming_frame_count_;
338 : unsigned int incoming_bit_count_;
339 : unsigned int incoming_bit_rate_;
340 : // Number of packets in a row that have been too old.
341 : int num_consecutive_old_packets_;
342 : // Number of packets received.
343 : int num_packets_ GUARDED_BY(crit_sect_);
344 : // Number of duplicated packets received.
345 : int num_duplicated_packets_ GUARDED_BY(crit_sect_);
346 : // Number of packets discarded by the jitter buffer.
347 : int num_discarded_packets_ GUARDED_BY(crit_sect_);
348 : // Time when first packet is received.
349 : int64_t time_first_packet_ms_ GUARDED_BY(crit_sect_);
350 :
351 : // Jitter estimation.
352 : // Filter for estimating jitter.
353 : VCMJitterEstimator jitter_estimate_;
354 : // Calculates network delays used for jitter calculations.
355 : VCMInterFrameDelay inter_frame_delay_;
356 : VCMJitterSample waiting_for_completion_;
357 : int64_t rtt_ms_;
358 :
359 : // NACK and retransmissions.
360 : VCMNackMode nack_mode_;
361 : int64_t low_rtt_nack_threshold_ms_;
362 : int64_t high_rtt_nack_threshold_ms_;
363 : // Holds the internal NACK list (the missing sequence numbers).
364 : SequenceNumberSet missing_sequence_numbers_;
365 : uint16_t latest_received_sequence_number_;
366 : size_t max_nack_list_size_;
367 : int max_packet_age_to_nack_; // Measured in sequence numbers.
368 : int max_incomplete_time_ms_;
369 :
370 : VCMDecodeErrorMode decode_error_mode_;
371 : // Estimated rolling average of packets per frame
372 : float average_packets_per_frame_;
373 : // average_packets_per_frame converges fast if we have fewer than this many
374 : // frames.
375 : int frame_counter_;
376 :
377 : RTC_DISALLOW_COPY_AND_ASSIGN(VCMJitterBuffer);
378 : };
379 : } // namespace webrtc
380 :
381 : #endif // WEBRTC_MODULES_VIDEO_CODING_JITTER_BUFFER_H_
|
