Line data Source code
1 : /*
2 : * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 :
11 : #ifndef WEBRTC_MODULES_VIDEO_CODING_MEDIA_OPT_UTIL_H_
12 : #define WEBRTC_MODULES_VIDEO_CODING_MEDIA_OPT_UTIL_H_
13 :
14 : #include <math.h>
15 : #include <stdlib.h>
16 :
17 : #include <memory>
18 :
19 : #include "webrtc/base/numerics/exp_filter.h"
20 : #include "webrtc/modules/video_coding/internal_defines.h"
21 : #include "webrtc/system_wrappers/include/trace.h"
22 : #include "webrtc/typedefs.h"
23 :
24 : namespace webrtc {
25 : namespace media_optimization {
26 :
27 : // Number of time periods used for (max) window filter for packet loss
28 : // TODO(marpan): set reasonable window size for filtered packet loss,
29 : // adjustment should be based on logged/real data of loss stats/correlation.
30 : enum { kLossPrHistorySize = 10 };
31 :
32 : // 1000 ms, total filter length is (kLossPrHistorySize * 1000) ms
33 : enum { kLossPrShortFilterWinMs = 1000 };
34 :
35 : // The type of filter used on the received packet loss reports.
36 : enum FilterPacketLossMode {
37 : kNoFilter, // No filtering on received loss.
38 : kAvgFilter, // Recursive average filter.
39 : kMaxFilter // Max-window filter, over the time interval of:
40 : // (kLossPrHistorySize * kLossPrShortFilterWinMs) ms.
41 : };
42 :
43 : // Thresholds for hybrid NACK/FEC
44 : // common to media optimization and the jitter buffer.
45 : const int64_t kLowRttNackMs = 20;
46 :
47 : // If the RTT is higher than this an extra RTT wont be added to to the jitter
48 : // buffer delay.
49 : const int kMaxRttDelayThreshold = 500;
50 :
51 : struct VCMProtectionParameters {
52 0 : VCMProtectionParameters()
53 0 : : rtt(0),
54 : lossPr(0.0f),
55 : bitRate(0.0f),
56 : packetsPerFrame(0.0f),
57 : packetsPerFrameKey(0.0f),
58 : frameRate(0.0f),
59 : keyFrameSize(0.0f),
60 : fecRateDelta(0),
61 : fecRateKey(0),
62 : codecWidth(0),
63 : codecHeight(0),
64 0 : numLayers(1) {}
65 :
66 : int64_t rtt;
67 : float lossPr;
68 : float bitRate;
69 : float packetsPerFrame;
70 : float packetsPerFrameKey;
71 : float frameRate;
72 : float keyFrameSize;
73 : uint8_t fecRateDelta;
74 : uint8_t fecRateKey;
75 : uint16_t codecWidth;
76 : uint16_t codecHeight;
77 : int numLayers;
78 : };
79 :
80 : /******************************/
81 : /* VCMProtectionMethod class */
82 : /******************************/
83 :
84 : enum VCMProtectionMethodEnum { kNack, kFec, kNackFec, kNone };
85 :
86 : class VCMLossProbabilitySample {
87 : public:
88 0 : VCMLossProbabilitySample() : lossPr255(0), timeMs(-1) {}
89 :
90 : uint8_t lossPr255;
91 : int64_t timeMs;
92 : };
93 :
94 : class VCMProtectionMethod {
95 : public:
96 : VCMProtectionMethod();
97 : virtual ~VCMProtectionMethod();
98 :
99 : // Updates the efficiency of the method using the parameters provided
100 : //
101 : // Input:
102 : // - parameters : Parameters used to calculate efficiency
103 : //
104 : // Return value : True if this method is recommended in
105 : // the given conditions.
106 : virtual bool UpdateParameters(const VCMProtectionParameters* parameters) = 0;
107 :
108 : // Returns the protection type
109 : //
110 : // Return value : The protection type
111 0 : enum VCMProtectionMethodEnum Type() const { return _type; }
112 :
113 : // Returns the effective packet loss for ER, required by this protection
114 : // method
115 : //
116 : // Return value : Required effective packet loss
117 0 : virtual uint8_t RequiredPacketLossER() { return _effectivePacketLoss; }
118 :
119 : // Extracts the FEC protection factor for Key frame, required by this
120 : // protection method
121 : //
122 : // Return value : Required protectionFactor for Key frame
123 0 : virtual uint8_t RequiredProtectionFactorK() { return _protectionFactorK; }
124 :
125 : // Extracts the FEC protection factor for Delta frame, required by this
126 : // protection method
127 : //
128 : // Return value : Required protectionFactor for delta frame
129 0 : virtual uint8_t RequiredProtectionFactorD() { return _protectionFactorD; }
130 :
131 : // Extracts whether the FEC Unequal protection (UEP) is used for Key frame.
132 : //
133 : // Return value : Required Unequal protection on/off state.
134 0 : virtual bool RequiredUepProtectionK() { return _useUepProtectionK; }
135 :
136 : // Extracts whether the the FEC Unequal protection (UEP) is used for Delta
137 : // frame.
138 : //
139 : // Return value : Required Unequal protection on/off state.
140 0 : virtual bool RequiredUepProtectionD() { return _useUepProtectionD; }
141 :
142 0 : virtual int MaxFramesFec() const { return 1; }
143 :
144 : protected:
145 : uint8_t _effectivePacketLoss;
146 : uint8_t _protectionFactorK;
147 : uint8_t _protectionFactorD;
148 : // Estimation of residual loss after the FEC
149 : float _scaleProtKey;
150 : int32_t _maxPayloadSize;
151 :
152 : bool _useUepProtectionK;
153 : bool _useUepProtectionD;
154 : float _corrFecCost;
155 : enum VCMProtectionMethodEnum _type;
156 : };
157 :
158 : class VCMNackMethod : public VCMProtectionMethod {
159 : public:
160 : VCMNackMethod();
161 : virtual ~VCMNackMethod();
162 : virtual bool UpdateParameters(const VCMProtectionParameters* parameters);
163 : // Get the effective packet loss
164 : bool EffectivePacketLoss(const VCMProtectionParameters* parameter);
165 : };
166 :
167 : class VCMFecMethod : public VCMProtectionMethod {
168 : public:
169 : VCMFecMethod();
170 : virtual ~VCMFecMethod();
171 : virtual bool UpdateParameters(const VCMProtectionParameters* parameters);
172 : // Get the effective packet loss for ER
173 : bool EffectivePacketLoss(const VCMProtectionParameters* parameters);
174 : // Get the FEC protection factors
175 : bool ProtectionFactor(const VCMProtectionParameters* parameters);
176 : // Get the boost for key frame protection
177 : uint8_t BoostCodeRateKey(uint8_t packetFrameDelta,
178 : uint8_t packetFrameKey) const;
179 : // Convert the rates: defined relative to total# packets or source# packets
180 : uint8_t ConvertFECRate(uint8_t codeRate) const;
181 : // Get the average effective recovery from FEC: for random loss model
182 : float AvgRecoveryFEC(const VCMProtectionParameters* parameters) const;
183 : // Update FEC with protectionFactorD
184 : void UpdateProtectionFactorD(uint8_t protectionFactorD);
185 : // Update FEC with protectionFactorK
186 : void UpdateProtectionFactorK(uint8_t protectionFactorK);
187 : // Compute the bits per frame. Account for temporal layers when applicable.
188 : int BitsPerFrame(const VCMProtectionParameters* parameters);
189 :
190 : protected:
191 : enum { kUpperLimitFramesFec = 6 };
192 : // Thresholds values for the bytes/frame and round trip time, below which we
193 : // may turn off FEC, depending on |_numLayers| and |_maxFramesFec|.
194 : // Max bytes/frame for VGA, corresponds to ~140k at 25fps.
195 : enum { kMaxBytesPerFrameForFec = 700 };
196 : // Max bytes/frame for CIF and lower: corresponds to ~80k at 25fps.
197 : enum { kMaxBytesPerFrameForFecLow = 400 };
198 : // Max bytes/frame for frame size larger than VGA, ~200k at 25fps.
199 : enum { kMaxBytesPerFrameForFecHigh = 1000 };
200 : };
201 :
202 : class VCMNackFecMethod : public VCMFecMethod {
203 : public:
204 : VCMNackFecMethod(int64_t lowRttNackThresholdMs,
205 : int64_t highRttNackThresholdMs);
206 : virtual ~VCMNackFecMethod();
207 : virtual bool UpdateParameters(const VCMProtectionParameters* parameters);
208 : // Get the effective packet loss for ER
209 : bool EffectivePacketLoss(const VCMProtectionParameters* parameters);
210 : // Get the protection factors
211 : bool ProtectionFactor(const VCMProtectionParameters* parameters);
212 : // Get the max number of frames the FEC is allowed to be based on.
213 : int MaxFramesFec() const;
214 : // Turn off the FEC based on low bitrate and other factors.
215 : bool BitRateTooLowForFec(const VCMProtectionParameters* parameters);
216 :
217 : private:
218 : int ComputeMaxFramesFec(const VCMProtectionParameters* parameters);
219 :
220 : int64_t _lowRttNackMs;
221 : int64_t _highRttNackMs;
222 : int _maxFramesFec;
223 : };
224 :
225 : class VCMLossProtectionLogic {
226 : public:
227 : explicit VCMLossProtectionLogic(int64_t nowMs);
228 : ~VCMLossProtectionLogic();
229 :
230 : // Set the protection method to be used
231 : //
232 : // Input:
233 : // - newMethodType : New requested protection method type. If one
234 : // is already set, it will be deleted and replaced
235 : void SetMethod(VCMProtectionMethodEnum newMethodType);
236 :
237 : // Update the round-trip time
238 : //
239 : // Input:
240 : // - rtt : Round-trip time in seconds.
241 : void UpdateRtt(int64_t rtt);
242 :
243 : // Update the filtered packet loss.
244 : //
245 : // Input:
246 : // - packetLossEnc : The reported packet loss filtered
247 : // (max window or average)
248 : void UpdateFilteredLossPr(uint8_t packetLossEnc);
249 :
250 : // Update the current target bit rate.
251 : //
252 : // Input:
253 : // - bitRate : The current target bit rate in kbits/s
254 : void UpdateBitRate(float bitRate);
255 :
256 : // Update the number of packets per frame estimate, for delta frames
257 : //
258 : // Input:
259 : // - nPackets : Number of packets in the latest sent frame.
260 : void UpdatePacketsPerFrame(float nPackets, int64_t nowMs);
261 :
262 : // Update the number of packets per frame estimate, for key frames
263 : //
264 : // Input:
265 : // - nPackets : umber of packets in the latest sent frame.
266 : void UpdatePacketsPerFrameKey(float nPackets, int64_t nowMs);
267 :
268 : // Update the keyFrameSize estimate
269 : //
270 : // Input:
271 : // - keyFrameSize : The size of the latest sent key frame.
272 : void UpdateKeyFrameSize(float keyFrameSize);
273 :
274 : // Update the frame rate
275 : //
276 : // Input:
277 : // - frameRate : The current target frame rate.
278 0 : void UpdateFrameRate(float frameRate) { _frameRate = frameRate; }
279 :
280 : // Update the frame size
281 : //
282 : // Input:
283 : // - width : The codec frame width.
284 : // - height : The codec frame height.
285 : void UpdateFrameSize(size_t width, size_t height);
286 :
287 : // Update the number of active layers
288 : //
289 : // Input:
290 : // - numLayers : Number of layers used.
291 : void UpdateNumLayers(int numLayers);
292 :
293 : // The amount of packet loss to cover for with FEC.
294 : //
295 : // Input:
296 : // - fecRateKey : Packet loss to cover for with FEC when
297 : // sending key frames.
298 : // - fecRateDelta : Packet loss to cover for with FEC when
299 : // sending delta frames.
300 : void UpdateFECRates(uint8_t fecRateKey, uint8_t fecRateDelta) {
301 : _fecRateKey = fecRateKey;
302 : _fecRateDelta = fecRateDelta;
303 : }
304 :
305 : // Update the protection methods with the current VCMProtectionParameters
306 : // and set the requested protection settings.
307 : // Return value : Returns true on update
308 : bool UpdateMethod();
309 :
310 : // Returns the method currently selected.
311 : //
312 : // Return value : The protection method currently selected.
313 : VCMProtectionMethod* SelectedMethod() const;
314 :
315 : // Return the protection type of the currently selected method
316 : VCMProtectionMethodEnum SelectedType() const;
317 :
318 : // Updates the filtered loss for the average and max window packet loss,
319 : // and returns the filtered loss probability in the interval [0, 255].
320 : // The returned filtered loss value depends on the parameter |filter_mode|.
321 : // The input parameter |lossPr255| is the received packet loss.
322 :
323 : // Return value : The filtered loss probability
324 : uint8_t FilteredLoss(int64_t nowMs,
325 : FilterPacketLossMode filter_mode,
326 : uint8_t lossPr255);
327 :
328 : void Reset(int64_t nowMs);
329 :
330 : void Release();
331 :
332 : private:
333 : // Sets the available loss protection methods.
334 : void UpdateMaxLossHistory(uint8_t lossPr255, int64_t now);
335 : uint8_t MaxFilteredLossPr(int64_t nowMs) const;
336 : std::unique_ptr<VCMProtectionMethod> _selectedMethod;
337 : VCMProtectionParameters _currentParameters;
338 : int64_t _rtt;
339 : float _lossPr;
340 : float _bitRate;
341 : float _frameRate;
342 : float _keyFrameSize;
343 : uint8_t _fecRateKey;
344 : uint8_t _fecRateDelta;
345 : int64_t _lastPrUpdateT;
346 : int64_t _lastPacketPerFrameUpdateT;
347 : int64_t _lastPacketPerFrameUpdateTKey;
348 : rtc::ExpFilter _lossPr255;
349 : VCMLossProbabilitySample _lossPrHistory[kLossPrHistorySize];
350 : uint8_t _shortMaxLossPr255;
351 : rtc::ExpFilter _packetsPerFrame;
352 : rtc::ExpFilter _packetsPerFrameKey;
353 : size_t _codecWidth;
354 : size_t _codecHeight;
355 : int _numLayers;
356 : };
357 :
358 : } // namespace media_optimization
359 : } // namespace webrtc
360 :
361 : #endif // WEBRTC_MODULES_VIDEO_CODING_MEDIA_OPT_UTIL_H_
|
