Line data Source code
1 : /* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
2 : *
3 : * Use of this source code is governed by a BSD-style license
4 : * that can be found in the LICENSE file in the root of the source
5 : * tree. An additional intellectual property rights grant can be found
6 : * in the file PATENTS. All contributing project authors may
7 : * be found in the AUTHORS file in the root of the source tree.
8 : */
9 :
10 : #include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h"
11 :
12 : #include <stdlib.h>
13 :
14 : #include <algorithm>
15 :
16 : #include "webrtc/base/checks.h"
17 : #include "vpx/vpx_encoder.h"
18 : #include "vpx/vp8cx.h"
19 : #include "webrtc/modules/video_coding/include/video_codec_interface.h"
20 : #include "webrtc/system_wrappers/include/clock.h"
21 : #include "webrtc/system_wrappers/include/metrics.h"
22 :
23 : namespace webrtc {
24 :
25 : static const int kOneSecond90Khz = 90000;
26 : static const int kMinTimeBetweenSyncs = kOneSecond90Khz * 5;
27 : static const int kMaxTimeBetweenSyncs = kOneSecond90Khz * 10;
28 : static const int kQpDeltaThresholdForSync = 8;
29 :
30 : const double ScreenshareLayers::kMaxTL0FpsReduction = 2.5;
31 : const double ScreenshareLayers::kAcceptableTargetOvershoot = 2.0;
32 :
33 : constexpr int ScreenshareLayers::kMaxNumTemporalLayers;
34 :
35 : // Since this is TL0 we only allow updating and predicting from the LAST
36 : // reference frame.
37 : const int ScreenshareLayers::kTl0Flags =
38 : VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF |
39 : VP8_EFLAG_NO_REF_ARF;
40 :
41 : // Allow predicting from both TL0 and TL1.
42 : const int ScreenshareLayers::kTl1Flags =
43 : VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST;
44 :
45 : // Allow predicting from only TL0 to allow participants to switch to the high
46 : // bitrate stream. This means predicting only from the LAST reference frame, but
47 : // only updating GF to not corrupt TL0.
48 : const int ScreenshareLayers::kTl1SyncFlags =
49 : VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_ARF |
50 : VP8_EFLAG_NO_UPD_LAST;
51 :
52 : // Always emit a frame with certain interval, even if bitrate targets have
53 : // been exceeded.
54 : const int ScreenshareLayers::kMaxFrameIntervalMs = 2000;
55 :
56 0 : webrtc::TemporalLayers* ScreenshareTemporalLayersFactory::Create(
57 : int simulcast_id,
58 : int num_temporal_layers,
59 : uint8_t initial_tl0_pic_idx) const {
60 : webrtc::TemporalLayers* tl;
61 0 : if (simulcast_id == 0) {
62 0 : tl = new webrtc::ScreenshareLayers(num_temporal_layers, rand(),
63 0 : webrtc::Clock::GetRealTimeClock());
64 : } else {
65 0 : RealTimeTemporalLayersFactory rt_tl_factory;
66 0 : tl = rt_tl_factory.Create(simulcast_id, num_temporal_layers, rand());
67 : }
68 0 : if (listener_)
69 0 : listener_->OnTemporalLayersCreated(simulcast_id, tl);
70 0 : return tl;
71 : }
72 :
73 0 : ScreenshareLayers::ScreenshareLayers(int num_temporal_layers,
74 : uint8_t initial_tl0_pic_idx,
75 0 : Clock* clock)
76 : : clock_(clock),
77 : number_of_temporal_layers_(
78 0 : std::min(kMaxNumTemporalLayers, num_temporal_layers)),
79 : last_base_layer_sync_(false),
80 : tl0_pic_idx_(initial_tl0_pic_idx),
81 : active_layer_(-1),
82 : last_timestamp_(-1),
83 : last_sync_timestamp_(-1),
84 : last_emitted_tl0_timestamp_(-1),
85 : min_qp_(-1),
86 : max_qp_(-1),
87 : max_debt_bytes_(0),
88 : encode_framerate_(1000.0f, 1000.0f), // 1 second window, second scale.
89 0 : bitrate_updated_(false) {
90 0 : RTC_CHECK_GT(number_of_temporal_layers_, 0);
91 0 : RTC_CHECK_LE(number_of_temporal_layers_, kMaxNumTemporalLayers);
92 0 : }
93 :
94 0 : ScreenshareLayers::~ScreenshareLayers() {
95 0 : UpdateHistograms();
96 0 : }
97 :
98 0 : int ScreenshareLayers::CurrentLayerId() const {
99 : // Codec does not use temporal layers for screenshare.
100 0 : return 0;
101 : }
102 :
103 0 : int ScreenshareLayers::EncodeFlags(uint32_t timestamp) {
104 0 : if (number_of_temporal_layers_ <= 1) {
105 : // No flags needed for 1 layer screenshare.
106 0 : return 0;
107 : }
108 :
109 0 : const int64_t now_ms = clock_->TimeInMilliseconds();
110 0 : if (target_framerate_.value_or(0) > 0 &&
111 0 : encode_framerate_.Rate(now_ms).value_or(0) > *target_framerate_) {
112 : // Max framerate exceeded, drop frame.
113 0 : return -1;
114 : }
115 :
116 0 : if (stats_.first_frame_time_ms_ == -1)
117 0 : stats_.first_frame_time_ms_ = now_ms;
118 :
119 0 : int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp);
120 0 : int flags = 0;
121 0 : if (active_layer_ == -1 ||
122 0 : layers_[active_layer_].state != TemporalLayer::State::kDropped) {
123 0 : if (last_emitted_tl0_timestamp_ != -1 &&
124 0 : (unwrapped_timestamp - last_emitted_tl0_timestamp_) / 90 >
125 : kMaxFrameIntervalMs) {
126 : // Too long time has passed since the last frame was emitted, cancel
127 : // enough debt to allow a single frame.
128 0 : layers_[0].debt_bytes_ = max_debt_bytes_ - 1;
129 : }
130 0 : if (layers_[0].debt_bytes_ > max_debt_bytes_) {
131 : // Must drop TL0, encode TL1 instead.
132 0 : if (layers_[1].debt_bytes_ > max_debt_bytes_) {
133 : // Must drop both TL0 and TL1.
134 0 : active_layer_ = -1;
135 : } else {
136 0 : active_layer_ = 1;
137 : }
138 : } else {
139 0 : active_layer_ = 0;
140 : }
141 : }
142 :
143 0 : switch (active_layer_) {
144 : case 0:
145 0 : flags = kTl0Flags;
146 0 : last_emitted_tl0_timestamp_ = unwrapped_timestamp;
147 0 : break;
148 : case 1:
149 0 : if (TimeToSync(unwrapped_timestamp)) {
150 0 : last_sync_timestamp_ = unwrapped_timestamp;
151 0 : flags = kTl1SyncFlags;
152 : } else {
153 0 : flags = kTl1Flags;
154 : }
155 0 : break;
156 : case -1:
157 0 : flags = -1;
158 0 : ++stats_.num_dropped_frames_;
159 0 : break;
160 : default:
161 0 : flags = -1;
162 0 : RTC_NOTREACHED();
163 : }
164 :
165 : int64_t ts_diff;
166 0 : if (last_timestamp_ == -1) {
167 0 : ts_diff = kOneSecond90Khz / capture_framerate_.value_or(*target_framerate_);
168 : } else {
169 0 : ts_diff = unwrapped_timestamp - last_timestamp_;
170 : }
171 : // Make sure both frame droppers leak out bits.
172 0 : layers_[0].UpdateDebt(ts_diff / 90);
173 0 : layers_[1].UpdateDebt(ts_diff / 90);
174 0 : last_timestamp_ = timestamp;
175 0 : return flags;
176 : }
177 :
178 0 : std::vector<uint32_t> ScreenshareLayers::OnRatesUpdated(int bitrate_kbps,
179 : int max_bitrate_kbps,
180 : int framerate) {
181 0 : RTC_DCHECK_GT(framerate, 0);
182 0 : if (!target_framerate_) {
183 : // First OnRatesUpdated() is called during construction, with the configured
184 : // targets as parameters.
185 0 : target_framerate_.emplace(framerate);
186 0 : capture_framerate_ = target_framerate_;
187 0 : bitrate_updated_ = true;
188 : } else {
189 0 : bitrate_updated_ =
190 0 : bitrate_kbps != static_cast<int>(layers_[0].target_rate_kbps_) ||
191 0 : max_bitrate_kbps != static_cast<int>(layers_[1].target_rate_kbps_) ||
192 0 : (capture_framerate_ &&
193 0 : framerate != static_cast<int>(*capture_framerate_));
194 0 : if (framerate < 0) {
195 0 : capture_framerate_.reset();
196 : } else {
197 0 : capture_framerate_.emplace(framerate);
198 : }
199 : }
200 :
201 0 : layers_[0].target_rate_kbps_ = bitrate_kbps;
202 0 : layers_[1].target_rate_kbps_ = max_bitrate_kbps;
203 :
204 0 : std::vector<uint32_t> allocation;
205 0 : allocation.push_back(bitrate_kbps);
206 0 : if (max_bitrate_kbps > bitrate_kbps)
207 0 : allocation.push_back(max_bitrate_kbps - bitrate_kbps);
208 0 : return allocation;
209 : }
210 :
211 0 : void ScreenshareLayers::FrameEncoded(unsigned int size,
212 : uint32_t timestamp,
213 : int qp) {
214 0 : if (size > 0)
215 0 : encode_framerate_.Update(1, clock_->TimeInMilliseconds());
216 :
217 0 : if (number_of_temporal_layers_ == 1)
218 0 : return;
219 :
220 0 : RTC_DCHECK_NE(-1, active_layer_);
221 0 : if (size == 0) {
222 0 : layers_[active_layer_].state = TemporalLayer::State::kDropped;
223 0 : ++stats_.num_overshoots_;
224 0 : return;
225 : }
226 :
227 0 : if (layers_[active_layer_].state == TemporalLayer::State::kDropped) {
228 0 : layers_[active_layer_].state = TemporalLayer::State::kQualityBoost;
229 : }
230 :
231 0 : if (qp != -1)
232 0 : layers_[active_layer_].last_qp = qp;
233 :
234 0 : if (active_layer_ == 0) {
235 0 : layers_[0].debt_bytes_ += size;
236 0 : layers_[1].debt_bytes_ += size;
237 0 : ++stats_.num_tl0_frames_;
238 0 : stats_.tl0_target_bitrate_sum_ += layers_[0].target_rate_kbps_;
239 0 : stats_.tl0_qp_sum_ += qp;
240 0 : } else if (active_layer_ == 1) {
241 0 : layers_[1].debt_bytes_ += size;
242 0 : ++stats_.num_tl1_frames_;
243 0 : stats_.tl1_target_bitrate_sum_ += layers_[1].target_rate_kbps_;
244 0 : stats_.tl1_qp_sum_ += qp;
245 : }
246 : }
247 :
248 0 : void ScreenshareLayers::PopulateCodecSpecific(bool base_layer_sync,
249 : CodecSpecificInfoVP8* vp8_info,
250 : uint32_t timestamp) {
251 0 : int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp);
252 0 : if (number_of_temporal_layers_ == 1) {
253 0 : vp8_info->temporalIdx = kNoTemporalIdx;
254 0 : vp8_info->layerSync = false;
255 0 : vp8_info->tl0PicIdx = kNoTl0PicIdx;
256 : } else {
257 0 : RTC_DCHECK_NE(-1, active_layer_);
258 0 : vp8_info->temporalIdx = active_layer_;
259 0 : if (base_layer_sync) {
260 0 : vp8_info->temporalIdx = 0;
261 0 : last_sync_timestamp_ = unwrapped_timestamp;
262 0 : } else if (last_base_layer_sync_ && vp8_info->temporalIdx != 0) {
263 : // Regardless of pattern the frame after a base layer sync will always
264 : // be a layer sync.
265 0 : last_sync_timestamp_ = unwrapped_timestamp;
266 : }
267 0 : vp8_info->layerSync = last_sync_timestamp_ != -1 &&
268 0 : last_sync_timestamp_ == unwrapped_timestamp;
269 0 : if (vp8_info->temporalIdx == 0) {
270 0 : tl0_pic_idx_++;
271 : }
272 0 : last_base_layer_sync_ = base_layer_sync;
273 0 : vp8_info->tl0PicIdx = tl0_pic_idx_;
274 : }
275 0 : }
276 :
277 0 : bool ScreenshareLayers::TimeToSync(int64_t timestamp) const {
278 0 : RTC_DCHECK_EQ(1, active_layer_);
279 0 : RTC_DCHECK_NE(-1, layers_[0].last_qp);
280 0 : if (layers_[1].last_qp == -1) {
281 : // First frame in TL1 should only depend on TL0 since there are no
282 : // previous frames in TL1.
283 0 : return true;
284 : }
285 :
286 0 : RTC_DCHECK_NE(-1, last_sync_timestamp_);
287 0 : int64_t timestamp_diff = timestamp - last_sync_timestamp_;
288 0 : if (timestamp_diff > kMaxTimeBetweenSyncs) {
289 : // After a certain time, force a sync frame.
290 0 : return true;
291 0 : } else if (timestamp_diff < kMinTimeBetweenSyncs) {
292 : // If too soon from previous sync frame, don't issue a new one.
293 0 : return false;
294 : }
295 : // Issue a sync frame if difference in quality between TL0 and TL1 isn't too
296 : // large.
297 0 : if (layers_[0].last_qp - layers_[1].last_qp < kQpDeltaThresholdForSync)
298 0 : return true;
299 0 : return false;
300 : }
301 :
302 0 : uint32_t ScreenshareLayers::GetCodecTargetBitrateKbps() const {
303 0 : uint32_t target_bitrate_kbps = layers_[0].target_rate_kbps_;
304 :
305 0 : if (number_of_temporal_layers_ > 1) {
306 : // Calculate a codec target bitrate. This may be higher than TL0, gaining
307 : // quality at the expense of frame rate at TL0. Constraints:
308 : // - TL0 frame rate no less than framerate / kMaxTL0FpsReduction.
309 : // - Target rate * kAcceptableTargetOvershoot should not exceed TL1 rate.
310 0 : target_bitrate_kbps =
311 0 : std::min(layers_[0].target_rate_kbps_ * kMaxTL0FpsReduction,
312 0 : layers_[1].target_rate_kbps_ / kAcceptableTargetOvershoot);
313 : }
314 :
315 0 : return std::max(layers_[0].target_rate_kbps_, target_bitrate_kbps);
316 : }
317 :
318 0 : bool ScreenshareLayers::UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) {
319 0 : bool cfg_updated = false;
320 0 : uint32_t target_bitrate_kbps = GetCodecTargetBitrateKbps();
321 0 : if (bitrate_updated_ || cfg->rc_target_bitrate != target_bitrate_kbps) {
322 0 : cfg->rc_target_bitrate = target_bitrate_kbps;
323 :
324 : // Don't reconfigure qp limits during quality boost frames.
325 0 : if (active_layer_ == -1 ||
326 0 : layers_[active_layer_].state != TemporalLayer::State::kQualityBoost) {
327 0 : min_qp_ = cfg->rc_min_quantizer;
328 0 : max_qp_ = cfg->rc_max_quantizer;
329 : // After a dropped frame, a frame with max qp will be encoded and the
330 : // quality will then ramp up from there. To boost the speed of recovery,
331 : // encode the next frame with lower max qp. TL0 is the most important to
332 : // improve since the errors in this layer will propagate to TL1.
333 : // Currently, reduce max qp by 20% for TL0 and 15% for TL1.
334 0 : layers_[0].enhanced_max_qp = min_qp_ + (((max_qp_ - min_qp_) * 80) / 100);
335 0 : layers_[1].enhanced_max_qp = min_qp_ + (((max_qp_ - min_qp_) * 85) / 100);
336 : }
337 :
338 0 : if (capture_framerate_) {
339 : int avg_frame_size =
340 0 : (target_bitrate_kbps * 1000) / (8 * *capture_framerate_);
341 0 : max_debt_bytes_ = 4 * avg_frame_size;
342 : }
343 :
344 0 : bitrate_updated_ = false;
345 0 : cfg_updated = true;
346 : }
347 :
348 : // Don't try to update boosts state if not active yet.
349 0 : if (active_layer_ == -1)
350 0 : return cfg_updated;
351 :
352 0 : if (max_qp_ == -1 || number_of_temporal_layers_ <= 1)
353 0 : return cfg_updated;
354 :
355 : // If layer is in the quality boost state (following a dropped frame), update
356 : // the configuration with the adjusted (lower) qp and set the state back to
357 : // normal.
358 : unsigned int adjusted_max_qp;
359 0 : if (layers_[active_layer_].state == TemporalLayer::State::kQualityBoost &&
360 0 : layers_[active_layer_].enhanced_max_qp != -1) {
361 0 : adjusted_max_qp = layers_[active_layer_].enhanced_max_qp;
362 0 : layers_[active_layer_].state = TemporalLayer::State::kNormal;
363 : } else {
364 0 : if (max_qp_ == -1)
365 0 : return cfg_updated;
366 0 : adjusted_max_qp = max_qp_; // Set the normal max qp.
367 : }
368 :
369 0 : if (adjusted_max_qp == cfg->rc_max_quantizer)
370 0 : return cfg_updated;
371 :
372 0 : cfg->rc_max_quantizer = adjusted_max_qp;
373 0 : cfg_updated = true;
374 :
375 0 : return cfg_updated;
376 : }
377 :
378 0 : void ScreenshareLayers::TemporalLayer::UpdateDebt(int64_t delta_ms) {
379 0 : uint32_t debt_reduction_bytes = target_rate_kbps_ * delta_ms / 8;
380 0 : if (debt_reduction_bytes >= debt_bytes_) {
381 0 : debt_bytes_ = 0;
382 : } else {
383 0 : debt_bytes_ -= debt_reduction_bytes;
384 : }
385 0 : }
386 :
387 0 : void ScreenshareLayers::UpdateHistograms() {
388 0 : if (stats_.first_frame_time_ms_ == -1)
389 0 : return;
390 : int64_t duration_sec =
391 0 : (clock_->TimeInMilliseconds() - stats_.first_frame_time_ms_ + 500) / 1000;
392 0 : if (duration_sec >= metrics::kMinRunTimeInSeconds) {
393 0 : RTC_HISTOGRAM_COUNTS_10000(
394 : "WebRTC.Video.Screenshare.Layer0.FrameRate",
395 : (stats_.num_tl0_frames_ + (duration_sec / 2)) / duration_sec);
396 0 : RTC_HISTOGRAM_COUNTS_10000(
397 : "WebRTC.Video.Screenshare.Layer1.FrameRate",
398 : (stats_.num_tl1_frames_ + (duration_sec / 2)) / duration_sec);
399 0 : int total_frames = stats_.num_tl0_frames_ + stats_.num_tl1_frames_;
400 0 : RTC_HISTOGRAM_COUNTS_10000(
401 : "WebRTC.Video.Screenshare.FramesPerDrop",
402 : (stats_.num_dropped_frames_ == 0 ? 0 : total_frames /
403 : stats_.num_dropped_frames_));
404 0 : RTC_HISTOGRAM_COUNTS_10000(
405 : "WebRTC.Video.Screenshare.FramesPerOvershoot",
406 : (stats_.num_overshoots_ == 0 ? 0
407 : : total_frames / stats_.num_overshoots_));
408 0 : if (stats_.num_tl0_frames_ > 0) {
409 0 : RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.Screenshare.Layer0.Qp",
410 : stats_.tl0_qp_sum_ / stats_.num_tl0_frames_);
411 0 : RTC_HISTOGRAM_COUNTS_10000(
412 : "WebRTC.Video.Screenshare.Layer0.TargetBitrate",
413 : stats_.tl0_target_bitrate_sum_ / stats_.num_tl0_frames_);
414 : }
415 0 : if (stats_.num_tl1_frames_ > 0) {
416 0 : RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.Screenshare.Layer1.Qp",
417 : stats_.tl1_qp_sum_ / stats_.num_tl1_frames_);
418 0 : RTC_HISTOGRAM_COUNTS_10000(
419 : "WebRTC.Video.Screenshare.Layer1.TargetBitrate",
420 : stats_.tl1_target_bitrate_sum_ / stats_.num_tl1_frames_);
421 : }
422 : }
423 : }
424 :
425 : } // namespace webrtc
|
