Line data Source code
1 : /*
2 : * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 :
11 : #include "webrtc/modules/video_coding/codecs/vp8/vp8_impl.h"
12 :
13 : #include <stdlib.h>
14 : #include <string.h>
15 : #include <time.h>
16 : #include <algorithm>
17 :
18 : // NOTE(ajm): Path provided by gyp.
19 : #include "libyuv/scale.h" // NOLINT
20 : #include "libyuv/convert.h" // NOLINT
21 :
22 : #include "webrtc/base/checks.h"
23 : #include "webrtc/base/timeutils.h"
24 : #include "webrtc/base/trace_event.h"
25 : #include "webrtc/common_types.h"
26 : #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
27 : #include "webrtc/modules/include/module_common_types.h"
28 : #include "webrtc/modules/video_coding/include/video_codec_interface.h"
29 : #include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h"
30 : #include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h"
31 : #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h"
32 : #include "webrtc/modules/video_coding/utility/simulcast_rate_allocator.h"
33 : #include "webrtc/system_wrappers/include/clock.h"
34 : #include "webrtc/system_wrappers/include/metrics.h"
35 :
36 : namespace webrtc {
37 : namespace {
38 :
39 : enum { kVp8ErrorPropagationTh = 30 };
40 : enum { kVp832ByteAlign = 32 };
41 :
42 : // VP8 denoiser states.
43 : enum denoiserState {
44 : kDenoiserOff,
45 : kDenoiserOnYOnly,
46 : kDenoiserOnYUV,
47 : kDenoiserOnYUVAggressive,
48 : // Adaptive mode defaults to kDenoiserOnYUV on key frame, but may switch
49 : // to kDenoiserOnYUVAggressive based on a computed noise metric.
50 : kDenoiserOnAdaptive
51 : };
52 :
53 : // Greatest common divisior
54 0 : int GCD(int a, int b) {
55 0 : int c = a % b;
56 0 : while (c != 0) {
57 0 : a = b;
58 0 : b = c;
59 0 : c = a % b;
60 : }
61 0 : return b;
62 : }
63 :
64 0 : uint32_t SumStreamMaxBitrate(int streams, const VideoCodec& codec) {
65 0 : uint32_t bitrate_sum = 0;
66 0 : for (int i = 0; i < streams; ++i) {
67 0 : bitrate_sum += codec.simulcastStream[i].maxBitrate;
68 : }
69 0 : return bitrate_sum;
70 : }
71 :
72 0 : int NumberOfStreams(const VideoCodec& codec) {
73 : int streams =
74 0 : codec.numberOfSimulcastStreams < 1 ? 1 : codec.numberOfSimulcastStreams;
75 0 : uint32_t simulcast_max_bitrate = SumStreamMaxBitrate(streams, codec);
76 0 : if (simulcast_max_bitrate == 0) {
77 0 : streams = 1;
78 : }
79 0 : return streams;
80 : }
81 :
82 0 : bool ValidSimulcastResolutions(const VideoCodec& codec, int num_streams) {
83 0 : if (codec.width != codec.simulcastStream[num_streams - 1].width ||
84 0 : codec.height != codec.simulcastStream[num_streams - 1].height) {
85 0 : return false;
86 : }
87 0 : for (int i = 0; i < num_streams; ++i) {
88 0 : if (codec.width * codec.simulcastStream[i].height !=
89 0 : codec.height * codec.simulcastStream[i].width) {
90 0 : return false;
91 : }
92 : }
93 0 : return true;
94 : }
95 :
96 0 : int NumStreamsDisabled(const std::vector<bool>& streams) {
97 0 : int num_disabled = 0;
98 0 : for (bool stream : streams) {
99 0 : if (!stream)
100 0 : ++num_disabled;
101 : }
102 0 : return num_disabled;
103 : }
104 : } // namespace
105 :
106 0 : VP8Encoder* VP8Encoder::Create() {
107 0 : return new VP8EncoderImpl();
108 : }
109 :
110 0 : VP8Decoder* VP8Decoder::Create() {
111 0 : return new VP8DecoderImpl();
112 : }
113 :
114 0 : VP8EncoderImpl::VP8EncoderImpl()
115 : : encoded_complete_callback_(nullptr),
116 : inited_(false),
117 : timestamp_(0),
118 : feedback_mode_(false),
119 : qp_max_(56), // Setting for max quantizer.
120 : cpu_speed_default_(-6),
121 : number_of_cores_(0),
122 : rc_max_intra_target_(0),
123 : token_partitions_(VP8_ONE_TOKENPARTITION),
124 : down_scale_requested_(false),
125 : down_scale_bitrate_(0),
126 0 : key_frame_request_(kMaxSimulcastStreams, false) {
127 0 : uint32_t seed = rtc::Time32();
128 0 : srand(seed);
129 :
130 0 : picture_id_.reserve(kMaxSimulcastStreams);
131 0 : last_key_frame_picture_id_.reserve(kMaxSimulcastStreams);
132 0 : temporal_layers_.reserve(kMaxSimulcastStreams);
133 0 : raw_images_.reserve(kMaxSimulcastStreams);
134 0 : encoded_images_.reserve(kMaxSimulcastStreams);
135 0 : send_stream_.reserve(kMaxSimulcastStreams);
136 0 : cpu_speed_.assign(kMaxSimulcastStreams, -6); // Set default to -6.
137 0 : encoders_.reserve(kMaxSimulcastStreams);
138 0 : configurations_.reserve(kMaxSimulcastStreams);
139 0 : downsampling_factors_.reserve(kMaxSimulcastStreams);
140 0 : }
141 :
142 0 : VP8EncoderImpl::~VP8EncoderImpl() {
143 0 : Release();
144 0 : }
145 :
146 0 : int VP8EncoderImpl::Release() {
147 0 : int ret_val = WEBRTC_VIDEO_CODEC_OK;
148 :
149 0 : while (!encoded_images_.empty()) {
150 0 : EncodedImage& image = encoded_images_.back();
151 0 : delete[] image._buffer;
152 0 : encoded_images_.pop_back();
153 : }
154 0 : while (!encoders_.empty()) {
155 0 : vpx_codec_ctx_t& encoder = encoders_.back();
156 0 : if (vpx_codec_destroy(&encoder)) {
157 0 : ret_val = WEBRTC_VIDEO_CODEC_MEMORY;
158 : }
159 0 : encoders_.pop_back();
160 : }
161 0 : configurations_.clear();
162 0 : send_stream_.clear();
163 0 : cpu_speed_.clear();
164 0 : while (!raw_images_.empty()) {
165 0 : vpx_img_free(&raw_images_.back());
166 0 : raw_images_.pop_back();
167 : }
168 0 : while (!temporal_layers_.empty()) {
169 0 : delete temporal_layers_.back();
170 0 : temporal_layers_.pop_back();
171 : }
172 0 : inited_ = false;
173 0 : return ret_val;
174 : }
175 :
176 0 : int VP8EncoderImpl::SetRateAllocation(const BitrateAllocation& bitrate,
177 : uint32_t new_framerate) {
178 0 : if (!inited_)
179 0 : return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
180 :
181 0 : if (encoders_[0].err)
182 0 : return WEBRTC_VIDEO_CODEC_ERROR;
183 :
184 0 : if (new_framerate < 1)
185 0 : return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
186 :
187 0 : if (bitrate.get_sum_bps() == 0) {
188 : // Encoder paused, turn off all encoding.
189 0 : const int num_streams = static_cast<size_t>(encoders_.size());
190 0 : for (int i = 0; i < num_streams; ++i)
191 0 : SetStreamState(false, i);
192 0 : return WEBRTC_VIDEO_CODEC_OK;
193 : }
194 :
195 : // At this point, bitrate allocation should already match codec settings.
196 0 : if (codec_.maxBitrate > 0)
197 0 : RTC_DCHECK_LE(bitrate.get_sum_kbps(), codec_.maxBitrate);
198 0 : RTC_DCHECK_GE(bitrate.get_sum_kbps(), codec_.minBitrate);
199 0 : if (codec_.numberOfSimulcastStreams > 0)
200 0 : RTC_DCHECK_GE(bitrate.get_sum_kbps(), codec_.simulcastStream[0].minBitrate);
201 :
202 0 : codec_.maxFramerate = new_framerate;
203 :
204 0 : if (encoders_.size() == 1) {
205 : // 1:1.
206 : // Calculate a rough limit for when to trigger a potental down scale.
207 0 : uint32_t k_pixels_per_frame = codec_.width * codec_.height / 1000;
208 : // TODO(pwestin): we currently lack CAMA, this is a temporary fix to work
209 : // around the current limitations.
210 : // Only trigger keyframes if we are allowed to scale down.
211 0 : if (configurations_[0].rc_resize_allowed) {
212 0 : if (!down_scale_requested_) {
213 0 : if (k_pixels_per_frame > bitrate.get_sum_kbps()) {
214 0 : down_scale_requested_ = true;
215 0 : down_scale_bitrate_ = bitrate.get_sum_kbps();
216 0 : key_frame_request_[0] = true;
217 : }
218 : } else {
219 0 : if (bitrate.get_sum_kbps() > (2 * down_scale_bitrate_) ||
220 0 : bitrate.get_sum_kbps() < (down_scale_bitrate_ / 2)) {
221 0 : down_scale_requested_ = false;
222 : }
223 : }
224 : }
225 : } else {
226 : // If we have more than 1 stream, reduce the qp_max for the low resolution
227 : // stream if frame rate is not too low. The trade-off with lower qp_max is
228 : // possibly more dropped frames, so we only do this if the frame rate is
229 : // above some threshold (base temporal layer is down to 1/4 for 3 layers).
230 : // We may want to condition this on bitrate later.
231 0 : if (new_framerate > 20) {
232 0 : configurations_[encoders_.size() - 1].rc_max_quantizer = 45;
233 : } else {
234 : // Go back to default value set in InitEncode.
235 0 : configurations_[encoders_.size() - 1].rc_max_quantizer = qp_max_;
236 : }
237 : }
238 :
239 0 : size_t stream_idx = encoders_.size() - 1;
240 0 : for (size_t i = 0; i < encoders_.size(); ++i, --stream_idx) {
241 : unsigned int target_bitrate_kbps =
242 0 : bitrate.GetSpatialLayerSum(stream_idx) / 1000;
243 :
244 0 : bool send_stream = target_bitrate_kbps > 0;
245 0 : if (send_stream || encoders_.size() > 1)
246 0 : SetStreamState(send_stream, stream_idx);
247 :
248 0 : configurations_[i].rc_target_bitrate = target_bitrate_kbps;
249 0 : temporal_layers_[stream_idx]->UpdateConfiguration(&configurations_[i]);
250 :
251 0 : if (vpx_codec_enc_config_set(&encoders_[i], &configurations_[i])) {
252 0 : return WEBRTC_VIDEO_CODEC_ERROR;
253 : }
254 : }
255 0 : return WEBRTC_VIDEO_CODEC_OK;
256 : }
257 :
258 0 : const char* VP8EncoderImpl::ImplementationName() const {
259 0 : return "libvpx";
260 : }
261 :
262 0 : void VP8EncoderImpl::SetStreamState(bool send_stream,
263 : int stream_idx) {
264 0 : if (send_stream && !send_stream_[stream_idx]) {
265 : // Need a key frame if we have not sent this stream before.
266 0 : key_frame_request_[stream_idx] = true;
267 : }
268 0 : send_stream_[stream_idx] = send_stream;
269 0 : }
270 :
271 0 : void VP8EncoderImpl::SetupTemporalLayers(int num_streams,
272 : int num_temporal_layers,
273 : const VideoCodec& codec) {
274 0 : RTC_DCHECK(codec.VP8().tl_factory != nullptr);
275 0 : const TemporalLayersFactory* tl_factory = codec.VP8().tl_factory;
276 0 : if (num_streams == 1) {
277 0 : temporal_layers_.push_back(
278 0 : tl_factory->Create(0, num_temporal_layers, rand()));
279 : } else {
280 0 : for (int i = 0; i < num_streams; ++i) {
281 0 : RTC_CHECK_GT(num_temporal_layers, 0);
282 0 : int layers = std::max(static_cast<uint8_t>(1),
283 0 : codec.simulcastStream[i].numberOfTemporalLayers);
284 0 : temporal_layers_.push_back(tl_factory->Create(i, layers, rand()));
285 : }
286 : }
287 0 : }
288 :
289 0 : int VP8EncoderImpl::InitEncode(const VideoCodec* inst,
290 : int number_of_cores,
291 : size_t /*maxPayloadSize */) {
292 0 : if (inst == NULL) {
293 0 : return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
294 : }
295 0 : if (inst->maxFramerate < 1) {
296 0 : return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
297 : }
298 : // allow zero to represent an unspecified maxBitRate
299 0 : if (inst->maxBitrate > 0 && inst->startBitrate > inst->maxBitrate) {
300 0 : return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
301 : }
302 0 : if (inst->width <= 1 || inst->height <= 1) {
303 0 : return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
304 : }
305 0 : if (number_of_cores < 1) {
306 0 : return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
307 : }
308 0 : if (inst->VP8().feedbackModeOn && inst->numberOfSimulcastStreams > 1) {
309 0 : return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
310 : }
311 0 : if (inst->VP8().automaticResizeOn && inst->numberOfSimulcastStreams > 1) {
312 0 : return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
313 : }
314 0 : int retVal = Release();
315 0 : if (retVal < 0) {
316 0 : return retVal;
317 : }
318 :
319 0 : int number_of_streams = NumberOfStreams(*inst);
320 0 : bool doing_simulcast = (number_of_streams > 1);
321 :
322 0 : if (doing_simulcast && !ValidSimulcastResolutions(*inst, number_of_streams)) {
323 0 : return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
324 : }
325 :
326 : int num_temporal_layers =
327 0 : doing_simulcast ? inst->simulcastStream[0].numberOfTemporalLayers
328 0 : : inst->VP8().numberOfTemporalLayers;
329 0 : RTC_DCHECK_GT(num_temporal_layers, 0);
330 :
331 0 : SetupTemporalLayers(number_of_streams, num_temporal_layers, *inst);
332 :
333 0 : feedback_mode_ = inst->VP8().feedbackModeOn;
334 :
335 0 : number_of_cores_ = number_of_cores;
336 0 : timestamp_ = 0;
337 0 : codec_ = *inst;
338 :
339 : // Code expects simulcastStream resolutions to be correct, make sure they are
340 : // filled even when there are no simulcast layers.
341 0 : if (codec_.numberOfSimulcastStreams == 0) {
342 0 : codec_.simulcastStream[0].width = codec_.width;
343 0 : codec_.simulcastStream[0].height = codec_.height;
344 : }
345 :
346 0 : picture_id_.resize(number_of_streams);
347 0 : last_key_frame_picture_id_.resize(number_of_streams);
348 0 : encoded_images_.resize(number_of_streams);
349 0 : encoders_.resize(number_of_streams);
350 0 : configurations_.resize(number_of_streams);
351 0 : downsampling_factors_.resize(number_of_streams);
352 0 : raw_images_.resize(number_of_streams);
353 0 : send_stream_.resize(number_of_streams);
354 0 : send_stream_[0] = true; // For non-simulcast case.
355 0 : cpu_speed_.resize(number_of_streams);
356 0 : std::fill(key_frame_request_.begin(), key_frame_request_.end(), false);
357 :
358 0 : int idx = number_of_streams - 1;
359 0 : for (int i = 0; i < (number_of_streams - 1); ++i, --idx) {
360 0 : int gcd = GCD(inst->simulcastStream[idx].width,
361 0 : inst->simulcastStream[idx - 1].width);
362 0 : downsampling_factors_[i].num = inst->simulcastStream[idx].width / gcd;
363 0 : downsampling_factors_[i].den = inst->simulcastStream[idx - 1].width / gcd;
364 0 : send_stream_[i] = false;
365 : }
366 0 : if (number_of_streams > 1) {
367 0 : send_stream_[number_of_streams - 1] = false;
368 0 : downsampling_factors_[number_of_streams - 1].num = 1;
369 0 : downsampling_factors_[number_of_streams - 1].den = 1;
370 : }
371 0 : for (int i = 0; i < number_of_streams; ++i) {
372 : // Random start, 16 bits is enough.
373 0 : picture_id_[i] = static_cast<uint16_t>(rand()) & 0x7FFF; // NOLINT
374 0 : last_key_frame_picture_id_[i] = -1;
375 : // allocate memory for encoded image
376 0 : if (encoded_images_[i]._buffer != NULL) {
377 0 : delete[] encoded_images_[i]._buffer;
378 : }
379 : // Reserve 100 extra bytes for overhead at small resolutions.
380 0 : encoded_images_[i]._size = CalcBufferSize(kI420, codec_.width, codec_.height)
381 0 : + 100;
382 0 : encoded_images_[i]._buffer = new uint8_t[encoded_images_[i]._size];
383 0 : encoded_images_[i]._completeFrame = true;
384 : }
385 : // populate encoder configuration with default values
386 0 : if (vpx_codec_enc_config_default(vpx_codec_vp8_cx(), &configurations_[0],
387 : 0)) {
388 0 : return WEBRTC_VIDEO_CODEC_ERROR;
389 : }
390 : // setting the time base of the codec
391 0 : configurations_[0].g_timebase.num = 1;
392 0 : configurations_[0].g_timebase.den = 90000;
393 0 : configurations_[0].g_lag_in_frames = 0; // 0- no frame lagging
394 :
395 : // Set the error resilience mode according to user settings.
396 0 : switch (inst->VP8().resilience) {
397 : case kResilienceOff:
398 0 : configurations_[0].g_error_resilient = 0;
399 0 : break;
400 : case kResilientStream:
401 0 : configurations_[0].g_error_resilient = 1; // TODO(holmer): Replace with
402 : // VPX_ERROR_RESILIENT_DEFAULT when we
403 : // drop support for libvpx 9.6.0.
404 0 : break;
405 : case kResilientFrames:
406 0 : return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; // Not supported
407 : }
408 :
409 : // rate control settings
410 0 : configurations_[0].rc_dropframe_thresh = inst->VP8().frameDroppingOn ? 30 : 0;
411 0 : configurations_[0].rc_end_usage = VPX_CBR;
412 0 : configurations_[0].g_pass = VPX_RC_ONE_PASS;
413 : // TODO(hellner): investigate why the following two lines produce
414 : // automaticResizeOn value of 3 when running
415 : // WebRtcVideoMediaChannelTest.GetStatsMultipleSendStreams inside the talk
416 : // framework.
417 : // configurations_[0].rc_resize_allowed =
418 : // inst->codecSpecific.VP8.automaticResizeOn ? 1 : 0;
419 0 : configurations_[0].rc_resize_allowed = 0;
420 : // Handle resizing outside of libvpx when doing single-stream.
421 0 : if (inst->VP8().automaticResizeOn && number_of_streams > 1) {
422 0 : configurations_[0].rc_resize_allowed = 1;
423 : }
424 0 : configurations_[0].rc_min_quantizer = 2;
425 0 : if (inst->qpMax >= configurations_[0].rc_min_quantizer) {
426 0 : qp_max_ = inst->qpMax;
427 : }
428 0 : configurations_[0].rc_max_quantizer = qp_max_;
429 0 : configurations_[0].rc_undershoot_pct = 100;
430 0 : configurations_[0].rc_overshoot_pct = 15;
431 0 : configurations_[0].rc_buf_initial_sz = 500;
432 0 : configurations_[0].rc_buf_optimal_sz = 600;
433 0 : configurations_[0].rc_buf_sz = 1000;
434 :
435 : // Set the maximum target size of any key-frame.
436 0 : rc_max_intra_target_ = MaxIntraTarget(configurations_[0].rc_buf_optimal_sz);
437 :
438 0 : if (feedback_mode_) {
439 : // Disable periodic key frames if we get feedback from the decoder
440 : // through SLI and RPSI.
441 0 : configurations_[0].kf_mode = VPX_KF_DISABLED;
442 0 : } else if (inst->VP8().keyFrameInterval > 0) {
443 0 : configurations_[0].kf_mode = VPX_KF_AUTO;
444 0 : configurations_[0].kf_max_dist = inst->VP8().keyFrameInterval;
445 : } else {
446 0 : configurations_[0].kf_mode = VPX_KF_DISABLED;
447 : }
448 :
449 : // Allow the user to set the complexity for the base stream.
450 0 : switch (inst->VP8().complexity) {
451 : case kComplexityHigh:
452 0 : cpu_speed_[0] = -5;
453 0 : break;
454 : case kComplexityHigher:
455 0 : cpu_speed_[0] = -4;
456 0 : break;
457 : case kComplexityMax:
458 0 : cpu_speed_[0] = -3;
459 0 : break;
460 : default:
461 0 : cpu_speed_[0] = -6;
462 0 : break;
463 : }
464 0 : cpu_speed_default_ = cpu_speed_[0];
465 : // Set encoding complexity (cpu_speed) based on resolution and/or platform.
466 0 : cpu_speed_[0] = SetCpuSpeed(inst->width, inst->height);
467 0 : for (int i = 1; i < number_of_streams; ++i) {
468 0 : cpu_speed_[i] =
469 0 : SetCpuSpeed(inst->simulcastStream[number_of_streams - 1 - i].width,
470 0 : inst->simulcastStream[number_of_streams - 1 - i].height);
471 : }
472 0 : configurations_[0].g_w = inst->width;
473 0 : configurations_[0].g_h = inst->height;
474 :
475 : // Determine number of threads based on the image size and #cores.
476 : // TODO(fbarchard): Consider number of Simulcast layers.
477 0 : configurations_[0].g_threads = NumberOfThreads(
478 0 : configurations_[0].g_w, configurations_[0].g_h, number_of_cores);
479 :
480 : // Creating a wrapper to the image - setting image data to NULL.
481 : // Actual pointer will be set in encode. Setting align to 1, as it
482 : // is meaningless (no memory allocation is done here).
483 0 : vpx_img_wrap(&raw_images_[0], VPX_IMG_FMT_I420, inst->width, inst->height, 1,
484 0 : NULL);
485 :
486 : // Note the order we use is different from webm, we have lowest resolution
487 : // at position 0 and they have highest resolution at position 0.
488 0 : int stream_idx = encoders_.size() - 1;
489 0 : SimulcastRateAllocator init_allocator(codec_, nullptr);
490 : BitrateAllocation allocation = init_allocator.GetAllocation(
491 0 : inst->startBitrate * 1000, inst->maxFramerate);
492 0 : std::vector<uint32_t> stream_bitrates;
493 0 : for (int i = 0; i == 0 || i < inst->numberOfSimulcastStreams; ++i) {
494 0 : uint32_t bitrate = allocation.GetSpatialLayerSum(i) / 1000;
495 0 : stream_bitrates.push_back(bitrate);
496 : }
497 :
498 0 : configurations_[0].rc_target_bitrate = stream_bitrates[stream_idx];
499 0 : temporal_layers_[stream_idx]->OnRatesUpdated(
500 0 : stream_bitrates[stream_idx], inst->maxBitrate, inst->maxFramerate);
501 0 : temporal_layers_[stream_idx]->UpdateConfiguration(&configurations_[0]);
502 0 : --stream_idx;
503 0 : for (size_t i = 1; i < encoders_.size(); ++i, --stream_idx) {
504 0 : memcpy(&configurations_[i], &configurations_[0],
505 0 : sizeof(configurations_[0]));
506 :
507 0 : configurations_[i].g_w = inst->simulcastStream[stream_idx].width;
508 0 : configurations_[i].g_h = inst->simulcastStream[stream_idx].height;
509 :
510 : // Use 1 thread for lower resolutions.
511 0 : configurations_[i].g_threads = 1;
512 :
513 : // Setting alignment to 32 - as that ensures at least 16 for all
514 : // planes (32 for Y, 16 for U,V). Libvpx sets the requested stride for
515 : // the y plane, but only half of it to the u and v planes.
516 0 : vpx_img_alloc(&raw_images_[i], VPX_IMG_FMT_I420,
517 0 : inst->simulcastStream[stream_idx].width,
518 0 : inst->simulcastStream[stream_idx].height, kVp832ByteAlign);
519 0 : SetStreamState(stream_bitrates[stream_idx] > 0, stream_idx);
520 0 : configurations_[i].rc_target_bitrate = stream_bitrates[stream_idx];
521 0 : temporal_layers_[stream_idx]->OnRatesUpdated(
522 0 : stream_bitrates[stream_idx], inst->maxBitrate, inst->maxFramerate);
523 0 : temporal_layers_[stream_idx]->UpdateConfiguration(&configurations_[i]);
524 : }
525 :
526 0 : rps_.Init();
527 0 : return InitAndSetControlSettings();
528 : }
529 :
530 0 : int VP8EncoderImpl::SetCpuSpeed(int width, int height) {
531 : #if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || defined(ANDROID)
532 : // On mobile platform, use a lower speed setting for lower resolutions for
533 : // CPUs with 4 or more cores.
534 : RTC_DCHECK_GT(number_of_cores_, 0);
535 : if (number_of_cores_ <= 3)
536 : return -12;
537 :
538 : if (width * height <= 352 * 288)
539 : return -8;
540 : else if (width * height <= 640 * 480)
541 : return -10;
542 : else
543 : return -12;
544 : #else
545 : // For non-ARM, increase encoding complexity (i.e., use lower speed setting)
546 : // if resolution is below CIF. Otherwise, keep the default/user setting
547 : // (|cpu_speed_default_|) set on InitEncode via VP8().complexity.
548 0 : if (width * height < 352 * 288)
549 0 : return (cpu_speed_default_ < -4) ? -4 : cpu_speed_default_;
550 : else
551 0 : return cpu_speed_default_;
552 : #endif
553 : }
554 :
555 0 : int VP8EncoderImpl::NumberOfThreads(int width, int height, int cpus) {
556 : #if defined(ANDROID)
557 : if (width * height >= 320 * 180) {
558 : if (cpus >= 4) {
559 : // 3 threads for CPUs with 4 and more cores since most of times only 4
560 : // cores will be active.
561 : return 3;
562 : } else if (cpus == 3 || cpus == 2) {
563 : return 2;
564 : } else {
565 : return 1;
566 : }
567 : }
568 : return 1;
569 : #else
570 0 : if (width * height >= 1920 * 1080 && cpus > 8) {
571 0 : return 8; // 8 threads for 1080p on high perf machines.
572 0 : } else if (width * height > 1280 * 960 && cpus >= 6) {
573 : // 3 threads for 1080p.
574 0 : return 3;
575 0 : } else if (width * height > 640 * 480 && cpus >= 3) {
576 : // 2 threads for qHD/HD.
577 0 : return 2;
578 : } else {
579 : // 1 thread for VGA or less.
580 0 : return 1;
581 : }
582 : #endif
583 : }
584 :
585 0 : int VP8EncoderImpl::InitAndSetControlSettings() {
586 0 : vpx_codec_flags_t flags = 0;
587 0 : flags |= VPX_CODEC_USE_OUTPUT_PARTITION;
588 :
589 0 : if (encoders_.size() > 1) {
590 0 : int error = vpx_codec_enc_init_multi(&encoders_[0], vpx_codec_vp8_cx(),
591 : &configurations_[0], encoders_.size(),
592 : flags, &downsampling_factors_[0]);
593 0 : if (error) {
594 0 : return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
595 : }
596 : } else {
597 0 : if (vpx_codec_enc_init(&encoders_[0], vpx_codec_vp8_cx(),
598 : &configurations_[0], flags)) {
599 0 : return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
600 : }
601 : }
602 : // Enable denoising for the highest resolution stream, and for
603 : // the second highest resolution if we are doing more than 2
604 : // spatial layers/streams.
605 : // TODO(holmer): Investigate possibility of adding a libvpx API
606 : // for getting the denoised frame from the encoder and using that
607 : // when encoding lower resolution streams. Would it work with the
608 : // multi-res encoding feature?
609 0 : denoiserState denoiser_state = kDenoiserOnYOnly;
610 : #if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || defined(ANDROID)
611 : denoiser_state = kDenoiserOnYOnly;
612 : #else
613 0 : denoiser_state = kDenoiserOnAdaptive;
614 : #endif
615 0 : vpx_codec_control(&encoders_[0], VP8E_SET_NOISE_SENSITIVITY,
616 0 : codec_.VP8()->denoisingOn ? denoiser_state : kDenoiserOff);
617 0 : if (encoders_.size() > 2) {
618 0 : vpx_codec_control(
619 : &encoders_[1], VP8E_SET_NOISE_SENSITIVITY,
620 0 : codec_.VP8()->denoisingOn ? denoiser_state : kDenoiserOff);
621 : }
622 0 : for (size_t i = 0; i < encoders_.size(); ++i) {
623 : // Allow more screen content to be detected as static.
624 0 : vpx_codec_control(&(encoders_[i]), VP8E_SET_STATIC_THRESHOLD,
625 0 : codec_.mode == kScreensharing ? 300 : 1);
626 0 : vpx_codec_control(&(encoders_[i]), VP8E_SET_CPUUSED, cpu_speed_[i]);
627 0 : vpx_codec_control(&(encoders_[i]), VP8E_SET_TOKEN_PARTITIONS,
628 0 : static_cast<vp8e_token_partitions>(token_partitions_));
629 0 : vpx_codec_control(&(encoders_[i]), VP8E_SET_MAX_INTRA_BITRATE_PCT,
630 0 : rc_max_intra_target_);
631 : // VP8E_SET_SCREEN_CONTENT_MODE 2 = screen content with more aggressive
632 : // rate control (drop frames on large target bitrate overshoot)
633 0 : vpx_codec_control(&(encoders_[i]), VP8E_SET_SCREEN_CONTENT_MODE,
634 0 : codec_.mode == kScreensharing ? 2 : 0);
635 : }
636 0 : inited_ = true;
637 0 : return WEBRTC_VIDEO_CODEC_OK;
638 : }
639 :
640 0 : uint32_t VP8EncoderImpl::MaxIntraTarget(uint32_t optimalBuffersize) {
641 : // Set max to the optimal buffer level (normalized by target BR),
642 : // and scaled by a scalePar.
643 : // Max target size = scalePar * optimalBufferSize * targetBR[Kbps].
644 : // This values is presented in percentage of perFrameBw:
645 : // perFrameBw = targetBR[Kbps] * 1000 / frameRate.
646 : // The target in % is as follows:
647 :
648 0 : float scalePar = 0.5;
649 0 : uint32_t targetPct = optimalBuffersize * scalePar * codec_.maxFramerate / 10;
650 :
651 : // Don't go below 3 times the per frame bandwidth.
652 0 : const uint32_t minIntraTh = 300;
653 0 : return (targetPct < minIntraTh) ? minIntraTh : targetPct;
654 : }
655 :
656 0 : int VP8EncoderImpl::Encode(const VideoFrame& frame,
657 : const CodecSpecificInfo* codec_specific_info,
658 : const std::vector<FrameType>* frame_types) {
659 0 : RTC_DCHECK_EQ(frame.width(), codec_.width);
660 0 : RTC_DCHECK_EQ(frame.height(), codec_.height);
661 :
662 0 : if (!inited_)
663 0 : return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
664 0 : if (encoded_complete_callback_ == NULL)
665 0 : return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
666 :
667 0 : rtc::scoped_refptr<VideoFrameBuffer> input_image = frame.video_frame_buffer();
668 : // Since we are extracting raw pointers from |input_image| to
669 : // |raw_images_[0]|, the resolution of these frames must match. Note that
670 : // |input_image| might be scaled from |frame|. In that case, the resolution of
671 : // |raw_images_[0]| should have been updated in UpdateCodecFrameSize.
672 0 : RTC_DCHECK_EQ(input_image->width(), raw_images_[0].d_w);
673 0 : RTC_DCHECK_EQ(input_image->height(), raw_images_[0].d_h);
674 :
675 : // Image in vpx_image_t format.
676 : // Input image is const. VP8's raw image is not defined as const.
677 0 : raw_images_[0].planes[VPX_PLANE_Y] =
678 0 : const_cast<uint8_t*>(input_image->DataY());
679 0 : raw_images_[0].planes[VPX_PLANE_U] =
680 0 : const_cast<uint8_t*>(input_image->DataU());
681 0 : raw_images_[0].planes[VPX_PLANE_V] =
682 0 : const_cast<uint8_t*>(input_image->DataV());
683 :
684 0 : raw_images_[0].stride[VPX_PLANE_Y] = input_image->StrideY();
685 0 : raw_images_[0].stride[VPX_PLANE_U] = input_image->StrideU();
686 0 : raw_images_[0].stride[VPX_PLANE_V] = input_image->StrideV();
687 :
688 0 : for (size_t i = 1; i < encoders_.size(); ++i) {
689 : // Scale the image down a number of times by downsampling factor
690 0 : libyuv::I420Scale(
691 0 : raw_images_[i - 1].planes[VPX_PLANE_Y],
692 0 : raw_images_[i - 1].stride[VPX_PLANE_Y],
693 0 : raw_images_[i - 1].planes[VPX_PLANE_U],
694 0 : raw_images_[i - 1].stride[VPX_PLANE_U],
695 0 : raw_images_[i - 1].planes[VPX_PLANE_V],
696 0 : raw_images_[i - 1].stride[VPX_PLANE_V], raw_images_[i - 1].d_w,
697 0 : raw_images_[i - 1].d_h, raw_images_[i].planes[VPX_PLANE_Y],
698 0 : raw_images_[i].stride[VPX_PLANE_Y], raw_images_[i].planes[VPX_PLANE_U],
699 0 : raw_images_[i].stride[VPX_PLANE_U], raw_images_[i].planes[VPX_PLANE_V],
700 0 : raw_images_[i].stride[VPX_PLANE_V], raw_images_[i].d_w,
701 0 : raw_images_[i].d_h, libyuv::kFilterBilinear);
702 : }
703 : vpx_enc_frame_flags_t flags[kMaxSimulcastStreams];
704 0 : for (size_t i = 0; i < encoders_.size(); ++i) {
705 0 : int ret = temporal_layers_[i]->EncodeFlags(frame.timestamp());
706 0 : if (ret < 0) {
707 : // Drop this frame.
708 0 : return WEBRTC_VIDEO_CODEC_OK;
709 : }
710 0 : flags[i] = ret;
711 : }
712 0 : bool send_key_frame = false;
713 0 : for (size_t i = 0; i < key_frame_request_.size() && i < send_stream_.size();
714 : ++i) {
715 0 : if (key_frame_request_[i] && send_stream_[i]) {
716 0 : send_key_frame = true;
717 0 : break;
718 : }
719 : }
720 0 : if (!send_key_frame && frame_types) {
721 0 : for (size_t i = 0; i < frame_types->size() && i < send_stream_.size();
722 : ++i) {
723 0 : if ((*frame_types)[i] == kVideoFrameKey && send_stream_[i]) {
724 0 : send_key_frame = true;
725 0 : break;
726 : }
727 : }
728 : }
729 : // The flag modification below (due to forced key frame, RPS, etc.,) for now
730 : // will be the same for all encoders/spatial layers.
731 : // TODO(marpan/holmer): Allow for key frame request to be set per encoder.
732 0 : bool only_predict_from_key_frame = false;
733 0 : if (send_key_frame) {
734 : // Adapt the size of the key frame when in screenshare with 1 temporal
735 : // layer.
736 0 : if (encoders_.size() == 1 && codec_.mode == kScreensharing &&
737 0 : codec_.VP8()->numberOfTemporalLayers <= 1) {
738 0 : const uint32_t forceKeyFrameIntraTh = 100;
739 0 : vpx_codec_control(&(encoders_[0]), VP8E_SET_MAX_INTRA_BITRATE_PCT,
740 0 : forceKeyFrameIntraTh);
741 : }
742 : // Key frame request from caller.
743 : // Will update both golden and alt-ref.
744 0 : for (size_t i = 0; i < encoders_.size(); ++i) {
745 0 : flags[i] = VPX_EFLAG_FORCE_KF;
746 : }
747 0 : std::fill(key_frame_request_.begin(), key_frame_request_.end(), false);
748 0 : } else if (codec_specific_info &&
749 0 : codec_specific_info->codecType == kVideoCodecVP8) {
750 0 : if (feedback_mode_) {
751 : // Handle RPSI and SLI messages and set up the appropriate encode flags.
752 0 : bool sendRefresh = false;
753 0 : if (codec_specific_info->codecSpecific.VP8.hasReceivedRPSI) {
754 0 : rps_.ReceivedRPSI(codec_specific_info->codecSpecific.VP8.pictureIdRPSI);
755 : }
756 0 : if (codec_specific_info->codecSpecific.VP8.hasReceivedSLI) {
757 0 : sendRefresh = rps_.ReceivedSLI(frame.timestamp());
758 : }
759 0 : for (size_t i = 0; i < encoders_.size(); ++i) {
760 0 : flags[i] = rps_.EncodeFlags(picture_id_[i], sendRefresh,
761 : frame.timestamp());
762 : }
763 : } else {
764 0 : if (codec_specific_info->codecSpecific.VP8.hasReceivedRPSI) {
765 : // Is this our last key frame? If not ignore.
766 : // |picture_id_| is defined per spatial stream/layer, so check that
767 : // |RPSI| matches the last key frame from any of the spatial streams.
768 : // If so, then all spatial streams for this encoding will predict from
769 : // its long-term reference (last key frame).
770 0 : int RPSI = codec_specific_info->codecSpecific.VP8.pictureIdRPSI;
771 0 : for (size_t i = 0; i < encoders_.size(); ++i) {
772 0 : if (last_key_frame_picture_id_[i] == RPSI) {
773 : // Request for a long term reference frame.
774 : // Note 1: overwrites any temporal settings.
775 : // Note 2: VP8_EFLAG_NO_UPD_ENTROPY is not needed as that flag is
776 : // set by error_resilient mode.
777 0 : for (size_t j = 0; j < encoders_.size(); ++j) {
778 0 : flags[j] = VP8_EFLAG_NO_UPD_ARF;
779 0 : flags[j] |= VP8_EFLAG_NO_REF_GF;
780 0 : flags[j] |= VP8_EFLAG_NO_REF_LAST;
781 : }
782 0 : only_predict_from_key_frame = true;
783 0 : break;
784 : }
785 : }
786 : }
787 : }
788 : }
789 : // Set the encoder frame flags and temporal layer_id for each spatial stream.
790 : // Note that |temporal_layers_| are defined starting from lowest resolution at
791 : // position 0 to highest resolution at position |encoders_.size() - 1|,
792 : // whereas |encoder_| is from highest to lowest resolution.
793 0 : size_t stream_idx = encoders_.size() - 1;
794 0 : for (size_t i = 0; i < encoders_.size(); ++i, --stream_idx) {
795 : // Allow the layers adapter to temporarily modify the configuration. This
796 : // change isn't stored in configurations_ so change will be discarded at
797 : // the next update.
798 : vpx_codec_enc_cfg_t temp_config;
799 0 : memcpy(&temp_config, &configurations_[i], sizeof(vpx_codec_enc_cfg_t));
800 0 : if (temporal_layers_[stream_idx]->UpdateConfiguration(&temp_config)) {
801 0 : if (vpx_codec_enc_config_set(&encoders_[i], &temp_config))
802 0 : return WEBRTC_VIDEO_CODEC_ERROR;
803 : }
804 :
805 0 : vpx_codec_control(&encoders_[i], VP8E_SET_FRAME_FLAGS, flags[stream_idx]);
806 0 : vpx_codec_control(&encoders_[i], VP8E_SET_TEMPORAL_LAYER_ID,
807 0 : temporal_layers_[stream_idx]->CurrentLayerId());
808 : }
809 : // TODO(holmer): Ideally the duration should be the timestamp diff of this
810 : // frame and the next frame to be encoded, which we don't have. Instead we
811 : // would like to use the duration of the previous frame. Unfortunately the
812 : // rate control seems to be off with that setup. Using the average input
813 : // frame rate to calculate an average duration for now.
814 0 : assert(codec_.maxFramerate > 0);
815 0 : uint32_t duration = 90000 / codec_.maxFramerate;
816 :
817 : // Note we must pass 0 for |flags| field in encode call below since they are
818 : // set above in |vpx_codec_control| function for each encoder/spatial layer.
819 0 : int error = vpx_codec_encode(&encoders_[0], &raw_images_[0], timestamp_,
820 0 : duration, 0, VPX_DL_REALTIME);
821 : // Reset specific intra frame thresholds, following the key frame.
822 0 : if (send_key_frame) {
823 0 : vpx_codec_control(&(encoders_[0]), VP8E_SET_MAX_INTRA_BITRATE_PCT,
824 0 : rc_max_intra_target_);
825 : }
826 0 : if (error)
827 0 : return WEBRTC_VIDEO_CODEC_ERROR;
828 0 : timestamp_ += duration;
829 : // Examines frame timestamps only.
830 0 : return GetEncodedPartitions(frame, only_predict_from_key_frame);
831 : }
832 :
833 : // TODO(pbos): Make sure this works for properly for >1 encoders.
834 0 : int VP8EncoderImpl::UpdateCodecFrameSize(int width, int height) {
835 0 : codec_.width = width;
836 0 : codec_.height = height;
837 0 : if (codec_.numberOfSimulcastStreams <= 1) {
838 : // For now scaling is only used for single-layer streams.
839 0 : codec_.simulcastStream[0].width = width;
840 0 : codec_.simulcastStream[0].height = height;
841 : }
842 : // Update the cpu_speed setting for resolution change.
843 0 : vpx_codec_control(&(encoders_[0]), VP8E_SET_CPUUSED,
844 0 : SetCpuSpeed(codec_.width, codec_.height));
845 0 : raw_images_[0].w = codec_.width;
846 0 : raw_images_[0].h = codec_.height;
847 0 : raw_images_[0].d_w = codec_.width;
848 0 : raw_images_[0].d_h = codec_.height;
849 0 : vpx_img_set_rect(&raw_images_[0], 0, 0, codec_.width, codec_.height);
850 :
851 : // Update encoder context for new frame size.
852 : // Change of frame size will automatically trigger a key frame.
853 0 : configurations_[0].g_w = codec_.width;
854 0 : configurations_[0].g_h = codec_.height;
855 0 : if (vpx_codec_enc_config_set(&encoders_[0], &configurations_[0])) {
856 0 : return WEBRTC_VIDEO_CODEC_ERROR;
857 : }
858 0 : return WEBRTC_VIDEO_CODEC_OK;
859 : }
860 :
861 0 : void VP8EncoderImpl::PopulateCodecSpecific(
862 : CodecSpecificInfo* codec_specific,
863 : const vpx_codec_cx_pkt_t& pkt,
864 : int stream_idx,
865 : uint32_t timestamp,
866 : bool only_predicting_from_key_frame) {
867 0 : assert(codec_specific != NULL);
868 0 : codec_specific->codecType = kVideoCodecVP8;
869 0 : codec_specific->codec_name = ImplementationName();
870 0 : CodecSpecificInfoVP8* vp8Info = &(codec_specific->codecSpecific.VP8);
871 0 : vp8Info->pictureId = picture_id_[stream_idx];
872 0 : if (pkt.data.frame.flags & VPX_FRAME_IS_KEY) {
873 0 : last_key_frame_picture_id_[stream_idx] = picture_id_[stream_idx];
874 : }
875 0 : vp8Info->simulcastIdx = stream_idx;
876 0 : vp8Info->keyIdx = kNoKeyIdx; // TODO(hlundin) populate this
877 0 : vp8Info->nonReference =
878 0 : (pkt.data.frame.flags & VPX_FRAME_IS_DROPPABLE) ? true : false;
879 0 : bool base_layer_sync_point = (pkt.data.frame.flags & VPX_FRAME_IS_KEY) ||
880 0 : only_predicting_from_key_frame;
881 0 : temporal_layers_[stream_idx]->PopulateCodecSpecific(base_layer_sync_point,
882 0 : vp8Info, timestamp);
883 : // Prepare next.
884 0 : picture_id_[stream_idx] = (picture_id_[stream_idx] + 1) & 0x7FFF;
885 0 : }
886 :
887 0 : int VP8EncoderImpl::GetEncodedPartitions(const VideoFrame& input_image,
888 : bool only_predicting_from_key_frame) {
889 : int bw_resolutions_disabled =
890 0 : (encoders_.size() > 1) ? NumStreamsDisabled(send_stream_) : -1;
891 :
892 0 : int stream_idx = static_cast<int>(encoders_.size()) - 1;
893 0 : int result = WEBRTC_VIDEO_CODEC_OK;
894 0 : for (size_t encoder_idx = 0; encoder_idx < encoders_.size();
895 : ++encoder_idx, --stream_idx) {
896 0 : vpx_codec_iter_t iter = NULL;
897 0 : int part_idx = 0;
898 0 : encoded_images_[encoder_idx]._length = 0;
899 0 : encoded_images_[encoder_idx]._frameType = kVideoFrameDelta;
900 0 : RTPFragmentationHeader frag_info;
901 : // token_partitions_ is number of bits used.
902 0 : frag_info.VerifyAndAllocateFragmentationHeader((1 << token_partitions_) +
903 0 : 1);
904 0 : CodecSpecificInfo codec_specific;
905 0 : const vpx_codec_cx_pkt_t* pkt = NULL;
906 0 : while ((pkt = vpx_codec_get_cx_data(&encoders_[encoder_idx], &iter)) !=
907 : NULL) {
908 0 : switch (pkt->kind) {
909 : case VPX_CODEC_CX_FRAME_PKT: {
910 0 : size_t length = encoded_images_[encoder_idx]._length;
911 0 : if (pkt->data.frame.sz + length >
912 0 : encoded_images_[encoder_idx]._size) {
913 0 : uint8_t* buffer = new uint8_t[pkt->data.frame.sz + length];
914 0 : memcpy(buffer, encoded_images_[encoder_idx]._buffer, length);
915 0 : delete[] encoded_images_[encoder_idx]._buffer;
916 0 : encoded_images_[encoder_idx]._buffer = buffer;
917 0 : encoded_images_[encoder_idx]._size = pkt->data.frame.sz + length;
918 : }
919 0 : memcpy(&encoded_images_[encoder_idx]._buffer[length],
920 0 : pkt->data.frame.buf, pkt->data.frame.sz);
921 0 : frag_info.fragmentationOffset[part_idx] = length;
922 0 : frag_info.fragmentationLength[part_idx] = pkt->data.frame.sz;
923 0 : frag_info.fragmentationPlType[part_idx] = 0; // not known here
924 0 : frag_info.fragmentationTimeDiff[part_idx] = 0;
925 0 : encoded_images_[encoder_idx]._length += pkt->data.frame.sz;
926 0 : assert(length <= encoded_images_[encoder_idx]._size);
927 0 : ++part_idx;
928 0 : break;
929 : }
930 : default:
931 0 : break;
932 : }
933 : // End of frame
934 0 : if ((pkt->data.frame.flags & VPX_FRAME_IS_FRAGMENT) == 0) {
935 : // check if encoded frame is a key frame
936 0 : if (pkt->data.frame.flags & VPX_FRAME_IS_KEY) {
937 0 : encoded_images_[encoder_idx]._frameType = kVideoFrameKey;
938 0 : rps_.EncodedKeyFrame(picture_id_[stream_idx]);
939 : }
940 0 : PopulateCodecSpecific(&codec_specific, *pkt, stream_idx,
941 : input_image.timestamp(),
942 0 : only_predicting_from_key_frame);
943 0 : break;
944 : }
945 : }
946 0 : encoded_images_[encoder_idx]._timeStamp = input_image.timestamp();
947 0 : encoded_images_[encoder_idx].capture_time_ms_ =
948 0 : input_image.render_time_ms();
949 0 : encoded_images_[encoder_idx].rotation_ = input_image.rotation();
950 :
951 0 : int qp = -1;
952 0 : vpx_codec_control(&encoders_[encoder_idx], VP8E_GET_LAST_QUANTIZER_64, &qp);
953 0 : temporal_layers_[stream_idx]->FrameEncoded(
954 0 : encoded_images_[encoder_idx]._length,
955 0 : encoded_images_[encoder_idx]._timeStamp, qp);
956 0 : if (send_stream_[stream_idx]) {
957 0 : if (encoded_images_[encoder_idx]._length > 0) {
958 0 : TRACE_COUNTER_ID1("webrtc", "EncodedFrameSize", encoder_idx,
959 : encoded_images_[encoder_idx]._length);
960 0 : encoded_images_[encoder_idx]._encodedHeight =
961 0 : codec_.simulcastStream[stream_idx].height;
962 0 : encoded_images_[encoder_idx]._encodedWidth =
963 0 : codec_.simulcastStream[stream_idx].width;
964 : // Report once per frame (lowest stream always sent).
965 0 : encoded_images_[encoder_idx].adapt_reason_.bw_resolutions_disabled =
966 0 : (stream_idx == 0) ? bw_resolutions_disabled : -1;
967 0 : int qp_128 = -1;
968 0 : vpx_codec_control(&encoders_[encoder_idx], VP8E_GET_LAST_QUANTIZER,
969 0 : &qp_128);
970 0 : encoded_images_[encoder_idx].qp_ = qp_128;
971 0 : encoded_complete_callback_->OnEncodedImage(encoded_images_[encoder_idx],
972 0 : &codec_specific, &frag_info);
973 0 : } else if (codec_.mode == kScreensharing) {
974 0 : result = WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT;
975 : }
976 : }
977 : }
978 0 : return result;
979 : }
980 :
981 0 : VideoEncoder::ScalingSettings VP8EncoderImpl::GetScalingSettings() const {
982 0 : const bool enable_scaling = encoders_.size() == 1 &&
983 0 : configurations_[0].rc_dropframe_thresh > 0 &&
984 0 : codec_.VP8().automaticResizeOn;
985 0 : return VideoEncoder::ScalingSettings(enable_scaling);
986 : }
987 :
988 0 : int VP8EncoderImpl::SetChannelParameters(uint32_t packetLoss, int64_t rtt) {
989 0 : rps_.SetRtt(rtt);
990 0 : return WEBRTC_VIDEO_CODEC_OK;
991 : }
992 :
993 0 : int VP8EncoderImpl::RegisterEncodeCompleteCallback(
994 : EncodedImageCallback* callback) {
995 0 : encoded_complete_callback_ = callback;
996 0 : return WEBRTC_VIDEO_CODEC_OK;
997 : }
998 :
999 0 : VP8DecoderImpl::VP8DecoderImpl()
1000 : : buffer_pool_(false, 300 /* max_number_of_buffers*/),
1001 : decode_complete_callback_(NULL),
1002 : inited_(false),
1003 : feedback_mode_(false),
1004 : decoder_(NULL),
1005 : image_format_(VPX_IMG_FMT_NONE),
1006 : ref_frame_(NULL),
1007 : propagation_cnt_(-1),
1008 : last_frame_width_(0),
1009 : last_frame_height_(0),
1010 0 : key_frame_required_(true) {}
1011 :
1012 0 : VP8DecoderImpl::~VP8DecoderImpl() {
1013 0 : inited_ = true; // in order to do the actual release
1014 0 : Release();
1015 0 : }
1016 :
1017 0 : int VP8DecoderImpl::InitDecode(const VideoCodec* inst, int number_of_cores) {
1018 0 : int ret_val = Release();
1019 0 : if (ret_val < 0) {
1020 0 : return ret_val;
1021 : }
1022 0 : if (decoder_ == NULL) {
1023 0 : decoder_ = new vpx_codec_ctx_t;
1024 0 : memset(decoder_, 0, sizeof(*decoder_));
1025 : }
1026 0 : if (inst && inst->codecType == kVideoCodecVP8) {
1027 0 : feedback_mode_ = inst->VP8().feedbackModeOn;
1028 : }
1029 : vpx_codec_dec_cfg_t cfg;
1030 : // Setting number of threads to a constant value (1)
1031 0 : cfg.threads = 1;
1032 0 : cfg.h = cfg.w = 0; // set after decode
1033 :
1034 0 : vpx_codec_flags_t flags = 0;
1035 : #if !defined(WEBRTC_ARCH_ARM) && !defined(WEBRTC_ARCH_ARM64) && \
1036 : !defined(ANDROID)
1037 0 : flags = VPX_CODEC_USE_POSTPROC;
1038 : #endif
1039 :
1040 0 : if (vpx_codec_dec_init(decoder_, vpx_codec_vp8_dx(), &cfg, flags)) {
1041 0 : delete decoder_;
1042 0 : decoder_ = nullptr;
1043 0 : return WEBRTC_VIDEO_CODEC_MEMORY;
1044 : }
1045 :
1046 : // Save VideoCodec instance for later; mainly for duplicating the decoder.
1047 0 : if (&codec_ != inst)
1048 0 : codec_ = *inst;
1049 0 : propagation_cnt_ = -1;
1050 :
1051 0 : inited_ = true;
1052 :
1053 : // Always start with a complete key frame.
1054 0 : key_frame_required_ = true;
1055 0 : return WEBRTC_VIDEO_CODEC_OK;
1056 : }
1057 :
1058 0 : int VP8DecoderImpl::Decode(const EncodedImage& input_image,
1059 : bool missing_frames,
1060 : const RTPFragmentationHeader* fragmentation,
1061 : const CodecSpecificInfo* codec_specific_info,
1062 : int64_t /*render_time_ms*/) {
1063 0 : if (!inited_) {
1064 0 : return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
1065 : }
1066 0 : if (decode_complete_callback_ == NULL) {
1067 0 : return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
1068 : }
1069 0 : if (input_image._buffer == NULL && input_image._length > 0) {
1070 : // Reset to avoid requesting key frames too often.
1071 0 : if (propagation_cnt_ > 0)
1072 0 : propagation_cnt_ = 0;
1073 0 : return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
1074 : }
1075 :
1076 : #if !defined(WEBRTC_ARCH_ARM) && !defined(WEBRTC_ARCH_ARM64) && \
1077 : !defined(ANDROID)
1078 : vp8_postproc_cfg_t ppcfg;
1079 : // MFQE enabled to reduce key frame popping.
1080 0 : ppcfg.post_proc_flag = VP8_MFQE | VP8_DEBLOCK;
1081 : // For VGA resolutions and lower, enable the demacroblocker postproc.
1082 0 : if (last_frame_width_ * last_frame_height_ <= 640 * 360) {
1083 0 : ppcfg.post_proc_flag |= VP8_DEMACROBLOCK;
1084 : }
1085 : // Strength of deblocking filter. Valid range:[0,16]
1086 0 : ppcfg.deblocking_level = 3;
1087 0 : vpx_codec_control(decoder_, VP8_SET_POSTPROC, &ppcfg);
1088 : #endif
1089 :
1090 : // Always start with a complete key frame.
1091 0 : if (key_frame_required_) {
1092 0 : if (input_image._frameType != kVideoFrameKey)
1093 0 : return WEBRTC_VIDEO_CODEC_ERROR;
1094 : // We have a key frame - is it complete?
1095 0 : if (input_image._completeFrame) {
1096 0 : key_frame_required_ = false;
1097 : } else {
1098 0 : return WEBRTC_VIDEO_CODEC_ERROR;
1099 : }
1100 : }
1101 : // Restrict error propagation using key frame requests. Disabled when
1102 : // the feedback mode is enabled (RPS).
1103 : // Reset on a key frame refresh.
1104 0 : if (!feedback_mode_) {
1105 0 : if (input_image._frameType == kVideoFrameKey &&
1106 0 : input_image._completeFrame) {
1107 0 : propagation_cnt_ = -1;
1108 : // Start count on first loss.
1109 0 : } else if ((!input_image._completeFrame || missing_frames) &&
1110 0 : propagation_cnt_ == -1) {
1111 0 : propagation_cnt_ = 0;
1112 : }
1113 0 : if (propagation_cnt_ >= 0) {
1114 0 : propagation_cnt_++;
1115 : }
1116 : }
1117 :
1118 0 : vpx_codec_iter_t iter = NULL;
1119 : vpx_image_t* img;
1120 : int ret;
1121 :
1122 : // Check for missing frames.
1123 0 : if (missing_frames) {
1124 : // Call decoder with zero data length to signal missing frames.
1125 0 : if (vpx_codec_decode(decoder_, NULL, 0, 0, VPX_DL_REALTIME)) {
1126 : // Reset to avoid requesting key frames too often.
1127 0 : if (propagation_cnt_ > 0)
1128 0 : propagation_cnt_ = 0;
1129 0 : return WEBRTC_VIDEO_CODEC_ERROR;
1130 : }
1131 0 : img = vpx_codec_get_frame(decoder_, &iter);
1132 0 : iter = NULL;
1133 : }
1134 :
1135 0 : uint8_t* buffer = input_image._buffer;
1136 0 : if (input_image._length == 0) {
1137 0 : buffer = NULL; // Triggers full frame concealment.
1138 : }
1139 0 : if (vpx_codec_decode(decoder_, buffer, input_image._length, 0,
1140 : VPX_DL_REALTIME)) {
1141 : // Reset to avoid requesting key frames too often.
1142 0 : if (propagation_cnt_ > 0) {
1143 0 : propagation_cnt_ = 0;
1144 : }
1145 0 : return WEBRTC_VIDEO_CODEC_ERROR;
1146 : }
1147 :
1148 0 : img = vpx_codec_get_frame(decoder_, &iter);
1149 0 : ret = ReturnFrame(img, input_image._timeStamp, input_image.ntp_time_ms_);
1150 0 : if (ret != 0) {
1151 : // Reset to avoid requesting key frames too often.
1152 0 : if (ret < 0 && propagation_cnt_ > 0)
1153 0 : propagation_cnt_ = 0;
1154 0 : return ret;
1155 : }
1156 0 : if (feedback_mode_) {
1157 : // Whenever we receive an incomplete key frame all reference buffers will
1158 : // be corrupt. If that happens we must request new key frames until we
1159 : // decode a complete key frame.
1160 0 : if (input_image._frameType == kVideoFrameKey && !input_image._completeFrame)
1161 0 : return WEBRTC_VIDEO_CODEC_ERROR;
1162 : // Check for reference updates and last reference buffer corruption and
1163 : // signal successful reference propagation or frame corruption to the
1164 : // encoder.
1165 0 : int reference_updates = 0;
1166 0 : if (vpx_codec_control(decoder_, VP8D_GET_LAST_REF_UPDATES,
1167 : &reference_updates)) {
1168 : // Reset to avoid requesting key frames too often.
1169 0 : if (propagation_cnt_ > 0) {
1170 0 : propagation_cnt_ = 0;
1171 : }
1172 0 : return WEBRTC_VIDEO_CODEC_ERROR;
1173 : }
1174 0 : int corrupted = 0;
1175 0 : if (vpx_codec_control(decoder_, VP8D_GET_FRAME_CORRUPTED, &corrupted)) {
1176 : // Reset to avoid requesting key frames too often.
1177 0 : if (propagation_cnt_ > 0)
1178 0 : propagation_cnt_ = 0;
1179 0 : return WEBRTC_VIDEO_CODEC_ERROR;
1180 : }
1181 0 : int16_t picture_id = -1;
1182 0 : if (codec_specific_info) {
1183 0 : picture_id = codec_specific_info->codecSpecific.VP8.pictureId;
1184 : }
1185 0 : if (picture_id > -1) {
1186 0 : if (((reference_updates & VP8_GOLD_FRAME) ||
1187 0 : (reference_updates & VP8_ALTR_FRAME)) &&
1188 0 : !corrupted) {
1189 0 : decode_complete_callback_->ReceivedDecodedReferenceFrame(picture_id);
1190 : }
1191 0 : decode_complete_callback_->ReceivedDecodedFrame(picture_id);
1192 : }
1193 0 : if (corrupted) {
1194 : // we can decode but with artifacts
1195 0 : return WEBRTC_VIDEO_CODEC_REQUEST_SLI;
1196 : }
1197 : }
1198 : // Check Vs. threshold
1199 0 : if (propagation_cnt_ > kVp8ErrorPropagationTh) {
1200 : // Reset to avoid requesting key frames too often.
1201 0 : propagation_cnt_ = 0;
1202 0 : return WEBRTC_VIDEO_CODEC_ERROR;
1203 : }
1204 0 : return WEBRTC_VIDEO_CODEC_OK;
1205 : }
1206 :
1207 0 : int VP8DecoderImpl::ReturnFrame(const vpx_image_t* img,
1208 : uint32_t timestamp,
1209 : int64_t ntp_time_ms) {
1210 0 : if (img == NULL) {
1211 : // Decoder OK and NULL image => No show frame
1212 0 : return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
1213 : }
1214 0 : last_frame_width_ = img->d_w;
1215 0 : last_frame_height_ = img->d_h;
1216 : // Allocate memory for decoded image.
1217 : rtc::scoped_refptr<I420Buffer> buffer =
1218 0 : buffer_pool_.CreateBuffer(img->d_w, img->d_h);
1219 0 : if (!buffer.get()) {
1220 : // Pool has too many pending frames.
1221 0 : RTC_HISTOGRAM_BOOLEAN("WebRTC.Video.VP8DecoderImpl.TooManyPendingFrames",
1222 : 1);
1223 0 : return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
1224 : }
1225 :
1226 0 : libyuv::I420Copy(img->planes[VPX_PLANE_Y], img->stride[VPX_PLANE_Y],
1227 0 : img->planes[VPX_PLANE_U], img->stride[VPX_PLANE_U],
1228 0 : img->planes[VPX_PLANE_V], img->stride[VPX_PLANE_V],
1229 0 : buffer->MutableDataY(), buffer->StrideY(),
1230 0 : buffer->MutableDataU(), buffer->StrideU(),
1231 0 : buffer->MutableDataV(), buffer->StrideV(),
1232 0 : img->d_w, img->d_h);
1233 :
1234 0 : VideoFrame decoded_image(buffer, timestamp, 0, kVideoRotation_0);
1235 0 : decoded_image.set_ntp_time_ms(ntp_time_ms);
1236 0 : int ret = decode_complete_callback_->Decoded(decoded_image);
1237 0 : if (ret != 0)
1238 0 : return ret;
1239 :
1240 : // Remember image format for later
1241 0 : image_format_ = img->fmt;
1242 0 : return WEBRTC_VIDEO_CODEC_OK;
1243 : }
1244 :
1245 0 : int VP8DecoderImpl::RegisterDecodeCompleteCallback(
1246 : DecodedImageCallback* callback) {
1247 0 : decode_complete_callback_ = callback;
1248 0 : return WEBRTC_VIDEO_CODEC_OK;
1249 : }
1250 :
1251 0 : int VP8DecoderImpl::Release() {
1252 0 : if (decoder_ != NULL) {
1253 0 : if (vpx_codec_destroy(decoder_)) {
1254 0 : return WEBRTC_VIDEO_CODEC_MEMORY;
1255 : }
1256 0 : delete decoder_;
1257 0 : decoder_ = NULL;
1258 : }
1259 0 : if (ref_frame_ != NULL) {
1260 0 : vpx_img_free(&ref_frame_->img);
1261 0 : delete ref_frame_;
1262 0 : ref_frame_ = NULL;
1263 : }
1264 0 : buffer_pool_.Release();
1265 0 : inited_ = false;
1266 0 : return WEBRTC_VIDEO_CODEC_OK;
1267 : }
1268 :
1269 0 : const char* VP8DecoderImpl::ImplementationName() const {
1270 0 : return "libvpx";
1271 : }
1272 :
1273 0 : int VP8DecoderImpl::CopyReference(VP8DecoderImpl* copy) {
1274 : // The type of frame to copy should be set in ref_frame_->frame_type
1275 : // before the call to this function.
1276 0 : if (vpx_codec_control(decoder_, VP8_COPY_REFERENCE, ref_frame_) !=
1277 : VPX_CODEC_OK) {
1278 0 : return -1;
1279 : }
1280 0 : if (vpx_codec_control(copy->decoder_, VP8_SET_REFERENCE, ref_frame_) !=
1281 : VPX_CODEC_OK) {
1282 0 : return -1;
1283 : }
1284 0 : return 0;
1285 : }
1286 :
1287 : } // namespace webrtc
|
