Line data Source code
1 : /*
2 : * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 :
11 : #include "webrtc/video/vie_encoder.h"
12 :
13 : #include <algorithm>
14 : #include <limits>
15 : #include <utility>
16 :
17 : #include "webrtc/modules/video_coding/include/video_codec_initializer.h"
18 : #include "webrtc/base/arraysize.h"
19 : #include "webrtc/base/checks.h"
20 : #include "webrtc/base/logging.h"
21 : #include "webrtc/base/trace_event.h"
22 : #include "webrtc/base/timeutils.h"
23 : #include "webrtc/common_video/include/video_bitrate_allocator.h"
24 : #include "webrtc/modules/pacing/paced_sender.h"
25 : #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h"
26 : #include "webrtc/modules/video_coding/include/video_coding.h"
27 : #include "webrtc/modules/video_coding/include/video_coding_defines.h"
28 : #include "webrtc/video/overuse_frame_detector.h"
29 : #include "webrtc/video/send_statistics_proxy.h"
30 : #include "webrtc/video_frame.h"
31 : namespace webrtc {
32 :
33 : namespace {
34 : // Time interval for logging frame counts.
35 : const int64_t kFrameLogIntervalMs = 60000;
36 : // We will never ask for a resolution lower than this.
37 : #if defined(WEBRTC_ANDROID)
38 : // TODO(kthelgason): Lower this limit when better testing
39 : // on MediaCodec and fallback implementations are in place.
40 : const int kMinPixelsPerFrame = 320 * 180;
41 : #else
42 : const int kMinPixelsPerFrame = 120 * 90;
43 : #endif
44 :
45 : // TODO(pbos): Lower these thresholds (to closer to 100%) when we handle
46 : // pipelining encoders better (multiple input frames before something comes
47 : // out). This should effectively turn off CPU adaptations for systems that
48 : // remotely cope with the load right now.
49 0 : CpuOveruseOptions GetCpuOveruseOptions(bool full_overuse_time) {
50 0 : CpuOveruseOptions options;
51 0 : if (full_overuse_time) {
52 0 : options.low_encode_usage_threshold_percent = 150;
53 0 : options.high_encode_usage_threshold_percent = 200;
54 : }
55 0 : return options;
56 : }
57 :
58 : } // namespace
59 :
60 0 : class ViEEncoder::ConfigureEncoderTask : public rtc::QueuedTask {
61 : public:
62 0 : ConfigureEncoderTask(ViEEncoder* vie_encoder,
63 : VideoEncoderConfig config,
64 : size_t max_data_payload_length,
65 : bool nack_enabled)
66 0 : : vie_encoder_(vie_encoder),
67 0 : config_(std::move(config)),
68 : max_data_payload_length_(max_data_payload_length),
69 0 : nack_enabled_(nack_enabled) {}
70 :
71 : private:
72 0 : bool Run() override {
73 0 : vie_encoder_->ConfigureEncoderOnTaskQueue(
74 0 : std::move(config_), max_data_payload_length_, nack_enabled_);
75 0 : return true;
76 : }
77 :
78 : ViEEncoder* const vie_encoder_;
79 : VideoEncoderConfig config_;
80 : size_t max_data_payload_length_;
81 : bool nack_enabled_;
82 : };
83 :
84 0 : class ViEEncoder::EncodeTask : public rtc::QueuedTask {
85 : public:
86 0 : EncodeTask(const VideoFrame& frame,
87 : ViEEncoder* vie_encoder,
88 : int64_t time_when_posted_in_ms,
89 : bool log_stats)
90 0 : : frame_(frame),
91 : vie_encoder_(vie_encoder),
92 : time_when_posted_ms_(time_when_posted_in_ms),
93 0 : log_stats_(log_stats) {
94 0 : ++vie_encoder_->posted_frames_waiting_for_encode_;
95 0 : }
96 :
97 : private:
98 0 : bool Run() override {
99 0 : RTC_DCHECK_RUN_ON(&vie_encoder_->encoder_queue_);
100 0 : RTC_DCHECK_GT(vie_encoder_->posted_frames_waiting_for_encode_.Value(), 0);
101 0 : vie_encoder_->stats_proxy_->OnIncomingFrame(frame_.width(),
102 0 : frame_.height());
103 0 : ++vie_encoder_->captured_frame_count_;
104 0 : if (--vie_encoder_->posted_frames_waiting_for_encode_ == 0) {
105 0 : vie_encoder_->EncodeVideoFrame(frame_, time_when_posted_ms_);
106 : } else {
107 : // There is a newer frame in flight. Do not encode this frame.
108 0 : LOG(LS_VERBOSE)
109 0 : << "Incoming frame dropped due to that the encoder is blocked (captured="
110 0 : << vie_encoder_->captured_frame_count_ << " dropped="
111 0 : << vie_encoder_->dropped_frame_count_ << " queued="
112 0 : << vie_encoder_->posted_frames_waiting_for_encode_.Value();
113 0 : ++vie_encoder_->dropped_frame_count_;
114 : }
115 0 : if (log_stats_) {
116 0 : LOG(LS_INFO) << "Number of frames: captured "
117 0 : << vie_encoder_->captured_frame_count_
118 0 : << ", dropped (due to encoder blocked) "
119 0 : << vie_encoder_->dropped_frame_count_ << ", interval_ms "
120 0 : << kFrameLogIntervalMs;
121 0 : vie_encoder_->captured_frame_count_ = 0;
122 0 : vie_encoder_->dropped_frame_count_ = 0;
123 : }
124 0 : return true;
125 : }
126 : VideoFrame frame_;
127 : ViEEncoder* const vie_encoder_;
128 : const int64_t time_when_posted_ms_;
129 : const bool log_stats_;
130 : };
131 :
132 : // VideoSourceProxy is responsible ensuring thread safety between calls to
133 : // ViEEncoder::SetSource that will happen on libjingle's worker thread when a
134 : // video capturer is connected to the encoder and the encoder task queue
135 : // (encoder_queue_) where the encoder reports its VideoSinkWants.
136 0 : class ViEEncoder::VideoSourceProxy {
137 : public:
138 0 : explicit VideoSourceProxy(ViEEncoder* vie_encoder)
139 0 : : vie_encoder_(vie_encoder),
140 : degradation_preference_(
141 : VideoSendStream::DegradationPreference::kMaintainResolution),
142 0 : source_(nullptr) {}
143 :
144 0 : void SetSource(
145 : rtc::VideoSourceInterface<VideoFrame>* source,
146 : const VideoSendStream::DegradationPreference& degradation_preference) {
147 : // Called on libjingle's worker thread.
148 0 : RTC_DCHECK_CALLED_SEQUENTIALLY(&main_checker_);
149 0 : rtc::VideoSourceInterface<VideoFrame>* old_source = nullptr;
150 0 : rtc::VideoSinkWants wants;
151 : {
152 0 : rtc::CritScope lock(&crit_);
153 0 : old_source = source_;
154 0 : source_ = source;
155 0 : degradation_preference_ = degradation_preference;
156 0 : wants = current_wants();
157 : }
158 :
159 0 : if (old_source != source && old_source != nullptr) {
160 0 : old_source->RemoveSink(vie_encoder_);
161 : }
162 :
163 0 : if (!source) {
164 0 : return;
165 : }
166 :
167 0 : source->AddOrUpdateSink(vie_encoder_, wants);
168 : }
169 :
170 0 : void SetWantsRotationApplied(bool rotation_applied) {
171 0 : rtc::CritScope lock(&crit_);
172 0 : sink_wants_.rotation_applied = rotation_applied;
173 0 : disabled_scaling_sink_wants_.rotation_applied = rotation_applied;
174 0 : if (source_) {
175 0 : source_->AddOrUpdateSink(vie_encoder_, current_wants());
176 : }
177 0 : }
178 :
179 0 : void RequestResolutionLowerThan(int pixel_count) {
180 : // Called on the encoder task queue.
181 0 : rtc::CritScope lock(&crit_);
182 0 : if (!IsResolutionScalingEnabledLocked()) {
183 : // This can happen since |degradation_preference_| is set on
184 : // libjingle's worker thread but the adaptation is done on the encoder
185 : // task queue.
186 0 : return;
187 : }
188 : // The input video frame size will have a resolution with less than or
189 : // equal to |max_pixel_count| depending on how the source can scale the
190 : // input frame size.
191 0 : const int pixels_wanted = (pixel_count * 3) / 5;
192 0 : if (pixels_wanted < kMinPixelsPerFrame)
193 0 : return;
194 0 : sink_wants_.max_pixel_count = rtc::Optional<int>(pixels_wanted);
195 0 : sink_wants_.max_pixel_count_step_up = rtc::Optional<int>();
196 0 : if (source_)
197 0 : source_->AddOrUpdateSink(vie_encoder_, sink_wants_);
198 : }
199 :
200 0 : void RequestHigherResolutionThan(int pixel_count) {
201 0 : rtc::CritScope lock(&crit_);
202 0 : if (!IsResolutionScalingEnabledLocked()) {
203 : // This can happen since |degradation_preference_| is set on
204 : // libjingle's worker thread but the adaptation is done on the encoder
205 : // task
206 : // queue.
207 0 : return;
208 : }
209 : // The input video frame size will have a resolution with "one step up"
210 : // pixels than |max_pixel_count_step_up| where "one step up" depends on
211 : // how the source can scale the input frame size.
212 0 : sink_wants_.max_pixel_count = rtc::Optional<int>();
213 0 : sink_wants_.max_pixel_count_step_up = rtc::Optional<int>(pixel_count);
214 0 : if (source_)
215 0 : source_->AddOrUpdateSink(vie_encoder_, sink_wants_);
216 : }
217 :
218 : private:
219 0 : bool IsResolutionScalingEnabledLocked() const
220 : EXCLUSIVE_LOCKS_REQUIRED(&crit_) {
221 0 : return degradation_preference_ !=
222 0 : VideoSendStream::DegradationPreference::kMaintainResolution;
223 : }
224 :
225 0 : const rtc::VideoSinkWants& current_wants() const
226 : EXCLUSIVE_LOCKS_REQUIRED(&crit_) {
227 0 : return IsResolutionScalingEnabledLocked() ? sink_wants_
228 0 : : disabled_scaling_sink_wants_;
229 : }
230 :
231 : rtc::CriticalSection crit_;
232 : rtc::SequencedTaskChecker main_checker_;
233 : ViEEncoder* const vie_encoder_;
234 : rtc::VideoSinkWants sink_wants_ GUARDED_BY(&crit_);
235 : rtc::VideoSinkWants disabled_scaling_sink_wants_ GUARDED_BY(&crit_);
236 : VideoSendStream::DegradationPreference degradation_preference_
237 : GUARDED_BY(&crit_);
238 : rtc::VideoSourceInterface<VideoFrame>* source_ GUARDED_BY(&crit_);
239 :
240 : RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceProxy);
241 : };
242 :
243 0 : ViEEncoder::ViEEncoder(uint32_t number_of_cores,
244 : SendStatisticsProxy* stats_proxy,
245 : const VideoSendStream::Config::EncoderSettings& settings,
246 : rtc::VideoSinkInterface<VideoFrame>* pre_encode_callback,
247 0 : EncodedFrameObserver* encoder_timing)
248 : : shutdown_event_(true /* manual_reset */, false),
249 : number_of_cores_(number_of_cores),
250 0 : source_proxy_(new VideoSourceProxy(this)),
251 : sink_(nullptr),
252 : settings_(settings),
253 0 : codec_type_(PayloadNameToCodecType(settings.payload_name)
254 0 : .value_or(VideoCodecType::kVideoCodecUnknown)),
255 : video_sender_(Clock::GetRealTimeClock(), this, this),
256 : overuse_detector_(Clock::GetRealTimeClock(),
257 0 : GetCpuOveruseOptions(settings.full_overuse_time),
258 : this,
259 : encoder_timing,
260 : stats_proxy),
261 : stats_proxy_(stats_proxy),
262 : pre_encode_callback_(pre_encode_callback),
263 : module_process_thread_(nullptr),
264 : pending_encoder_reconfiguration_(false),
265 : encoder_start_bitrate_bps_(0),
266 : max_data_payload_length_(0),
267 : nack_enabled_(false),
268 : last_observed_bitrate_bps_(0),
269 : encoder_paused_and_dropped_frame_(false),
270 : has_received_sli_(false),
271 : picture_id_sli_(0),
272 : has_received_rpsi_(false),
273 : picture_id_rpsi_(0),
274 0 : clock_(Clock::GetRealTimeClock()),
275 : scale_counter_(kScaleReasonSize, 0),
276 : last_captured_timestamp_(0),
277 0 : delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() -
278 0 : clock_->TimeInMilliseconds()),
279 0 : last_frame_log_ms_(clock_->TimeInMilliseconds()),
280 : captured_frame_count_(0),
281 : dropped_frame_count_(0),
282 : bitrate_observer_(nullptr),
283 0 : encoder_queue_("EncoderQueue") {
284 0 : encoder_queue_.PostTask([this] {
285 0 : RTC_DCHECK_RUN_ON(&encoder_queue_);
286 0 : overuse_detector_.StartCheckForOveruse();
287 0 : video_sender_.RegisterExternalEncoder(
288 0 : settings_.encoder, settings_.payload_type, settings_.internal_source);
289 0 : });
290 0 : }
291 :
292 0 : ViEEncoder::~ViEEncoder() {
293 0 : RTC_DCHECK_RUN_ON(&thread_checker_);
294 0 : RTC_DCHECK(shutdown_event_.Wait(0))
295 0 : << "Must call ::Stop() before destruction.";
296 0 : }
297 :
298 0 : void ViEEncoder::Stop() {
299 0 : RTC_DCHECK_RUN_ON(&thread_checker_);
300 0 : source_proxy_->SetSource(nullptr, VideoSendStream::DegradationPreference());
301 0 : encoder_queue_.PostTask([this] {
302 0 : RTC_DCHECK_RUN_ON(&encoder_queue_);
303 0 : overuse_detector_.StopCheckForOveruse();
304 0 : rate_allocator_.reset();
305 0 : bitrate_observer_ = nullptr;
306 0 : video_sender_.RegisterExternalEncoder(nullptr, settings_.payload_type,
307 0 : false);
308 0 : quality_scaler_ = nullptr;
309 0 : shutdown_event_.Set();
310 0 : });
311 :
312 0 : shutdown_event_.Wait(rtc::Event::kForever);
313 0 : }
314 :
315 0 : void ViEEncoder::RegisterProcessThread(ProcessThread* module_process_thread) {
316 0 : RTC_DCHECK_RUN_ON(&thread_checker_);
317 0 : RTC_DCHECK(!module_process_thread_);
318 0 : module_process_thread_ = module_process_thread;
319 0 : module_process_thread_->RegisterModule(&video_sender_);
320 0 : module_process_thread_checker_.DetachFromThread();
321 0 : }
322 :
323 0 : void ViEEncoder::DeRegisterProcessThread() {
324 0 : RTC_DCHECK_RUN_ON(&thread_checker_);
325 0 : module_process_thread_->DeRegisterModule(&video_sender_);
326 0 : }
327 :
328 0 : void ViEEncoder::SetBitrateObserver(
329 : VideoBitrateAllocationObserver* bitrate_observer) {
330 0 : RTC_DCHECK_RUN_ON(&thread_checker_);
331 0 : encoder_queue_.PostTask([this, bitrate_observer] {
332 0 : RTC_DCHECK_RUN_ON(&encoder_queue_);
333 0 : RTC_DCHECK(!bitrate_observer_);
334 0 : bitrate_observer_ = bitrate_observer;
335 0 : });
336 0 : }
337 :
338 0 : void ViEEncoder::SetSource(
339 : rtc::VideoSourceInterface<VideoFrame>* source,
340 : const VideoSendStream::DegradationPreference& degradation_preference) {
341 0 : RTC_DCHECK_RUN_ON(&thread_checker_);
342 0 : source_proxy_->SetSource(source, degradation_preference);
343 0 : encoder_queue_.PostTask([this, degradation_preference] {
344 0 : RTC_DCHECK_RUN_ON(&encoder_queue_);
345 0 : scaling_enabled_ = (degradation_preference !=
346 : VideoSendStream::DegradationPreference::kMaintainResolution);
347 0 : stats_proxy_->SetResolutionRestrictionStats(
348 0 : scaling_enabled_, scale_counter_[kCpu] > 0, scale_counter_[kQuality]);
349 0 : });
350 0 : }
351 :
352 0 : void ViEEncoder::SetSink(EncoderSink* sink, bool rotation_applied) {
353 0 : source_proxy_->SetWantsRotationApplied(rotation_applied);
354 0 : encoder_queue_.PostTask([this, sink] {
355 0 : RTC_DCHECK_RUN_ON(&encoder_queue_);
356 0 : sink_ = sink;
357 0 : });
358 0 : }
359 :
360 0 : void ViEEncoder::SetStartBitrate(int start_bitrate_bps) {
361 0 : encoder_queue_.PostTask([this, start_bitrate_bps] {
362 0 : RTC_DCHECK_RUN_ON(&encoder_queue_);
363 0 : encoder_start_bitrate_bps_ = start_bitrate_bps;
364 0 : });
365 0 : }
366 :
367 0 : void ViEEncoder::ConfigureEncoder(VideoEncoderConfig config,
368 : size_t max_data_payload_length,
369 : bool nack_enabled) {
370 0 : encoder_queue_.PostTask(
371 0 : std::unique_ptr<rtc::QueuedTask>(new ConfigureEncoderTask(
372 0 : this, std::move(config), max_data_payload_length, nack_enabled)));
373 0 : }
374 :
375 0 : void ViEEncoder::ConfigureEncoderOnTaskQueue(VideoEncoderConfig config,
376 : size_t max_data_payload_length,
377 : bool nack_enabled) {
378 0 : RTC_DCHECK_RUN_ON(&encoder_queue_);
379 0 : RTC_DCHECK(sink_);
380 0 : LOG(LS_INFO) << "ConfigureEncoder requested.";
381 :
382 0 : max_data_payload_length_ = max_data_payload_length;
383 0 : nack_enabled_ = nack_enabled;
384 0 : encoder_config_ = std::move(config);
385 0 : pending_encoder_reconfiguration_ = true;
386 :
387 : // Reconfigure the encoder now if the encoder has an internal source or
388 : // if the frame resolution is known. Otherwise, the reconfiguration is
389 : // deferred until the next frame to minimize the number of reconfigurations.
390 : // The codec configuration depends on incoming video frame size.
391 0 : if (last_frame_info_) {
392 0 : ReconfigureEncoder();
393 0 : } else if (settings_.internal_source) {
394 0 : last_frame_info_ = rtc::Optional<VideoFrameInfo>(
395 0 : VideoFrameInfo(176, 144, kVideoRotation_0, false));
396 0 : ReconfigureEncoder();
397 : }
398 0 : }
399 :
400 0 : void ViEEncoder::ReconfigureEncoder() {
401 0 : RTC_DCHECK_RUN_ON(&encoder_queue_);
402 0 : RTC_DCHECK(pending_encoder_reconfiguration_);
403 : std::vector<VideoStream> streams =
404 0 : encoder_config_.video_stream_factory->CreateEncoderStreams(
405 0 : last_frame_info_->width, last_frame_info_->height, encoder_config_);
406 :
407 0 : VideoCodec codec;
408 0 : if (!VideoCodecInitializer::SetupCodec(encoder_config_, settings_, streams,
409 0 : nack_enabled_, &codec,
410 : &rate_allocator_)) {
411 0 : LOG(LS_ERROR) << "Failed to create encoder configuration.";
412 : }
413 :
414 0 : codec.startBitrate =
415 0 : std::max(encoder_start_bitrate_bps_ / 1000, codec.minBitrate);
416 0 : codec.startBitrate = std::min(codec.startBitrate, codec.maxBitrate);
417 0 : codec.expect_encode_from_texture = last_frame_info_->is_texture;
418 :
419 0 : bool success = video_sender_.RegisterSendCodec(
420 0 : &codec, number_of_cores_,
421 0 : static_cast<uint32_t>(max_data_payload_length_)) == VCM_OK;
422 0 : if (!success) {
423 0 : LOG(LS_ERROR) << "Failed to configure encoder.";
424 0 : RTC_DCHECK(success);
425 : }
426 :
427 0 : video_sender_.UpdateChannelParemeters(rate_allocator_.get(),
428 0 : bitrate_observer_);
429 :
430 0 : if (stats_proxy_) {
431 0 : int framerate = stats_proxy_->GetSendFrameRate();
432 0 : if (framerate == 0)
433 0 : framerate = codec.maxFramerate;
434 0 : stats_proxy_->OnEncoderReconfigured(
435 0 : encoder_config_, rate_allocator_->GetPreferredBitrateBps(framerate));
436 : }
437 :
438 0 : pending_encoder_reconfiguration_ = false;
439 :
440 0 : sink_->OnEncoderConfigurationChanged(
441 0 : std::move(streams), encoder_config_.min_transmit_bitrate_bps);
442 :
443 0 : const auto scaling_settings = settings_.encoder->GetScalingSettings();
444 0 : if (scaling_enabled_ && scaling_settings.enabled) {
445 0 : if (scaling_settings.thresholds) {
446 0 : quality_scaler_.reset(
447 0 : new QualityScaler(this, *(scaling_settings.thresholds)));
448 : } else {
449 0 : quality_scaler_.reset(new QualityScaler(this, codec_type_));
450 : }
451 : } else {
452 0 : quality_scaler_.reset(nullptr);
453 0 : stats_proxy_->SetResolutionRestrictionStats(
454 0 : false, scale_counter_[kCpu] > 0, scale_counter_[kQuality]);
455 : }
456 0 : }
457 :
458 0 : void ViEEncoder::OnFrame(const VideoFrame& video_frame) {
459 0 : RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_);
460 0 : VideoFrame incoming_frame = video_frame;
461 :
462 : // Local time in webrtc time base.
463 0 : int64_t current_time = clock_->TimeInMilliseconds();
464 0 : incoming_frame.set_render_time_ms(current_time);
465 :
466 : // Capture time may come from clock with an offset and drift from clock_.
467 : int64_t capture_ntp_time_ms;
468 0 : if (video_frame.ntp_time_ms() > 0) {
469 0 : capture_ntp_time_ms = video_frame.ntp_time_ms();
470 0 : } else if (video_frame.render_time_ms() != 0) {
471 0 : capture_ntp_time_ms = video_frame.render_time_ms() + delta_ntp_internal_ms_;
472 : } else {
473 0 : capture_ntp_time_ms = current_time + delta_ntp_internal_ms_;
474 : }
475 0 : incoming_frame.set_ntp_time_ms(capture_ntp_time_ms);
476 :
477 : // Convert NTP time, in ms, to RTP timestamp.
478 0 : const int kMsToRtpTimestamp = 90;
479 0 : incoming_frame.set_timestamp(
480 0 : kMsToRtpTimestamp * static_cast<uint32_t>(incoming_frame.ntp_time_ms()));
481 :
482 0 : if (incoming_frame.ntp_time_ms() <= last_captured_timestamp_) {
483 : // We don't allow the same capture time for two frames, drop this one.
484 0 : LOG(LS_WARNING) << "Same/old NTP timestamp ("
485 0 : << incoming_frame.ntp_time_ms()
486 0 : << " <= " << last_captured_timestamp_
487 0 : << ") for incoming frame. Dropping.";
488 0 : return;
489 : }
490 :
491 0 : bool log_stats = false;
492 0 : if (current_time - last_frame_log_ms_ > kFrameLogIntervalMs) {
493 0 : last_frame_log_ms_ = current_time;
494 0 : log_stats = true;
495 : }
496 :
497 0 : last_captured_timestamp_ = incoming_frame.ntp_time_ms();
498 0 : encoder_queue_.PostTask(std::unique_ptr<rtc::QueuedTask>(new EncodeTask(
499 0 : incoming_frame, this, clock_->TimeInMilliseconds(), log_stats)));
500 : }
501 :
502 0 : bool ViEEncoder::EncoderPaused() const {
503 0 : RTC_DCHECK_RUN_ON(&encoder_queue_);
504 : // Pause video if paused by caller or as long as the network is down or the
505 : // pacer queue has grown too large in buffered mode.
506 : // If the pacer queue has grown too large or the network is down,
507 : // last_observed_bitrate_bps_ will be 0.
508 0 : return last_observed_bitrate_bps_ == 0;
509 : }
510 :
511 0 : void ViEEncoder::TraceFrameDropStart() {
512 0 : RTC_DCHECK_RUN_ON(&encoder_queue_);
513 : // Start trace event only on the first frame after encoder is paused.
514 0 : if (!encoder_paused_and_dropped_frame_) {
515 0 : TRACE_EVENT_ASYNC_BEGIN0("webrtc", "EncoderPaused", this);
516 : }
517 0 : encoder_paused_and_dropped_frame_ = true;
518 0 : return;
519 : }
520 :
521 0 : void ViEEncoder::TraceFrameDropEnd() {
522 0 : RTC_DCHECK_RUN_ON(&encoder_queue_);
523 : // End trace event on first frame after encoder resumes, if frame was dropped.
524 0 : if (encoder_paused_and_dropped_frame_) {
525 0 : TRACE_EVENT_ASYNC_END0("webrtc", "EncoderPaused", this);
526 : }
527 0 : encoder_paused_and_dropped_frame_ = false;
528 0 : }
529 :
530 0 : void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame,
531 : int64_t time_when_posted_in_ms) {
532 0 : RTC_DCHECK_RUN_ON(&encoder_queue_);
533 :
534 0 : if (pre_encode_callback_)
535 0 : pre_encode_callback_->OnFrame(video_frame);
536 :
537 0 : if (!last_frame_info_ || video_frame.width() != last_frame_info_->width ||
538 0 : video_frame.height() != last_frame_info_->height ||
539 0 : video_frame.rotation() != last_frame_info_->rotation ||
540 0 : video_frame.is_texture() != last_frame_info_->is_texture) {
541 0 : pending_encoder_reconfiguration_ = true;
542 0 : last_frame_info_ = rtc::Optional<VideoFrameInfo>(
543 0 : VideoFrameInfo(video_frame.width(), video_frame.height(),
544 0 : video_frame.rotation(), video_frame.is_texture()));
545 0 : LOG(LS_INFO) << "Video frame parameters changed: dimensions="
546 0 : << last_frame_info_->width << "x" << last_frame_info_->height
547 0 : << ", rotation=" << last_frame_info_->rotation
548 0 : << ", texture=" << last_frame_info_->is_texture;
549 : }
550 :
551 0 : int64_t now_ms = clock_->TimeInMilliseconds();
552 0 : if (pending_encoder_reconfiguration_) {
553 0 : ReconfigureEncoder();
554 0 : } else if (!last_parameters_update_ms_ ||
555 0 : now_ms - *last_parameters_update_ms_ >=
556 : vcm::VCMProcessTimer::kDefaultProcessIntervalMs) {
557 0 : video_sender_.UpdateChannelParemeters(rate_allocator_.get(),
558 0 : bitrate_observer_);
559 : }
560 0 : last_parameters_update_ms_.emplace(now_ms);
561 :
562 0 : if (EncoderPaused()) {
563 0 : TraceFrameDropStart();
564 0 : return;
565 : }
566 0 : TraceFrameDropEnd();
567 :
568 0 : TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(),
569 : "Encode");
570 :
571 0 : overuse_detector_.FrameCaptured(video_frame, time_when_posted_in_ms);
572 :
573 0 : if (codec_type_ == webrtc::kVideoCodecVP8) {
574 0 : webrtc::CodecSpecificInfo codec_specific_info;
575 0 : codec_specific_info.codecType = webrtc::kVideoCodecVP8;
576 :
577 0 : codec_specific_info.codecSpecific.VP8.hasReceivedRPSI = has_received_rpsi_;
578 0 : codec_specific_info.codecSpecific.VP8.hasReceivedSLI = has_received_sli_;
579 0 : codec_specific_info.codecSpecific.VP8.pictureIdRPSI = picture_id_rpsi_;
580 0 : codec_specific_info.codecSpecific.VP8.pictureIdSLI = picture_id_sli_;
581 0 : has_received_sli_ = false;
582 0 : has_received_rpsi_ = false;
583 :
584 0 : video_sender_.AddVideoFrame(video_frame, &codec_specific_info);
585 0 : return;
586 : }
587 0 : video_sender_.AddVideoFrame(video_frame, nullptr);
588 : }
589 :
590 0 : void ViEEncoder::SendKeyFrame() {
591 0 : if (!encoder_queue_.IsCurrent()) {
592 0 : encoder_queue_.PostTask([this] { SendKeyFrame(); });
593 0 : return;
594 : }
595 0 : RTC_DCHECK_RUN_ON(&encoder_queue_);
596 0 : video_sender_.IntraFrameRequest(0);
597 : }
598 :
599 0 : EncodedImageCallback::Result ViEEncoder::OnEncodedImage(
600 : const EncodedImage& encoded_image,
601 : const CodecSpecificInfo* codec_specific_info,
602 : const RTPFragmentationHeader* fragmentation) {
603 : // Encoded is called on whatever thread the real encoder implementation run
604 : // on. In the case of hardware encoders, there might be several encoders
605 : // running in parallel on different threads.
606 0 : if (stats_proxy_)
607 0 : stats_proxy_->OnSendEncodedImage(encoded_image, codec_specific_info);
608 :
609 : EncodedImageCallback::Result result =
610 0 : sink_->OnEncodedImage(encoded_image, codec_specific_info, fragmentation);
611 :
612 0 : int64_t time_sent = clock_->TimeInMilliseconds();
613 0 : uint32_t timestamp = encoded_image._timeStamp;
614 0 : const int qp = encoded_image.qp_;
615 0 : encoder_queue_.PostTask([this, timestamp, time_sent, qp] {
616 0 : RTC_DCHECK_RUN_ON(&encoder_queue_);
617 0 : overuse_detector_.FrameSent(timestamp, time_sent);
618 0 : if (quality_scaler_)
619 0 : quality_scaler_->ReportQP(qp);
620 0 : });
621 :
622 0 : return result;
623 : }
624 :
625 0 : void ViEEncoder::OnDroppedFrame() {
626 0 : encoder_queue_.PostTask([this] {
627 0 : RTC_DCHECK_RUN_ON(&encoder_queue_);
628 0 : if (quality_scaler_)
629 0 : quality_scaler_->ReportDroppedFrame();
630 0 : });
631 0 : }
632 :
633 0 : void ViEEncoder::SendStatistics(uint32_t bit_rate, uint32_t frame_rate) {
634 0 : RTC_DCHECK(module_process_thread_checker_.CalledOnValidThread());
635 0 : if (stats_proxy_)
636 0 : stats_proxy_->OnEncoderStatsUpdate(frame_rate, bit_rate);
637 0 : }
638 :
639 0 : void ViEEncoder::OnReceivedSLI(uint8_t picture_id) {
640 0 : if (!encoder_queue_.IsCurrent()) {
641 0 : encoder_queue_.PostTask([this, picture_id] { OnReceivedSLI(picture_id); });
642 0 : return;
643 : }
644 0 : RTC_DCHECK_RUN_ON(&encoder_queue_);
645 0 : picture_id_sli_ = picture_id;
646 0 : has_received_sli_ = true;
647 : }
648 :
649 0 : void ViEEncoder::OnReceivedRPSI(uint64_t picture_id) {
650 0 : if (!encoder_queue_.IsCurrent()) {
651 0 : encoder_queue_.PostTask([this, picture_id] { OnReceivedRPSI(picture_id); });
652 0 : return;
653 : }
654 0 : RTC_DCHECK_RUN_ON(&encoder_queue_);
655 0 : picture_id_rpsi_ = picture_id;
656 0 : has_received_rpsi_ = true;
657 : }
658 :
659 0 : void ViEEncoder::OnReceivedIntraFrameRequest(size_t stream_index) {
660 0 : if (!encoder_queue_.IsCurrent()) {
661 0 : encoder_queue_.PostTask(
662 0 : [this, stream_index] { OnReceivedIntraFrameRequest(stream_index); });
663 0 : return;
664 : }
665 0 : RTC_DCHECK_RUN_ON(&encoder_queue_);
666 : // Key frame request from remote side, signal to VCM.
667 0 : TRACE_EVENT0("webrtc", "OnKeyFrameRequest");
668 0 : video_sender_.IntraFrameRequest(stream_index);
669 : }
670 :
671 0 : void ViEEncoder::OnBitrateUpdated(uint32_t bitrate_bps,
672 : uint8_t fraction_lost,
673 : int64_t round_trip_time_ms) {
674 0 : if (!encoder_queue_.IsCurrent()) {
675 0 : encoder_queue_.PostTask(
676 0 : [this, bitrate_bps, fraction_lost, round_trip_time_ms] {
677 0 : OnBitrateUpdated(bitrate_bps, fraction_lost, round_trip_time_ms);
678 0 : });
679 0 : return;
680 : }
681 0 : RTC_DCHECK_RUN_ON(&encoder_queue_);
682 0 : RTC_DCHECK(sink_) << "sink_ must be set before the encoder is active.";
683 :
684 0 : LOG(LS_VERBOSE) << "OnBitrateUpdated, bitrate " << bitrate_bps
685 0 : << " packet loss " << static_cast<int>(fraction_lost)
686 0 : << " rtt " << round_trip_time_ms;
687 :
688 0 : video_sender_.SetChannelParameters(bitrate_bps, fraction_lost,
689 : round_trip_time_ms, rate_allocator_.get(),
690 0 : bitrate_observer_);
691 :
692 0 : encoder_start_bitrate_bps_ =
693 0 : bitrate_bps != 0 ? bitrate_bps : encoder_start_bitrate_bps_;
694 0 : bool video_is_suspended = bitrate_bps == 0;
695 0 : bool video_suspension_changed = video_is_suspended != EncoderPaused();
696 0 : last_observed_bitrate_bps_ = bitrate_bps;
697 :
698 0 : if (stats_proxy_ && video_suspension_changed) {
699 0 : LOG(LS_INFO) << "Video suspend state changed to: "
700 0 : << (video_is_suspended ? "suspended" : "not suspended");
701 0 : stats_proxy_->OnSuspendChange(video_is_suspended);
702 : }
703 : }
704 :
705 0 : void ViEEncoder::onLoadStateChanged(CPULoadState state) {
706 0 : video_sender_.SetCPULoadState(state);
707 0 : }
708 :
709 0 : void ViEEncoder::ScaleDown(ScaleReason reason) {
710 0 : RTC_DCHECK_RUN_ON(&encoder_queue_);
711 0 : if (!scaling_enabled_)
712 0 : return;
713 : // Request lower resolution if the current resolution is lower than last time
714 : // we asked for the resolution to be lowered.
715 : int current_pixel_count =
716 0 : last_frame_info_ ? last_frame_info_->pixel_count() : 0;
717 0 : if (max_pixel_count_ && current_pixel_count >= *max_pixel_count_)
718 0 : return;
719 0 : switch (reason) {
720 : case kQuality:
721 0 : stats_proxy_->OnQualityRestrictedResolutionChanged(
722 0 : scale_counter_[reason] + 1);
723 0 : break;
724 : case kCpu:
725 0 : if (scale_counter_[reason] >= kMaxCpuDowngrades)
726 0 : return;
727 : // Update stats accordingly.
728 0 : stats_proxy_->OnCpuRestrictedResolutionChanged(true);
729 0 : break;
730 : }
731 0 : max_pixel_count_ = rtc::Optional<int>(current_pixel_count);
732 0 : max_pixel_count_step_up_ = rtc::Optional<int>();
733 0 : ++scale_counter_[reason];
734 0 : source_proxy_->RequestResolutionLowerThan(current_pixel_count);
735 0 : LOG(LS_INFO) << "Scaling down resolution.";
736 0 : for (size_t i = 0; i < kScaleReasonSize; ++i) {
737 0 : LOG(LS_INFO) << "Scaled " << scale_counter_[i]
738 0 : << " times for reason: " << (i ? "cpu" : "quality");
739 : }
740 : }
741 :
742 0 : void ViEEncoder::ScaleUp(ScaleReason reason) {
743 0 : RTC_DCHECK_RUN_ON(&encoder_queue_);
744 0 : if (scale_counter_[reason] == 0 || !scaling_enabled_)
745 0 : return;
746 : // Only scale if resolution is higher than last time
747 : // we requested higher resolution.
748 : int current_pixel_count =
749 0 : last_frame_info_ ? last_frame_info_->pixel_count() : 0;
750 0 : if (current_pixel_count <= max_pixel_count_step_up_.value_or(0))
751 0 : return;
752 0 : switch (reason) {
753 : case kQuality:
754 0 : stats_proxy_->OnQualityRestrictedResolutionChanged(
755 0 : scale_counter_[reason] - 1);
756 0 : break;
757 : case kCpu:
758 : // Update stats accordingly.
759 0 : stats_proxy_->OnCpuRestrictedResolutionChanged(scale_counter_[reason] >
760 0 : 1);
761 0 : break;
762 : }
763 0 : max_pixel_count_ = rtc::Optional<int>();
764 0 : max_pixel_count_step_up_ = rtc::Optional<int>(current_pixel_count);
765 0 : --scale_counter_[reason];
766 0 : source_proxy_->RequestHigherResolutionThan(current_pixel_count);
767 0 : LOG(LS_INFO) << "Scaling up resolution.";
768 0 : for (size_t i = 0; i < kScaleReasonSize; ++i) {
769 0 : LOG(LS_INFO) << "Scaled " << scale_counter_[i]
770 0 : << " times for reason: " << (i ? "cpu" : "quality");
771 : }
772 : }
773 :
774 : } // namespace webrtc
|
