Line data Source code
1 : /*
2 : * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 :
11 :
12 : #include <algorithm> // std::max
13 :
14 : #include "webrtc/base/checks.h"
15 : #include "webrtc/base/logging.h"
16 : #include "webrtc/common_types.h"
17 : #include "webrtc/common_video/include/video_bitrate_allocator.h"
18 : #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
19 : #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h"
20 : #include "webrtc/modules/video_coding/include/video_codec_interface.h"
21 : #include "webrtc/modules/video_coding/encoded_frame.h"
22 : #include "webrtc/modules/video_coding/utility/default_video_bitrate_allocator.h"
23 : #include "webrtc/modules/video_coding/utility/quality_scaler.h"
24 : #include "webrtc/modules/video_coding/video_coding_impl.h"
25 : #include "webrtc/system_wrappers/include/clock.h"
26 :
27 : namespace webrtc {
28 : namespace vcm {
29 :
30 0 : VideoSender::VideoSender(Clock* clock,
31 : EncodedImageCallback* post_encode_callback,
32 0 : VCMSendStatisticsCallback* send_stats_callback)
33 : : clock_(clock),
34 : _encoder(nullptr),
35 0 : _mediaOpt(clock_),
36 : _encodedFrameCallback(post_encode_callback, &_mediaOpt),
37 : post_encode_callback_(post_encode_callback),
38 : send_stats_callback_(send_stats_callback),
39 : _codecDataBase(&_encodedFrameCallback),
40 : frame_dropper_enabled_(true),
41 0 : _sendStatsTimer(VCMProcessTimer::kDefaultProcessIntervalMs, clock_),
42 : current_codec_(),
43 : encoder_params_({BitrateAllocation(), 0, 0, 0}),
44 : encoder_has_internal_source_(false),
45 0 : next_frame_types_(1, kVideoFrameDelta) {
46 0 : _mediaOpt.Reset();
47 : // Allow VideoSender to be created on one thread but used on another, post
48 : // construction. This is currently how this class is being used by at least
49 : // one external project (diffractor).
50 0 : sequenced_checker_.Detach();
51 0 : }
52 :
53 0 : VideoSender::~VideoSender() {}
54 :
55 0 : void VideoSender::Process() {
56 0 : if (_sendStatsTimer.TimeUntilProcess() == 0) {
57 : // |_sendStatsTimer.Processed()| must be called. Otherwise
58 : // VideoSender::Process() will be called in an infinite loop.
59 0 : _sendStatsTimer.Processed();
60 0 : if (send_stats_callback_) {
61 0 : uint32_t bitRate = _mediaOpt.SentBitRate();
62 0 : uint32_t frameRate = _mediaOpt.SentFrameRate();
63 0 : send_stats_callback_->SendStatistics(bitRate, frameRate);
64 : }
65 : }
66 0 : }
67 :
68 0 : int64_t VideoSender::TimeUntilNextProcess() {
69 0 : return _sendStatsTimer.TimeUntilProcess();
70 : }
71 :
72 : // Register the send codec to be used.
73 0 : int32_t VideoSender::RegisterSendCodec(const VideoCodec* sendCodec,
74 : uint32_t numberOfCores,
75 : uint32_t maxPayloadSize) {
76 0 : RTC_DCHECK(sequenced_checker_.CalledSequentially());
77 0 : rtc::CritScope lock(&encoder_crit_);
78 0 : if (sendCodec == nullptr) {
79 0 : return VCM_PARAMETER_ERROR;
80 : }
81 :
82 : bool ret =
83 0 : _codecDataBase.SetSendCodec(sendCodec, numberOfCores, maxPayloadSize);
84 :
85 : // Update encoder regardless of result to make sure that we're not holding on
86 : // to a deleted instance.
87 0 : _encoder = _codecDataBase.GetEncoder();
88 : // Cache the current codec here so they can be fetched from this thread
89 : // without requiring the _sendCritSect lock.
90 0 : current_codec_ = *sendCodec;
91 :
92 0 : if (!ret) {
93 0 : LOG(LS_ERROR) << "Failed to initialize set encoder with payload name '"
94 0 : << sendCodec->plName << "'.";
95 0 : return VCM_CODEC_ERROR;
96 : }
97 :
98 : // SetSendCodec succeeded, _encoder should be set.
99 0 : RTC_DCHECK(_encoder);
100 :
101 : int numLayers;
102 0 : if (sendCodec->codecType == kVideoCodecVP8) {
103 0 : numLayers = sendCodec->VP8().numberOfTemporalLayers;
104 0 : } else if (sendCodec->codecType == kVideoCodecVP9) {
105 0 : numLayers = sendCodec->VP9().numberOfTemporalLayers;
106 : } else {
107 0 : numLayers = 1;
108 : }
109 :
110 : // If we have screensharing and we have layers, we disable frame dropper.
111 : bool disable_frame_dropper =
112 0 : numLayers > 1 && sendCodec->mode == kScreensharing;
113 0 : if (disable_frame_dropper) {
114 0 : _mediaOpt.EnableFrameDropper(false);
115 0 : } else if (frame_dropper_enabled_) {
116 0 : _mediaOpt.EnableFrameDropper(true);
117 : }
118 :
119 : {
120 0 : rtc::CritScope cs(¶ms_crit_);
121 0 : next_frame_types_.clear();
122 0 : next_frame_types_.resize(VCM_MAX(sendCodec->numberOfSimulcastStreams, 1),
123 0 : kVideoFrameKey);
124 : // Cache InternalSource() to have this available from IntraFrameRequest()
125 : // without having to acquire encoder_crit_ (avoid blocking on encoder use).
126 0 : encoder_has_internal_source_ = _encoder->InternalSource();
127 : }
128 :
129 0 : LOG(LS_VERBOSE) << " max bitrate " << sendCodec->maxBitrate
130 0 : << " start bitrate " << sendCodec->startBitrate
131 0 : << " max frame rate " << sendCodec->maxFramerate
132 0 : << " max payload size " << maxPayloadSize;
133 0 : _mediaOpt.SetEncodingData(sendCodec->maxBitrate * 1000,
134 0 : sendCodec->startBitrate * 1000, sendCodec->width,
135 0 : sendCodec->height, sendCodec->maxFramerate * 1000,
136 0 : sendCodec->resolution_divisor,
137 0 : numLayers, maxPayloadSize);
138 0 : return VCM_OK;
139 : }
140 :
141 : // Register an external decoder object.
142 : // This can not be used together with external decoder callbacks.
143 0 : void VideoSender::RegisterExternalEncoder(VideoEncoder* externalEncoder,
144 : uint8_t payloadType,
145 : bool internalSource /*= false*/) {
146 0 : RTC_DCHECK(sequenced_checker_.CalledSequentially());
147 :
148 0 : rtc::CritScope lock(&encoder_crit_);
149 :
150 0 : if (externalEncoder == nullptr) {
151 0 : bool wasSendCodec = false;
152 0 : RTC_CHECK(
153 0 : _codecDataBase.DeregisterExternalEncoder(payloadType, &wasSendCodec));
154 0 : if (wasSendCodec) {
155 : // Make sure the VCM doesn't use the de-registered codec
156 0 : rtc::CritScope params_lock(¶ms_crit_);
157 0 : _encoder = nullptr;
158 0 : encoder_has_internal_source_ = false;
159 : }
160 0 : return;
161 : }
162 0 : _codecDataBase.RegisterExternalEncoder(externalEncoder, payloadType,
163 0 : internalSource);
164 : }
165 :
166 : // Get encode bitrate
167 0 : int VideoSender::Bitrate(unsigned int* bitrate) const {
168 0 : RTC_DCHECK(sequenced_checker_.CalledSequentially());
169 : // Since we're running on the thread that's the only thread known to modify
170 : // the value of _encoder, we don't need to grab the lock here.
171 :
172 0 : if (!_encoder)
173 0 : return VCM_UNINITIALIZED;
174 0 : *bitrate = _encoder->GetEncoderParameters().target_bitrate.get_sum_bps();
175 0 : return 0;
176 : }
177 :
178 : // Get encode frame rate
179 0 : int VideoSender::FrameRate(unsigned int* framerate) const {
180 0 : RTC_DCHECK(sequenced_checker_.CalledSequentially());
181 : // Since we're running on the thread that's the only thread known to modify
182 : // the value of _encoder, we don't need to grab the lock here.
183 :
184 0 : if (!_encoder)
185 0 : return VCM_UNINITIALIZED;
186 :
187 0 : *framerate = _encoder->GetEncoderParameters().input_frame_rate;
188 0 : return 0;
189 : }
190 :
191 0 : EncoderParameters VideoSender::UpdateEncoderParameters(
192 : const EncoderParameters& params,
193 : VideoBitrateAllocator* bitrate_allocator,
194 : uint32_t target_bitrate_bps) {
195 0 : uint32_t video_target_rate_bps = _mediaOpt.SetTargetRates(target_bitrate_bps);
196 0 : uint32_t input_frame_rate = _mediaOpt.InputFrameRate();
197 0 : if (input_frame_rate == 0)
198 0 : input_frame_rate = current_codec_.maxFramerate;
199 :
200 0 : BitrateAllocation bitrate_allocation;
201 0 : if (bitrate_allocator) {
202 : bitrate_allocation = bitrate_allocator->GetAllocation(video_target_rate_bps,
203 0 : input_frame_rate);
204 : } else {
205 0 : DefaultVideoBitrateAllocator default_allocator(current_codec_);
206 : bitrate_allocation = default_allocator.GetAllocation(video_target_rate_bps,
207 0 : input_frame_rate);
208 : }
209 0 : EncoderParameters new_encoder_params = {bitrate_allocation, params.loss_rate,
210 0 : params.rtt, input_frame_rate};
211 0 : return new_encoder_params;
212 : }
213 :
214 0 : void VideoSender::UpdateChannelParemeters(
215 : VideoBitrateAllocator* bitrate_allocator,
216 : VideoBitrateAllocationObserver* bitrate_updated_callback) {
217 0 : BitrateAllocation target_rate;
218 : {
219 0 : rtc::CritScope cs(¶ms_crit_);
220 : encoder_params_ =
221 0 : UpdateEncoderParameters(encoder_params_, bitrate_allocator,
222 0 : encoder_params_.target_bitrate.get_sum_bps());
223 0 : target_rate = encoder_params_.target_bitrate;
224 : }
225 0 : if (bitrate_updated_callback)
226 0 : bitrate_updated_callback->OnBitrateAllocationUpdated(target_rate);
227 0 : }
228 :
229 0 : int32_t VideoSender::SetChannelParameters(
230 : uint32_t target_bitrate_bps,
231 : uint8_t loss_rate,
232 : int64_t rtt,
233 : VideoBitrateAllocator* bitrate_allocator,
234 : VideoBitrateAllocationObserver* bitrate_updated_callback) {
235 0 : EncoderParameters encoder_params;
236 0 : encoder_params.loss_rate = loss_rate;
237 0 : encoder_params.rtt = rtt;
238 0 : encoder_params = UpdateEncoderParameters(encoder_params, bitrate_allocator,
239 0 : target_bitrate_bps);
240 0 : if (bitrate_updated_callback) {
241 : bitrate_updated_callback->OnBitrateAllocationUpdated(
242 0 : encoder_params.target_bitrate);
243 : }
244 :
245 : bool encoder_has_internal_source;
246 : {
247 0 : rtc::CritScope cs(¶ms_crit_);
248 0 : encoder_params_ = encoder_params;
249 0 : encoder_has_internal_source = encoder_has_internal_source_;
250 : }
251 :
252 : // For encoders with internal sources, we need to tell the encoder directly,
253 : // instead of waiting for an AddVideoFrame that will never come (internal
254 : // source encoders don't get input frames).
255 0 : if (encoder_has_internal_source) {
256 0 : rtc::CritScope cs(&encoder_crit_);
257 0 : if (_encoder) {
258 0 : SetEncoderParameters(encoder_params, encoder_has_internal_source);
259 : }
260 : }
261 :
262 0 : return VCM_OK;
263 : }
264 :
265 0 : void VideoSender::SetEncoderParameters(EncoderParameters params,
266 : bool has_internal_source) {
267 : // |target_bitrate == 0 | means that the network is down or the send pacer is
268 : // full. We currently only report this if the encoder has an internal source.
269 : // If the encoder does not have an internal source, higher levels are expected
270 : // to not call AddVideoFrame. We do this since its unclear how current
271 : // encoder implementations behave when given a zero target bitrate.
272 : // TODO(perkj): Make sure all known encoder implementations handle zero
273 : // target bitrate and remove this check.
274 0 : if (!has_internal_source && params.target_bitrate.get_sum_bps() == 0)
275 0 : return;
276 :
277 0 : if (params.input_frame_rate == 0) {
278 : // No frame rate estimate available, use default.
279 0 : params.input_frame_rate = current_codec_.maxFramerate;
280 : }
281 0 : if (_encoder != nullptr)
282 0 : _encoder->SetEncoderParameters(params);
283 : }
284 :
285 : // Deprecated:
286 : // TODO(perkj): Remove once no projects call this method. It currently do
287 : // nothing.
288 0 : int32_t VideoSender::RegisterProtectionCallback(
289 : VCMProtectionCallback* protection_callback) {
290 : // Deprecated:
291 : // TODO(perkj): Remove once no projects call this method. It currently do
292 : // nothing.
293 0 : return VCM_OK;
294 : }
295 :
296 : // Add one raw video frame to the encoder, blocking.
297 0 : int32_t VideoSender::AddVideoFrame(const VideoFrame& videoFrame,
298 : const CodecSpecificInfo* codecSpecificInfo) {
299 0 : EncoderParameters encoder_params;
300 0 : std::vector<FrameType> next_frame_types;
301 0 : bool encoder_has_internal_source = false;
302 : {
303 0 : rtc::CritScope lock(¶ms_crit_);
304 0 : encoder_params = encoder_params_;
305 0 : next_frame_types = next_frame_types_;
306 0 : encoder_has_internal_source = encoder_has_internal_source_;
307 : }
308 0 : rtc::CritScope lock(&encoder_crit_);
309 0 : if (_encoder == nullptr)
310 0 : return VCM_UNINITIALIZED;
311 0 : SetEncoderParameters(encoder_params, encoder_has_internal_source);
312 0 : if (_mediaOpt.DropFrame()) {
313 0 : LOG(LS_VERBOSE) << "Drop Frame "
314 0 : << "target bitrate "
315 0 : << encoder_params.target_bitrate.get_sum_bps()
316 0 : << " loss rate " << encoder_params.loss_rate << " rtt "
317 0 : << encoder_params.rtt << " input frame rate "
318 0 : << encoder_params.input_frame_rate;
319 0 : post_encode_callback_->OnDroppedFrame();
320 0 : return VCM_OK;
321 : }
322 : // TODO(pbos): Make sure setting send codec is synchronized with video
323 : // processing so frame size always matches.
324 : #ifdef VERIFY_FRAME_SIZE_VS_DATABASE
325 : if (!_codecDataBase.MatchesCurrentResolution(videoFrame.width(),
326 : videoFrame.height())) {
327 : LOG(LS_ERROR) << "Incoming frame doesn't match set resolution. Dropping.";
328 : return VCM_PARAMETER_ERROR;
329 : }
330 : #endif
331 0 : VideoFrame converted_frame = videoFrame;
332 0 : if (converted_frame.video_frame_buffer()->native_handle() &&
333 0 : !_encoder->SupportsNativeHandle()) {
334 : // This module only supports software encoding.
335 : // TODO(pbos): Offload conversion from the encoder thread.
336 : rtc::scoped_refptr<VideoFrameBuffer> converted_buffer(
337 0 : converted_frame.video_frame_buffer()->NativeToI420Buffer());
338 :
339 0 : if (!converted_buffer) {
340 0 : LOG(LS_ERROR) << "Frame conversion failed, dropping frame.";
341 0 : return VCM_PARAMETER_ERROR;
342 : }
343 0 : converted_frame = VideoFrame(converted_buffer,
344 : converted_frame.timestamp(),
345 : converted_frame.render_time_ms(),
346 0 : converted_frame.rotation());
347 : }
348 : int32_t ret =
349 0 : _encoder->Encode(converted_frame, codecSpecificInfo, next_frame_types);
350 0 : if (ret < 0) {
351 0 : LOG(LS_ERROR) << "Failed to encode frame. Error code: " << ret;
352 0 : return ret;
353 : }
354 :
355 : {
356 0 : rtc::CritScope lock(¶ms_crit_);
357 : // Change all keyframe requests to encode delta frames the next time.
358 0 : for (size_t i = 0; i < next_frame_types_.size(); ++i) {
359 : // Check for equality (same requested as before encoding) to not
360 : // accidentally drop a keyframe request while encoding.
361 0 : if (next_frame_types[i] == next_frame_types_[i])
362 0 : next_frame_types_[i] = kVideoFrameDelta;
363 : }
364 : }
365 0 : return VCM_OK;
366 : }
367 :
368 0 : int32_t VideoSender::IntraFrameRequest(size_t stream_index) {
369 : {
370 0 : rtc::CritScope lock(¶ms_crit_);
371 0 : if (stream_index >= next_frame_types_.size()) {
372 0 : return -1;
373 : }
374 0 : next_frame_types_[stream_index] = kVideoFrameKey;
375 0 : if (!encoder_has_internal_source_)
376 0 : return VCM_OK;
377 : }
378 : // TODO(pbos): Remove when InternalSource() is gone. Both locks have to be
379 : // held here for internal consistency, since _encoder could be removed while
380 : // not holding encoder_crit_. Checks have to be performed again since
381 : // params_crit_ was dropped to not cause lock-order inversions with
382 : // encoder_crit_.
383 0 : rtc::CritScope lock(&encoder_crit_);
384 0 : rtc::CritScope params_lock(¶ms_crit_);
385 0 : if (stream_index >= next_frame_types_.size())
386 0 : return -1;
387 0 : if (_encoder != nullptr && _encoder->InternalSource()) {
388 : // Try to request the frame if we have an external encoder with
389 : // internal source since AddVideoFrame never will be called.
390 0 : if (_encoder->RequestFrame(next_frame_types_) == WEBRTC_VIDEO_CODEC_OK) {
391 : // Try to remove just-performed keyframe request, if stream still exists.
392 0 : next_frame_types_[stream_index] = kVideoFrameDelta;
393 : }
394 : }
395 0 : return VCM_OK;
396 : }
397 :
398 0 : int32_t VideoSender::EnableFrameDropper(bool enable) {
399 0 : rtc::CritScope lock(&encoder_crit_);
400 0 : frame_dropper_enabled_ = enable;
401 0 : _mediaOpt.EnableFrameDropper(enable);
402 0 : return VCM_OK;
403 : }
404 :
405 0 : void VideoSender::SetCPULoadState(CPULoadState state) {
406 0 : rtc::CritScope lock(&encoder_crit_);
407 0 : _mediaOpt.SetCPULoadState(state);
408 0 : }
409 :
410 : } // namespace vcm
411 : } // namespace webrtc
|
