Line data Source code
1 : /*
2 : * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 : #include "webrtc/video/video_send_stream.h"
11 :
12 : #include <algorithm>
13 : #include <cmath>
14 : #include <sstream>
15 : #include <string>
16 : #include <utility>
17 : #include <vector>
18 :
19 : #include "webrtc/common_types.h"
20 : #include "webrtc/common_video/include/video_bitrate_allocator.h"
21 : #include "webrtc/base/checks.h"
22 : #include "webrtc/base/file.h"
23 : #include "webrtc/base/logging.h"
24 : #include "webrtc/base/trace_event.h"
25 : #include "webrtc/base/weak_ptr.h"
26 : #include "webrtc/modules/bitrate_controller/include/bitrate_controller.h"
27 : #include "webrtc/modules/congestion_controller/include/congestion_controller.h"
28 : #include "webrtc/modules/pacing/packet_router.h"
29 : #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h"
30 : #include "webrtc/modules/utility/include/process_thread.h"
31 : #include "webrtc/modules/video_coding/utility/ivf_file_writer.h"
32 : #include "webrtc/system_wrappers/include/field_trial.h"
33 : #include "webrtc/video/call_stats.h"
34 : #include "webrtc/video/vie_remb.h"
35 : #include "webrtc/video_send_stream.h"
36 :
37 : namespace webrtc {
38 :
39 : static const int kMinSendSidePacketHistorySize = 600;
40 : namespace {
41 :
42 : // We don't do MTU discovery, so assume that we have the standard ethernet MTU.
43 : const size_t kPathMTU = 1500;
44 :
45 0 : std::vector<RtpRtcp*> CreateRtpRtcpModules(
46 : Transport* outgoing_transport,
47 : RtcpIntraFrameObserver* intra_frame_callback,
48 : RtcpBandwidthObserver* bandwidth_callback,
49 : TransportFeedbackObserver* transport_feedback_callback,
50 : RtcpRttStats* rtt_stats,
51 : RtpPacketSender* paced_sender,
52 : TransportSequenceNumberAllocator* transport_sequence_number_allocator,
53 : FlexfecSender* flexfec_sender,
54 : SendStatisticsProxy* stats_proxy,
55 : SendDelayStats* send_delay_stats,
56 : RtcEventLog* event_log,
57 : RateLimiter* retransmission_rate_limiter,
58 : OverheadObserver* overhead_observer,
59 : size_t num_modules) {
60 0 : RTC_DCHECK_GT(num_modules, 0);
61 0 : RtpRtcp::Configuration configuration;
62 0 : ReceiveStatistics* null_receive_statistics = configuration.receive_statistics;
63 0 : configuration.audio = false;
64 0 : configuration.receiver_only = false;
65 0 : configuration.flexfec_sender = flexfec_sender;
66 0 : configuration.receive_statistics = null_receive_statistics;
67 0 : configuration.outgoing_transport = outgoing_transport;
68 0 : configuration.intra_frame_callback = intra_frame_callback;
69 0 : configuration.bandwidth_callback = bandwidth_callback;
70 0 : configuration.transport_feedback_callback = transport_feedback_callback;
71 0 : configuration.rtt_stats = rtt_stats;
72 0 : configuration.rtcp_packet_type_counter_observer = stats_proxy;
73 0 : configuration.paced_sender = paced_sender;
74 0 : configuration.transport_sequence_number_allocator =
75 : transport_sequence_number_allocator;
76 0 : configuration.send_bitrate_observer = stats_proxy;
77 0 : configuration.send_frame_count_observer = stats_proxy;
78 0 : configuration.send_side_delay_observer = stats_proxy;
79 0 : configuration.send_packet_observer = send_delay_stats;
80 0 : configuration.event_log = event_log;
81 0 : configuration.retransmission_rate_limiter = retransmission_rate_limiter;
82 0 : configuration.overhead_observer = overhead_observer;
83 0 : std::vector<RtpRtcp*> modules;
84 0 : for (size_t i = 0; i < num_modules; ++i) {
85 0 : RtpRtcp* rtp_rtcp = RtpRtcp::CreateRtpRtcp(configuration);
86 0 : rtp_rtcp->SetSendingStatus(false);
87 0 : rtp_rtcp->SetSendingMediaStatus(false);
88 0 : rtp_rtcp->SetRTCPStatus(RtcpMode::kCompound);
89 0 : modules.push_back(rtp_rtcp);
90 : }
91 0 : return modules;
92 : }
93 :
94 : // TODO(brandtr): Update this function when we support multistream protection.
95 0 : std::unique_ptr<FlexfecSender> MaybeCreateFlexfecSender(
96 : const VideoSendStream::Config& config) {
97 0 : if (config.rtp.flexfec.payload_type < 0) {
98 0 : return nullptr;
99 : }
100 0 : RTC_DCHECK_GE(config.rtp.flexfec.payload_type, 0);
101 0 : RTC_DCHECK_LE(config.rtp.flexfec.payload_type, 127);
102 0 : if (config.rtp.flexfec.ssrc == 0) {
103 0 : LOG(LS_WARNING) << "FlexFEC is enabled, but no FlexFEC SSRC given. "
104 0 : "Therefore disabling FlexFEC.";
105 0 : return nullptr;
106 : }
107 0 : if (config.rtp.flexfec.protected_media_ssrcs.empty()) {
108 0 : LOG(LS_WARNING) << "FlexFEC is enabled, but no protected media SSRC given. "
109 0 : "Therefore disabling FlexFEC.";
110 0 : return nullptr;
111 : }
112 :
113 0 : if (config.rtp.ssrcs.size() > 1) {
114 0 : LOG(LS_WARNING) << "Both FlexFEC and simulcast are enabled. This "
115 : "combination is however not supported by our current "
116 0 : "FlexFEC implementation. Therefore disabling FlexFEC.";
117 0 : return nullptr;
118 : }
119 :
120 0 : if (config.rtp.flexfec.protected_media_ssrcs.size() > 1) {
121 0 : LOG(LS_WARNING)
122 : << "The supplied FlexfecConfig contained multiple protected "
123 : "media streams, but our implementation currently only "
124 : "supports protecting a single media stream. "
125 0 : "To avoid confusion, disabling FlexFEC completely.";
126 0 : return nullptr;
127 : }
128 :
129 0 : RTC_DCHECK_EQ(1U, config.rtp.flexfec.protected_media_ssrcs.size());
130 : return std::unique_ptr<FlexfecSender>(new FlexfecSender(
131 0 : config.rtp.flexfec.payload_type, config.rtp.flexfec.ssrc,
132 0 : config.rtp.flexfec.protected_media_ssrcs[0], config.rtp.extensions,
133 0 : Clock::GetRealTimeClock()));
134 : }
135 :
136 : } // namespace
137 :
138 : std::string
139 0 : VideoSendStream::Config::EncoderSettings::ToString() const {
140 0 : std::stringstream ss;
141 0 : ss << "{payload_name: " << payload_name;
142 0 : ss << ", payload_type: " << payload_type;
143 0 : ss << ", encoder: " << (encoder ? "(VideoEncoder)" : "nullptr");
144 0 : ss << '}';
145 0 : return ss.str();
146 : }
147 :
148 0 : std::string VideoSendStream::Config::Rtp::Rtx::ToString()
149 : const {
150 0 : std::stringstream ss;
151 0 : ss << "{ssrcs: [";
152 0 : for (size_t i = 0; i < ssrcs.size(); ++i) {
153 0 : ss << ssrcs[i];
154 0 : if (i != ssrcs.size() - 1)
155 0 : ss << ", ";
156 : }
157 0 : ss << ']';
158 :
159 0 : ss << ", payload_type: " << payload_type;
160 0 : ss << '}';
161 0 : return ss.str();
162 : }
163 :
164 0 : std::string VideoSendStream::Config::Rtp::ToString() const {
165 0 : std::stringstream ss;
166 0 : ss << "{ssrcs: [";
167 0 : for (size_t i = 0; i < ssrcs.size(); ++i) {
168 0 : ss << ssrcs[i];
169 0 : if (i != ssrcs.size() - 1)
170 0 : ss << ", ";
171 : }
172 0 : ss << ']';
173 : ss << ", rtcp_mode: "
174 0 : << (rtcp_mode == RtcpMode::kCompound ? "RtcpMode::kCompound"
175 0 : : "RtcpMode::kReducedSize");
176 0 : ss << ", max_packet_size: " << max_packet_size;
177 0 : ss << ", extensions: [";
178 0 : for (size_t i = 0; i < extensions.size(); ++i) {
179 0 : ss << extensions[i].ToString();
180 0 : if (i != extensions.size() - 1)
181 0 : ss << ", ";
182 : }
183 0 : ss << ']';
184 :
185 0 : ss << ", nack: {rtp_history_ms: " << nack.rtp_history_ms << '}';
186 0 : ss << ", ulpfec: " << ulpfec.ToString();
187 :
188 0 : ss << ", flexfec: {payload_type: " << flexfec.payload_type;
189 0 : ss << ", ssrc: " << flexfec.ssrc;
190 0 : ss << ", protected_media_ssrcs: [";
191 0 : for (size_t i = 0; i < flexfec.protected_media_ssrcs.size(); ++i) {
192 0 : ss << flexfec.protected_media_ssrcs[i];
193 0 : if (i != flexfec.protected_media_ssrcs.size() - 1)
194 0 : ss << ", ";
195 : }
196 0 : ss << ']';
197 :
198 0 : ss << ", rtx: " << rtx.ToString();
199 0 : ss << ", c_name: " << c_name;
200 0 : ss << '}';
201 0 : return ss.str();
202 : }
203 :
204 0 : std::string VideoSendStream::Config::ToString() const {
205 0 : std::stringstream ss;
206 0 : ss << "{encoder_settings: " << encoder_settings.ToString();
207 0 : ss << ", rtp: " << rtp.ToString();
208 : ss << ", pre_encode_callback: "
209 0 : << (pre_encode_callback ? "(I420FrameCallback)" : "nullptr");
210 : ss << ", post_encode_callback: "
211 0 : << (post_encode_callback ? "(EncodedFrameObserver)" : "nullptr");
212 0 : ss << ", render_delay_ms: " << render_delay_ms;
213 0 : ss << ", target_delay_ms: " << target_delay_ms;
214 0 : ss << ", suspend_below_min_bitrate: " << (suspend_below_min_bitrate ? "on"
215 0 : : "off");
216 0 : ss << '}';
217 0 : return ss.str();
218 : }
219 :
220 0 : std::string VideoSendStream::Stats::ToString(int64_t time_ms) const {
221 0 : std::stringstream ss;
222 0 : ss << "VideoSendStream stats: " << time_ms << ", {";
223 0 : ss << "input_fps: " << input_frame_rate << ", ";
224 0 : ss << "encode_fps: " << encode_frame_rate << ", ";
225 0 : ss << "encode_ms: " << avg_encode_time_ms << ", ";
226 0 : ss << "encode_usage_perc: " << encode_usage_percent << ", ";
227 0 : ss << "target_bps: " << target_media_bitrate_bps << ", ";
228 0 : ss << "media_bps: " << media_bitrate_bps << ", ";
229 0 : ss << "preferred_media_bitrate_bps: " << preferred_media_bitrate_bps << ", ";
230 0 : ss << "suspended: " << (suspended ? "true" : "false") << ", ";
231 0 : ss << "bw_adapted: " << (bw_limited_resolution ? "true" : "false");
232 0 : ss << '}';
233 0 : for (const auto& substream : substreams) {
234 0 : if (!substream.second.is_rtx && !substream.second.is_flexfec) {
235 0 : ss << " {ssrc: " << substream.first << ", ";
236 0 : ss << substream.second.ToString();
237 0 : ss << '}';
238 : }
239 : }
240 0 : return ss.str();
241 : }
242 :
243 0 : std::string VideoSendStream::StreamStats::ToString() const {
244 0 : std::stringstream ss;
245 0 : ss << "width: " << width << ", ";
246 0 : ss << "height: " << height << ", ";
247 0 : ss << "key: " << frame_counts.key_frames << ", ";
248 0 : ss << "delta: " << frame_counts.delta_frames << ", ";
249 0 : ss << "total_bps: " << total_bitrate_bps << ", ";
250 0 : ss << "retransmit_bps: " << retransmit_bitrate_bps << ", ";
251 0 : ss << "avg_delay_ms: " << avg_delay_ms << ", ";
252 0 : ss << "max_delay_ms: " << max_delay_ms << ", ";
253 0 : ss << "cum_loss: " << rtcp_stats.cumulative_lost << ", ";
254 0 : ss << "max_ext_seq: " << rtcp_stats.extended_max_sequence_number << ", ";
255 0 : ss << "nack: " << rtcp_packet_type_counts.nack_packets << ", ";
256 0 : ss << "fir: " << rtcp_packet_type_counts.fir_packets << ", ";
257 0 : ss << "pli: " << rtcp_packet_type_counts.pli_packets;
258 0 : return ss.str();
259 : }
260 :
261 : namespace {
262 :
263 0 : bool PayloadTypeSupportsSkippingFecPackets(const std::string& payload_name) {
264 0 : if (payload_name == "VP8" || payload_name == "VP9")
265 0 : return true;
266 0 : RTC_DCHECK(payload_name == "H264" || payload_name == "FAKE")
267 0 : << "unknown payload_name " << payload_name;
268 0 : return false;
269 : }
270 :
271 0 : int CalculateMaxPadBitrateBps(std::vector<VideoStream> streams,
272 : int min_transmit_bitrate_bps,
273 : bool pad_to_min_bitrate) {
274 0 : int pad_up_to_bitrate_bps = 0;
275 : // Calculate max padding bitrate for a multi layer codec.
276 0 : if (streams.size() > 1) {
277 : // Pad to min bitrate of the highest layer.
278 0 : pad_up_to_bitrate_bps = streams[streams.size() - 1].min_bitrate_bps;
279 : // Add target_bitrate_bps of the lower layers.
280 0 : for (size_t i = 0; i < streams.size() - 1; ++i)
281 0 : pad_up_to_bitrate_bps += streams[i].target_bitrate_bps;
282 0 : } else if (pad_to_min_bitrate) {
283 0 : pad_up_to_bitrate_bps = streams[0].min_bitrate_bps;
284 : }
285 :
286 0 : pad_up_to_bitrate_bps =
287 0 : std::max(pad_up_to_bitrate_bps, min_transmit_bitrate_bps);
288 :
289 0 : return pad_up_to_bitrate_bps;
290 : }
291 :
292 : } // namespace
293 :
294 : namespace internal {
295 :
296 : // VideoSendStreamImpl implements internal::VideoSendStream.
297 : // It is created and destroyed on |worker_queue|. The intent is to decrease the
298 : // need for locking and to ensure methods are called in sequence.
299 : // Public methods except |DeliverRtcp| must be called on |worker_queue|.
300 : // DeliverRtcp is called on the libjingle worker thread or a network thread.
301 : // An encoder may deliver frames through the EncodedImageCallback on an
302 : // arbitrary thread.
303 : class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver,
304 : public webrtc::OverheadObserver,
305 : public webrtc::VCMProtectionCallback,
306 : public ViEEncoder::EncoderSink,
307 : public VideoBitrateAllocationObserver {
308 : public:
309 : VideoSendStreamImpl(SendStatisticsProxy* stats_proxy,
310 : rtc::TaskQueue* worker_queue,
311 : CallStats* call_stats,
312 : CongestionController* congestion_controller,
313 : PacketRouter* packet_router,
314 : BitrateAllocator* bitrate_allocator,
315 : SendDelayStats* send_delay_stats,
316 : VieRemb* remb,
317 : ViEEncoder* vie_encoder,
318 : RtcEventLog* event_log,
319 : const VideoSendStream::Config* config,
320 : int initial_encoder_max_bitrate,
321 : std::map<uint32_t, RtpState> suspended_ssrcs);
322 : ~VideoSendStreamImpl() override;
323 :
324 : // RegisterProcessThread register |module_process_thread| with those objects
325 : // that use it. Registration has to happen on the thread were
326 : // |module_process_thread| was created (libjingle's worker thread).
327 : // TODO(perkj): Replace the use of |module_process_thread| with a TaskQueue,
328 : // maybe |worker_queue|.
329 : void RegisterProcessThread(ProcessThread* module_process_thread);
330 : void DeRegisterProcessThread();
331 :
332 : void SignalNetworkState(NetworkState state);
333 : bool DeliverRtcp(const uint8_t* packet, size_t length);
334 : void Start();
335 : void Stop();
336 :
337 : VideoSendStream::RtpStateMap GetRtpStates() const;
338 :
339 : void EnableEncodedFrameRecording(const std::vector<rtc::PlatformFile>& files,
340 : size_t byte_limit);
341 :
342 : void SetTransportOverhead(size_t transport_overhead_per_packet);
343 :
344 : private:
345 : class CheckEncoderActivityTask;
346 : class EncoderReconfiguredTask;
347 :
348 : // Implements BitrateAllocatorObserver.
349 : uint32_t OnBitrateUpdated(uint32_t bitrate_bps,
350 : uint8_t fraction_loss,
351 : int64_t rtt,
352 : int64_t probing_interval_ms) override;
353 :
354 : // Implements webrtc::VCMProtectionCallback.
355 : int ProtectionRequest(const FecProtectionParams* delta_params,
356 : const FecProtectionParams* key_params,
357 : uint32_t* sent_video_rate_bps,
358 : uint32_t* sent_nack_rate_bps,
359 : uint32_t* sent_fec_rate_bps) override;
360 :
361 : // Implements OverheadObserver.
362 : void OnOverheadChanged(size_t overhead_bytes_per_packet) override;
363 :
364 : void OnEncoderConfigurationChanged(std::vector<VideoStream> streams,
365 : int min_transmit_bitrate_bps) override;
366 :
367 : // Implements EncodedImageCallback. The implementation routes encoded frames
368 : // to the |payload_router_| and |config.pre_encode_callback| if set.
369 : // Called on an arbitrary encoder callback thread.
370 : EncodedImageCallback::Result OnEncodedImage(
371 : const EncodedImage& encoded_image,
372 : const CodecSpecificInfo* codec_specific_info,
373 : const RTPFragmentationHeader* fragmentation) override;
374 :
375 : // Implements VideoBitrateAllocationObserver.
376 : void OnBitrateAllocationUpdated(const BitrateAllocation& allocation) override;
377 :
378 : void ConfigureProtection();
379 : void ConfigureSsrcs();
380 : void SignalEncoderTimedOut();
381 : void SignalEncoderActive();
382 :
383 : SendStatisticsProxy* const stats_proxy_;
384 : const VideoSendStream::Config* const config_;
385 : std::map<uint32_t, RtpState> suspended_ssrcs_;
386 :
387 : ProcessThread* module_process_thread_;
388 : rtc::ThreadChecker module_process_thread_checker_;
389 : rtc::TaskQueue* const worker_queue_;
390 :
391 : rtc::CriticalSection encoder_activity_crit_sect_;
392 : CheckEncoderActivityTask* check_encoder_activity_task_
393 : GUARDED_BY(encoder_activity_crit_sect_);
394 :
395 : CallStats* const call_stats_;
396 : CongestionController* const congestion_controller_;
397 : PacketRouter* const packet_router_;
398 : BitrateAllocator* const bitrate_allocator_;
399 : VieRemb* const remb_;
400 :
401 : // TODO(brandtr): Consider moving this to a new FlexfecSendStream class.
402 : std::unique_ptr<FlexfecSender> flexfec_sender_;
403 :
404 : rtc::CriticalSection ivf_writers_crit_;
405 : std::unique_ptr<IvfFileWriter> file_writers_[kMaxSimulcastStreams] GUARDED_BY(
406 : ivf_writers_crit_);
407 :
408 : int max_padding_bitrate_;
409 : int encoder_min_bitrate_bps_;
410 : uint32_t encoder_max_bitrate_bps_;
411 : uint32_t encoder_target_rate_bps_;
412 :
413 : ViEEncoder* const vie_encoder_;
414 : EncoderRtcpFeedback encoder_feedback_;
415 : ProtectionBitrateCalculator protection_bitrate_calculator_;
416 :
417 : const std::unique_ptr<RtcpBandwidthObserver> bandwidth_observer_;
418 : // RtpRtcp modules, declared here as they use other members on construction.
419 : const std::vector<RtpRtcp*> rtp_rtcp_modules_;
420 : PayloadRouter payload_router_;
421 :
422 : // |weak_ptr_| to our self. This is used since we can not call
423 : // |weak_ptr_factory_.GetWeakPtr| from multiple sequences but it is ok to copy
424 : // an existing WeakPtr.
425 : rtc::WeakPtr<VideoSendStreamImpl> weak_ptr_;
426 : // |weak_ptr_factory_| must be declared last to make sure all WeakPtr's are
427 : // invalidated before any other members are destroyed.
428 : rtc::WeakPtrFactory<VideoSendStreamImpl> weak_ptr_factory_;
429 :
430 : rtc::CriticalSection overhead_bytes_per_packet_crit_;
431 : size_t overhead_bytes_per_packet_ GUARDED_BY(overhead_bytes_per_packet_crit_);
432 : size_t transport_overhead_bytes_per_packet_;
433 : };
434 :
435 : // TODO(tommi): See if there's a more elegant way to create a task that creates
436 : // an object on the correct task queue.
437 : class VideoSendStream::ConstructionTask : public rtc::QueuedTask {
438 : public:
439 0 : ConstructionTask(std::unique_ptr<VideoSendStreamImpl>* send_stream,
440 : rtc::Event* done_event,
441 : SendStatisticsProxy* stats_proxy,
442 : ViEEncoder* vie_encoder,
443 : ProcessThread* module_process_thread,
444 : CallStats* call_stats,
445 : CongestionController* congestion_controller,
446 : PacketRouter* packet_router,
447 : BitrateAllocator* bitrate_allocator,
448 : SendDelayStats* send_delay_stats,
449 : VieRemb* remb,
450 : RtcEventLog* event_log,
451 : const VideoSendStream::Config* config,
452 : int initial_encoder_max_bitrate,
453 : const std::map<uint32_t, RtpState>& suspended_ssrcs)
454 0 : : send_stream_(send_stream),
455 : done_event_(done_event),
456 : stats_proxy_(stats_proxy),
457 : vie_encoder_(vie_encoder),
458 : call_stats_(call_stats),
459 : congestion_controller_(congestion_controller),
460 : packet_router_(packet_router),
461 : bitrate_allocator_(bitrate_allocator),
462 : send_delay_stats_(send_delay_stats),
463 : remb_(remb),
464 : event_log_(event_log),
465 : config_(config),
466 : initial_encoder_max_bitrate_(initial_encoder_max_bitrate),
467 0 : suspended_ssrcs_(suspended_ssrcs) {}
468 :
469 0 : ~ConstructionTask() override { done_event_->Set(); }
470 :
471 : private:
472 0 : bool Run() override {
473 0 : send_stream_->reset(new VideoSendStreamImpl(
474 0 : stats_proxy_, rtc::TaskQueue::Current(), call_stats_,
475 0 : congestion_controller_, packet_router_, bitrate_allocator_,
476 0 : send_delay_stats_, remb_, vie_encoder_, event_log_, config_,
477 0 : initial_encoder_max_bitrate_, std::move(suspended_ssrcs_)));
478 0 : return true;
479 : }
480 :
481 : std::unique_ptr<VideoSendStreamImpl>* const send_stream_;
482 : rtc::Event* const done_event_;
483 : SendStatisticsProxy* const stats_proxy_;
484 : ViEEncoder* const vie_encoder_;
485 : CallStats* const call_stats_;
486 : CongestionController* const congestion_controller_;
487 : PacketRouter* const packet_router_;
488 : BitrateAllocator* const bitrate_allocator_;
489 : SendDelayStats* const send_delay_stats_;
490 : VieRemb* const remb_;
491 : RtcEventLog* const event_log_;
492 : const VideoSendStream::Config* config_;
493 : int initial_encoder_max_bitrate_;
494 : std::map<uint32_t, RtpState> suspended_ssrcs_;
495 : };
496 :
497 : class VideoSendStream::DestructAndGetRtpStateTask : public rtc::QueuedTask {
498 : public:
499 0 : DestructAndGetRtpStateTask(VideoSendStream::RtpStateMap* state_map,
500 : std::unique_ptr<VideoSendStreamImpl> send_stream,
501 : rtc::Event* done_event)
502 0 : : state_map_(state_map),
503 0 : send_stream_(std::move(send_stream)),
504 0 : done_event_(done_event) {}
505 :
506 0 : ~DestructAndGetRtpStateTask() override { RTC_CHECK(!send_stream_); }
507 :
508 : private:
509 0 : bool Run() override {
510 0 : send_stream_->Stop();
511 0 : *state_map_ = send_stream_->GetRtpStates();
512 0 : send_stream_.reset();
513 0 : done_event_->Set();
514 0 : return true;
515 : }
516 :
517 : VideoSendStream::RtpStateMap* state_map_;
518 : std::unique_ptr<VideoSendStreamImpl> send_stream_;
519 : rtc::Event* done_event_;
520 : };
521 :
522 : // CheckEncoderActivityTask is used for tracking when the encoder last produced
523 : // and encoded video frame. If the encoder has not produced anything the last
524 : // kEncoderTimeOutMs we also want to stop sending padding.
525 0 : class VideoSendStreamImpl::CheckEncoderActivityTask : public rtc::QueuedTask {
526 : public:
527 : static const int kEncoderTimeOutMs = 2000;
528 0 : explicit CheckEncoderActivityTask(
529 : const rtc::WeakPtr<VideoSendStreamImpl>& send_stream)
530 0 : : activity_(0), send_stream_(std::move(send_stream)), timed_out_(false) {}
531 :
532 0 : void Stop() {
533 0 : RTC_CHECK(task_checker_.CalledSequentially());
534 0 : send_stream_.reset();
535 0 : }
536 :
537 0 : void UpdateEncoderActivity() {
538 : // UpdateEncoderActivity is called from VideoSendStreamImpl::Encoded on
539 : // whatever thread the real encoder implementation run on. In the case of
540 : // hardware encoders, there might be several encoders
541 : // running in parallel on different threads.
542 0 : rtc::AtomicOps::ReleaseStore(&activity_, 1);
543 0 : }
544 :
545 : private:
546 0 : bool Run() override {
547 0 : RTC_CHECK(task_checker_.CalledSequentially());
548 0 : if (!send_stream_)
549 0 : return true;
550 0 : if (!rtc::AtomicOps::AcquireLoad(&activity_)) {
551 0 : if (!timed_out_) {
552 0 : send_stream_->SignalEncoderTimedOut();
553 : }
554 0 : timed_out_ = true;
555 0 : } else if (timed_out_) {
556 0 : send_stream_->SignalEncoderActive();
557 0 : timed_out_ = false;
558 : }
559 0 : rtc::AtomicOps::ReleaseStore(&activity_, 0);
560 :
561 0 : rtc::TaskQueue::Current()->PostDelayedTask(
562 0 : std::unique_ptr<rtc::QueuedTask>(this), kEncoderTimeOutMs);
563 : // Return false to prevent this task from being deleted. Ownership has been
564 : // transferred to the task queue when PostDelayedTask was called.
565 0 : return false;
566 : }
567 : volatile int activity_;
568 :
569 : rtc::SequencedTaskChecker task_checker_;
570 : rtc::WeakPtr<VideoSendStreamImpl> send_stream_;
571 : bool timed_out_;
572 : };
573 :
574 0 : class VideoSendStreamImpl::EncoderReconfiguredTask : public rtc::QueuedTask {
575 : public:
576 0 : EncoderReconfiguredTask(const rtc::WeakPtr<VideoSendStreamImpl>& send_stream,
577 : std::vector<VideoStream> streams,
578 : int min_transmit_bitrate_bps)
579 0 : : send_stream_(std::move(send_stream)),
580 0 : streams_(std::move(streams)),
581 0 : min_transmit_bitrate_bps_(min_transmit_bitrate_bps) {}
582 :
583 : private:
584 0 : bool Run() override {
585 0 : if (send_stream_)
586 0 : send_stream_->OnEncoderConfigurationChanged(std::move(streams_),
587 0 : min_transmit_bitrate_bps_);
588 0 : return true;
589 : }
590 :
591 : rtc::WeakPtr<VideoSendStreamImpl> send_stream_;
592 : std::vector<VideoStream> streams_;
593 : int min_transmit_bitrate_bps_;
594 : };
595 :
596 0 : VideoSendStream::VideoSendStream(
597 : int num_cpu_cores,
598 : ProcessThread* module_process_thread,
599 : rtc::TaskQueue* worker_queue,
600 : CallStats* call_stats,
601 : CongestionController* congestion_controller,
602 : PacketRouter* packet_router,
603 : BitrateAllocator* bitrate_allocator,
604 : SendDelayStats* send_delay_stats,
605 : VieRemb* remb,
606 : RtcEventLog* event_log,
607 : VideoSendStream::Config config,
608 : VideoEncoderConfig encoder_config,
609 0 : const std::map<uint32_t, RtpState>& suspended_ssrcs)
610 : : worker_queue_(worker_queue),
611 : thread_sync_event_(false /* manual_reset */, false),
612 : stats_proxy_(Clock::GetRealTimeClock(),
613 : config,
614 : encoder_config.content_type),
615 0 : config_(std::move(config)) {
616 0 : vie_encoder_.reset(new ViEEncoder(
617 : num_cpu_cores, &stats_proxy_, config_.encoder_settings,
618 0 : config_.pre_encode_callback, config_.post_encode_callback));
619 0 : worker_queue_->PostTask(std::unique_ptr<rtc::QueuedTask>(new ConstructionTask(
620 0 : &send_stream_, &thread_sync_event_, &stats_proxy_, vie_encoder_.get(),
621 : module_process_thread, call_stats, congestion_controller, packet_router,
622 : bitrate_allocator, send_delay_stats, remb, event_log, &config_,
623 0 : encoder_config.max_bitrate_bps, suspended_ssrcs)));
624 :
625 : // Wait for ConstructionTask to complete so that |send_stream_| can be used.
626 : // |module_process_thread| must be registered and deregistered on the thread
627 : // it was created on.
628 0 : thread_sync_event_.Wait(rtc::Event::kForever);
629 0 : send_stream_->RegisterProcessThread(module_process_thread);
630 : // TODO(sprang): Enable this also for regular video calls if it works well.
631 0 : if (encoder_config.content_type == VideoEncoderConfig::ContentType::kScreen) {
632 : // Only signal target bitrate for screenshare streams, for now.
633 0 : vie_encoder_->SetBitrateObserver(send_stream_.get());
634 : }
635 0 : vie_encoder_->RegisterProcessThread(module_process_thread);
636 :
637 0 : ReconfigureVideoEncoder(std::move(encoder_config));
638 0 : }
639 :
640 0 : VideoSendStream::~VideoSendStream() {
641 0 : RTC_DCHECK_RUN_ON(&thread_checker_);
642 0 : RTC_DCHECK(!send_stream_);
643 0 : }
644 :
645 0 : CPULoadStateObserver* VideoSendStream::LoadStateObserver() {
646 0 : return vie_encoder_.get();
647 : }
648 :
649 0 : void VideoSendStream::Start() {
650 0 : RTC_DCHECK_RUN_ON(&thread_checker_);
651 0 : LOG(LS_INFO) << "VideoSendStream::Start";
652 0 : VideoSendStreamImpl* send_stream = send_stream_.get();
653 0 : worker_queue_->PostTask([this, send_stream] {
654 0 : send_stream->Start();
655 0 : thread_sync_event_.Set();
656 0 : });
657 :
658 : // It is expected that after VideoSendStream::Start has been called, incoming
659 : // frames are not dropped in ViEEncoder. To ensure this, Start has to be
660 : // synchronized.
661 0 : thread_sync_event_.Wait(rtc::Event::kForever);
662 0 : }
663 :
664 0 : void VideoSendStream::Stop() {
665 0 : RTC_DCHECK_RUN_ON(&thread_checker_);
666 0 : LOG(LS_INFO) << "VideoSendStream::Stop";
667 0 : VideoSendStreamImpl* send_stream = send_stream_.get();
668 0 : worker_queue_->PostTask([send_stream] { send_stream->Stop(); });
669 0 : }
670 :
671 0 : void VideoSendStream::SetSource(
672 : rtc::VideoSourceInterface<webrtc::VideoFrame>* source,
673 : const DegradationPreference& degradation_preference) {
674 0 : RTC_DCHECK_RUN_ON(&thread_checker_);
675 0 : vie_encoder_->SetSource(source, degradation_preference);
676 0 : }
677 :
678 0 : void VideoSendStream::ReconfigureVideoEncoder(VideoEncoderConfig config) {
679 : // TODO(perkj): Some test cases in VideoSendStreamTest call
680 : // ReconfigureVideoEncoder from the network thread.
681 : // RTC_DCHECK_RUN_ON(&thread_checker_);
682 0 : vie_encoder_->ConfigureEncoder(std::move(config), config_.rtp.max_packet_size,
683 0 : config_.rtp.nack.rtp_history_ms > 0);
684 0 : }
685 :
686 0 : VideoSendStream::Stats VideoSendStream::GetStats() {
687 : // TODO(perkj, solenberg): Some test cases in EndToEndTest call GetStats from
688 : // a network thread. See comment in Call::GetStats().
689 : // RTC_DCHECK_RUN_ON(&thread_checker_);
690 0 : return stats_proxy_.GetStats();
691 : }
692 :
693 0 : void VideoSendStream::SignalNetworkState(NetworkState state) {
694 0 : RTC_DCHECK_RUN_ON(&thread_checker_);
695 0 : VideoSendStreamImpl* send_stream = send_stream_.get();
696 0 : worker_queue_->PostTask(
697 0 : [send_stream, state] { send_stream->SignalNetworkState(state); });
698 0 : }
699 :
700 0 : VideoSendStream::RtpStateMap VideoSendStream::StopPermanentlyAndGetRtpStates() {
701 0 : RTC_DCHECK_RUN_ON(&thread_checker_);
702 0 : vie_encoder_->Stop();
703 0 : vie_encoder_->DeRegisterProcessThread();
704 0 : VideoSendStream::RtpStateMap state_map;
705 0 : send_stream_->DeRegisterProcessThread();
706 0 : worker_queue_->PostTask(
707 0 : std::unique_ptr<rtc::QueuedTask>(new DestructAndGetRtpStateTask(
708 0 : &state_map, std::move(send_stream_), &thread_sync_event_)));
709 0 : thread_sync_event_.Wait(rtc::Event::kForever);
710 0 : return state_map;
711 : }
712 :
713 0 : void VideoSendStream::SetTransportOverhead(
714 : size_t transport_overhead_per_packet) {
715 0 : RTC_DCHECK_RUN_ON(&thread_checker_);
716 0 : VideoSendStreamImpl* send_stream = send_stream_.get();
717 0 : worker_queue_->PostTask([send_stream, transport_overhead_per_packet] {
718 0 : send_stream->SetTransportOverhead(transport_overhead_per_packet);
719 0 : });
720 0 : }
721 :
722 0 : bool VideoSendStream::DeliverRtcp(const uint8_t* packet, size_t length) {
723 : // Called on a network thread.
724 0 : return send_stream_->DeliverRtcp(packet, length);
725 : }
726 :
727 0 : void VideoSendStream::EnableEncodedFrameRecording(
728 : const std::vector<rtc::PlatformFile>& files,
729 : size_t byte_limit) {
730 0 : send_stream_->EnableEncodedFrameRecording(files, byte_limit);
731 0 : }
732 :
733 0 : VideoSendStreamImpl::VideoSendStreamImpl(
734 : SendStatisticsProxy* stats_proxy,
735 : rtc::TaskQueue* worker_queue,
736 : CallStats* call_stats,
737 : CongestionController* congestion_controller,
738 : PacketRouter* packet_router,
739 : BitrateAllocator* bitrate_allocator,
740 : SendDelayStats* send_delay_stats,
741 : VieRemb* remb,
742 : ViEEncoder* vie_encoder,
743 : RtcEventLog* event_log,
744 : const VideoSendStream::Config* config,
745 : int initial_encoder_max_bitrate,
746 0 : std::map<uint32_t, RtpState> suspended_ssrcs)
747 : : stats_proxy_(stats_proxy),
748 : config_(config),
749 0 : suspended_ssrcs_(std::move(suspended_ssrcs)),
750 : module_process_thread_(nullptr),
751 : worker_queue_(worker_queue),
752 : check_encoder_activity_task_(nullptr),
753 : call_stats_(call_stats),
754 : congestion_controller_(congestion_controller),
755 : packet_router_(packet_router),
756 : bitrate_allocator_(bitrate_allocator),
757 : remb_(remb),
758 0 : flexfec_sender_(MaybeCreateFlexfecSender(*config_)),
759 : max_padding_bitrate_(0),
760 : encoder_min_bitrate_bps_(0),
761 : encoder_max_bitrate_bps_(initial_encoder_max_bitrate),
762 : encoder_target_rate_bps_(0),
763 : vie_encoder_(vie_encoder),
764 : encoder_feedback_(Clock::GetRealTimeClock(),
765 0 : config_->rtp.ssrcs,
766 : vie_encoder),
767 : protection_bitrate_calculator_(Clock::GetRealTimeClock(), this),
768 0 : bandwidth_observer_(congestion_controller_->GetBitrateController()
769 0 : ->CreateRtcpBandwidthObserver()),
770 : rtp_rtcp_modules_(CreateRtpRtcpModules(
771 0 : config_->send_transport,
772 : &encoder_feedback_,
773 : bandwidth_observer_.get(),
774 0 : congestion_controller_->GetTransportFeedbackObserver(),
775 0 : call_stats_->rtcp_rtt_stats(),
776 0 : congestion_controller_->pacer(),
777 0 : packet_router_,
778 : flexfec_sender_.get(),
779 0 : stats_proxy_,
780 : send_delay_stats,
781 : event_log,
782 0 : congestion_controller_->GetRetransmissionRateLimiter(),
783 : this,
784 0 : config_->rtp.ssrcs.size())),
785 : payload_router_(rtp_rtcp_modules_,
786 0 : config_->encoder_settings.payload_type),
787 : weak_ptr_factory_(this),
788 : overhead_bytes_per_packet_(0),
789 0 : transport_overhead_bytes_per_packet_(0) {
790 0 : RTC_DCHECK_RUN_ON(worker_queue_);
791 0 : LOG(LS_INFO) << "VideoSendStreamInternal: " << config_->ToString();
792 0 : weak_ptr_ = weak_ptr_factory_.GetWeakPtr();
793 0 : module_process_thread_checker_.DetachFromThread();
794 :
795 0 : RTC_DCHECK(!config_->rtp.ssrcs.empty());
796 0 : RTC_DCHECK(call_stats_);
797 0 : RTC_DCHECK(congestion_controller_);
798 0 : RTC_DCHECK(remb_);
799 :
800 0 : congestion_controller_->EnablePeriodicAlrProbing(
801 0 : config_->periodic_alr_bandwidth_probing);
802 :
803 : // RTP/RTCP initialization.
804 0 : for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
805 0 : packet_router_->AddRtpModule(rtp_rtcp);
806 : }
807 :
808 0 : for (size_t i = 0; i < config_->rtp.extensions.size(); ++i) {
809 0 : const std::string& extension = config_->rtp.extensions[i].uri;
810 0 : int id = config_->rtp.extensions[i].id;
811 : // One-byte-extension local identifiers are in the range 1-14 inclusive.
812 0 : RTC_DCHECK_GE(id, 1);
813 0 : RTC_DCHECK_LE(id, 14);
814 0 : RTC_DCHECK(RtpExtension::IsSupportedForVideo(extension));
815 0 : for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
816 0 : RTC_CHECK_EQ(0, rtp_rtcp->RegisterSendRtpHeaderExtension(
817 0 : StringToRtpExtensionType(extension), id));
818 : }
819 : }
820 :
821 0 : remb_->AddRembSender(rtp_rtcp_modules_[0]);
822 0 : rtp_rtcp_modules_[0]->SetREMBStatus(true);
823 :
824 0 : ConfigureProtection();
825 0 : ConfigureSsrcs();
826 :
827 : // TODO(pbos): Should we set CNAME on all RTP modules?
828 0 : rtp_rtcp_modules_.front()->SetCNAME(config_->rtp.c_name.c_str());
829 :
830 0 : for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
831 0 : rtp_rtcp->RegisterRtcpStatisticsCallback(stats_proxy_);
832 0 : rtp_rtcp->RegisterSendChannelRtpStatisticsCallback(stats_proxy_);
833 0 : rtp_rtcp->SetMaxRtpPacketSize(config_->rtp.max_packet_size);
834 0 : rtp_rtcp->RegisterVideoSendPayload(
835 0 : config_->encoder_settings.payload_type,
836 0 : config_->encoder_settings.payload_name.c_str());
837 : }
838 :
839 0 : RTC_DCHECK(config_->encoder_settings.encoder);
840 0 : RTC_DCHECK_GE(config_->encoder_settings.payload_type, 0);
841 0 : RTC_DCHECK_LE(config_->encoder_settings.payload_type, 127);
842 :
843 0 : vie_encoder_->SetStartBitrate(bitrate_allocator_->GetStartBitrate(this));
844 :
845 : // Only request rotation at the source when we positively know that the remote
846 : // side doesn't support the rotation extension. This allows us to prepare the
847 : // encoder in the expectation that rotation is supported - which is the common
848 : // case.
849 : bool rotation_applied =
850 0 : std::find_if(config_->rtp.extensions.begin(),
851 0 : config_->rtp.extensions.end(),
852 0 : [](const RtpExtension& extension) {
853 0 : return extension.uri == RtpExtension::kVideoRotationUri;
854 0 : }) == config_->rtp.extensions.end();
855 :
856 0 : vie_encoder_->SetSink(this, rotation_applied);
857 0 : }
858 :
859 0 : void VideoSendStreamImpl::RegisterProcessThread(
860 : ProcessThread* module_process_thread) {
861 0 : RTC_DCHECK_RUN_ON(&module_process_thread_checker_);
862 0 : RTC_DCHECK(!module_process_thread_);
863 0 : module_process_thread_ = module_process_thread;
864 :
865 0 : for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_)
866 0 : module_process_thread_->RegisterModule(rtp_rtcp);
867 0 : }
868 :
869 0 : void VideoSendStreamImpl::DeRegisterProcessThread() {
870 0 : RTC_DCHECK_RUN_ON(&module_process_thread_checker_);
871 0 : for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_)
872 0 : module_process_thread_->DeRegisterModule(rtp_rtcp);
873 0 : }
874 :
875 0 : VideoSendStreamImpl::~VideoSendStreamImpl() {
876 0 : RTC_DCHECK_RUN_ON(worker_queue_);
877 0 : RTC_DCHECK(!payload_router_.IsActive())
878 0 : << "VideoSendStreamImpl::Stop not called";
879 0 : LOG(LS_INFO) << "~VideoSendStreamInternal: " << config_->ToString();
880 :
881 0 : rtp_rtcp_modules_[0]->SetREMBStatus(false);
882 0 : remb_->RemoveRembSender(rtp_rtcp_modules_[0]);
883 :
884 0 : for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
885 0 : packet_router_->RemoveRtpModule(rtp_rtcp);
886 0 : delete rtp_rtcp;
887 : }
888 0 : }
889 :
890 0 : bool VideoSendStreamImpl::DeliverRtcp(const uint8_t* packet, size_t length) {
891 : // Runs on a network thread.
892 0 : RTC_DCHECK(!worker_queue_->IsCurrent());
893 0 : for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_)
894 0 : rtp_rtcp->IncomingRtcpPacket(packet, length);
895 0 : return true;
896 : }
897 :
898 0 : void VideoSendStreamImpl::Start() {
899 0 : RTC_DCHECK_RUN_ON(worker_queue_);
900 0 : LOG(LS_INFO) << "VideoSendStream::Start";
901 0 : if (payload_router_.IsActive())
902 0 : return;
903 0 : TRACE_EVENT_INSTANT0("webrtc", "VideoSendStream::Start");
904 0 : payload_router_.SetActive(true);
905 :
906 0 : bitrate_allocator_->AddObserver(
907 0 : this, encoder_min_bitrate_bps_, encoder_max_bitrate_bps_,
908 0 : max_padding_bitrate_, !config_->suspend_below_min_bitrate);
909 :
910 : // Start monitoring encoder activity.
911 : {
912 0 : rtc::CritScope lock(&encoder_activity_crit_sect_);
913 0 : RTC_DCHECK(!check_encoder_activity_task_);
914 0 : check_encoder_activity_task_ = new CheckEncoderActivityTask(weak_ptr_);
915 0 : worker_queue_->PostDelayedTask(
916 0 : std::unique_ptr<rtc::QueuedTask>(check_encoder_activity_task_),
917 0 : CheckEncoderActivityTask::kEncoderTimeOutMs);
918 : }
919 :
920 0 : vie_encoder_->SendKeyFrame();
921 : }
922 :
923 0 : void VideoSendStreamImpl::Stop() {
924 0 : RTC_DCHECK_RUN_ON(worker_queue_);
925 0 : LOG(LS_INFO) << "VideoSendStream::Stop";
926 0 : if (!payload_router_.IsActive())
927 0 : return;
928 0 : TRACE_EVENT_INSTANT0("webrtc", "VideoSendStream::Stop");
929 0 : payload_router_.SetActive(false);
930 0 : bitrate_allocator_->RemoveObserver(this);
931 : {
932 0 : rtc::CritScope lock(&encoder_activity_crit_sect_);
933 0 : check_encoder_activity_task_->Stop();
934 0 : check_encoder_activity_task_ = nullptr;
935 : }
936 0 : vie_encoder_->OnBitrateUpdated(0, 0, 0);
937 0 : stats_proxy_->OnSetEncoderTargetRate(0);
938 : }
939 :
940 0 : void VideoSendStreamImpl::SignalEncoderTimedOut() {
941 0 : RTC_DCHECK_RUN_ON(worker_queue_);
942 : // If the encoder has not produced anything the last kEncoderTimeOutMs and it
943 : // is supposed to, deregister as BitrateAllocatorObserver. This can happen
944 : // if a camera stops producing frames.
945 0 : if (encoder_target_rate_bps_ > 0) {
946 0 : LOG(LS_INFO) << "SignalEncoderTimedOut, Encoder timed out.";
947 0 : bitrate_allocator_->RemoveObserver(this);
948 : }
949 0 : }
950 :
951 0 : void VideoSendStreamImpl::OnBitrateAllocationUpdated(
952 : const BitrateAllocation& allocation) {
953 0 : payload_router_.OnBitrateAllocationUpdated(allocation);
954 0 : }
955 :
956 0 : void VideoSendStreamImpl::SignalEncoderActive() {
957 0 : RTC_DCHECK_RUN_ON(worker_queue_);
958 0 : LOG(LS_INFO) << "SignalEncoderActive, Encoder is active.";
959 0 : bitrate_allocator_->AddObserver(
960 0 : this, encoder_min_bitrate_bps_, encoder_max_bitrate_bps_,
961 0 : max_padding_bitrate_, !config_->suspend_below_min_bitrate);
962 0 : }
963 :
964 0 : void VideoSendStreamImpl::OnEncoderConfigurationChanged(
965 : std::vector<VideoStream> streams,
966 : int min_transmit_bitrate_bps) {
967 0 : if (!worker_queue_->IsCurrent()) {
968 0 : worker_queue_->PostTask(
969 0 : std::unique_ptr<rtc::QueuedTask>(new EncoderReconfiguredTask(
970 0 : weak_ptr_, std::move(streams), min_transmit_bitrate_bps)));
971 0 : return;
972 : }
973 0 : RTC_DCHECK_GE(config_->rtp.ssrcs.size(), streams.size());
974 0 : TRACE_EVENT0("webrtc", "VideoSendStream::OnEncoderConfigurationChanged");
975 0 : RTC_DCHECK_GE(config_->rtp.ssrcs.size(), streams.size());
976 0 : RTC_DCHECK_RUN_ON(worker_queue_);
977 :
978 0 : const int kEncoderMinBitrateBps = 30000;
979 0 : encoder_min_bitrate_bps_ =
980 0 : std::max(streams[0].min_bitrate_bps, kEncoderMinBitrateBps);
981 0 : encoder_max_bitrate_bps_ = 0;
982 0 : for (const auto& stream : streams)
983 0 : encoder_max_bitrate_bps_ += stream.max_bitrate_bps;
984 0 : max_padding_bitrate_ = CalculateMaxPadBitrateBps(
985 0 : streams, min_transmit_bitrate_bps, config_->suspend_below_min_bitrate);
986 :
987 : // Clear stats for disabled layers.
988 0 : for (size_t i = streams.size(); i < config_->rtp.ssrcs.size(); ++i) {
989 0 : stats_proxy_->OnInactiveSsrc(config_->rtp.ssrcs[i]);
990 : }
991 :
992 : size_t number_of_temporal_layers =
993 0 : streams.back().temporal_layer_thresholds_bps.size() + 1;
994 0 : protection_bitrate_calculator_.SetEncodingData(
995 0 : streams[0].width, streams[0].height, number_of_temporal_layers,
996 0 : config_->rtp.max_packet_size);
997 :
998 0 : if (payload_router_.IsActive()) {
999 : // The send stream is started already. Update the allocator with new bitrate
1000 : // limits.
1001 0 : bitrate_allocator_->AddObserver(
1002 0 : this, encoder_min_bitrate_bps_, encoder_max_bitrate_bps_,
1003 0 : max_padding_bitrate_, !config_->suspend_below_min_bitrate);
1004 : }
1005 : }
1006 :
1007 0 : EncodedImageCallback::Result VideoSendStreamImpl::OnEncodedImage(
1008 : const EncodedImage& encoded_image,
1009 : const CodecSpecificInfo* codec_specific_info,
1010 : const RTPFragmentationHeader* fragmentation) {
1011 : // Encoded is called on whatever thread the real encoder implementation run
1012 : // on. In the case of hardware encoders, there might be several encoders
1013 : // running in parallel on different threads.
1014 0 : if (config_->post_encode_callback) {
1015 0 : config_->post_encode_callback->EncodedFrameCallback(
1016 0 : EncodedFrame(encoded_image._buffer, encoded_image._length,
1017 0 : encoded_image._frameType));
1018 : }
1019 : {
1020 0 : rtc::CritScope lock(&encoder_activity_crit_sect_);
1021 0 : if (check_encoder_activity_task_)
1022 0 : check_encoder_activity_task_->UpdateEncoderActivity();
1023 : }
1024 :
1025 0 : protection_bitrate_calculator_.UpdateWithEncodedData(encoded_image);
1026 : EncodedImageCallback::Result result = payload_router_.OnEncodedImage(
1027 0 : encoded_image, codec_specific_info, fragmentation);
1028 :
1029 0 : RTC_DCHECK(codec_specific_info);
1030 :
1031 0 : int layer = codec_specific_info->codecType == kVideoCodecVP8
1032 0 : ? codec_specific_info->codecSpecific.VP8.simulcastIdx
1033 0 : : 0;
1034 : {
1035 0 : rtc::CritScope lock(&ivf_writers_crit_);
1036 0 : if (file_writers_[layer].get()) {
1037 0 : bool ok = file_writers_[layer]->WriteFrame(
1038 0 : encoded_image, codec_specific_info->codecType);
1039 0 : RTC_DCHECK(ok);
1040 : }
1041 : }
1042 :
1043 0 : return result;
1044 : }
1045 :
1046 0 : void VideoSendStreamImpl::ConfigureProtection() {
1047 0 : RTC_DCHECK_RUN_ON(worker_queue_);
1048 :
1049 : // Consistency of FlexFEC parameters is checked in MaybeCreateFlexfecSender.
1050 0 : const bool flexfec_enabled = (flexfec_sender_ != nullptr);
1051 :
1052 : // Consistency of NACK and RED+ULPFEC parameters is checked in this function.
1053 0 : const bool nack_enabled = config_->rtp.nack.rtp_history_ms > 0;
1054 0 : int red_payload_type = config_->rtp.ulpfec.red_payload_type;
1055 0 : int ulpfec_payload_type = config_->rtp.ulpfec.ulpfec_payload_type;
1056 :
1057 : // Shorthands.
1058 0 : auto IsRedEnabled = [&]() { return red_payload_type >= 0; };
1059 0 : auto DisableRed = [&]() { red_payload_type = -1; };
1060 0 : auto IsUlpfecEnabled = [&]() { return ulpfec_payload_type >= 0; };
1061 0 : auto DisableUlpfec = [&]() { ulpfec_payload_type = -1; };
1062 :
1063 : // If enabled, FlexFEC takes priority over RED+ULPFEC.
1064 0 : if (flexfec_enabled) {
1065 : // We can safely disable RED here, because if the remote supports FlexFEC,
1066 : // we know that it has a receiver without the RED/RTX workaround.
1067 : // See http://crbug.com/webrtc/6650 for more information.
1068 0 : if (IsRedEnabled()) {
1069 0 : LOG(LS_INFO) << "Both FlexFEC and RED are configured. Disabling RED.";
1070 0 : DisableRed();
1071 : }
1072 0 : if (IsUlpfecEnabled()) {
1073 0 : LOG(LS_INFO)
1074 0 : << "Both FlexFEC and ULPFEC are configured. Disabling ULPFEC.";
1075 0 : DisableUlpfec();
1076 : }
1077 : }
1078 :
1079 : // Payload types without picture ID cannot determine that a stream is complete
1080 : // without retransmitting FEC, so using ULPFEC + NACK for H.264 (for instance)
1081 : // is a waste of bandwidth since FEC packets still have to be transmitted.
1082 : // Note that this is not the case with FlexFEC.
1083 0 : if (nack_enabled && IsUlpfecEnabled() &&
1084 0 : !PayloadTypeSupportsSkippingFecPackets(
1085 0 : config_->encoder_settings.payload_name)) {
1086 0 : LOG(LS_WARNING)
1087 : << "Transmitting payload type without picture ID using "
1088 : "NACK+ULPFEC is a waste of bandwidth since ULPFEC packets "
1089 0 : "also have to be retransmitted. Disabling ULPFEC.";
1090 0 : DisableUlpfec();
1091 : }
1092 :
1093 : // Verify payload types.
1094 : //
1095 : // Due to how old receivers work, we need to always send RED if it has been
1096 : // negotiated. This is a remnant of an old RED/RTX workaround, see
1097 : // https://codereview.webrtc.org/2469093003.
1098 : // TODO(brandtr): This change went into M56, so we can remove it in ~M59.
1099 : // At that time, we can disable RED whenever ULPFEC is disabled, as there is
1100 : // no point in using RED without ULPFEC.
1101 0 : if (IsRedEnabled()) {
1102 0 : RTC_DCHECK_GE(red_payload_type, 0);
1103 0 : RTC_DCHECK_LE(red_payload_type, 127);
1104 : }
1105 0 : if (IsUlpfecEnabled()) {
1106 0 : RTC_DCHECK_GE(ulpfec_payload_type, 0);
1107 0 : RTC_DCHECK_LE(ulpfec_payload_type, 127);
1108 0 : if (!IsRedEnabled()) {
1109 0 : LOG(LS_WARNING)
1110 0 : << "ULPFEC is enabled but RED is disabled. Disabling ULPFEC.";
1111 0 : DisableUlpfec();
1112 : }
1113 : }
1114 :
1115 0 : for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
1116 : // Set NACK.
1117 : rtp_rtcp->SetStorePacketsStatus(
1118 : true,
1119 0 : kMinSendSidePacketHistorySize);
1120 : // Set RED/ULPFEC information.
1121 0 : for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
1122 0 : rtp_rtcp->SetUlpfecConfig(red_payload_type, ulpfec_payload_type);
1123 : }
1124 : }
1125 :
1126 : // Currently, both ULPFEC and FlexFEC use the same FEC rate calculation logic,
1127 : // so enable that logic if either of those FEC schemes are enabled.
1128 0 : protection_bitrate_calculator_.SetProtectionMethod(
1129 0 : flexfec_enabled || IsUlpfecEnabled(), nack_enabled);
1130 0 : }
1131 :
1132 0 : void VideoSendStreamImpl::ConfigureSsrcs() {
1133 0 : RTC_DCHECK_RUN_ON(worker_queue_);
1134 : // Configure regular SSRCs.
1135 0 : bool has_rids = false;
1136 0 : if (config_->rtp.rids.size() != 0) {
1137 0 : has_rids = true;
1138 : // if we have rids, we must have a rid entry for every ssrc (even if it's "")
1139 0 : RTC_DCHECK(config_->rtp.rids.size() == config_->rtp.ssrcs.size());
1140 : }
1141 0 : for (size_t i = 0; i < config_->rtp.ssrcs.size(); ++i) {
1142 0 : uint32_t ssrc = config_->rtp.ssrcs[i];
1143 0 : RtpRtcp* const rtp_rtcp = rtp_rtcp_modules_[i];
1144 0 : rtp_rtcp->SetSSRC(ssrc);
1145 0 : if (has_rids && config_->rtp.rids[i] != "") {
1146 0 : rtp_rtcp->SetRID(config_->rtp.rids[i].c_str());
1147 : }
1148 :
1149 : // Restore RTP state if previous existed.
1150 0 : VideoSendStream::RtpStateMap::iterator it = suspended_ssrcs_.find(ssrc);
1151 0 : if (it != suspended_ssrcs_.end())
1152 0 : rtp_rtcp->SetRtpState(it->second);
1153 : }
1154 :
1155 : // Set up RTX if available.
1156 0 : if (config_->rtp.rtx.ssrcs.empty())
1157 0 : return;
1158 :
1159 : // Configure RTX SSRCs.
1160 0 : RTC_DCHECK_EQ(config_->rtp.rtx.ssrcs.size(), config_->rtp.ssrcs.size());
1161 0 : for (size_t i = 0; i < config_->rtp.rtx.ssrcs.size(); ++i) {
1162 0 : uint32_t ssrc = config_->rtp.rtx.ssrcs[i];
1163 0 : RtpRtcp* const rtp_rtcp = rtp_rtcp_modules_[i];
1164 0 : rtp_rtcp->SetRtxSsrc(ssrc);
1165 0 : VideoSendStream::RtpStateMap::iterator it = suspended_ssrcs_.find(ssrc);
1166 0 : if (it != suspended_ssrcs_.end())
1167 0 : rtp_rtcp->SetRtxState(it->second);
1168 : }
1169 :
1170 : // Configure RTX payload types.
1171 0 : RTC_DCHECK_GE(config_->rtp.rtx.payload_type, 0);
1172 0 : for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
1173 0 : rtp_rtcp->SetRtxSendPayloadType(config_->rtp.rtx.payload_type,
1174 0 : config_->encoder_settings.payload_type);
1175 0 : rtp_rtcp->SetRtxSendStatus(kRtxRetransmitted | kRtxRedundantPayloads);
1176 : }
1177 0 : if (config_->rtp.ulpfec.red_payload_type != -1 &&
1178 0 : config_->rtp.ulpfec.red_rtx_payload_type != -1) {
1179 0 : for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
1180 0 : rtp_rtcp->SetRtxSendPayloadType(config_->rtp.ulpfec.red_rtx_payload_type,
1181 0 : config_->rtp.ulpfec.red_payload_type);
1182 : }
1183 : }
1184 : }
1185 :
1186 0 : std::map<uint32_t, RtpState> VideoSendStreamImpl::GetRtpStates() const {
1187 0 : RTC_DCHECK_RUN_ON(worker_queue_);
1188 0 : std::map<uint32_t, RtpState> rtp_states;
1189 0 : for (size_t i = 0; i < config_->rtp.ssrcs.size(); ++i) {
1190 0 : uint32_t ssrc = config_->rtp.ssrcs[i];
1191 0 : RTC_DCHECK_EQ(ssrc, rtp_rtcp_modules_[i]->SSRC());
1192 0 : rtp_states[ssrc] = rtp_rtcp_modules_[i]->GetRtpState();
1193 : }
1194 :
1195 0 : for (size_t i = 0; i < config_->rtp.rtx.ssrcs.size(); ++i) {
1196 0 : uint32_t ssrc = config_->rtp.rtx.ssrcs[i];
1197 0 : rtp_states[ssrc] = rtp_rtcp_modules_[i]->GetRtxState();
1198 : }
1199 :
1200 0 : return rtp_states;
1201 : }
1202 :
1203 0 : void VideoSendStreamImpl::SignalNetworkState(NetworkState state) {
1204 0 : RTC_DCHECK_RUN_ON(worker_queue_);
1205 0 : for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
1206 0 : rtp_rtcp->SetRTCPStatus(state == kNetworkUp ? config_->rtp.rtcp_mode
1207 0 : : RtcpMode::kOff);
1208 : }
1209 0 : }
1210 :
1211 0 : uint32_t VideoSendStreamImpl::OnBitrateUpdated(uint32_t bitrate_bps,
1212 : uint8_t fraction_loss,
1213 : int64_t rtt,
1214 : int64_t probing_interval_ms) {
1215 0 : RTC_DCHECK_RUN_ON(worker_queue_);
1216 0 : RTC_DCHECK(payload_router_.IsActive())
1217 0 : << "VideoSendStream::Start has not been called.";
1218 :
1219 0 : if (webrtc::field_trial::FindFullName("WebRTC-SendSideBwe-WithOverhead") ==
1220 : "Enabled") {
1221 : // Subtract total overhead (transport + rtp) from bitrate.
1222 : size_t rtp_overhead;
1223 : {
1224 0 : rtc::CritScope lock(&overhead_bytes_per_packet_crit_);
1225 0 : rtp_overhead = overhead_bytes_per_packet_;
1226 : }
1227 0 : RTC_CHECK_GE(rtp_overhead, 0);
1228 0 : RTC_DCHECK_LT(rtp_overhead, config_->rtp.max_packet_size);
1229 0 : if (rtp_overhead >= config_->rtp.max_packet_size) {
1230 0 : LOG(LS_WARNING) << "RTP overhead (" << rtp_overhead << " bytes)"
1231 0 : << "exceeds maximum packet size ("
1232 0 : << config_->rtp.max_packet_size << " bytes)";
1233 :
1234 0 : bitrate_bps = 0;
1235 : } else {
1236 0 : bitrate_bps =
1237 0 : static_cast<uint32_t>(static_cast<uint64_t>(bitrate_bps) *
1238 0 : (config_->rtp.max_packet_size - rtp_overhead) /
1239 0 : (config_->rtp.max_packet_size +
1240 0 : transport_overhead_bytes_per_packet_));
1241 : }
1242 : }
1243 :
1244 : // Get the encoder target rate. It is the estimated network rate -
1245 : // protection overhead.
1246 0 : encoder_target_rate_bps_ = protection_bitrate_calculator_.SetTargetRates(
1247 0 : bitrate_bps, stats_proxy_->GetSendFrameRate(), fraction_loss, rtt);
1248 0 : uint32_t protection_bitrate = bitrate_bps - encoder_target_rate_bps_;
1249 :
1250 0 : encoder_target_rate_bps_ =
1251 0 : std::min(encoder_max_bitrate_bps_, encoder_target_rate_bps_);
1252 0 : vie_encoder_->OnBitrateUpdated(encoder_target_rate_bps_, fraction_loss, rtt);
1253 0 : stats_proxy_->OnSetEncoderTargetRate(encoder_target_rate_bps_);
1254 0 : return protection_bitrate;
1255 : }
1256 :
1257 0 : void VideoSendStreamImpl::EnableEncodedFrameRecording(
1258 : const std::vector<rtc::PlatformFile>& files,
1259 : size_t byte_limit) {
1260 : {
1261 0 : rtc::CritScope lock(&ivf_writers_crit_);
1262 0 : for (unsigned int i = 0; i < kMaxSimulcastStreams; ++i) {
1263 0 : if (i < files.size()) {
1264 0 : file_writers_[i] = IvfFileWriter::Wrap(rtc::File(files[i]), byte_limit);
1265 : } else {
1266 0 : file_writers_[i].reset();
1267 : }
1268 : }
1269 : }
1270 :
1271 0 : if (!files.empty()) {
1272 : // Make a keyframe appear as early as possible in the logs, to give actually
1273 : // decodable output.
1274 0 : vie_encoder_->SendKeyFrame();
1275 : }
1276 0 : }
1277 :
1278 0 : int VideoSendStreamImpl::ProtectionRequest(
1279 : const FecProtectionParams* delta_params,
1280 : const FecProtectionParams* key_params,
1281 : uint32_t* sent_video_rate_bps,
1282 : uint32_t* sent_nack_rate_bps,
1283 : uint32_t* sent_fec_rate_bps) {
1284 0 : RTC_DCHECK_RUN_ON(worker_queue_);
1285 0 : *sent_video_rate_bps = 0;
1286 0 : *sent_nack_rate_bps = 0;
1287 0 : *sent_fec_rate_bps = 0;
1288 0 : for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
1289 0 : uint32_t not_used = 0;
1290 0 : uint32_t module_video_rate = 0;
1291 0 : uint32_t module_fec_rate = 0;
1292 0 : uint32_t module_nack_rate = 0;
1293 0 : rtp_rtcp->SetFecParameters(*delta_params, *key_params);
1294 : rtp_rtcp->BitrateSent(¬_used, &module_video_rate, &module_fec_rate,
1295 0 : &module_nack_rate);
1296 0 : *sent_video_rate_bps += module_video_rate;
1297 0 : *sent_nack_rate_bps += module_nack_rate;
1298 0 : *sent_fec_rate_bps += module_fec_rate;
1299 : }
1300 0 : return 0;
1301 : }
1302 :
1303 0 : void VideoSendStreamImpl::OnOverheadChanged(size_t overhead_bytes_per_packet) {
1304 0 : rtc::CritScope lock(&overhead_bytes_per_packet_crit_);
1305 0 : overhead_bytes_per_packet_ = overhead_bytes_per_packet;
1306 0 : }
1307 :
1308 0 : void VideoSendStreamImpl::SetTransportOverhead(
1309 : size_t transport_overhead_bytes_per_packet) {
1310 0 : if (transport_overhead_bytes_per_packet >= static_cast<int>(kPathMTU)) {
1311 0 : LOG(LS_ERROR) << "Transport overhead exceeds size of ethernet frame";
1312 0 : return;
1313 : }
1314 :
1315 0 : transport_overhead_bytes_per_packet_ = transport_overhead_bytes_per_packet;
1316 :
1317 0 : congestion_controller_->SetTransportOverhead(
1318 0 : transport_overhead_bytes_per_packet_);
1319 :
1320 : size_t rtp_packet_size =
1321 0 : std::min(config_->rtp.max_packet_size,
1322 0 : kPathMTU - transport_overhead_bytes_per_packet_);
1323 :
1324 0 : for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
1325 0 : rtp_rtcp->SetMaxRtpPacketSize(rtp_packet_size);
1326 : }
1327 : }
1328 :
1329 : } // namespace internal
1330 : } // namespace webrtc
|
