Line data Source code
1 : /*
2 : * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 :
11 : #include "webrtc/modules/rtp_rtcp/source/rtp_sender_video.h"
12 :
13 : #include <stdlib.h>
14 : #include <string.h>
15 :
16 : #include <memory>
17 : #include <vector>
18 : #include <utility>
19 :
20 : #include "webrtc/common_types.h"
21 : #include "webrtc/base/checks.h"
22 : #include "webrtc/base/logging.h"
23 : #include "webrtc/base/trace_event.h"
24 : #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h"
25 : #include "webrtc/modules/rtp_rtcp/source/byte_io.h"
26 : #include "webrtc/modules/rtp_rtcp/source/rtp_format_video_generic.h"
27 : #include "webrtc/modules/rtp_rtcp/source/rtp_format_vp8.h"
28 : #include "webrtc/modules/rtp_rtcp/source/rtp_format_vp9.h"
29 : #include "webrtc/modules/rtp_rtcp/source/rtp_header_extensions.h"
30 : #include "webrtc/modules/rtp_rtcp/source/rtp_packet_to_send.h"
31 :
32 : namespace webrtc {
33 :
34 : namespace {
35 : constexpr size_t kRedForFecHeaderLength = 1;
36 :
37 0 : void BuildRedPayload(const RtpPacketToSend& media_packet,
38 : RtpPacketToSend* red_packet) {
39 0 : uint8_t* red_payload = red_packet->AllocatePayload(
40 0 : kRedForFecHeaderLength + media_packet.payload_size());
41 0 : RTC_DCHECK(red_payload);
42 0 : red_payload[0] = media_packet.PayloadType();
43 :
44 0 : auto media_payload = media_packet.payload();
45 0 : memcpy(&red_payload[kRedForFecHeaderLength], media_payload.data(),
46 0 : media_payload.size());
47 0 : }
48 : } // namespace
49 :
50 0 : RTPSenderVideo::RTPSenderVideo(Clock* clock,
51 : RTPSender* rtp_sender,
52 0 : FlexfecSender* flexfec_sender)
53 : : rtp_sender_(rtp_sender),
54 : clock_(clock),
55 : video_type_(kRtpVideoGeneric),
56 : retransmission_settings_(kRetransmitBaseLayer),
57 : last_rotation_(kVideoRotation_0),
58 : red_payload_type_(-1),
59 : ulpfec_payload_type_(-1),
60 : flexfec_sender_(flexfec_sender),
61 : delta_fec_params_{0, 1, kFecMaskRandom},
62 : key_fec_params_{0, 1, kFecMaskRandom},
63 : fec_bitrate_(1000, RateStatistics::kBpsScale),
64 0 : video_bitrate_(1000, RateStatistics::kBpsScale) {}
65 :
66 0 : RTPSenderVideo::~RTPSenderVideo() {}
67 :
68 0 : void RTPSenderVideo::SetVideoCodecType(RtpVideoCodecTypes video_type) {
69 0 : video_type_ = video_type;
70 0 : }
71 :
72 0 : RtpVideoCodecTypes RTPSenderVideo::VideoCodecType() const {
73 0 : return video_type_;
74 : }
75 :
76 : // Static.
77 0 : RtpUtility::Payload* RTPSenderVideo::CreateVideoPayload(
78 : const char payload_name[RTP_PAYLOAD_NAME_SIZE],
79 : int8_t payload_type) {
80 0 : RtpVideoCodecTypes video_type = kRtpVideoGeneric;
81 0 : if (RtpUtility::StringCompare(payload_name, "VP8", 3)) {
82 0 : video_type = kRtpVideoVp8;
83 0 : } else if (RtpUtility::StringCompare(payload_name, "VP9", 3)) {
84 0 : video_type = kRtpVideoVp9;
85 0 : } else if (RtpUtility::StringCompare(payload_name, "H264", 4)) {
86 0 : video_type = kRtpVideoH264;
87 0 : } else if (RtpUtility::StringCompare(payload_name, "I420", 4)) {
88 0 : video_type = kRtpVideoGeneric;
89 : } else {
90 0 : video_type = kRtpVideoGeneric;
91 : }
92 0 : RtpUtility::Payload* payload = new RtpUtility::Payload();
93 0 : payload->name[RTP_PAYLOAD_NAME_SIZE - 1] = 0;
94 0 : strncpy(payload->name, payload_name, RTP_PAYLOAD_NAME_SIZE - 1);
95 0 : payload->typeSpecific.Video.videoCodecType = video_type;
96 0 : payload->audio = false;
97 0 : return payload;
98 : }
99 :
100 0 : void RTPSenderVideo::SendVideoPacket(std::unique_ptr<RtpPacketToSend> packet,
101 : StorageType storage) {
102 : // Remember some values about the packet before sending it away.
103 0 : size_t packet_size = packet->size();
104 0 : uint16_t seq_num = packet->SequenceNumber();
105 0 : uint32_t rtp_timestamp = packet->Timestamp();
106 0 : if (!rtp_sender_->SendToNetwork(std::move(packet), storage,
107 : RtpPacketSender::kLowPriority)) {
108 0 : LOG(LS_WARNING) << "Failed to send video packet " << seq_num;
109 0 : return;
110 : }
111 0 : rtc::CritScope cs(&stats_crit_);
112 0 : video_bitrate_.Update(packet_size, clock_->TimeInMilliseconds());
113 0 : TRACE_EVENT_INSTANT2(TRACE_DISABLED_BY_DEFAULT("webrtc_rtp"),
114 : "Video::PacketNormal", "timestamp", rtp_timestamp,
115 : "seqnum", seq_num);
116 : }
117 :
118 0 : void RTPSenderVideo::SendVideoPacketAsRedMaybeWithUlpfec(
119 : std::unique_ptr<RtpPacketToSend> media_packet,
120 : StorageType media_packet_storage,
121 : bool protect_media_packet) {
122 0 : uint32_t rtp_timestamp = media_packet->Timestamp();
123 0 : uint16_t media_seq_num = media_packet->SequenceNumber();
124 :
125 : std::unique_ptr<RtpPacketToSend> red_packet(
126 0 : new RtpPacketToSend(*media_packet));
127 0 : BuildRedPayload(*media_packet, red_packet.get());
128 :
129 0 : std::vector<std::unique_ptr<RedPacket>> fec_packets;
130 0 : StorageType fec_storage = kDontRetransmit;
131 : {
132 : // Only protect while creating RED and FEC packets, not when sending.
133 0 : rtc::CritScope cs(&crit_);
134 0 : red_packet->SetPayloadType(red_payload_type_);
135 0 : if (ulpfec_enabled()) {
136 0 : if (protect_media_packet) {
137 0 : ulpfec_generator_.AddRtpPacketAndGenerateFec(
138 0 : media_packet->data(), media_packet->payload_size(),
139 0 : media_packet->headers_size());
140 : }
141 0 : uint16_t num_fec_packets = ulpfec_generator_.NumAvailableFecPackets();
142 0 : if (num_fec_packets > 0) {
143 : uint16_t first_fec_sequence_number =
144 0 : rtp_sender_->AllocateSequenceNumber(num_fec_packets);
145 0 : fec_packets = ulpfec_generator_.GetUlpfecPacketsAsRed(
146 : red_payload_type_, ulpfec_payload_type_, first_fec_sequence_number,
147 0 : media_packet->headers_size());
148 0 : RTC_DCHECK_EQ(num_fec_packets, fec_packets.size());
149 0 : if (retransmission_settings_ & kRetransmitFECPackets)
150 0 : fec_storage = kAllowRetransmission;
151 : }
152 : }
153 : }
154 : // Send |red_packet| instead of |packet| for allocated sequence number.
155 0 : size_t red_packet_size = red_packet->size();
156 0 : if (rtp_sender_->SendToNetwork(std::move(red_packet), media_packet_storage,
157 : RtpPacketSender::kLowPriority)) {
158 0 : rtc::CritScope cs(&stats_crit_);
159 0 : video_bitrate_.Update(red_packet_size, clock_->TimeInMilliseconds());
160 0 : TRACE_EVENT_INSTANT2(TRACE_DISABLED_BY_DEFAULT("webrtc_rtp"),
161 : "Video::PacketRed", "timestamp", rtp_timestamp,
162 : "seqnum", media_seq_num);
163 : } else {
164 0 : LOG(LS_WARNING) << "Failed to send RED packet " << media_seq_num;
165 : }
166 0 : for (const auto& fec_packet : fec_packets) {
167 : // TODO(danilchap): Make ulpfec_generator_ generate RtpPacketToSend to avoid
168 : // reparsing them.
169 : std::unique_ptr<RtpPacketToSend> rtp_packet(
170 0 : new RtpPacketToSend(*media_packet));
171 0 : RTC_CHECK(rtp_packet->Parse(fec_packet->data(), fec_packet->length()));
172 0 : rtp_packet->set_capture_time_ms(media_packet->capture_time_ms());
173 0 : uint16_t fec_sequence_number = rtp_packet->SequenceNumber();
174 0 : if (rtp_sender_->SendToNetwork(std::move(rtp_packet), fec_storage,
175 : RtpPacketSender::kLowPriority)) {
176 0 : rtc::CritScope cs(&stats_crit_);
177 0 : fec_bitrate_.Update(fec_packet->length(), clock_->TimeInMilliseconds());
178 0 : TRACE_EVENT_INSTANT2(TRACE_DISABLED_BY_DEFAULT("webrtc_rtp"),
179 : "Video::PacketUlpfec", "timestamp", rtp_timestamp,
180 : "seqnum", fec_sequence_number);
181 : } else {
182 0 : LOG(LS_WARNING) << "Failed to send ULPFEC packet " << fec_sequence_number;
183 : }
184 : }
185 0 : }
186 :
187 0 : void RTPSenderVideo::SendVideoPacketWithFlexfec(
188 : std::unique_ptr<RtpPacketToSend> media_packet,
189 : StorageType media_packet_storage,
190 : bool protect_media_packet) {
191 0 : RTC_DCHECK(flexfec_sender_);
192 :
193 0 : if (protect_media_packet)
194 0 : flexfec_sender_->AddRtpPacketAndGenerateFec(*media_packet);
195 :
196 0 : SendVideoPacket(std::move(media_packet), media_packet_storage);
197 :
198 0 : if (flexfec_sender_->FecAvailable()) {
199 : std::vector<std::unique_ptr<RtpPacketToSend>> fec_packets =
200 0 : flexfec_sender_->GetFecPackets();
201 0 : for (auto& fec_packet : fec_packets) {
202 0 : uint32_t timestamp = fec_packet->Timestamp();
203 0 : uint16_t seq_num = fec_packet->SequenceNumber();
204 0 : if (rtp_sender_->SendToNetwork(std::move(fec_packet), kDontRetransmit,
205 : RtpPacketSender::kLowPriority)) {
206 : // TODO(brandtr): Wire up stats here.
207 0 : TRACE_EVENT_INSTANT2(TRACE_DISABLED_BY_DEFAULT("webrtc_rtp"),
208 : "Video::PacketFlexfec", "timestamp", timestamp,
209 : "seqnum", seq_num);
210 : } else {
211 0 : LOG(LS_WARNING) << "Failed to send FlexFEC packet " << seq_num;
212 : }
213 : }
214 : }
215 0 : }
216 :
217 0 : void RTPSenderVideo::SetUlpfecConfig(int red_payload_type,
218 : int ulpfec_payload_type) {
219 : // Sanity check. Per the definition of UlpfecConfig (see config.h),
220 : // a payload type of -1 means that the corresponding feature is
221 : // turned off.
222 0 : RTC_DCHECK_GE(red_payload_type, -1);
223 0 : RTC_DCHECK_LE(red_payload_type, 127);
224 0 : RTC_DCHECK_GE(ulpfec_payload_type, -1);
225 0 : RTC_DCHECK_LE(ulpfec_payload_type, 127);
226 :
227 0 : rtc::CritScope cs(&crit_);
228 0 : red_payload_type_ = red_payload_type;
229 0 : ulpfec_payload_type_ = ulpfec_payload_type;
230 :
231 : // Must not enable ULPFEC without RED.
232 : // TODO(brandtr): We currently support enabling RED without ULPFEC. Change
233 : // this when we have removed the RED/RTX send-side workaround, so that we
234 : // ensure that RED and ULPFEC are only enabled together.
235 0 : RTC_DCHECK(red_enabled() || !ulpfec_enabled());
236 :
237 : // Reset FEC parameters.
238 0 : delta_fec_params_ = FecProtectionParams{0, 1, kFecMaskRandom};
239 0 : key_fec_params_ = FecProtectionParams{0, 1, kFecMaskRandom};
240 0 : }
241 :
242 0 : void RTPSenderVideo::GetUlpfecConfig(int* red_payload_type,
243 : int* ulpfec_payload_type) const {
244 0 : rtc::CritScope cs(&crit_);
245 0 : *red_payload_type = red_payload_type_;
246 0 : *ulpfec_payload_type = ulpfec_payload_type_;
247 0 : }
248 :
249 0 : size_t RTPSenderVideo::CalculateFecPacketOverhead() const {
250 0 : if (flexfec_enabled())
251 0 : return flexfec_sender_->MaxPacketOverhead();
252 :
253 0 : size_t overhead = 0;
254 0 : if (red_enabled()) {
255 : // The RED overhead is due to a small header.
256 0 : overhead += kRedForFecHeaderLength;
257 : }
258 0 : if (ulpfec_enabled()) {
259 : // For ULPFEC, the overhead is the FEC headers plus RED for FEC header
260 : // (see above) plus anything in RTP header beyond the 12 bytes base header
261 : // (CSRC list, extensions...)
262 : // This reason for the header extensions to be included here is that
263 : // from an FEC viewpoint, they are part of the payload to be protected.
264 : // (The base RTP header is already protected by the FEC header.)
265 0 : overhead += ulpfec_generator_.MaxPacketOverhead() +
266 0 : (rtp_sender_->RtpHeaderLength() - kRtpHeaderSize);
267 : }
268 0 : return overhead;
269 : }
270 :
271 0 : void RTPSenderVideo::SetFecParameters(const FecProtectionParams& delta_params,
272 : const FecProtectionParams& key_params) {
273 0 : rtc::CritScope cs(&crit_);
274 0 : delta_fec_params_ = delta_params;
275 0 : key_fec_params_ = key_params;
276 0 : }
277 :
278 0 : rtc::Optional<uint32_t> RTPSenderVideo::FlexfecSsrc() const {
279 0 : if (flexfec_sender_) {
280 0 : return rtc::Optional<uint32_t>(flexfec_sender_->ssrc());
281 : }
282 0 : return rtc::Optional<uint32_t>();
283 : }
284 :
285 0 : bool RTPSenderVideo::SendVideo(RtpVideoCodecTypes video_type,
286 : FrameType frame_type,
287 : int8_t payload_type,
288 : uint32_t rtp_timestamp,
289 : int64_t capture_time_ms,
290 : const uint8_t* payload_data,
291 : size_t payload_size,
292 : const RTPFragmentationHeader* fragmentation,
293 : const RTPVideoHeader* video_header,
294 : const char* rid) {
295 0 : if (payload_size == 0)
296 0 : return false;
297 :
298 : // Create header that will be reused in all packets.
299 0 : std::unique_ptr<RtpPacketToSend> rtp_header = rtp_sender_->AllocatePacket();
300 0 : rtp_header->SetPayloadType(payload_type);
301 0 : rtp_header->SetTimestamp(rtp_timestamp);
302 0 : rtp_header->set_capture_time_ms(capture_time_ms);
303 :
304 : size_t fec_packet_overhead;
305 : bool red_enabled;
306 : int32_t retransmission_settings;
307 : {
308 0 : rtc::CritScope cs(&crit_);
309 : // According to
310 : // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/
311 : // ts_126114v120700p.pdf Section 7.4.5:
312 : // The MTSI client shall add the payload bytes as defined in this clause
313 : // onto the last RTP packet in each group of packets which make up a key
314 : // frame (I-frame or IDR frame in H.264 (AVC), or an IRAP picture in H.265
315 : // (HEVC)). The MTSI client may also add the payload bytes onto the last RTP
316 : // packet in each group of packets which make up another type of frame
317 : // (e.g. a P-Frame) only if the current value is different from the previous
318 : // value sent.
319 0 : if (video_header) {
320 : // Set rotation when key frame or when changed (to follow standard).
321 : // Or when different from 0 (to follow current receiver implementation).
322 0 : VideoRotation current_rotation = video_header->rotation;
323 0 : if (frame_type == kVideoFrameKey || current_rotation != last_rotation_ ||
324 : current_rotation != kVideoRotation_0)
325 0 : rtp_header->SetExtension<VideoOrientation>(current_rotation);
326 0 : last_rotation_ = current_rotation;
327 : }
328 0 : if (rid && rid[0]) {
329 0 : const size_t len = strlen(rid);
330 0 : if (len) {
331 0 : rtp_header->SetExtensionWithLength<StreamId>(len - 1, rid);
332 : }
333 : }
334 :
335 : // FEC settings.
336 : const FecProtectionParams& fec_params =
337 0 : frame_type == kVideoFrameKey ? key_fec_params_ : delta_fec_params_;
338 0 : if (flexfec_enabled())
339 0 : flexfec_sender_->SetFecParameters(fec_params);
340 0 : if (ulpfec_enabled())
341 0 : ulpfec_generator_.SetFecParameters(fec_params);
342 :
343 0 : fec_packet_overhead = CalculateFecPacketOverhead();
344 0 : red_enabled = this->red_enabled();
345 0 : retransmission_settings = retransmission_settings_;
346 : }
347 :
348 0 : size_t packet_capacity = rtp_sender_->MaxRtpPacketSize() -
349 0 : fec_packet_overhead -
350 0 : (rtp_sender_->RtxStatus() ? kRtxHeaderSize : 0);
351 0 : RTC_DCHECK_LE(packet_capacity, rtp_header->capacity());
352 0 : RTC_DCHECK_GT(packet_capacity, rtp_header->headers_size());
353 0 : size_t max_data_payload_length = packet_capacity - rtp_header->headers_size();
354 :
355 : std::unique_ptr<RtpPacketizer> packetizer(RtpPacketizer::Create(
356 : video_type, max_data_payload_length,
357 0 : video_header ? &(video_header->codecHeader) : nullptr, frame_type));
358 : // Media packet storage.
359 0 : StorageType storage = packetizer->GetStorageType(retransmission_settings);
360 :
361 : // TODO(changbin): we currently don't support to configure the codec to
362 : // output multiple partitions for VP8. Should remove below check after the
363 : // issue is fixed.
364 : const RTPFragmentationHeader* frag =
365 0 : (video_type == kRtpVideoVp8) ? nullptr : fragmentation;
366 0 : packetizer->SetPayloadData(payload_data, payload_size, frag);
367 :
368 0 : bool first_frame = first_frame_sent_();
369 0 : bool first = true;
370 0 : bool last = false;
371 0 : while (!last) {
372 0 : std::unique_ptr<RtpPacketToSend> packet(new RtpPacketToSend(*rtp_header));
373 :
374 0 : if (!packetizer->NextPacket(packet.get(), &last))
375 0 : return false;
376 0 : RTC_DCHECK_LE(packet->payload_size(), max_data_payload_length);
377 :
378 0 : if (!rtp_sender_->AssignSequenceNumber(packet.get()))
379 0 : return false;
380 :
381 : const bool protect_packet =
382 0 : (packetizer->GetProtectionType() == kProtectedPacket);
383 0 : if (flexfec_enabled()) {
384 : // TODO(brandtr): Remove the FlexFEC code path when FlexfecSender
385 : // is wired up to PacedSender instead.
386 0 : SendVideoPacketWithFlexfec(std::move(packet), storage, protect_packet);
387 0 : } else if (red_enabled) {
388 0 : SendVideoPacketAsRedMaybeWithUlpfec(std::move(packet), storage,
389 0 : protect_packet);
390 : } else {
391 0 : SendVideoPacket(std::move(packet), storage);
392 : }
393 :
394 0 : if (first_frame) {
395 0 : if (first) {
396 0 : LOG(LS_INFO)
397 0 : << "Sent first RTP packet of the first video frame (pre-pacer)";
398 : }
399 0 : if (last) {
400 0 : LOG(LS_INFO)
401 0 : << "Sent last RTP packet of the first video frame (pre-pacer)";
402 : }
403 : }
404 0 : first = false;
405 : }
406 :
407 0 : TRACE_EVENT_ASYNC_END1("webrtc", "Video", capture_time_ms, "timestamp",
408 : rtp_timestamp);
409 0 : return true;
410 : }
411 :
412 0 : uint32_t RTPSenderVideo::VideoBitrateSent() const {
413 0 : rtc::CritScope cs(&stats_crit_);
414 0 : return video_bitrate_.Rate(clock_->TimeInMilliseconds()).value_or(0);
415 : }
416 :
417 0 : uint32_t RTPSenderVideo::FecOverheadRate() const {
418 0 : rtc::CritScope cs(&stats_crit_);
419 0 : return fec_bitrate_.Rate(clock_->TimeInMilliseconds()).value_or(0);
420 : }
421 :
422 0 : int RTPSenderVideo::SelectiveRetransmissions() const {
423 0 : rtc::CritScope cs(&crit_);
424 0 : return retransmission_settings_;
425 : }
426 :
427 0 : void RTPSenderVideo::SetSelectiveRetransmissions(uint8_t settings) {
428 0 : rtc::CritScope cs(&crit_);
429 0 : retransmission_settings_ = settings;
430 0 : }
431 :
432 : } // namespace webrtc
|
