Line data Source code
1 : /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 : /* This Source Code Form is subject to the terms of the Mozilla Public
3 : * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 : * You can obtain one at http://mozilla.org/MPL/2.0/. */
5 :
6 : // Original author: ekr@rtfm.com
7 :
8 : #ifndef mediapipeline_h__
9 : #define mediapipeline_h__
10 :
11 : #include <map>
12 :
13 : #include "sigslot.h"
14 :
15 : #include "MediaConduitInterface.h"
16 : #include "mozilla/ReentrantMonitor.h"
17 : #include "mozilla/Atomics.h"
18 : #include "SrtpFlow.h"
19 : #include "databuffer.h"
20 : #include "runnable_utils.h"
21 : #include "transportflow.h"
22 : #include "AudioPacketizer.h"
23 : #include "StreamTracks.h"
24 :
25 : #include "webrtc/modules/rtp_rtcp/include/rtp_header_parser.h"
26 :
27 : // Should come from MediaEngine.h, but that's a pain to include here
28 : // because of the MOZILLA_EXTERNAL_LINKAGE stuff.
29 : #define WEBRTC_DEFAULT_SAMPLE_RATE 32000
30 :
31 : class nsIPrincipal;
32 :
33 : namespace mozilla {
34 : class MediaPipelineFilter;
35 : class PeerIdentity;
36 : class AudioProxyThread;
37 : class VideoFrameConverter;
38 :
39 : namespace dom {
40 : class MediaStreamTrack;
41 : struct RTCRTPContributingSourceStats;
42 : } // namespace dom
43 :
44 : class SourceMediaStream;
45 :
46 : // A class that represents the pipeline of audio and video
47 : // The dataflow looks like:
48 : //
49 : // TRANSMIT
50 : // CaptureDevice -> stream -> [us] -> conduit -> [us] -> transport -> network
51 : //
52 : // RECEIVE
53 : // network -> transport -> [us] -> conduit -> [us] -> stream -> Playout
54 : //
55 : // The boxes labeled [us] are just bridge logic implemented in this class
56 : //
57 : // We have to deal with a number of threads:
58 : //
59 : // GSM:
60 : // * Assembles the pipeline
61 : // SocketTransportService
62 : // * Receives notification that ICE and DTLS have completed
63 : // * Processes incoming network data and passes it to the conduit
64 : // * Processes outgoing RTP and RTCP
65 : // MediaStreamGraph
66 : // * Receives outgoing data from the MediaStreamGraph
67 : // * Receives pull requests for more data from the
68 : // MediaStreamGraph
69 : // One or another GIPS threads
70 : // * Receives RTCP messages to send to the other side
71 : // * Processes video frames GIPS wants to render
72 : //
73 : // For a transmitting conduit, "output" is RTP and "input" is RTCP.
74 : // For a receiving conduit, "input" is RTP and "output" is RTCP.
75 : //
76 :
77 : class MediaPipeline : public sigslot::has_slots<> {
78 : public:
79 : enum Direction { TRANSMIT, RECEIVE };
80 : enum State { MP_CONNECTING, MP_OPEN, MP_CLOSED };
81 : MediaPipeline(const std::string& pc,
82 : Direction direction,
83 : nsCOMPtr<nsIEventTarget> main_thread,
84 : nsCOMPtr<nsIEventTarget> sts_thread,
85 : const std::string& track_id,
86 : int level,
87 : RefPtr<MediaSessionConduit> conduit,
88 : RefPtr<TransportFlow> rtp_transport,
89 : RefPtr<TransportFlow> rtcp_transport,
90 : nsAutoPtr<MediaPipelineFilter> filter);
91 :
92 : // Must be called on the STS thread. Must be called after ShutdownMedia_m().
93 : void DetachTransport_s();
94 :
95 : // Must be called on the main thread.
96 0 : void ShutdownMedia_m()
97 : {
98 0 : ASSERT_ON_THREAD(main_thread_);
99 :
100 0 : if (direction_ == RECEIVE) {
101 0 : conduit_->StopReceiving();
102 : } else {
103 0 : conduit_->StopTransmitting();
104 : }
105 0 : DetachMedia();
106 0 : }
107 :
108 : virtual nsresult Init();
109 :
110 : void UpdateTransport_m(int level,
111 : RefPtr<TransportFlow> rtp_transport,
112 : RefPtr<TransportFlow> rtcp_transport,
113 : nsAutoPtr<MediaPipelineFilter> filter);
114 :
115 : void UpdateTransport_s(int level,
116 : RefPtr<TransportFlow> rtp_transport,
117 : RefPtr<TransportFlow> rtcp_transport,
118 : nsAutoPtr<MediaPipelineFilter> filter);
119 :
120 : // Used only for testing; adds RTP header extension for RTP Stream Id with
121 : // the given id.
122 : void AddRIDExtension_m(size_t extension_id);
123 : void AddRIDExtension_s(size_t extension_id);
124 : // Used only for testing; installs a MediaPipelineFilter that filters
125 : // everything but the given RID
126 : void AddRIDFilter_m(const std::string& rid);
127 : void AddRIDFilter_s(const std::string& rid);
128 :
129 0 : virtual Direction direction() const { return direction_; }
130 0 : virtual const std::string& trackid() const { return track_id_; }
131 0 : virtual int level() const { return level_; }
132 : virtual bool IsVideo() const = 0;
133 :
134 : bool IsDoingRtcpMux() const {
135 : return (rtp_.type_ == MUX);
136 : }
137 :
138 : class RtpCSRCStats {
139 : public:
140 : // Gets an expiration time for CRC info given a reference time,
141 : // this reference time would normally be the time of calling.
142 : // This value can then be used to check if a RtpCSRCStats
143 : // has expired via Expired(...)
144 : static DOMHighResTimeStamp
145 : GetExpiryFromTime(const DOMHighResTimeStamp aTime);
146 :
147 : RtpCSRCStats(const uint32_t aCsrc,
148 : const DOMHighResTimeStamp aTime);
149 0 : ~RtpCSRCStats() {};
150 : // Initialize a webidl representation suitable for adding to a report.
151 : // This assumes that the webidl object is empty.
152 : // @param aWebidlObj the webidl binding object to popluate
153 : // @param aRtpInboundStreamId the associated RTCInboundRTPStreamStats.id
154 : void
155 : GetWebidlInstance(dom::RTCRTPContributingSourceStats& aWebidlObj,
156 : const nsString &aInboundRtpStreamId) const;
157 0 : void SetTimestamp(const DOMHighResTimeStamp aTime) { mTimestamp = aTime; }
158 : // Check if the RtpCSRCStats has expired, checks against a
159 : // given expiration time.
160 0 : bool Expired(const DOMHighResTimeStamp aExpiry) const {
161 0 : return mTimestamp < aExpiry;
162 : }
163 : private:
164 : static const double constexpr EXPIRY_TIME_MILLISECONDS = 10 * 1000;
165 : uint32_t mCsrc;
166 : DOMHighResTimeStamp mTimestamp;
167 : };
168 :
169 : // Gets the gathered contributing source stats for the last expiration period.
170 : // @param aId the stream id to use for populating inboundRtpStreamId field
171 : // @param aArr the array to append the stats objects to
172 : void
173 : GetContributingSourceStats(
174 : const nsString& aInboundStreamId,
175 : FallibleTArray<dom::RTCRTPContributingSourceStats>& aArr) const;
176 :
177 0 : int32_t rtp_packets_sent() const { return rtp_packets_sent_; }
178 0 : int64_t rtp_bytes_sent() const { return rtp_bytes_sent_; }
179 : int32_t rtcp_packets_sent() const { return rtcp_packets_sent_; }
180 0 : int32_t rtp_packets_received() const { return rtp_packets_received_; }
181 0 : int64_t rtp_bytes_received() const { return rtp_bytes_received_; }
182 : int32_t rtcp_packets_received() const { return rtcp_packets_received_; }
183 :
184 0 : MediaSessionConduit *Conduit() const { return conduit_; }
185 :
186 : // Thread counting
187 0 : NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaPipeline)
188 :
189 : typedef enum {
190 : RTP,
191 : RTCP,
192 : MUX,
193 : MAX_RTP_TYPE
194 : } RtpType;
195 :
196 : // Separate class to allow ref counting
197 0 : class PipelineTransport : public TransportInterface {
198 : public:
199 : // Implement the TransportInterface functions
200 0 : explicit PipelineTransport(MediaPipeline *pipeline)
201 0 : : pipeline_(pipeline),
202 0 : sts_thread_(pipeline->sts_thread_) {}
203 :
204 0 : void Attach(MediaPipeline *pipeline) { pipeline_ = pipeline; }
205 0 : void Detach() { pipeline_ = nullptr; }
206 0 : MediaPipeline *pipeline() const { return pipeline_; }
207 :
208 : virtual nsresult SendRtpPacket(const uint8_t* data, size_t len);
209 : virtual nsresult SendRtcpPacket(const uint8_t* data, size_t len);
210 :
211 : private:
212 : nsresult SendRtpRtcpPacket_s(nsAutoPtr<DataBuffer> data,
213 : bool is_rtp);
214 :
215 : MediaPipeline *pipeline_; // Raw pointer to avoid cycles
216 : nsCOMPtr<nsIEventTarget> sts_thread_;
217 : };
218 :
219 : RefPtr<PipelineTransport> GetPiplelineTransport() {
220 : return transport_;
221 : }
222 :
223 : protected:
224 : virtual ~MediaPipeline();
225 0 : virtual void DetachMedia() {}
226 : nsresult AttachTransport_s();
227 : friend class PipelineTransport;
228 :
229 0 : class TransportInfo {
230 : public:
231 0 : TransportInfo(RefPtr<TransportFlow> flow, RtpType type) :
232 : transport_(flow),
233 : state_(MP_CONNECTING),
234 0 : type_(type) {
235 0 : MOZ_ASSERT(flow);
236 0 : }
237 :
238 0 : void Detach()
239 : {
240 0 : transport_ = nullptr;
241 0 : send_srtp_ = nullptr;
242 0 : recv_srtp_ = nullptr;
243 0 : }
244 :
245 : RefPtr<TransportFlow> transport_;
246 : State state_;
247 : RefPtr<SrtpFlow> send_srtp_;
248 : RefPtr<SrtpFlow> recv_srtp_;
249 : RtpType type_;
250 : };
251 :
252 : // The transport is down
253 : virtual nsresult TransportFailed_s(TransportInfo &info);
254 : // The transport is ready
255 : virtual nsresult TransportReady_s(TransportInfo &info);
256 : void UpdateRtcpMuxState(TransportInfo &info);
257 :
258 : // Unhooks from signals
259 : void DisconnectTransport_s(TransportInfo &info);
260 : nsresult ConnectTransport_s(TransportInfo &info);
261 :
262 : TransportInfo* GetTransportInfo_s(TransportFlow *flow);
263 :
264 : void increment_rtp_packets_sent(int bytes);
265 : void increment_rtcp_packets_sent();
266 : void increment_rtp_packets_received(int bytes);
267 : void increment_rtcp_packets_received();
268 :
269 : virtual nsresult SendPacket(TransportFlow *flow, const void *data, int len);
270 :
271 : // Process slots on transports
272 : void StateChange(TransportFlow *flow, TransportLayer::State);
273 : void RtpPacketReceived(TransportLayer *layer, const unsigned char *data,
274 : size_t len);
275 : void RtcpPacketReceived(TransportLayer *layer, const unsigned char *data,
276 : size_t len);
277 : void PacketReceived(TransportLayer *layer, const unsigned char *data,
278 : size_t len);
279 :
280 : Direction direction_;
281 : std::string track_id_; // The track on the stream.
282 : // Written on the main thread.
283 : // Used on STS and MediaStreamGraph threads.
284 : // Not used outside initialization in MediaPipelineTransmit
285 : // The m-line index (starting at 0, to match convention) Atomic because
286 : // this value is updated from STS, but read on main, and we don't want to
287 : // bother with dispatches just to get an int occasionally.
288 : Atomic<int> level_;
289 : RefPtr<MediaSessionConduit> conduit_; // Our conduit. Written on the main
290 : // thread. Read on STS thread.
291 :
292 : // The transport objects. Read/written on STS thread.
293 : TransportInfo rtp_;
294 : TransportInfo rtcp_;
295 :
296 : // Pointers to the threads we need. Initialized at creation
297 : // and used all over the place.
298 : nsCOMPtr<nsIEventTarget> main_thread_;
299 : nsCOMPtr<nsIEventTarget> sts_thread_;
300 :
301 : // Created on Init. Referenced by the conduit and eventually
302 : // destroyed on the STS thread.
303 : RefPtr<PipelineTransport> transport_;
304 :
305 : // Only safe to access from STS thread.
306 : // Build into TransportInfo?
307 : int32_t rtp_packets_sent_;
308 : int32_t rtcp_packets_sent_;
309 : int32_t rtp_packets_received_;
310 : int32_t rtcp_packets_received_;
311 : int64_t rtp_bytes_sent_;
312 : int64_t rtp_bytes_received_;
313 :
314 : // Only safe to access from STS thread.
315 : std::map<uint32_t, RtpCSRCStats> csrc_stats_;
316 :
317 : // Written on Init. Read on STS thread.
318 : std::string pc_;
319 : std::string description_;
320 :
321 : // Written on Init, all following accesses are on the STS thread.
322 : nsAutoPtr<MediaPipelineFilter> filter_;
323 : nsAutoPtr<webrtc::RtpHeaderParser> rtp_parser_;
324 :
325 : private:
326 : // Gets the current time as a DOMHighResTimeStamp
327 : static DOMHighResTimeStamp GetNow();
328 : nsresult Init_s();
329 :
330 : bool IsRtp(const unsigned char *data, size_t len);
331 : };
332 :
333 0 : class ConduitDeleteEvent: public Runnable
334 : {
335 : public:
336 0 : explicit ConduitDeleteEvent(already_AddRefed<MediaSessionConduit> aConduit) :
337 : Runnable("ConduitDeleteEvent"),
338 0 : mConduit(aConduit) {}
339 :
340 : /* we exist solely to proxy release of the conduit */
341 0 : NS_IMETHOD Run() override { return NS_OK; }
342 : private:
343 : RefPtr<MediaSessionConduit> mConduit;
344 : };
345 :
346 : // A specialization of pipeline for reading from an input device
347 : // and transmitting to the network.
348 : class MediaPipelineTransmit : public MediaPipeline {
349 : public:
350 : // Set rtcp_transport to nullptr to use rtcp-mux
351 : MediaPipelineTransmit(const std::string& pc,
352 : nsCOMPtr<nsIEventTarget> main_thread,
353 : nsCOMPtr<nsIEventTarget> sts_thread,
354 : dom::MediaStreamTrack* domtrack,
355 : const std::string& track_id,
356 : int level,
357 : RefPtr<MediaSessionConduit> conduit,
358 : RefPtr<TransportFlow> rtp_transport,
359 : RefPtr<TransportFlow> rtcp_transport,
360 : nsAutoPtr<MediaPipelineFilter> filter);
361 :
362 : // Initialize (stuff here may fail)
363 : nsresult Init() override;
364 :
365 : virtual void AttachToTrack(const std::string& track_id);
366 :
367 : // written and used from MainThread
368 : bool IsVideo() const override;
369 :
370 : // When the principal of the domtrack changes, it calls through to here
371 : // so that we can determine whether to enable track transmission.
372 : // `track` has to be null or equal `domtrack_` for us to apply the update.
373 : virtual void UpdateSinkIdentity_m(dom::MediaStreamTrack* track,
374 : nsIPrincipal* principal,
375 : const PeerIdentity* sinkIdentity);
376 :
377 : // Called on the main thread.
378 : void DetachMedia() override;
379 :
380 : // Override MediaPipeline::TransportReady.
381 : nsresult TransportReady_s(TransportInfo &info) override;
382 :
383 : // Replace a track with a different one
384 : // In non-compliance with the likely final spec, allow the new
385 : // track to be part of a different stream (since we don't support
386 : // multiple tracks of a type in a stream yet). bug 1056650
387 : virtual nsresult ReplaceTrack(dom::MediaStreamTrack& domtrack);
388 :
389 : // Separate classes to allow ref counting
390 : class PipelineListener;
391 : class VideoFrameFeeder;
392 :
393 : protected:
394 : ~MediaPipelineTransmit();
395 :
396 : private:
397 : RefPtr<PipelineListener> listener_;
398 : RefPtr<AudioProxyThread> audio_processing_;
399 : RefPtr<VideoFrameFeeder> feeder_;
400 : RefPtr<VideoFrameConverter> converter_;
401 : dom::MediaStreamTrack* domtrack_;
402 : };
403 :
404 :
405 : // A specialization of pipeline for reading from the network and
406 : // rendering video.
407 : class MediaPipelineReceive : public MediaPipeline {
408 : public:
409 : // Set rtcp_transport to nullptr to use rtcp-mux
410 : MediaPipelineReceive(const std::string& pc,
411 : nsCOMPtr<nsIEventTarget> main_thread,
412 : nsCOMPtr<nsIEventTarget> sts_thread,
413 : SourceMediaStream *stream,
414 : const std::string& track_id,
415 : int level,
416 : RefPtr<MediaSessionConduit> conduit,
417 : RefPtr<TransportFlow> rtp_transport,
418 : RefPtr<TransportFlow> rtcp_transport,
419 : nsAutoPtr<MediaPipelineFilter> filter);
420 :
421 : int segments_added() const { return segments_added_; }
422 :
423 : // Sets the PrincipalHandle we set on the media chunks produced by this
424 : // pipeline. Must be called on the main thread.
425 : virtual void SetPrincipalHandle_m(const PrincipalHandle& principal_handle) = 0;
426 : protected:
427 : ~MediaPipelineReceive();
428 :
429 : RefPtr<SourceMediaStream> stream_;
430 : int segments_added_;
431 :
432 : private:
433 : };
434 :
435 :
436 : // A specialization of pipeline for reading from the network and
437 : // rendering audio.
438 0 : class MediaPipelineReceiveAudio : public MediaPipelineReceive {
439 : public:
440 : MediaPipelineReceiveAudio(const std::string& pc,
441 : nsCOMPtr<nsIEventTarget> main_thread,
442 : nsCOMPtr<nsIEventTarget> sts_thread,
443 : SourceMediaStream* stream,
444 : // This comes from an msid attribute. Everywhere
445 : // but MediaStreamGraph uses this.
446 : const std::string& media_stream_track_id,
447 : // This is an integer identifier that is only
448 : // unique within a single DOMMediaStream, which is
449 : // used by MediaStreamGraph
450 : TrackID numeric_track_id,
451 : int level,
452 : RefPtr<AudioSessionConduit> conduit,
453 : RefPtr<TransportFlow> rtp_transport,
454 : RefPtr<TransportFlow> rtcp_transport,
455 : nsAutoPtr<MediaPipelineFilter> filter);
456 :
457 : void DetachMedia() override;
458 :
459 : nsresult Init() override;
460 0 : bool IsVideo() const override { return false; }
461 :
462 : void SetPrincipalHandle_m(const PrincipalHandle& principal_handle) override;
463 :
464 : private:
465 : // Separate class to allow ref counting
466 : class PipelineListener;
467 :
468 : RefPtr<PipelineListener> listener_;
469 : };
470 :
471 :
472 : // A specialization of pipeline for reading from the network and
473 : // rendering video.
474 0 : class MediaPipelineReceiveVideo : public MediaPipelineReceive {
475 : public:
476 : MediaPipelineReceiveVideo(const std::string& pc,
477 : nsCOMPtr<nsIEventTarget> main_thread,
478 : nsCOMPtr<nsIEventTarget> sts_thread,
479 : SourceMediaStream *stream,
480 : // This comes from an msid attribute. Everywhere
481 : // but MediaStreamGraph uses this.
482 : const std::string& media_stream_track_id,
483 : // This is an integer identifier that is only
484 : // unique within a single DOMMediaStream, which is
485 : // used by MediaStreamGraph
486 : TrackID numeric_track_id,
487 : int level,
488 : RefPtr<VideoSessionConduit> conduit,
489 : RefPtr<TransportFlow> rtp_transport,
490 : RefPtr<TransportFlow> rtcp_transport,
491 : nsAutoPtr<MediaPipelineFilter> filter);
492 :
493 : // Called on the main thread.
494 : void DetachMedia() override;
495 :
496 : nsresult Init() override;
497 0 : bool IsVideo() const override { return true; }
498 :
499 : void SetPrincipalHandle_m(const PrincipalHandle& principal_handle) override;
500 :
501 : private:
502 : class PipelineRenderer;
503 : friend class PipelineRenderer;
504 :
505 : // Separate class to allow ref counting
506 : class PipelineListener;
507 :
508 : RefPtr<PipelineRenderer> renderer_;
509 : RefPtr<PipelineListener> listener_;
510 : };
511 :
512 :
513 : } // namespace mozilla
514 : #endif
|
