Line data Source code
1 : /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
2 : /* This Source Code Form is subject to the terms of the Mozilla Public
3 : * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 : * You can obtain one at http://mozilla.org/MPL/2.0/. */
5 :
6 : #ifndef MOZILLA_MEDIASTREAMGRAPH_H_
7 : #define MOZILLA_MEDIASTREAMGRAPH_H_
8 :
9 : #include "AudioStream.h"
10 : #include "MainThreadUtils.h"
11 : #include "MediaStreamTypes.h"
12 : #include "StreamTracks.h"
13 : #include "VideoSegment.h"
14 : #include "mozilla/LinkedList.h"
15 : #include "mozilla/Mutex.h"
16 : #include "mozilla/TaskQueue.h"
17 : #include "mozilla/dom/AudioChannelBinding.h"
18 : #include "nsAutoPtr.h"
19 : #include "nsAutoRef.h"
20 : #include "nsIRunnable.h"
21 : #include "nsTArray.h"
22 : #include <speex/speex_resampler.h>
23 :
24 : class nsIRunnable;
25 : class nsIGlobalObject;
26 : class nsPIDOMWindowInner;
27 :
28 : template <>
29 0 : class nsAutoRefTraits<SpeexResamplerState> : public nsPointerRefTraits<SpeexResamplerState>
30 : {
31 : public:
32 0 : static void Release(SpeexResamplerState* aState) { speex_resampler_destroy(aState); }
33 : };
34 :
35 : namespace mozilla {
36 :
37 : extern LazyLogModule gMediaStreamGraphLog;
38 :
39 : namespace dom {
40 : enum class AudioContextOperation;
41 : }
42 :
43 : namespace media {
44 : template<typename V, typename E> class Pledge;
45 : }
46 :
47 : /*
48 : * MediaStreamGraph is a framework for synchronized audio/video processing
49 : * and playback. It is designed to be used by other browser components such as
50 : * HTML media elements, media capture APIs, real-time media streaming APIs,
51 : * multitrack media APIs, and advanced audio APIs.
52 : *
53 : * The MediaStreamGraph uses a dedicated thread to process media --- the media
54 : * graph thread. This ensures that we can process media through the graph
55 : * without blocking on main-thread activity. The media graph is only modified
56 : * on the media graph thread, to ensure graph changes can be processed without
57 : * interfering with media processing. All interaction with the media graph
58 : * thread is done with message passing.
59 : *
60 : * APIs that modify the graph or its properties are described as "control APIs".
61 : * These APIs are asynchronous; they queue graph changes internally and
62 : * those changes are processed all-at-once by the MediaStreamGraph. The
63 : * MediaStreamGraph monitors the main thread event loop via nsIAppShell::RunInStableState
64 : * to ensure that graph changes from a single event loop task are always
65 : * processed all together. Control APIs should only be used on the main thread,
66 : * currently; we may be able to relax that later.
67 : *
68 : * To allow precise synchronization of times in the control API, the
69 : * MediaStreamGraph maintains a "media timeline". Control APIs that take or
70 : * return times use that timeline. Those times never advance during
71 : * an event loop task. This time is returned by MediaStreamGraph::GetCurrentTime().
72 : *
73 : * Media decoding, audio processing and media playback use thread-safe APIs to
74 : * the media graph to ensure they can continue while the main thread is blocked.
75 : *
76 : * When the graph is changed, we may need to throw out buffered data and
77 : * reprocess it. This is triggered automatically by the MediaStreamGraph.
78 : */
79 :
80 : class AudioNodeEngine;
81 : class AudioNodeExternalInputStream;
82 : class AudioNodeStream;
83 : class MediaInputPort;
84 : class MediaStream;
85 : class MediaStreamGraph;
86 : class MediaStreamGraphImpl;
87 : class ProcessedMediaStream;
88 : class SourceMediaStream;
89 :
90 0 : class AudioDataListenerInterface {
91 : protected:
92 : // Protected destructor, to discourage deletion outside of Release():
93 0 : virtual ~AudioDataListenerInterface() {}
94 :
95 : public:
96 : /* These are for cubeb audio input & output streams: */
97 : /**
98 : * Output data to speakers, for use as the "far-end" data for echo
99 : * cancellation. This is not guaranteed to be in any particular size
100 : * chunks.
101 : */
102 : virtual void NotifyOutputData(MediaStreamGraph* aGraph,
103 : AudioDataValue* aBuffer, size_t aFrames,
104 : TrackRate aRate, uint32_t aChannels) = 0;
105 : /**
106 : * Input data from a microphone (or other audio source. This is not
107 : * guaranteed to be in any particular size chunks.
108 : */
109 : virtual void NotifyInputData(MediaStreamGraph* aGraph,
110 : const AudioDataValue* aBuffer, size_t aFrames,
111 : TrackRate aRate, uint32_t aChannels) = 0;
112 :
113 : /**
114 : * Called when the underlying audio device has changed.
115 : */
116 : virtual void DeviceChanged() = 0;
117 : };
118 :
119 0 : class AudioDataListener : public AudioDataListenerInterface {
120 : protected:
121 : // Protected destructor, to discourage deletion outside of Release():
122 0 : virtual ~AudioDataListener() {}
123 :
124 : public:
125 0 : NS_INLINE_DECL_THREADSAFE_REFCOUNTING(AudioDataListener)
126 : };
127 :
128 : /**
129 : * This is a base class for main-thread listener callbacks.
130 : * This callback is invoked on the main thread when the main-thread-visible
131 : * state of a stream has changed.
132 : *
133 : * These methods are called with the media graph monitor held, so
134 : * reentry into general media graph methods is not possible.
135 : * You should do something non-blocking and non-reentrant (e.g. dispatch an
136 : * event) and return. DispatchFromMainThreadAfterNextStreamStateUpdate
137 : * would be a good choice.
138 : * The listener is allowed to synchronously remove itself from the stream, but
139 : * not add or remove any other listeners.
140 : */
141 0 : class MainThreadMediaStreamListener {
142 : public:
143 : virtual void NotifyMainThreadStreamFinished() = 0;
144 : };
145 :
146 : /**
147 : * Helper struct used to keep track of memory usage by AudioNodes.
148 : */
149 : struct AudioNodeSizes
150 : {
151 0 : AudioNodeSizes() : mStream(0), mEngine(0), mNodeType() {}
152 : size_t mStream;
153 : size_t mEngine;
154 : const char* mNodeType;
155 : };
156 :
157 : class AudioNodeEngine;
158 : class AudioNodeExternalInputStream;
159 : class AudioNodeStream;
160 : class AudioSegment;
161 : class DirectMediaStreamListener;
162 : class DirectMediaStreamTrackListener;
163 : class MediaInputPort;
164 : class MediaStreamGraphImpl;
165 : class MediaStreamListener;
166 : class MediaStreamTrackListener;
167 : class MediaStreamVideoSink;
168 : class ProcessedMediaStream;
169 : class SourceMediaStream;
170 : class TrackUnionStream;
171 :
172 : /**
173 : * Helper struct for binding a track listener to a specific TrackID.
174 : */
175 : template<typename Listener>
176 0 : struct TrackBound
177 : {
178 : RefPtr<Listener> mListener;
179 : TrackID mTrackID;
180 : };
181 :
182 : /**
183 : * A stream of synchronized audio and video data. All (not blocked) streams
184 : * progress at the same rate --- "real time". Streams cannot seek. The only
185 : * operation readers can perform on a stream is to read the next data.
186 : *
187 : * Consumers of a stream can be reading from it at different offsets, but that
188 : * should only happen due to the order in which consumers are being run.
189 : * Those offsets must not diverge in the long term, otherwise we would require
190 : * unbounded buffering.
191 : *
192 : * Streams can be in a "blocked" state. While blocked, a stream does not
193 : * produce data. A stream can be explicitly blocked via the control API,
194 : * or implicitly blocked by whatever's generating it (e.g. an underrun in the
195 : * source resource), or implicitly blocked because something consuming it
196 : * blocks, or implicitly because it has finished.
197 : *
198 : * A stream can be in a "finished" state. "Finished" streams are permanently
199 : * blocked.
200 : *
201 : * Transitions into and out of the "blocked" and "finished" states are managed
202 : * by the MediaStreamGraph on the media graph thread.
203 : *
204 : * We buffer media data ahead of the consumers' reading offsets. It is possible
205 : * to have buffered data but still be blocked.
206 : *
207 : * Any stream can have its audio and video playing when requested. The media
208 : * stream graph plays audio by constructing audio output streams as necessary.
209 : * Video is played by setting video frames into an MediaStreamVideoSink at the right
210 : * time. To ensure video plays in sync with audio, make sure that the same
211 : * stream is playing both the audio and video.
212 : *
213 : * The data in a stream is managed by StreamTracks. It consists of a set of
214 : * tracks of various types that can start and end over time.
215 : *
216 : * Streams are explicitly managed. The client creates them via
217 : * MediaStreamGraph::CreateInput/ProcessedMediaStream, and releases them by calling
218 : * Destroy() when no longer needed (actual destruction will be deferred).
219 : * The actual object is owned by the MediaStreamGraph. The basic idea is that
220 : * main thread objects will keep Streams alive as long as necessary (using the
221 : * cycle collector to clean up whenever needed).
222 : *
223 : * We make them refcounted only so that stream-related messages with MediaStream*
224 : * pointers can be sent to the main thread safely.
225 : *
226 : * The lifetimes of MediaStreams are controlled from the main thread.
227 : * For MediaStreams exposed to the DOM, the lifetime is controlled by the DOM
228 : * wrapper; the DOM wrappers own their associated MediaStreams. When a DOM
229 : * wrapper is destroyed, it sends a Destroy message for the associated
230 : * MediaStream and clears its reference (the last main-thread reference to
231 : * the object). When the Destroy message is processed on the graph manager
232 : * thread we immediately release the affected objects (disentangling them
233 : * from other objects as necessary).
234 : *
235 : * This could cause problems for media processing if a MediaStream is
236 : * destroyed while a downstream MediaStream is still using it. Therefore
237 : * the DOM wrappers must keep upstream MediaStreams alive as long as they
238 : * could be being used in the media graph.
239 : *
240 : * At any time, however, a set of MediaStream wrappers could be
241 : * collected via cycle collection. Destroy messages will be sent
242 : * for those objects in arbitrary order and the MediaStreamGraph has to be able
243 : * to handle this.
244 : */
245 :
246 : // GetCurrentTime is defined in winbase.h as zero argument macro forwarding to
247 : // GetTickCount() and conflicts with MediaStream::GetCurrentTime.
248 : #ifdef GetCurrentTime
249 : #undef GetCurrentTime
250 : #endif
251 :
252 : class MediaStream : public mozilla::LinkedListElement<MediaStream>
253 : {
254 : public:
255 0 : NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaStream)
256 :
257 : explicit MediaStream();
258 :
259 : protected:
260 : // Protected destructor, to discourage deletion outside of Release():
261 : virtual ~MediaStream();
262 :
263 : public:
264 : /**
265 : * Returns the graph that owns this stream.
266 : */
267 : MediaStreamGraphImpl* GraphImpl();
268 : MediaStreamGraph* Graph();
269 : /**
270 : * Sets the graph that owns this stream. Should only be called once.
271 : */
272 : void SetGraphImpl(MediaStreamGraphImpl* aGraph);
273 : void SetGraphImpl(MediaStreamGraph* aGraph);
274 :
275 : /**
276 : * Returns sample rate of the graph.
277 : */
278 0 : TrackRate GraphRate() { return mTracks.GraphRate(); }
279 :
280 : // Control API.
281 : // Since a stream can be played multiple ways, we need to combine independent
282 : // volume settings. The aKey parameter is used to keep volume settings
283 : // separate. Since the stream is always playing the same contents, only
284 : // a single audio output stream is used; the volumes are combined.
285 : // Currently only the first enabled audio track is played.
286 : // XXX change this so all enabled audio tracks are mixed and played.
287 : virtual void AddAudioOutput(void* aKey);
288 : virtual void SetAudioOutputVolume(void* aKey, float aVolume);
289 : virtual void RemoveAudioOutput(void* aKey);
290 : // Since a stream can be played multiple ways, we need to be able to
291 : // play to multiple MediaStreamVideoSinks.
292 : // Only the first enabled video track is played.
293 : virtual void AddVideoOutput(MediaStreamVideoSink* aSink,
294 : TrackID aID = TRACK_ANY);
295 : virtual void RemoveVideoOutput(MediaStreamVideoSink* aSink,
296 : TrackID aID = TRACK_ANY);
297 : // Explicitly suspend. Useful for example if a media element is pausing
298 : // and we need to stop its stream emitting its buffered data. As soon as the
299 : // Suspend message reaches the graph, the stream stops processing. It
300 : // ignores its inputs and produces silence/no video until Resumed. Its
301 : // current time does not advance.
302 : virtual void Suspend();
303 : virtual void Resume();
304 : // Events will be dispatched by calling methods of aListener.
305 : virtual void AddListener(MediaStreamListener* aListener);
306 : virtual void RemoveListener(MediaStreamListener* aListener);
307 : virtual void AddTrackListener(MediaStreamTrackListener* aListener,
308 : TrackID aTrackID);
309 : virtual void RemoveTrackListener(MediaStreamTrackListener* aListener,
310 : TrackID aTrackID);
311 :
312 : /**
313 : * Adds aListener to the source stream of track aTrackID in this stream.
314 : * When the MediaStreamGraph processes the added listener, it will traverse
315 : * the graph and add it to the track's source stream (remapping the TrackID
316 : * along the way).
317 : * Note that the listener will be notified on the MediaStreamGraph thread
318 : * with whether the installation of it at the source was successful or not.
319 : */
320 : virtual void AddDirectTrackListener(DirectMediaStreamTrackListener* aListener,
321 : TrackID aTrackID);
322 :
323 : /**
324 : * Removes aListener from the source stream of track aTrackID in this stream.
325 : * Note that the listener has already been removed if the link between the
326 : * source of track aTrackID and this stream has been broken (and made track
327 : * aTrackID end). The caller doesn't have to care about this, removing when
328 : * the source cannot be found, or when the listener had already been removed
329 : * does nothing.
330 : */
331 : virtual void RemoveDirectTrackListener(DirectMediaStreamTrackListener* aListener,
332 : TrackID aTrackID);
333 :
334 : // A disabled track has video replaced by black, and audio replaced by
335 : // silence.
336 : void SetTrackEnabled(TrackID aTrackID, DisabledTrackMode aMode);
337 :
338 : // Finish event will be notified by calling methods of aListener. It is the
339 : // responsibility of the caller to remove aListener before it is destroyed.
340 : void AddMainThreadListener(MainThreadMediaStreamListener* aListener);
341 : // It's safe to call this even if aListener is not currently a listener;
342 : // the call will be ignored.
343 0 : void RemoveMainThreadListener(MainThreadMediaStreamListener* aListener)
344 : {
345 0 : MOZ_ASSERT(NS_IsMainThread());
346 0 : MOZ_ASSERT(aListener);
347 0 : mMainThreadListeners.RemoveElement(aListener);
348 0 : }
349 :
350 : /**
351 : * Ensure a runnable will run on the main thread after running all pending
352 : * updates that were sent from the graph thread or will be sent before the
353 : * graph thread receives the next graph update.
354 : *
355 : * If the graph has been shut down or destroyed, then the runnable will be
356 : * dispatched to the event queue immediately. If the graph is non-realtime
357 : * and has not started, then the runnable will be run
358 : * synchronously/immediately. (There are no pending updates in these
359 : * situations.)
360 : *
361 : * Main thread only.
362 : */
363 : void RunAfterPendingUpdates(already_AddRefed<nsIRunnable> aRunnable);
364 :
365 : // Signal that the client is done with this MediaStream. It will be deleted
366 : // later. Do not mix usage of Destroy() with RegisterUser()/UnregisterUser().
367 : // That will cause the MediaStream to be destroyed twice, which will cause
368 : // some assertions to fail.
369 : virtual void Destroy();
370 : // Signal that a client is using this MediaStream. Useful to not have to
371 : // explicitly manage ownership (responsibility to Destroy()) when there are
372 : // multiple clients using a MediaStream.
373 : void RegisterUser();
374 : // Signal that a client no longer needs this MediaStream. When the number of
375 : // clients using this MediaStream reaches 0, it will be destroyed.
376 : void UnregisterUser();
377 :
378 : // Returns the main-thread's view of how much data has been processed by
379 : // this stream.
380 0 : StreamTime GetCurrentTime()
381 : {
382 0 : NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
383 0 : return mMainThreadCurrentTime;
384 : }
385 : // Return the main thread's view of whether this stream has finished.
386 0 : bool IsFinished()
387 : {
388 0 : NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
389 0 : return mMainThreadFinished;
390 : }
391 :
392 0 : bool IsDestroyed()
393 : {
394 0 : NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
395 0 : return mMainThreadDestroyed;
396 : }
397 :
398 : friend class MediaStreamGraphImpl;
399 : friend class MediaInputPort;
400 : friend class AudioNodeExternalInputStream;
401 :
402 0 : virtual SourceMediaStream* AsSourceStream() { return nullptr; }
403 0 : virtual ProcessedMediaStream* AsProcessedStream() { return nullptr; }
404 0 : virtual AudioNodeStream* AsAudioNodeStream() { return nullptr; }
405 0 : virtual TrackUnionStream* AsTrackUnionStream() { return nullptr; }
406 :
407 : // These Impl methods perform the core functionality of the control methods
408 : // above, on the media graph thread.
409 : /**
410 : * Stop all stream activity and disconnect it from all inputs and outputs.
411 : * This must be idempotent.
412 : */
413 : virtual void DestroyImpl();
414 0 : StreamTime GetTracksEnd() { return mTracks.GetEnd(); }
415 : #ifdef DEBUG
416 0 : void DumpTrackInfo() { return mTracks.DumpTrackInfo(); }
417 : #endif
418 : void SetAudioOutputVolumeImpl(void* aKey, float aVolume);
419 : void AddAudioOutputImpl(void* aKey);
420 : // Returns true if this stream has an audio output.
421 : bool HasAudioOutput()
422 : {
423 : return !mAudioOutputs.IsEmpty();
424 : }
425 : void RemoveAudioOutputImpl(void* aKey);
426 : void AddVideoOutputImpl(already_AddRefed<MediaStreamVideoSink> aSink,
427 : TrackID aID);
428 : void RemoveVideoOutputImpl(MediaStreamVideoSink* aSink, TrackID aID);
429 : void AddListenerImpl(already_AddRefed<MediaStreamListener> aListener);
430 : void RemoveListenerImpl(MediaStreamListener* aListener);
431 : void RemoveAllListenersImpl();
432 : virtual void AddTrackListenerImpl(already_AddRefed<MediaStreamTrackListener> aListener,
433 : TrackID aTrackID);
434 : virtual void RemoveTrackListenerImpl(MediaStreamTrackListener* aListener,
435 : TrackID aTrackID);
436 : virtual void AddDirectTrackListenerImpl(already_AddRefed<DirectMediaStreamTrackListener> aListener,
437 : TrackID aTrackID);
438 : virtual void RemoveDirectTrackListenerImpl(DirectMediaStreamTrackListener* aListener,
439 : TrackID aTrackID);
440 : virtual void SetTrackEnabledImpl(TrackID aTrackID, DisabledTrackMode aMode);
441 : DisabledTrackMode GetDisabledTrackMode(TrackID aTrackID);
442 :
443 0 : void AddConsumer(MediaInputPort* aPort)
444 : {
445 0 : mConsumers.AppendElement(aPort);
446 0 : }
447 0 : void RemoveConsumer(MediaInputPort* aPort)
448 : {
449 0 : mConsumers.RemoveElement(aPort);
450 0 : }
451 : uint32_t ConsumerCount()
452 : {
453 : return mConsumers.Length();
454 : }
455 0 : StreamTracks& GetStreamTracks() { return mTracks; }
456 : GraphTime GetStreamTracksStartTime() { return mTracksStartTime; }
457 :
458 0 : double StreamTimeToSeconds(StreamTime aTime)
459 : {
460 0 : NS_ASSERTION(0 <= aTime && aTime <= STREAM_TIME_MAX, "Bad time");
461 0 : return static_cast<double>(aTime)/mTracks.GraphRate();
462 : }
463 0 : int64_t StreamTimeToMicroseconds(StreamTime aTime)
464 : {
465 0 : NS_ASSERTION(0 <= aTime && aTime <= STREAM_TIME_MAX, "Bad time");
466 0 : return (aTime*1000000)/mTracks.GraphRate();
467 : }
468 0 : StreamTime SecondsToNearestStreamTime(double aSeconds)
469 : {
470 0 : NS_ASSERTION(0 <= aSeconds && aSeconds <= TRACK_TICKS_MAX/TRACK_RATE_MAX,
471 : "Bad seconds");
472 0 : return mTracks.GraphRate() * aSeconds + 0.5;
473 : }
474 0 : StreamTime MicrosecondsToStreamTimeRoundDown(int64_t aMicroseconds) {
475 0 : return (aMicroseconds*mTracks.GraphRate())/1000000;
476 : }
477 :
478 0 : TrackTicks TimeToTicksRoundUp(TrackRate aRate, StreamTime aTime)
479 : {
480 0 : return RateConvertTicksRoundUp(aRate, mTracks.GraphRate(), aTime);
481 : }
482 0 : StreamTime TicksToTimeRoundDown(TrackRate aRate, TrackTicks aTicks)
483 : {
484 0 : return RateConvertTicksRoundDown(mTracks.GraphRate(), aRate, aTicks);
485 : }
486 : /**
487 : * Convert graph time to stream time. aTime must be <= mStateComputedTime
488 : * to ensure we know exactly how much time this stream will be blocked during
489 : * the interval.
490 : */
491 : StreamTime GraphTimeToStreamTimeWithBlocking(GraphTime aTime);
492 : /**
493 : * Convert graph time to stream time. This assumes there is no blocking time
494 : * to take account of, which is always true except between a stream
495 : * having its blocking time calculated in UpdateGraph and its blocking time
496 : * taken account of in UpdateCurrentTimeForStreams.
497 : */
498 : StreamTime GraphTimeToStreamTime(GraphTime aTime);
499 : /**
500 : * Convert stream time to graph time. This assumes there is no blocking time
501 : * to take account of, which is always true except between a stream
502 : * having its blocking time calculated in UpdateGraph and its blocking time
503 : * taken account of in UpdateCurrentTimeForStreams.
504 : */
505 : GraphTime StreamTimeToGraphTime(StreamTime aTime);
506 :
507 0 : bool IsFinishedOnGraphThread() { return mFinished; }
508 : void FinishOnGraphThread();
509 :
510 0 : bool HasCurrentData() { return mHasCurrentData; }
511 :
512 : /**
513 : * Find track by track id.
514 : */
515 : StreamTracks::Track* FindTrack(TrackID aID);
516 :
517 : StreamTracks::Track* EnsureTrack(TrackID aTrack);
518 :
519 : virtual void ApplyTrackDisabling(TrackID aTrackID, MediaSegment* aSegment, MediaSegment* aRawSegment = nullptr);
520 :
521 : // Return true if the main thread needs to observe updates from this stream.
522 0 : virtual bool MainThreadNeedsUpdates() const
523 : {
524 0 : return true;
525 : }
526 :
527 : virtual size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const;
528 : virtual size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const;
529 :
530 0 : void SetAudioChannelType(dom::AudioChannel aType) { mAudioChannelType = aType; }
531 : dom::AudioChannel AudioChannelType() const { return mAudioChannelType; }
532 :
533 0 : bool IsSuspended() { return mSuspendedCount > 0; }
534 : void IncrementSuspendCount();
535 : void DecrementSuspendCount();
536 :
537 : protected:
538 : // |AdvanceTimeVaryingValuesToCurrentTime| will be override in SourceMediaStream.
539 0 : virtual void AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime,
540 : GraphTime aBlockedTime)
541 : {
542 0 : mTracksStartTime += aBlockedTime;
543 0 : mTracks.ForgetUpTo(aCurrentTime - mTracksStartTime);
544 0 : }
545 :
546 0 : void NotifyMainThreadListeners()
547 : {
548 0 : NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
549 :
550 0 : for (int32_t i = mMainThreadListeners.Length() - 1; i >= 0; --i) {
551 0 : mMainThreadListeners[i]->NotifyMainThreadStreamFinished();
552 : }
553 0 : mMainThreadListeners.Clear();
554 0 : }
555 :
556 0 : bool ShouldNotifyStreamFinished()
557 : {
558 0 : NS_ASSERTION(NS_IsMainThread(), "Call only on main thread");
559 0 : if (!mMainThreadFinished || mFinishedNotificationSent) {
560 0 : return false;
561 : }
562 :
563 0 : mFinishedNotificationSent = true;
564 0 : return true;
565 : }
566 :
567 : // This state is all initialized on the main thread but
568 : // otherwise modified only on the media graph thread.
569 :
570 : // Buffered data. The start of the buffer corresponds to mTracksStartTime.
571 : // Conceptually the buffer contains everything this stream has ever played,
572 : // but we forget some prefix of the buffered data to bound the space usage.
573 : StreamTracks mTracks;
574 : // The time when the buffered data could be considered to have started playing.
575 : // This increases over time to account for time the stream was blocked before
576 : // mCurrentTime.
577 : GraphTime mTracksStartTime;
578 :
579 : // Client-set volume of this stream
580 : struct AudioOutput {
581 0 : explicit AudioOutput(void* aKey) : mKey(aKey), mVolume(1.0f) {}
582 : void* mKey;
583 : float mVolume;
584 : };
585 : nsTArray<AudioOutput> mAudioOutputs;
586 : nsTArray<TrackBound<MediaStreamVideoSink>> mVideoOutputs;
587 : // We record the last played video frame to avoid playing the frame again
588 : // with a different frame id.
589 : VideoFrame mLastPlayedVideoFrame;
590 : nsTArray<RefPtr<MediaStreamListener> > mListeners;
591 : nsTArray<TrackBound<MediaStreamTrackListener>> mTrackListeners;
592 : nsTArray<MainThreadMediaStreamListener*> mMainThreadListeners;
593 : // List of disabled TrackIDs and their associated disabled mode.
594 : // They can either by disabled by frames being replaced by black, or by
595 : // retaining the previous frame.
596 : nsTArray<DisabledTrack> mDisabledTracks;
597 :
598 : // GraphTime at which this stream starts blocking.
599 : // This is only valid up to mStateComputedTime. The stream is considered to
600 : // have not been blocked before mCurrentTime (its mTracksStartTime is increased
601 : // as necessary to account for that time instead).
602 : GraphTime mStartBlocking;
603 :
604 : // MediaInputPorts to which this is connected
605 : nsTArray<MediaInputPort*> mConsumers;
606 :
607 : // Where audio output is going. There is one AudioOutputStream per
608 : // audio track.
609 : struct AudioOutputStream
610 : {
611 : // When we started audio playback for this track.
612 : // Add mStream->GetPosition() to find the current audio playback position.
613 : GraphTime mAudioPlaybackStartTime;
614 : // Amount of time that we've wanted to play silence because of the stream
615 : // blocking.
616 : MediaTime mBlockedAudioTime;
617 : // Last tick written to the audio output.
618 : StreamTime mLastTickWritten;
619 : TrackID mTrackID;
620 : };
621 : nsTArray<AudioOutputStream> mAudioOutputStreams;
622 :
623 : /**
624 : * Number of outstanding suspend operations on this stream. Stream is
625 : * suspended when this is > 0.
626 : */
627 : int32_t mSuspendedCount;
628 :
629 : /**
630 : * When true, this means the stream will be finished once all
631 : * buffered data has been consumed.
632 : */
633 : bool mFinished;
634 : /**
635 : * When true, mFinished is true and we've played all the data in this stream
636 : * and fired NotifyFinished notifications.
637 : */
638 : bool mNotifiedFinished;
639 : /**
640 : * When true, the last NotifyBlockingChanged delivered to the listeners
641 : * indicated that the stream is blocked.
642 : */
643 : bool mNotifiedBlocked;
644 : /**
645 : * True if some data can be present by this stream if/when it's unblocked.
646 : * Set by the stream itself on the MediaStreamGraph thread. Only changes
647 : * from false to true once a stream has data, since we won't
648 : * unblock it until there's more data.
649 : */
650 : bool mHasCurrentData;
651 : /**
652 : * True if mHasCurrentData is true and we've notified listeners.
653 : */
654 : bool mNotifiedHasCurrentData;
655 :
656 : // Main-thread views of state
657 : StreamTime mMainThreadCurrentTime;
658 : bool mMainThreadFinished;
659 : bool mFinishedNotificationSent;
660 : bool mMainThreadDestroyed;
661 : int mNrOfMainThreadUsers;
662 :
663 : // Our media stream graph. null if destroyed on the graph thread.
664 : MediaStreamGraphImpl* mGraph;
665 :
666 : dom::AudioChannel mAudioChannelType;
667 : };
668 :
669 : /**
670 : * This is a stream into which a decoder can write audio and video.
671 : *
672 : * Audio and video can be written on any thread, but you probably want to
673 : * always write from the same thread to avoid unexpected interleavings.
674 : */
675 : class SourceMediaStream : public MediaStream
676 : {
677 : public:
678 : explicit SourceMediaStream();
679 :
680 0 : SourceMediaStream* AsSourceStream() override { return this; }
681 :
682 : // Media graph thread only
683 :
684 : // Users of audio inputs go through the stream so it can track when the
685 : // last stream referencing an input goes away, so it can close the cubeb
686 : // input. Also note: callable on any thread (though it bounces through
687 : // MainThread to set the command if needed).
688 : nsresult OpenAudioInput(int aID,
689 : AudioDataListener *aListener);
690 : // Note: also implied when Destroy() happens
691 : void CloseAudioInput();
692 :
693 : void DestroyImpl() override;
694 :
695 : // Call these on any thread.
696 : /**
697 : * Enable or disable pulling. When pulling is enabled, NotifyPull
698 : * gets called on MediaStreamListeners for this stream during the
699 : * MediaStreamGraph control loop. Pulling is initially disabled.
700 : * Due to unavoidable race conditions, after a call to SetPullEnabled(false)
701 : * it is still possible for a NotifyPull to occur.
702 : */
703 : void SetPullEnabled(bool aEnabled);
704 :
705 : /**
706 : * These add/remove DirectListeners, which allow bypassing the graph and any
707 : * synchronization delays for e.g. PeerConnection, which wants the data ASAP
708 : * and lets the far-end handle sync and playout timing.
709 : */
710 : void NotifyListenersEventImpl(MediaStreamGraphEvent aEvent);
711 : void NotifyListenersEvent(MediaStreamGraphEvent aEvent);
712 : void AddDirectListener(DirectMediaStreamListener* aListener);
713 : void RemoveDirectListener(DirectMediaStreamListener* aListener);
714 :
715 : enum {
716 : ADDTRACK_QUEUED = 0x01 // Queue track add until FinishAddTracks()
717 : };
718 : /**
719 : * Add a new track to the stream starting at the given base time (which
720 : * must be greater than or equal to the last time passed to
721 : * AdvanceKnownTracksTime). Takes ownership of aSegment. aSegment should
722 : * contain data starting after aStart.
723 : */
724 0 : void AddTrack(TrackID aID, StreamTime aStart, MediaSegment* aSegment,
725 : uint32_t aFlags = 0)
726 : {
727 0 : AddTrackInternal(aID, GraphRate(), aStart, aSegment, aFlags);
728 0 : }
729 :
730 : /**
731 : * Like AddTrack, but resamples audio from aRate to the graph rate.
732 : */
733 : void AddAudioTrack(TrackID aID, TrackRate aRate, StreamTime aStart,
734 : AudioSegment* aSegment, uint32_t aFlags = 0);
735 :
736 : /**
737 : * Call after a series of AddTrack or AddAudioTrack calls to implement
738 : * any pending track adds.
739 : */
740 : void FinishAddTracks();
741 :
742 : /**
743 : * Append media data to a track. Ownership of aSegment remains with the caller,
744 : * but aSegment is emptied.
745 : * Returns false if the data was not appended because no such track exists
746 : * or the stream was already finished.
747 : */
748 : virtual bool AppendToTrack(TrackID aID, MediaSegment* aSegment, MediaSegment *aRawSegment = nullptr);
749 : /**
750 : * Get the stream time of the end of the data that has been appended so far.
751 : * Can be called from any thread but won't be useful if it can race with
752 : * an AppendToTrack call, so should probably just be called from the thread
753 : * that also calls AppendToTrack.
754 : */
755 : StreamTime GetEndOfAppendedData(TrackID aID);
756 : /**
757 : * Indicate that a track has ended. Do not do any more API calls
758 : * affecting this track.
759 : * Ignored if the track does not exist.
760 : */
761 : void EndTrack(TrackID aID);
762 : /**
763 : * Indicate that no tracks will be added starting before time aKnownTime.
764 : * aKnownTime must be >= its value at the last call to AdvanceKnownTracksTime.
765 : */
766 : void AdvanceKnownTracksTime(StreamTime aKnownTime);
767 : /**
768 : * Indicate that this stream should enter the "finished" state. All tracks
769 : * must have been ended via EndTrack. The finish time of the stream is
770 : * when all tracks have ended.
771 : */
772 : void FinishWithLockHeld();
773 0 : void Finish()
774 : {
775 0 : MutexAutoLock lock(mMutex);
776 0 : FinishWithLockHeld();
777 0 : }
778 :
779 : // Overriding allows us to hold the mMutex lock while changing the track enable status
780 : void SetTrackEnabledImpl(TrackID aTrackID, DisabledTrackMode aMode) override;
781 :
782 : // Overriding allows us to ensure mMutex is locked while changing the track enable status
783 : void
784 0 : ApplyTrackDisabling(TrackID aTrackID, MediaSegment* aSegment,
785 : MediaSegment* aRawSegment = nullptr) override {
786 0 : mMutex.AssertCurrentThreadOwns();
787 0 : MediaStream::ApplyTrackDisabling(aTrackID, aSegment, aRawSegment);
788 0 : }
789 :
790 : /**
791 : * End all tracks and Finish() this stream. Used to voluntarily revoke access
792 : * to a LocalMediaStream.
793 : */
794 : void EndAllTrackAndFinish();
795 :
796 : void RegisterForAudioMixing();
797 :
798 : /**
799 : * Returns true if this SourceMediaStream contains at least one audio track
800 : * that is in pending state.
801 : * This is thread safe, and takes the SourceMediaStream mutex.
802 : */
803 : bool HasPendingAudioTrack();
804 :
805 0 : TimeStamp GetStreamTracksStrartTimeStamp() {
806 0 : MutexAutoLock lock(mMutex);
807 0 : return mStreamTracksStartTimeStamp;
808 : }
809 :
810 : bool OpenNewAudioCallbackDriver(AudioDataListener *aListener);
811 :
812 : // XXX need a Reset API
813 :
814 : friend class MediaStreamGraphImpl;
815 :
816 : protected:
817 : enum TrackCommands : uint32_t;
818 :
819 : virtual ~SourceMediaStream();
820 :
821 : /**
822 : * Data for each track that hasn't ended.
823 : */
824 0 : struct TrackData {
825 : TrackID mID;
826 : // Sample rate of the input data.
827 : TrackRate mInputRate;
828 : // Resampler if the rate of the input track does not match the
829 : // MediaStreamGraph's.
830 : nsAutoRef<SpeexResamplerState> mResampler;
831 : int mResamplerChannelCount;
832 : StreamTime mStart;
833 : // End-time of data already flushed to the track (excluding mData)
834 : StreamTime mEndOfFlushedData;
835 : // Each time the track updates are flushed to the media graph thread,
836 : // the segment buffer is emptied.
837 : nsAutoPtr<MediaSegment> mData;
838 : // Each time the track updates are flushed to the media graph thread,
839 : // this is cleared.
840 : uint32_t mCommands;
841 : };
842 :
843 : bool NeedsMixing();
844 :
845 : void ResampleAudioToGraphSampleRate(TrackData* aTrackData, MediaSegment* aSegment);
846 :
847 : void AddDirectTrackListenerImpl(already_AddRefed<DirectMediaStreamTrackListener> aListener,
848 : TrackID aTrackID) override;
849 : void RemoveDirectTrackListenerImpl(DirectMediaStreamTrackListener* aListener,
850 : TrackID aTrackID) override;
851 :
852 : void AddTrackInternal(TrackID aID, TrackRate aRate,
853 : StreamTime aStart, MediaSegment* aSegment,
854 : uint32_t aFlags);
855 :
856 0 : TrackData* FindDataForTrack(TrackID aID)
857 : {
858 0 : mMutex.AssertCurrentThreadOwns();
859 0 : for (uint32_t i = 0; i < mUpdateTracks.Length(); ++i) {
860 0 : if (mUpdateTracks[i].mID == aID) {
861 0 : return &mUpdateTracks[i];
862 : }
863 : }
864 0 : return nullptr;
865 : }
866 :
867 : /**
868 : * Notify direct consumers of new data to one of the stream tracks.
869 : * The data doesn't have to be resampled (though it may be). This is called
870 : * from AppendToTrack on the thread providing the data, and will call
871 : * the Listeners on this thread.
872 : */
873 : void NotifyDirectConsumers(TrackData *aTrack,
874 : MediaSegment *aSegment);
875 :
876 : virtual void
877 : AdvanceTimeVaryingValuesToCurrentTime(GraphTime aCurrentTime,
878 : GraphTime aBlockedTime) override;
879 0 : void SetStreamTracksStartTimeStamp(const TimeStamp& aTimeStamp)
880 : {
881 0 : MutexAutoLock lock(mMutex);
882 0 : mStreamTracksStartTimeStamp = aTimeStamp;
883 0 : }
884 :
885 : // Only accessed on the MSG thread. Used so to ask the MSGImpl to usecount
886 : // users of a specific input.
887 : // XXX Should really be a CubebUtils::AudioDeviceID, but they aren't
888 : // copyable (opaque pointers)
889 : RefPtr<AudioDataListener> mInputListener;
890 :
891 : // This must be acquired *before* MediaStreamGraphImpl's lock, if they are
892 : // held together.
893 : Mutex mMutex;
894 : // protected by mMutex
895 : StreamTime mUpdateKnownTracksTime;
896 : // This time stamp will be updated in adding and blocked SourceMediaStream,
897 : // |AddStreamGraphThread| and |AdvanceTimeVaryingValuesToCurrentTime| in
898 : // particularly.
899 : TimeStamp mStreamTracksStartTimeStamp;
900 : nsTArray<TrackData> mUpdateTracks;
901 : nsTArray<TrackData> mPendingTracks;
902 : nsTArray<RefPtr<DirectMediaStreamListener>> mDirectListeners;
903 : nsTArray<TrackBound<DirectMediaStreamTrackListener>> mDirectTrackListeners;
904 : bool mPullEnabled;
905 : bool mUpdateFinished;
906 : bool mNeedsMixing;
907 : };
908 :
909 : /**
910 : * The blocking mode decides how a track should be blocked in a MediaInputPort.
911 : */
912 : enum class BlockingMode
913 : {
914 : /**
915 : * BlockingMode CREATION blocks the source track from being created
916 : * in the destination. It'll end if it already exists.
917 : */
918 : CREATION,
919 : /**
920 : * BlockingMode END_EXISTING allows a track to be created in the destination
921 : * but will end it before any data has been passed through.
922 : */
923 : END_EXISTING,
924 : };
925 :
926 : /**
927 : * Represents a connection between a ProcessedMediaStream and one of its
928 : * input streams.
929 : * We make these refcounted so that stream-related messages with MediaInputPort*
930 : * pointers can be sent to the main thread safely.
931 : *
932 : * A port can be locked to a specific track in the source stream, in which case
933 : * only this track will be forwarded to the destination stream. TRACK_ANY
934 : * can used to signal that all tracks shall be forwarded.
935 : *
936 : * When a port is locked to a specific track in the source stream, it may also
937 : * indicate a TrackID to map this source track to in the destination stream
938 : * by setting aDestTrack to an explicit ID. When we do this, we must know
939 : * that this TrackID in the destination stream is available. We assert during
940 : * processing that the ID is available and that there are no generic input
941 : * ports already attached to the destination stream.
942 : * Note that this is currently only handled by TrackUnionStreams.
943 : *
944 : * When a port's source or destination stream dies, the stream's DestroyImpl
945 : * calls MediaInputPort::Disconnect to disconnect the port from
946 : * the source and destination streams.
947 : *
948 : * The lifetimes of MediaInputPort are controlled from the main thread.
949 : * The media graph adds a reference to the port. When a MediaInputPort is no
950 : * longer needed, main-thread code sends a Destroy message for the port and
951 : * clears its reference (the last main-thread reference to the object). When
952 : * the Destroy message is processed on the graph manager thread we disconnect
953 : * the port and drop the graph's reference, destroying the object.
954 : */
955 : class MediaInputPort final
956 : {
957 : private:
958 : // Do not call this constructor directly. Instead call aDest->AllocateInputPort.
959 0 : MediaInputPort(MediaStream* aSource,
960 : TrackID& aSourceTrack,
961 : ProcessedMediaStream* aDest,
962 : TrackID& aDestTrack,
963 : uint16_t aInputNumber,
964 : uint16_t aOutputNumber)
965 0 : : mSource(aSource)
966 : , mSourceTrack(aSourceTrack)
967 : , mDest(aDest)
968 : , mDestTrack(aDestTrack)
969 : , mInputNumber(aInputNumber)
970 : , mOutputNumber(aOutputNumber)
971 0 : , mGraph(nullptr)
972 : {
973 0 : MOZ_COUNT_CTOR(MediaInputPort);
974 0 : }
975 :
976 : // Private destructor, to discourage deletion outside of Release():
977 0 : ~MediaInputPort()
978 0 : {
979 0 : MOZ_COUNT_DTOR(MediaInputPort);
980 0 : }
981 :
982 : public:
983 0 : NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaInputPort)
984 :
985 : // Called on graph manager thread
986 : // Do not call these from outside MediaStreamGraph.cpp!
987 : void Init();
988 : // Called during message processing to trigger removal of this stream.
989 : void Disconnect();
990 :
991 : // Control API
992 : /**
993 : * Disconnects and destroys the port. The caller must not reference this
994 : * object again.
995 : */
996 : void Destroy();
997 :
998 : // Any thread
999 0 : MediaStream* GetSource() { return mSource; }
1000 0 : TrackID GetSourceTrackId() { return mSourceTrack; }
1001 0 : ProcessedMediaStream* GetDestination() { return mDest; }
1002 0 : TrackID GetDestinationTrackId() { return mDestTrack; }
1003 :
1004 : /**
1005 : * Block aTrackId in the source stream from being passed through the port.
1006 : * Consumers will interpret this track as ended.
1007 : * Returns a pledge that resolves on the main thread after the track block has
1008 : * been applied by the MSG.
1009 : */
1010 : already_AddRefed<media::Pledge<bool, nsresult>> BlockSourceTrackId(TrackID aTrackId,
1011 : BlockingMode aBlockingMode);
1012 : private:
1013 : void BlockSourceTrackIdImpl(TrackID aTrackId, BlockingMode aBlockingMode);
1014 :
1015 : public:
1016 : // Returns true if aTrackId has not been blocked for any reason and this port
1017 : // has not been locked to another track.
1018 0 : bool PassTrackThrough(TrackID aTrackId) {
1019 0 : bool blocked = false;
1020 0 : for (auto pair : mBlockedTracks) {
1021 0 : if (pair.first() == aTrackId &&
1022 0 : (pair.second() == BlockingMode::CREATION ||
1023 0 : pair.second() == BlockingMode::END_EXISTING)) {
1024 0 : blocked = true;
1025 0 : break;
1026 : }
1027 : }
1028 0 : return !blocked && (mSourceTrack == TRACK_ANY || mSourceTrack == aTrackId);
1029 : }
1030 :
1031 : // Returns true if aTrackId has not been blocked for track creation and this
1032 : // port has not been locked to another track.
1033 0 : bool AllowCreationOf(TrackID aTrackId) {
1034 0 : bool blocked = false;
1035 0 : for (auto pair : mBlockedTracks) {
1036 0 : if (pair.first() == aTrackId &&
1037 0 : pair.second() == BlockingMode::CREATION) {
1038 0 : blocked = true;
1039 0 : break;
1040 : }
1041 : }
1042 0 : return !blocked && (mSourceTrack == TRACK_ANY || mSourceTrack == aTrackId);
1043 : }
1044 :
1045 0 : uint16_t InputNumber() const { return mInputNumber; }
1046 0 : uint16_t OutputNumber() const { return mOutputNumber; }
1047 :
1048 : // Call on graph manager thread
1049 : struct InputInterval {
1050 : GraphTime mStart;
1051 : GraphTime mEnd;
1052 : bool mInputIsBlocked;
1053 : };
1054 : // Find the next time interval starting at or after aTime during which
1055 : // mDest is not blocked and mSource's blocking status does not change.
1056 : InputInterval GetNextInputInterval(GraphTime aTime);
1057 :
1058 : /**
1059 : * Returns the graph that owns this port.
1060 : */
1061 : MediaStreamGraphImpl* GraphImpl();
1062 : MediaStreamGraph* Graph();
1063 :
1064 : /**
1065 : * Sets the graph that owns this stream. Should only be called once.
1066 : */
1067 : void SetGraphImpl(MediaStreamGraphImpl* aGraph);
1068 :
1069 : /**
1070 : * Notify the port that the source MediaStream has been suspended.
1071 : */
1072 : void Suspended();
1073 :
1074 : /**
1075 : * Notify the port that the source MediaStream has been resumed.
1076 : */
1077 : void Resumed();
1078 :
1079 0 : size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const
1080 : {
1081 0 : size_t amount = 0;
1082 :
1083 : // Not owned:
1084 : // - mSource
1085 : // - mDest
1086 : // - mGraph
1087 0 : return amount;
1088 : }
1089 :
1090 0 : size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const
1091 : {
1092 0 : return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
1093 : }
1094 :
1095 : private:
1096 : friend class MediaStreamGraphImpl;
1097 : friend class MediaStream;
1098 : friend class ProcessedMediaStream;
1099 : // Never modified after Init()
1100 : MediaStream* mSource;
1101 : TrackID mSourceTrack;
1102 : ProcessedMediaStream* mDest;
1103 : TrackID mDestTrack;
1104 : // The input and output numbers are optional, and are currently only used by
1105 : // Web Audio.
1106 : const uint16_t mInputNumber;
1107 : const uint16_t mOutputNumber;
1108 :
1109 : typedef Pair<TrackID, BlockingMode> BlockedTrack;
1110 : nsTArray<BlockedTrack> mBlockedTracks;
1111 :
1112 : // Our media stream graph
1113 : MediaStreamGraphImpl* mGraph;
1114 : };
1115 :
1116 : /**
1117 : * This stream processes zero or more input streams in parallel to produce
1118 : * its output. The details of how the output is produced are handled by
1119 : * subclasses overriding the ProcessInput method.
1120 : */
1121 0 : class ProcessedMediaStream : public MediaStream
1122 : {
1123 : public:
1124 0 : explicit ProcessedMediaStream()
1125 0 : : MediaStream()
1126 : , mAutofinish(false)
1127 0 : , mCycleMarker(0)
1128 0 : {}
1129 :
1130 : // Control API.
1131 : /**
1132 : * Allocates a new input port attached to source aStream.
1133 : * This stream can be removed by calling MediaInputPort::Remove().
1134 : *
1135 : * The input port is tied to aTrackID in the source stream.
1136 : * aTrackID can be set to TRACK_ANY to automatically forward all tracks from
1137 : * aStream.
1138 : *
1139 : * If aTrackID is an explicit ID, aDestTrackID can also be made explicit
1140 : * to ensure that the track is assigned this ID in the destination stream.
1141 : * To avoid intermittent TrackID collisions the destination stream may not
1142 : * have any existing generic input ports (with TRACK_ANY source track) when
1143 : * you allocate an input port with a destination TrackID.
1144 : *
1145 : * To end a track in the destination stream forwarded with TRACK_ANY,
1146 : * it can be blocked in the input port through MediaInputPort::BlockTrackId().
1147 : *
1148 : * Tracks in aBlockedTracks will be blocked in the input port initially. This
1149 : * ensures that they don't get created by the MSG-thread before we can
1150 : * BlockTrackId() on the main thread.
1151 : */
1152 : already_AddRefed<MediaInputPort>
1153 : AllocateInputPort(MediaStream* aStream,
1154 : TrackID aTrackID = TRACK_ANY,
1155 : TrackID aDestTrackID = TRACK_ANY,
1156 : uint16_t aInputNumber = 0,
1157 : uint16_t aOutputNumber = 0,
1158 : nsTArray<TrackID>* aBlockedTracks = nullptr);
1159 : /**
1160 : * Force this stream into the finished state.
1161 : */
1162 : void Finish();
1163 : /**
1164 : * Set the autofinish flag on this stream (defaults to false). When this flag
1165 : * is set, and all input streams are in the finished state (including if there
1166 : * are no input streams), this stream automatically enters the finished state.
1167 : */
1168 : void SetAutofinish(bool aAutofinish);
1169 :
1170 0 : ProcessedMediaStream* AsProcessedStream() override { return this; }
1171 :
1172 : friend class MediaStreamGraphImpl;
1173 :
1174 : // Do not call these from outside MediaStreamGraph.cpp!
1175 : virtual void AddInput(MediaInputPort* aPort);
1176 0 : virtual void RemoveInput(MediaInputPort* aPort)
1177 : {
1178 0 : mInputs.RemoveElement(aPort) || mSuspendedInputs.RemoveElement(aPort);
1179 0 : }
1180 : bool HasInputPort(MediaInputPort* aPort)
1181 : {
1182 : return mInputs.Contains(aPort) || mSuspendedInputs.Contains(aPort);
1183 : }
1184 : uint32_t InputPortCount()
1185 : {
1186 : return mInputs.Length() + mSuspendedInputs.Length();
1187 : }
1188 : void InputSuspended(MediaInputPort* aPort);
1189 : void InputResumed(MediaInputPort* aPort);
1190 0 : virtual MediaStream* GetInputStreamFor(TrackID aTrackID) { return nullptr; }
1191 0 : virtual TrackID GetInputTrackIDFor(TrackID aTrackID) { return TRACK_NONE; }
1192 : void DestroyImpl() override;
1193 : /**
1194 : * This gets called after we've computed the blocking states for all
1195 : * streams (mBlocked is up to date up to mStateComputedTime).
1196 : * Also, we've produced output for all streams up to this one. If this stream
1197 : * is not in a cycle, then all its source streams have produced data.
1198 : * Generate output from aFrom to aTo.
1199 : * This will be called on streams that have finished. Most stream types should
1200 : * just return immediately if IsFinishedOnGraphThread(), but some may wish to
1201 : * update internal state (see AudioNodeStream).
1202 : * ProcessInput is allowed to call FinishOnGraphThread only if ALLOW_FINISH
1203 : * is in aFlags. (This flag will be set when aTo >= mStateComputedTime, i.e.
1204 : * when we've producing the last block of data we need to produce.) Otherwise
1205 : * we can get into a situation where we've determined the stream should not
1206 : * block before mStateComputedTime, but the stream finishes before
1207 : * mStateComputedTime, violating the invariant that finished streams are blocked.
1208 : */
1209 : enum {
1210 : ALLOW_FINISH = 0x01
1211 : };
1212 : virtual void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) = 0;
1213 0 : void SetAutofinishImpl(bool aAutofinish) { mAutofinish = aAutofinish; }
1214 :
1215 : // Only valid after MediaStreamGraphImpl::UpdateStreamOrder() has run.
1216 : // A DelayNode is considered to break a cycle and so this will not return
1217 : // true for echo loops, only for muted cycles.
1218 0 : bool InMutedCycle() const { return mCycleMarker; }
1219 :
1220 0 : size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const override
1221 : {
1222 0 : size_t amount = MediaStream::SizeOfExcludingThis(aMallocSizeOf);
1223 : // Not owned:
1224 : // - mInputs elements
1225 : // - mSuspendedInputs elements
1226 0 : amount += mInputs.ShallowSizeOfExcludingThis(aMallocSizeOf);
1227 0 : amount += mSuspendedInputs.ShallowSizeOfExcludingThis(aMallocSizeOf);
1228 0 : return amount;
1229 : }
1230 :
1231 0 : size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const override
1232 : {
1233 0 : return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf);
1234 : }
1235 :
1236 : protected:
1237 : // This state is all accessed only on the media graph thread.
1238 :
1239 : // The list of all inputs that are not currently suspended.
1240 : nsTArray<MediaInputPort*> mInputs;
1241 : // The list of all inputs that are currently suspended.
1242 : nsTArray<MediaInputPort*> mSuspendedInputs;
1243 : bool mAutofinish;
1244 : // After UpdateStreamOrder(), mCycleMarker is either 0 or 1 to indicate
1245 : // whether this stream is in a muted cycle. During ordering it can contain
1246 : // other marker values - see MediaStreamGraphImpl::UpdateStreamOrder().
1247 : uint32_t mCycleMarker;
1248 : };
1249 :
1250 : /**
1251 : * There is a single MediaStreamGraph per window.
1252 : * Additionaly, each OfflineAudioContext object creates its own MediaStreamGraph
1253 : * object too..
1254 : */
1255 : class MediaStreamGraph
1256 : {
1257 : public:
1258 :
1259 : // We ensure that the graph current time advances in multiples of
1260 : // IdealAudioBlockSize()/AudioStream::PreferredSampleRate(). A stream that
1261 : // never blocks and has a track with the ideal audio rate will produce audio
1262 : // in multiples of the block size.
1263 :
1264 : // Initializing an graph that outputs audio can be quite long on some
1265 : // platforms. Code that want to output audio at some point can express the
1266 : // fact that they will need an audio stream at some point by passing
1267 : // AUDIO_THREAD_DRIVER when getting an instance of MediaStreamGraph, so that
1268 : // the graph starts with the right driver.
1269 : enum GraphDriverType {
1270 : AUDIO_THREAD_DRIVER,
1271 : SYSTEM_THREAD_DRIVER,
1272 : OFFLINE_THREAD_DRIVER
1273 : };
1274 : static const uint32_t AUDIO_CALLBACK_DRIVER_SHUTDOWN_TIMEOUT = 20*1000;
1275 :
1276 : // Main thread only
1277 : static MediaStreamGraph* GetInstance(GraphDriverType aGraphDriverRequested,
1278 : dom::AudioChannel aChannel,
1279 : nsPIDOMWindowInner* aWindow);
1280 : static MediaStreamGraph* CreateNonRealtimeInstance(
1281 : TrackRate aSampleRate,
1282 : nsPIDOMWindowInner* aWindowId);
1283 :
1284 : // Return the correct main thread for this graph. This always returns
1285 : // something that is valid. Thread safe.
1286 : AbstractThread* AbstractMainThread();
1287 :
1288 : // Idempotent
1289 : static void DestroyNonRealtimeInstance(MediaStreamGraph* aGraph);
1290 :
1291 0 : virtual nsresult OpenAudioInput(int aID,
1292 : AudioDataListener *aListener) {
1293 0 : return NS_ERROR_FAILURE;
1294 : }
1295 0 : virtual void CloseAudioInput(AudioDataListener *aListener) {}
1296 :
1297 : // Control API.
1298 : /**
1299 : * Create a stream that a media decoder (or some other source of
1300 : * media data, such as a camera) can write to.
1301 : */
1302 : SourceMediaStream* CreateSourceStream();
1303 : /**
1304 : * Create a stream that will form the union of the tracks of its input
1305 : * streams.
1306 : * A TrackUnionStream contains all the tracks of all its input streams.
1307 : * Adding a new input stream makes that stream's tracks immediately appear as new
1308 : * tracks starting at the time the input stream was added.
1309 : * Removing an input stream makes the output tracks corresponding to the
1310 : * removed tracks immediately end.
1311 : * For each added track, the track ID of the output track is the track ID
1312 : * of the input track or one plus the maximum ID of all previously added
1313 : * tracks, whichever is greater.
1314 : * TODO at some point we will probably need to add API to select
1315 : * particular tracks of each input stream.
1316 : */
1317 : ProcessedMediaStream* CreateTrackUnionStream();
1318 : /**
1319 : * Create a stream that will mix all its audio input.
1320 : */
1321 : ProcessedMediaStream* CreateAudioCaptureStream(TrackID aTrackId);
1322 :
1323 : /**
1324 : * Add a new stream to the graph. Main thread.
1325 : */
1326 : void AddStream(MediaStream* aStream);
1327 :
1328 : /* From the main thread, ask the MSG to send back an event when the graph
1329 : * thread is running, and audio is being processed. */
1330 : void NotifyWhenGraphStarted(AudioNodeStream* aNodeStream);
1331 : /* From the main thread, suspend, resume or close an AudioContext.
1332 : * aStreams are the streams of all the AudioNodes of the AudioContext that
1333 : * need to be suspended or resumed. This can be empty if this is a second
1334 : * consecutive suspend call and all the nodes are already suspended.
1335 : *
1336 : * This can possibly pause the graph thread, releasing system resources, if
1337 : * all streams have been suspended/closed.
1338 : *
1339 : * When the operation is complete, aPromise is resolved.
1340 : */
1341 : void ApplyAudioContextOperation(MediaStream* aDestinationStream,
1342 : const nsTArray<MediaStream*>& aStreams,
1343 : dom::AudioContextOperation aState,
1344 : void* aPromise);
1345 :
1346 : bool IsNonRealtime() const;
1347 : /**
1348 : * Start processing non-realtime for a specific number of ticks.
1349 : */
1350 : void StartNonRealtimeProcessing(uint32_t aTicksToProcess);
1351 :
1352 : /**
1353 : * Media graph thread only.
1354 : * Dispatches a runnable that will run on the main thread after all
1355 : * main-thread stream state has been next updated.
1356 : *
1357 : * Should only be called during MediaStreamListener callbacks or during
1358 : * ProcessedMediaStream::ProcessInput().
1359 : */
1360 : virtual void DispatchToMainThreadAfterStreamStateUpdate(
1361 : already_AddRefed<nsIRunnable> aRunnable);
1362 :
1363 : /**
1364 : * Returns graph sample rate in Hz.
1365 : */
1366 0 : TrackRate GraphRate() const { return mSampleRate; }
1367 :
1368 : void RegisterCaptureStreamForWindow(uint64_t aWindowId,
1369 : ProcessedMediaStream* aCaptureStream);
1370 : void UnregisterCaptureStreamForWindow(uint64_t aWindowId);
1371 : already_AddRefed<MediaInputPort> ConnectToCaptureStream(
1372 : uint64_t aWindowId, MediaStream* aMediaStream);
1373 :
1374 : /**
1375 : * Data going to the speakers from the GraphDriver's DataCallback
1376 : * to notify any listeners (for echo cancellation).
1377 : */
1378 : void NotifyOutputData(AudioDataValue* aBuffer, size_t aFrames,
1379 : TrackRate aRate, uint32_t aChannels);
1380 :
1381 : void AssertOnGraphThreadOrNotRunning() const;
1382 :
1383 : protected:
1384 0 : explicit MediaStreamGraph(TrackRate aSampleRate)
1385 0 : : mSampleRate(aSampleRate)
1386 : {
1387 0 : MOZ_COUNT_CTOR(MediaStreamGraph);
1388 0 : }
1389 0 : virtual ~MediaStreamGraph()
1390 0 : {
1391 0 : MOZ_COUNT_DTOR(MediaStreamGraph);
1392 0 : }
1393 :
1394 : // Media graph thread only
1395 : nsTArray<nsCOMPtr<nsIRunnable> > mPendingUpdateRunnables;
1396 :
1397 : /**
1398 : * Sample rate at which this graph runs. For real time graphs, this is
1399 : * the rate of the audio mixer. For offline graphs, this is the rate specified
1400 : * at construction.
1401 : */
1402 : TrackRate mSampleRate;
1403 :
1404 : /**
1405 : * CloseAudioInput is async, so hold a reference here.
1406 : */
1407 : nsTArray<RefPtr<AudioDataListener>> mAudioInputs;
1408 : };
1409 :
1410 : } // namespace mozilla
1411 :
1412 : #endif /* MOZILLA_MEDIASTREAMGRAPH_H_ */
|
