Line data Source code
1 : /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
2 : /* This Source Code Form is subject to the terms of the Mozilla Public
3 : * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 : * You can obtain one at http://mozilla.org/MPL/2.0/. */
5 :
6 : #ifndef MOZILLA_MEDIASTREAMGRAPHIMPL_H_
7 : #define MOZILLA_MEDIASTREAMGRAPHIMPL_H_
8 :
9 : #include "MediaStreamGraph.h"
10 :
11 : #include "nsDataHashtable.h"
12 :
13 : #include "nsITimer.h"
14 : #include "mozilla/Monitor.h"
15 : #include "mozilla/TimeStamp.h"
16 : #include "nsIMemoryReporter.h"
17 : #include "nsIThread.h"
18 : #include "nsIRunnable.h"
19 : #include "nsIAsyncShutdown.h"
20 : #include "Latency.h"
21 : #include "mozilla/Services.h"
22 : #include "mozilla/UniquePtr.h"
23 : #include "mozilla/WeakPtr.h"
24 : #include "GraphDriver.h"
25 : #include "AudioMixer.h"
26 :
27 : namespace mozilla {
28 :
29 : template <typename T>
30 : class LinkedList;
31 : #ifdef MOZ_WEBRTC
32 : class AudioOutputObserver;
33 : #endif
34 :
35 : /**
36 : * A per-stream update message passed from the media graph thread to the
37 : * main thread.
38 : */
39 0 : struct StreamUpdate
40 : {
41 : RefPtr<MediaStream> mStream;
42 : StreamTime mNextMainThreadCurrentTime;
43 : bool mNextMainThreadFinished;
44 : };
45 :
46 : /**
47 : * This represents a message run on the graph thread to modify stream or graph
48 : * state. These are passed from main thread to graph thread through
49 : * AppendMessage(), or scheduled on the graph thread with
50 : * RunMessageAfterProcessing(). A ControlMessage
51 : * always has a weak reference to a particular affected stream.
52 : */
53 : class ControlMessage
54 : {
55 : public:
56 0 : explicit ControlMessage(MediaStream* aStream) : mStream(aStream)
57 : {
58 0 : MOZ_COUNT_CTOR(ControlMessage);
59 0 : }
60 : // All these run on the graph thread
61 0 : virtual ~ControlMessage()
62 0 : {
63 0 : MOZ_COUNT_DTOR(ControlMessage);
64 0 : }
65 : // Do the action of this message on the MediaStreamGraph thread. Any actions
66 : // affecting graph processing should take effect at mProcessedTime.
67 : // All stream data for times < mProcessedTime has already been
68 : // computed.
69 : virtual void Run() = 0;
70 : // RunDuringShutdown() is only relevant to messages generated on the main
71 : // thread (for AppendMessage()).
72 : // When we're shutting down the application, most messages are ignored but
73 : // some cleanup messages should still be processed (on the main thread).
74 : // This must not add new control messages to the graph.
75 0 : virtual void RunDuringShutdown() {}
76 0 : MediaStream* GetStream() { return mStream; }
77 :
78 : protected:
79 : // We do not hold a reference to mStream. The graph will be holding
80 : // a reference to the stream until the Destroy message is processed. The
81 : // last message referencing a stream is the Destroy message for that stream.
82 : MediaStream* mStream;
83 : };
84 :
85 0 : class MessageBlock
86 : {
87 : public:
88 : nsTArray<UniquePtr<ControlMessage>> mMessages;
89 : };
90 :
91 : /**
92 : * The implementation of a media stream graph. This class is private to this
93 : * file. It's not in the anonymous namespace because MediaStream needs to
94 : * be able to friend it.
95 : *
96 : * There can be multiple MediaStreamGraph per process: one per AudioChannel.
97 : * Additionaly, each OfflineAudioContext object creates its own MediaStreamGraph
98 : * object too.
99 : */
100 : class MediaStreamGraphImpl : public MediaStreamGraph,
101 : public nsIMemoryReporter,
102 : public nsITimerCallback
103 : {
104 : public:
105 : NS_DECL_THREADSAFE_ISUPPORTS
106 : NS_DECL_NSIMEMORYREPORTER
107 : NS_DECL_NSITIMERCALLBACK
108 :
109 : /**
110 : * Use aGraphDriverRequested with SYSTEM_THREAD_DRIVER or AUDIO_THREAD_DRIVER
111 : * to create a MediaStreamGraph which provides support for real-time audio
112 : * and/or video. Set it to OFFLINE_THREAD_DRIVER in order to create a
113 : * non-realtime instance which just churns through its inputs and produces
114 : * output. Those objects currently only support audio, and are used to
115 : * implement OfflineAudioContext. They do not support MediaStream inputs.
116 : */
117 : explicit MediaStreamGraphImpl(GraphDriverType aGraphDriverRequested,
118 : TrackRate aSampleRate,
119 : dom::AudioChannel aChannel,
120 : AbstractThread* aWindow);
121 :
122 : /**
123 : * Unregisters memory reporting and deletes this instance. This should be
124 : * called instead of calling the destructor directly.
125 : */
126 : void Destroy();
127 :
128 : // Main thread only.
129 : /**
130 : * This runs every time we need to sync state from the media graph thread
131 : * to the main thread while the main thread is not in the middle
132 : * of a script. It runs during a "stable state" (per HTML5) or during
133 : * an event posted to the main thread.
134 : * The boolean affects which boolean controlling runnable dispatch is cleared
135 : */
136 : void RunInStableState(bool aSourceIsMSG);
137 : /**
138 : * Ensure a runnable to run RunInStableState is posted to the appshell to
139 : * run at the next stable state (per HTML5).
140 : * See EnsureStableStateEventPosted.
141 : */
142 : void EnsureRunInStableState();
143 : /**
144 : * Called to apply a StreamUpdate to its stream.
145 : */
146 : void ApplyStreamUpdate(StreamUpdate* aUpdate);
147 : /**
148 : * Append a ControlMessage to the message queue. This queue is drained
149 : * during RunInStableState; the messages will run on the graph thread.
150 : */
151 : void AppendMessage(UniquePtr<ControlMessage> aMessage);
152 :
153 : /**
154 : * Dispatches a runnable from any thread to the correct main thread for this
155 : * MediaStreamGraph.
156 : */
157 : void Dispatch(already_AddRefed<nsIRunnable>&& aRunnable);
158 :
159 : // Shutdown helpers.
160 :
161 : static already_AddRefed<nsIAsyncShutdownClient>
162 0 : GetShutdownBarrier()
163 : {
164 0 : nsCOMPtr<nsIAsyncShutdownService> svc = services::GetAsyncShutdown();
165 0 : MOZ_RELEASE_ASSERT(svc);
166 :
167 0 : nsCOMPtr<nsIAsyncShutdownClient> barrier;
168 0 : nsresult rv = svc->GetProfileBeforeChange(getter_AddRefs(barrier));
169 0 : if (!barrier) {
170 : // We are probably in a content process. We need to do cleanup at
171 : // XPCOM shutdown in leakchecking builds.
172 0 : rv = svc->GetXpcomWillShutdown(getter_AddRefs(barrier));
173 : }
174 0 : MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv));
175 0 : MOZ_RELEASE_ASSERT(barrier);
176 0 : return barrier.forget();
177 : }
178 :
179 : class ShutdownTicket final
180 : {
181 : public:
182 0 : explicit ShutdownTicket(nsIAsyncShutdownBlocker* aBlocker) : mBlocker(aBlocker) {}
183 0 : NS_INLINE_DECL_REFCOUNTING(ShutdownTicket)
184 : private:
185 0 : ~ShutdownTicket()
186 0 : {
187 0 : nsCOMPtr<nsIAsyncShutdownClient> barrier = GetShutdownBarrier();
188 0 : barrier->RemoveBlocker(mBlocker);
189 0 : }
190 :
191 : nsCOMPtr<nsIAsyncShutdownBlocker> mBlocker;
192 : };
193 :
194 : /**
195 : * Make this MediaStreamGraph enter forced-shutdown state. This state
196 : * will be noticed by the media graph thread, which will shut down all streams
197 : * and other state controlled by the media graph thread.
198 : * This is called during application shutdown.
199 : */
200 : void ForceShutDown(ShutdownTicket* aShutdownTicket);
201 :
202 : /**
203 : * Called before the thread runs.
204 : */
205 : void Init();
206 :
207 : /**
208 : * Respond to CollectReports with sizes collected on the graph thread.
209 : */
210 : static void
211 : FinishCollectReports(nsIHandleReportCallback* aHandleReport,
212 : nsISupports* aData,
213 : const nsTArray<AudioNodeSizes>& aAudioStreamSizes);
214 :
215 : // The following methods run on the graph thread (or possibly the main thread
216 : // if mLifecycleState > LIFECYCLE_RUNNING)
217 : void CollectSizesForMemoryReport(
218 : already_AddRefed<nsIHandleReportCallback> aHandleReport,
219 : already_AddRefed<nsISupports> aHandlerData);
220 :
221 : /**
222 : * Returns true if this MediaStreamGraph should keep running
223 : */
224 : bool UpdateMainThreadState();
225 :
226 : /**
227 : * Returns true if this MediaStreamGraph should keep running
228 : */
229 : bool OneIteration(GraphTime aStateEnd);
230 :
231 : bool Running() const
232 : {
233 : mMonitor.AssertCurrentThreadOwns();
234 : return mLifecycleState == LIFECYCLE_RUNNING;
235 : }
236 :
237 : /* This is the end of the current iteration, that is, the current time of the
238 : * graph. */
239 : GraphTime IterationEnd() const;
240 :
241 : /**
242 : * Ensure there is an event posted to the main thread to run RunInStableState.
243 : * mMonitor must be held.
244 : * See EnsureRunInStableState
245 : */
246 : void EnsureStableStateEventPosted();
247 : /**
248 : * Generate messages to the main thread to update it for all state changes.
249 : * mMonitor must be held.
250 : */
251 : void PrepareUpdatesToMainThreadState(bool aFinalUpdate);
252 : /**
253 : * Returns false if there is any stream that has finished but not yet finished
254 : * playing out.
255 : */
256 : bool AllFinishedStreamsNotified();
257 : /**
258 : * If we are rendering in non-realtime mode, we don't want to send messages to
259 : * the main thread at each iteration for performance reasons. We instead
260 : * notify the main thread at the same rate
261 : */
262 : bool ShouldUpdateMainThread();
263 : // The following methods are the various stages of RunThread processing.
264 : /**
265 : * Advance all stream state to mStateComputedTime.
266 : */
267 : void UpdateCurrentTimeForStreams(GraphTime aPrevCurrentTime);
268 : /**
269 : * Process chunks for all streams and raise events for properties that have
270 : * changed, such as principalId.
271 : */
272 : void ProcessChunkMetadata(GraphTime aPrevCurrentTime);
273 : /**
274 : * Process chunks for the given stream and interval, and raise events for
275 : * properties that have changed, such as principalId.
276 : */
277 : template<typename C, typename Chunk>
278 : void ProcessChunkMetadataForInterval(MediaStream* aStream,
279 : TrackID aTrackID,
280 : C& aSegment,
281 : StreamTime aStart,
282 : StreamTime aEnd);
283 : /**
284 : * Process graph messages in mFrontMessageQueue.
285 : */
286 : void RunMessagesInQueue();
287 : /**
288 : * Update stream processing order and recompute stream blocking until
289 : * aEndBlockingDecisions.
290 : */
291 : void UpdateGraph(GraphTime aEndBlockingDecisions);
292 :
293 0 : void SwapMessageQueues()
294 : {
295 0 : MOZ_ASSERT(CurrentDriver()->OnThread());
296 0 : MOZ_ASSERT(mFrontMessageQueue.IsEmpty());
297 0 : mMonitor.AssertCurrentThreadOwns();
298 0 : mFrontMessageQueue.SwapElements(mBackMessageQueue);
299 0 : }
300 : /**
301 : * Do all the processing and play the audio and video, from
302 : * mProcessedTime to mStateComputedTime.
303 : */
304 : void Process();
305 : /**
306 : * Extract any state updates pending in aStream, and apply them.
307 : */
308 : void ExtractPendingInput(SourceMediaStream* aStream,
309 : GraphTime aDesiredUpToTime,
310 : bool* aEnsureNextIteration);
311 :
312 : /**
313 : * For use during ProcessedMediaStream::ProcessInput() or
314 : * MediaStreamListener callbacks, when graph state cannot be changed.
315 : * Schedules |aMessage| to run after processing, at a time when graph state
316 : * can be changed. Graph thread.
317 : */
318 : void RunMessageAfterProcessing(UniquePtr<ControlMessage> aMessage);
319 :
320 : /**
321 : * Called when a suspend/resume/close operation has been completed, on the
322 : * graph thread.
323 : */
324 : void AudioContextOperationCompleted(MediaStream* aStream,
325 : void* aPromise,
326 : dom::AudioContextOperation aOperation);
327 :
328 : /**
329 : * Apply and AudioContext operation (suspend/resume/closed), on the graph
330 : * thread.
331 : */
332 : void ApplyAudioContextOperationImpl(MediaStream* aDestinationStream,
333 : const nsTArray<MediaStream*>& aStreams,
334 : dom::AudioContextOperation aOperation,
335 : void* aPromise);
336 :
337 : /**
338 : * Increment suspend count on aStream and move it to mSuspendedStreams if
339 : * necessary.
340 : */
341 : void IncrementSuspendCount(MediaStream* aStream);
342 : /**
343 : * Increment suspend count on aStream and move it to mStreams if
344 : * necessary.
345 : */
346 : void DecrementSuspendCount(MediaStream* aStream);
347 :
348 : /*
349 : * Move streams from the mStreams to mSuspendedStream if suspending/closing an
350 : * AudioContext, or the inverse when resuming an AudioContext.
351 : */
352 : void SuspendOrResumeStreams(dom::AudioContextOperation aAudioContextOperation,
353 : const nsTArray<MediaStream*>& aStreamSet);
354 :
355 : /**
356 : * Determine if we have any audio tracks, or are about to add any audiotracks.
357 : * Also checks if we'll need the AEC running (i.e. microphone input tracks)
358 : */
359 : bool AudioTrackPresent(bool& aNeedsAEC);
360 :
361 : /**
362 : * Sort mStreams so that every stream not in a cycle is after any streams
363 : * it depends on, and every stream in a cycle is marked as being in a cycle.
364 : * Also sets mIsConsumed on every stream.
365 : */
366 : void UpdateStreamOrder();
367 :
368 : /**
369 : * Returns smallest value of t such that t is a multiple of
370 : * WEBAUDIO_BLOCK_SIZE and t > aTime.
371 : */
372 : GraphTime RoundUpToNextAudioBlock(GraphTime aTime);
373 : /**
374 : * Produce data for all streams >= aStreamIndex for the current time interval.
375 : * Advances block by block, each iteration producing data for all streams
376 : * for a single block.
377 : * This is called whenever we have an AudioNodeStream in the graph.
378 : */
379 : void ProduceDataForStreamsBlockByBlock(uint32_t aStreamIndex,
380 : TrackRate aSampleRate);
381 : /**
382 : * If aStream will underrun between aTime, and aEndBlockingDecisions, returns
383 : * the time at which the underrun will start. Otherwise return
384 : * aEndBlockingDecisions.
385 : */
386 : GraphTime WillUnderrun(MediaStream* aStream, GraphTime aEndBlockingDecisions);
387 :
388 : /**
389 : * Given a graph time aTime, convert it to a stream time taking into
390 : * account the time during which aStream is scheduled to be blocked.
391 : */
392 : StreamTime GraphTimeToStreamTimeWithBlocking(MediaStream* aStream, GraphTime aTime);
393 :
394 : /**
395 : * Call NotifyHaveCurrentData on aStream's listeners.
396 : */
397 : void NotifyHasCurrentData(MediaStream* aStream);
398 : /**
399 : * If aStream needs an audio stream but doesn't have one, create it.
400 : * If aStream doesn't need an audio stream but has one, destroy it.
401 : */
402 : void CreateOrDestroyAudioStreams(MediaStream* aStream);
403 : /**
404 : * Queue audio (mix of stream audio and silence for blocked intervals)
405 : * to the audio output stream. Returns the number of frames played.
406 : */
407 : StreamTime PlayAudio(MediaStream* aStream);
408 : /**
409 : * No more data will be forthcoming for aStream. The stream will end
410 : * at the current buffer end point. The StreamTracks's tracks must be
411 : * explicitly set to finished by the caller.
412 : */
413 : void OpenAudioInputImpl(int aID,
414 : AudioDataListener *aListener);
415 : virtual nsresult OpenAudioInput(int aID,
416 : AudioDataListener *aListener) override;
417 : void CloseAudioInputImpl(AudioDataListener *aListener);
418 : virtual void CloseAudioInput(AudioDataListener *aListener) override;
419 :
420 : void FinishStream(MediaStream* aStream);
421 : /**
422 : * Compute how much stream data we would like to buffer for aStream.
423 : */
424 : StreamTime GetDesiredBufferEnd(MediaStream* aStream);
425 : /**
426 : * Returns true when there are no active streams.
427 : */
428 0 : bool IsEmpty() const
429 : {
430 0 : return mStreams.IsEmpty() && mSuspendedStreams.IsEmpty() && mPortCount == 0;
431 : }
432 :
433 : /**
434 : * Add aStream to the graph and initializes its graph-specific state.
435 : */
436 : void AddStreamGraphThread(MediaStream* aStream);
437 : /**
438 : * Remove aStream from the graph. Ensures that pending messages about the
439 : * stream back to the main thread are flushed.
440 : */
441 : void RemoveStreamGraphThread(MediaStream* aStream);
442 : /**
443 : * Remove aPort from the graph and release it.
444 : */
445 : void DestroyPort(MediaInputPort* aPort);
446 : /**
447 : * Mark the media stream order as dirty.
448 : */
449 0 : void SetStreamOrderDirty()
450 : {
451 0 : mStreamOrderDirty = true;
452 0 : }
453 :
454 : // Always stereo for now.
455 0 : uint32_t AudioChannelCount() const { return 2; }
456 :
457 0 : double MediaTimeToSeconds(GraphTime aTime) const
458 : {
459 0 : NS_ASSERTION(aTime > -STREAM_TIME_MAX && aTime <= STREAM_TIME_MAX,
460 : "Bad time");
461 0 : return static_cast<double>(aTime)/GraphRate();
462 : }
463 :
464 0 : GraphTime SecondsToMediaTime(double aS) const
465 : {
466 0 : NS_ASSERTION(0 <= aS && aS <= TRACK_TICKS_MAX/TRACK_RATE_MAX,
467 : "Bad seconds");
468 0 : return GraphRate() * aS;
469 : }
470 :
471 0 : GraphTime MillisecondsToMediaTime(int32_t aMS) const
472 : {
473 0 : return RateConvertTicksRoundDown(GraphRate(), 1000, aMS);
474 : }
475 :
476 : /**
477 : * Signal to the graph that the thread has paused indefinitly,
478 : * or resumed.
479 : */
480 : void PausedIndefinitly();
481 : void ResumedFromPaused();
482 :
483 : /**
484 : * Not safe to call off the MediaStreamGraph thread unless monitor is held!
485 : */
486 0 : GraphDriver* CurrentDriver() const
487 : {
488 0 : AssertOnGraphThreadOrNotRunning();
489 0 : return mDriver;
490 : }
491 :
492 : bool RemoveMixerCallback(MixerCallbackReceiver* aReceiver)
493 : {
494 : return mMixer.RemoveCallback(aReceiver);
495 : }
496 :
497 : /**
498 : * Effectively set the new driver, while we are switching.
499 : * It is only safe to call this at the very end of an iteration, when there
500 : * has been a SwitchAtNextIteration call during the iteration. The driver
501 : * should return and pass the control to the new driver shortly after.
502 : * We can also switch from Revive() (on MainThread), in which case the
503 : * monitor is held
504 : */
505 0 : void SetCurrentDriver(GraphDriver* aDriver)
506 : {
507 0 : AssertOnGraphThreadOrNotRunning();
508 0 : mDriver = aDriver;
509 0 : }
510 :
511 0 : Monitor& GetMonitor()
512 : {
513 0 : return mMonitor;
514 : }
515 :
516 0 : void EnsureNextIteration()
517 : {
518 0 : mNeedAnotherIteration = true; // atomic
519 : // Note: GraphDriver must ensure that there's no race on setting
520 : // mNeedAnotherIteration and mGraphDriverAsleep -- see WaitForNextIteration()
521 0 : if (mGraphDriverAsleep) { // atomic
522 0 : MonitorAutoLock mon(mMonitor);
523 0 : CurrentDriver()->WakeUp(); // Might not be the same driver; might have woken already
524 : }
525 0 : }
526 :
527 0 : void EnsureNextIterationLocked()
528 : {
529 0 : mNeedAnotherIteration = true; // atomic
530 0 : if (mGraphDriverAsleep) { // atomic
531 0 : CurrentDriver()->WakeUp(); // Might not be the same driver; might have woken already
532 : }
533 0 : }
534 :
535 : // Capture Stream API. This allows to get a mixed-down output for a window.
536 : void RegisterCaptureStreamForWindow(uint64_t aWindowId,
537 : ProcessedMediaStream* aCaptureStream);
538 : void UnregisterCaptureStreamForWindow(uint64_t aWindowId);
539 : already_AddRefed<MediaInputPort>
540 : ConnectToCaptureStream(uint64_t aWindowId, MediaStream* aMediaStream);
541 :
542 : class StreamSet {
543 : public:
544 : class iterator {
545 : public:
546 0 : explicit iterator(MediaStreamGraphImpl& aGraph)
547 0 : : mGraph(&aGraph), mArrayNum(-1), mArrayIndex(0)
548 : {
549 0 : ++(*this);
550 0 : }
551 0 : iterator() : mGraph(nullptr), mArrayNum(2), mArrayIndex(0) {}
552 0 : MediaStream* operator*()
553 : {
554 0 : return Array()->ElementAt(mArrayIndex);
555 : }
556 0 : iterator operator++()
557 : {
558 0 : ++mArrayIndex;
559 0 : while (mArrayNum < 2 &&
560 0 : (mArrayNum < 0 || mArrayIndex >= Array()->Length())) {
561 0 : ++mArrayNum;
562 0 : mArrayIndex = 0;
563 : }
564 0 : return *this;
565 : }
566 0 : bool operator==(const iterator& aOther) const
567 : {
568 0 : return mArrayNum == aOther.mArrayNum && mArrayIndex == aOther.mArrayIndex;
569 : }
570 0 : bool operator!=(const iterator& aOther) const
571 : {
572 0 : return !(*this == aOther);
573 : }
574 : private:
575 0 : nsTArray<MediaStream*>* Array()
576 : {
577 0 : return mArrayNum == 0 ? &mGraph->mStreams : &mGraph->mSuspendedStreams;
578 : }
579 : MediaStreamGraphImpl* mGraph;
580 : int mArrayNum;
581 : uint32_t mArrayIndex;
582 : };
583 :
584 0 : explicit StreamSet(MediaStreamGraphImpl& aGraph) : mGraph(aGraph) {}
585 0 : iterator begin() { return iterator(mGraph); }
586 0 : iterator end() { return iterator(); }
587 : private:
588 : MediaStreamGraphImpl& mGraph;
589 : };
590 0 : StreamSet AllStreams() { return StreamSet(*this); }
591 :
592 : // Data members
593 : //
594 : /**
595 : * Graphs own owning references to their driver, until shutdown. When a driver
596 : * switch occur, previous driver is either deleted, or it's ownership is
597 : * passed to a event that will take care of the asynchronous cleanup, as
598 : * audio stream can take some time to shut down.
599 : */
600 : RefPtr<GraphDriver> mDriver;
601 :
602 : // The following state is managed on the graph thread only, unless
603 : // mLifecycleState > LIFECYCLE_RUNNING in which case the graph thread
604 : // is not running and this state can be used from the main thread.
605 :
606 : /**
607 : * The graph keeps a reference to each stream.
608 : * References are maintained manually to simplify reordering without
609 : * unnecessary thread-safe refcount changes.
610 : */
611 : nsTArray<MediaStream*> mStreams;
612 : /**
613 : * This stores MediaStreams that are part of suspended AudioContexts.
614 : * mStreams and mSuspendStream are disjoint sets: a stream is either suspended
615 : * or not suspended. Suspended streams are not ordered in UpdateStreamOrder,
616 : * and are therefore not doing any processing.
617 : */
618 : nsTArray<MediaStream*> mSuspendedStreams;
619 : /**
620 : * Suspended AudioContext IDs
621 : */
622 : nsTHashtable<nsUint64HashKey> mSuspendedContexts;
623 : /**
624 : * Streams from mFirstCycleBreaker to the end of mStreams produce output
625 : * before they receive input. They correspond to DelayNodes that are in
626 : * cycles.
627 : */
628 : uint32_t mFirstCycleBreaker;
629 : /**
630 : * Blocking decisions have been computed up to this time.
631 : * Between each iteration, this is the same as mProcessedTime.
632 : */
633 : GraphTime mStateComputedTime = 0;
634 : /**
635 : * All stream contents have been computed up to this time.
636 : * The next batch of updates from the main thread will be processed
637 : * at this time. This is behind mStateComputedTime during processing.
638 : */
639 : GraphTime mProcessedTime = 0;
640 : /**
641 : * Date of the last time we updated the main thread with the graph state.
642 : */
643 : TimeStamp mLastMainThreadUpdate;
644 : /**
645 : * Number of active MediaInputPorts
646 : */
647 : int32_t mPortCount;
648 :
649 : /**
650 : * Devices to use for cubeb input & output, or NULL for no input (void*),
651 : * and boolean to control if we want input/output
652 : */
653 : bool mInputWanted;
654 : int mInputDeviceID;
655 : bool mOutputWanted;
656 : int mOutputDeviceID;
657 : // Maps AudioDataListeners to a usecount of streams using the listener
658 : // so we can know when it's no longer in use.
659 : nsDataHashtable<nsPtrHashKey<AudioDataListener>, uint32_t> mInputDeviceUsers;
660 :
661 : // True if the graph needs another iteration after the current iteration.
662 : Atomic<bool> mNeedAnotherIteration;
663 : // GraphDriver may need a WakeUp() if something changes
664 : Atomic<bool> mGraphDriverAsleep;
665 :
666 : // mMonitor guards the data below.
667 : // MediaStreamGraph normally does its work without holding mMonitor, so it is
668 : // not safe to just grab mMonitor from some thread and start monkeying with
669 : // the graph. Instead, communicate with the graph thread using provided
670 : // mechanisms such as the ControlMessage queue.
671 : Monitor mMonitor;
672 :
673 : // Data guarded by mMonitor (must always be accessed with mMonitor held,
674 : // regardless of the value of mLifecycleState).
675 :
676 : /**
677 : * State to copy to main thread
678 : */
679 : nsTArray<StreamUpdate> mStreamUpdates;
680 : /**
681 : * Runnables to run after the next update to main thread state.
682 : */
683 : nsTArray<nsCOMPtr<nsIRunnable> > mUpdateRunnables;
684 : /**
685 : * A list of batches of messages to process. Each batch is processed
686 : * as an atomic unit.
687 : */
688 : /*
689 : * Message queue processed by the MSG thread during an iteration.
690 : * Accessed on graph thread only.
691 : */
692 : nsTArray<MessageBlock> mFrontMessageQueue;
693 : /*
694 : * Message queue in which the main thread appends messages.
695 : * Access guarded by mMonitor.
696 : */
697 : nsTArray<MessageBlock> mBackMessageQueue;
698 :
699 : /* True if there will messages to process if we swap the message queues. */
700 0 : bool MessagesQueued() const
701 : {
702 0 : mMonitor.AssertCurrentThreadOwns();
703 0 : return !mBackMessageQueue.IsEmpty();
704 : }
705 : /**
706 : * This enum specifies where this graph is in its lifecycle. This is used
707 : * to control shutdown.
708 : * Shutdown is tricky because it can happen in two different ways:
709 : * 1) Shutdown due to inactivity. RunThread() detects that it has no
710 : * pending messages and no streams, and exits. The next RunInStableState()
711 : * checks if there are new pending messages from the main thread (true only
712 : * if new stream creation raced with shutdown); if there are, it revives
713 : * RunThread(), otherwise it commits to shutting down the graph. New stream
714 : * creation after this point will create a new graph. An async event is
715 : * dispatched to Shutdown() the graph's threads and then delete the graph
716 : * object.
717 : * 2) Forced shutdown at application shutdown, or completion of a
718 : * non-realtime graph. A flag is set, RunThread() detects the flag and
719 : * exits, the next RunInStableState() detects the flag, and dispatches the
720 : * async event to Shutdown() the graph's threads. However the graph object
721 : * is not deleted. New messages for the graph are processed synchronously on
722 : * the main thread if necessary. When the last stream is destroyed, the
723 : * graph object is deleted.
724 : *
725 : * This should be kept in sync with the LifecycleState_str array in
726 : * MediaStreamGraph.cpp
727 : */
728 : enum LifecycleState
729 : {
730 : // The graph thread hasn't started yet.
731 : LIFECYCLE_THREAD_NOT_STARTED,
732 : // RunThread() is running normally.
733 : LIFECYCLE_RUNNING,
734 : // In the following states, the graph thread is not running so
735 : // all "graph thread only" state in this class can be used safely
736 : // on the main thread.
737 : // RunThread() has exited and we're waiting for the next
738 : // RunInStableState(), at which point we can clean up the main-thread
739 : // side of the graph.
740 : LIFECYCLE_WAITING_FOR_MAIN_THREAD_CLEANUP,
741 : // RunInStableState() posted a ShutdownRunnable, and we're waiting for it
742 : // to shut down the graph thread(s).
743 : LIFECYCLE_WAITING_FOR_THREAD_SHUTDOWN,
744 : // Graph threads have shut down but we're waiting for remaining streams
745 : // to be destroyed. Only happens during application shutdown and on
746 : // completed non-realtime graphs, since normally we'd only shut down a
747 : // realtime graph when it has no streams.
748 : LIFECYCLE_WAITING_FOR_STREAM_DESTRUCTION
749 : };
750 : /**
751 : * Modified only on the main thread in mMonitor.
752 : */
753 : LifecycleState mLifecycleState;
754 : /**
755 : * The graph should stop processing at or after this time.
756 : */
757 : GraphTime mEndTime;
758 :
759 : /**
760 : * True when we need to do a forced shutdown during application shutdown.
761 : */
762 : bool mForceShutDown;
763 :
764 : /**
765 : * Drop this reference during shutdown to unblock shutdown.
766 : **/
767 : RefPtr<ShutdownTicket> mForceShutdownTicket;
768 :
769 : /**
770 : * True when we have posted an event to the main thread to run
771 : * RunInStableState() and the event hasn't run yet.
772 : */
773 : bool mPostedRunInStableStateEvent;
774 :
775 : // Main thread only
776 :
777 : /**
778 : * Messages posted by the current event loop task. These are forwarded to
779 : * the media graph thread during RunInStableState. We can't forward them
780 : * immediately because we want all messages between stable states to be
781 : * processed as an atomic batch.
782 : */
783 : nsTArray<UniquePtr<ControlMessage>> mCurrentTaskMessageQueue;
784 : /**
785 : * True when RunInStableState has determined that mLifecycleState is >
786 : * LIFECYCLE_RUNNING. Since only the main thread can reset mLifecycleState to
787 : * LIFECYCLE_RUNNING, this can be relied on to not change unexpectedly.
788 : */
789 : bool mDetectedNotRunning;
790 : /**
791 : * True when a stable state runner has been posted to the appshell to run
792 : * RunInStableState at the next stable state.
793 : */
794 : bool mPostedRunInStableState;
795 : /**
796 : * True when processing real-time audio/video. False when processing non-realtime
797 : * audio.
798 : */
799 : bool mRealtime;
800 : /**
801 : * True when a non-realtime MediaStreamGraph has started to process input. This
802 : * value is only accessed on the main thread.
803 : */
804 : bool mNonRealtimeProcessing;
805 : /**
806 : * True when a change has happened which requires us to recompute the stream
807 : * blocking order.
808 : */
809 : bool mStreamOrderDirty;
810 : /**
811 : * Hold a ref to the Latency logger
812 : */
813 : RefPtr<AsyncLatencyLogger> mLatencyLog;
814 : AudioMixer mMixer;
815 : const RefPtr<AbstractThread> mAbstractMainThread;
816 : #ifdef MOZ_WEBRTC
817 : RefPtr<AudioOutputObserver> mFarendObserverRef;
818 : #endif
819 :
820 0 : dom::AudioChannel AudioChannel() const { return mAudioChannel; }
821 :
822 : // used to limit graph shutdown time
823 : nsCOMPtr<nsITimer> mShutdownTimer;
824 :
825 : private:
826 : virtual ~MediaStreamGraphImpl();
827 :
828 0 : MOZ_DEFINE_MALLOC_SIZE_OF(MallocSizeOf)
829 :
830 : /**
831 : * This class uses manual memory management, and all pointers to it are raw
832 : * pointers. However, in order for it to implement nsIMemoryReporter, it needs
833 : * to implement nsISupports and so be ref-counted. So it maintains a single
834 : * nsRefPtr to itself, giving it a ref-count of 1 during its entire lifetime,
835 : * and Destroy() nulls this self-reference in order to trigger self-deletion.
836 : */
837 : RefPtr<MediaStreamGraphImpl> mSelfRef;
838 :
839 0 : struct WindowAndStream
840 : {
841 : uint64_t mWindowId;
842 : RefPtr<ProcessedMediaStream> mCaptureStreamSink;
843 : };
844 : /**
845 : * Stream for window audio capture.
846 : */
847 : nsTArray<WindowAndStream> mWindowCaptureStreams;
848 :
849 : #ifdef DEBUG
850 : /**
851 : * Used to assert when AppendMessage() runs ControlMessages synchronously.
852 : */
853 : bool mCanRunMessagesSynchronously;
854 : #endif
855 :
856 : dom::AudioChannel mAudioChannel;
857 : };
858 :
859 : } // namespace mozilla
860 :
861 : #endif /* MEDIASTREAMGRAPHIMPL_H_ */
|
