Line data Source code
1 : /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 : /* This Source Code Form is subject to the terms of the Mozilla Public
3 : * License, v. 2.0. If a copy of the MPL was not distributed with this file,
4 : * You can obtain one at http://mozilla.org/MPL/2.0/. */
5 :
6 : #ifndef GRAPHDRIVER_H_
7 : #define GRAPHDRIVER_H_
8 :
9 : #include "nsAutoRef.h"
10 : #include "AudioBufferUtils.h"
11 : #include "AudioMixer.h"
12 : #include "AudioSegment.h"
13 : #include "SelfRef.h"
14 : #include "mozilla/Atomics.h"
15 : #include "mozilla/SharedThreadPool.h"
16 : #include "mozilla/StaticPtr.h"
17 :
18 : struct cubeb_stream;
19 :
20 : template <>
21 0 : class nsAutoRefTraits<cubeb_stream> : public nsPointerRefTraits<cubeb_stream>
22 : {
23 : public:
24 0 : static void Release(cubeb_stream* aStream) { cubeb_stream_destroy(aStream); }
25 : };
26 :
27 : namespace mozilla {
28 :
29 : /**
30 : * Assume we can run an iteration of the MediaStreamGraph loop in this much time
31 : * or less.
32 : * We try to run the control loop at this rate.
33 : */
34 : static const int MEDIA_GRAPH_TARGET_PERIOD_MS = 10;
35 :
36 : /**
37 : * Assume that we might miss our scheduled wakeup of the MediaStreamGraph by
38 : * this much.
39 : */
40 : static const int SCHEDULE_SAFETY_MARGIN_MS = 10;
41 :
42 : /**
43 : * Try have this much audio buffered in streams and queued to the hardware.
44 : * The maximum delay to the end of the next control loop
45 : * is 2*MEDIA_GRAPH_TARGET_PERIOD_MS + SCHEDULE_SAFETY_MARGIN_MS.
46 : * There is no point in buffering more audio than this in a stream at any
47 : * given time (until we add processing).
48 : * This is not optimal yet.
49 : */
50 : static const int AUDIO_TARGET_MS = 2*MEDIA_GRAPH_TARGET_PERIOD_MS +
51 : SCHEDULE_SAFETY_MARGIN_MS;
52 :
53 : class MediaStreamGraphImpl;
54 :
55 : class AudioCallbackDriver;
56 : class OfflineClockDriver;
57 : class SystemClockDriver;
58 :
59 : /**
60 : * A driver is responsible for the scheduling of the processing, the thread
61 : * management, and give the different clocks to a MediaStreamGraph. This is an
62 : * abstract base class. A MediaStreamGraph can be driven by an
63 : * OfflineClockDriver, if the graph is offline, or a SystemClockDriver, if the
64 : * graph is real time.
65 : * A MediaStreamGraph holds an owning reference to its driver.
66 : *
67 : * The lifetime of drivers is a complicated affair. Here are the different
68 : * scenarii that can happen:
69 : *
70 : * Starting a MediaStreamGraph with an AudioCallbackDriver
71 : * - A new thread T is created, from the main thread.
72 : * - On this thread T, cubeb is initialized if needed, and a cubeb_stream is
73 : * created and started
74 : * - The thread T posts a message to the main thread to terminate itself.
75 : * - The graph runs off the audio thread
76 : *
77 : * Starting a MediaStreamGraph with a SystemClockDriver:
78 : * - A new thread T is created from the main thread.
79 : * - The graph runs off this thread.
80 : *
81 : * Switching from a SystemClockDriver to an AudioCallbackDriver:
82 : * - A new AudioCallabackDriver is created and initialized on the graph thread
83 : * - At the end of the MSG iteration, the SystemClockDriver transfers its timing
84 : * info and a reference to itself to the AudioCallbackDriver. It then starts
85 : * the AudioCallbackDriver.
86 : * - When the AudioCallbackDriver starts, it checks if it has been switched from
87 : * a SystemClockDriver, and if that is the case, sends a message to the main
88 : * thread to shut the SystemClockDriver thread down.
89 : * - The graph now runs off an audio callback
90 : *
91 : * Switching from an AudioCallbackDriver to a SystemClockDriver:
92 : * - A new SystemClockDriver is created, and set as mNextDriver.
93 : * - At the end of the MSG iteration, the AudioCallbackDriver transfers its
94 : * timing info and a reference to itself to the SystemClockDriver. A new
95 : * SystemClockDriver is started from the current audio thread.
96 : * - When starting, the SystemClockDriver checks if it has been switched from an
97 : * AudioCallbackDriver. If yes, it creates a new temporary thread to release
98 : * the cubeb_streams. This temporary thread closes the cubeb_stream, and
99 : * then dispatches a message to the main thread to be terminated.
100 : * - The graph now runs off a normal thread.
101 : *
102 : * Two drivers cannot run at the same time for the same graph. The thread safety
103 : * of the different attributes of drivers, and they access pattern is documented
104 : * next to the members themselves.
105 : *
106 : */
107 : class GraphDriver
108 : {
109 : public:
110 : explicit GraphDriver(MediaStreamGraphImpl* aGraphImpl);
111 :
112 0 : NS_INLINE_DECL_THREADSAFE_REFCOUNTING(GraphDriver);
113 : /* For real-time graphs, this waits until it's time to process more data. For
114 : * offline graphs, this is a no-op. */
115 : virtual void WaitForNextIteration() = 0;
116 : /* Wakes up the graph if it is waiting. */
117 : virtual void WakeUp() = 0;
118 0 : virtual void Destroy() {}
119 : /* Start the graph, init the driver, start the thread. */
120 : virtual void Start() = 0;
121 : /* Stop the graph, shutting down the thread. */
122 : virtual void Stop() = 0;
123 : /* Resume after a stop */
124 : virtual void Resume() = 0;
125 : /* Revive this driver, as more messages just arrived. */
126 : virtual void Revive() = 0;
127 : /* Remove Mixer callbacks when switching */
128 : virtual void RemoveCallback() = 0;
129 : /* Shutdown GraphDriver (synchronously) */
130 : void Shutdown();
131 : /* Rate at which the GraphDriver runs, in ms. This can either be user
132 : * controlled (because we are using a {System,Offline}ClockDriver, and decide
133 : * how often we want to wakeup/how much we want to process per iteration), or
134 : * it can be indirectly set by the latency of the audio backend, and the
135 : * number of buffers of this audio backend: say we have four buffers, and 40ms
136 : * latency, we will get a callback approximately every 10ms. */
137 : virtual uint32_t IterationDuration() = 0;
138 :
139 : /* Return whether we are switching or not. */
140 : bool Switching();
141 :
142 : // Those are simply or setting the associated pointer, but assert that the
143 : // lock is held.
144 : GraphDriver* NextDriver();
145 : GraphDriver* PreviousDriver();
146 : void SetNextDriver(GraphDriver* aNextDriver);
147 : void SetPreviousDriver(GraphDriver* aPreviousDriver);
148 :
149 : /**
150 : * If we are running a real time graph, get the current time stamp to schedule
151 : * video frames. This has to be reimplemented by real time drivers.
152 : */
153 0 : virtual TimeStamp GetCurrentTimeStamp() {
154 0 : return mCurrentTimeStamp;
155 : }
156 :
157 0 : GraphTime IterationEnd() {
158 0 : return mIterationEnd;
159 : }
160 :
161 0 : virtual AudioCallbackDriver* AsAudioCallbackDriver() {
162 0 : return nullptr;
163 : }
164 :
165 0 : virtual OfflineClockDriver* AsOfflineClockDriver() {
166 0 : return nullptr;
167 : }
168 :
169 0 : virtual SystemClockDriver* AsSystemClockDriver() {
170 0 : return nullptr;
171 : }
172 :
173 : /**
174 : * Tell the driver it has to stop and return the current time of the graph, so
175 : * another driver can start from the right point in time.
176 : */
177 : virtual void SwitchAtNextIteration(GraphDriver* aDriver);
178 :
179 : /**
180 : * Set the time for a graph, on a driver. This is used so a new driver just
181 : * created can start at the right point in time.
182 : */
183 : void SetGraphTime(GraphDriver* aPreviousDriver,
184 : GraphTime aLastSwitchNextIterationStart,
185 : GraphTime aLastSwitchNextIterationEnd);
186 : /**
187 : * Call this to indicate that another iteration of the control loop is
188 : * required on its regular schedule. The monitor must not be held.
189 : * This function has to be idempotent.
190 : */
191 : void EnsureNextIteration();
192 :
193 : /**
194 : * Same thing, but not locked.
195 : */
196 : void EnsureNextIterationLocked();
197 :
198 0 : MediaStreamGraphImpl* GraphImpl() {
199 0 : return mGraphImpl;
200 : }
201 :
202 : virtual bool OnThread() = 0;
203 :
204 : protected:
205 : GraphTime StateComputedTime() const;
206 :
207 : // Time of the start of this graph iteration. This must be accessed while
208 : // having the monitor.
209 : GraphTime mIterationStart;
210 : // Time of the end of this graph iteration. This must be accessed while having
211 : // the monitor.
212 : GraphTime mIterationEnd;
213 : // The MediaStreamGraphImpl that owns this driver. This has a lifetime longer
214 : // than the driver, and will never be null. Hence, it can be accesed without
215 : // monitor.
216 : MediaStreamGraphImpl* mGraphImpl;
217 :
218 : // This enum specifies the wait state of the driver.
219 : enum WaitState {
220 : // RunThread() is running normally
221 : WAITSTATE_RUNNING,
222 : // RunThread() is paused waiting for its next iteration, which will
223 : // happen soon
224 : WAITSTATE_WAITING_FOR_NEXT_ITERATION,
225 : // RunThread() is paused indefinitely waiting for something to change
226 : WAITSTATE_WAITING_INDEFINITELY,
227 : // Something has signaled RunThread() to wake up immediately,
228 : // but it hasn't done so yet
229 : WAITSTATE_WAKING_UP
230 : };
231 : // This must be access with the monitor.
232 : WaitState mWaitState;
233 :
234 : // This is used on the main thread (during initialization), and the graph
235 : // thread. No monitor needed because we know the graph thread does not run
236 : // during the initialization.
237 : TimeStamp mCurrentTimeStamp;
238 : // This is non-null only when this driver has recently switched from an other
239 : // driver, and has not cleaned it up yet (for example because the audio stream
240 : // is currently calling the callback during initialization).
241 : //
242 : // This is written to when changing driver, from the previous driver's thread,
243 : // or a thread created for the occasion. This is read each time we need to
244 : // check whether we're changing driver (in Switching()), from the graph
245 : // thread.
246 : // This must be accessed using the {Set,Get}PreviousDriver methods.
247 : RefPtr<GraphDriver> mPreviousDriver;
248 : // This is non-null only when this driver is going to switch to an other
249 : // driver at the end of this iteration.
250 : // This must be accessed using the {Set,Get}NextDriver methods.
251 : RefPtr<GraphDriver> mNextDriver;
252 0 : virtual ~GraphDriver()
253 0 : { }
254 : };
255 :
256 : class MediaStreamGraphInitThreadRunnable;
257 :
258 : /**
259 : * This class is a driver that manages its own thread.
260 : */
261 : class ThreadedDriver : public GraphDriver
262 : {
263 : public:
264 : explicit ThreadedDriver(MediaStreamGraphImpl* aGraphImpl);
265 : virtual ~ThreadedDriver();
266 : void Start() override;
267 : void Stop() override;
268 : void Resume() override;
269 : void Revive() override;
270 : void RemoveCallback() override;
271 : /**
272 : * Runs main control loop on the graph thread. Normally a single invocation
273 : * of this runs for the entire lifetime of the graph thread.
274 : */
275 : void RunThread();
276 : friend class MediaStreamGraphInitThreadRunnable;
277 0 : uint32_t IterationDuration() override {
278 0 : return MEDIA_GRAPH_TARGET_PERIOD_MS;
279 : }
280 :
281 0 : bool OnThread() override { return !mThread || mThread->EventTarget()->IsOnCurrentThread(); }
282 :
283 : /* When the graph wakes up to do an iteration, implementations return the
284 : * range of time that will be processed. This is called only once per
285 : * iteration; it may determine the interval from state in a previous
286 : * call. */
287 : virtual MediaTime GetIntervalForIteration() = 0;
288 : protected:
289 : nsCOMPtr<nsIThread> mThread;
290 : };
291 :
292 : /**
293 : * A SystemClockDriver drives a MediaStreamGraph using a system clock, and waits
294 : * using a monitor, between each iteration.
295 : */
296 : class SystemClockDriver : public ThreadedDriver
297 : {
298 : public:
299 : explicit SystemClockDriver(MediaStreamGraphImpl* aGraphImpl);
300 : virtual ~SystemClockDriver();
301 : MediaTime GetIntervalForIteration() override;
302 : void WaitForNextIteration() override;
303 : void WakeUp() override;
304 : void MarkAsFallback();
305 : bool IsFallback();
306 0 : SystemClockDriver* AsSystemClockDriver() override {
307 0 : return this;
308 : }
309 :
310 : private:
311 : // Those are only modified (after initialization) on the graph thread. The
312 : // graph thread does not run during the initialization.
313 : TimeStamp mInitialTimeStamp;
314 : TimeStamp mLastTimeStamp;
315 : // This is true if this SystemClockDriver runs the graph because we could not
316 : // open an audio stream.
317 : bool mIsFallback;
318 : };
319 :
320 : /**
321 : * An OfflineClockDriver runs the graph as fast as possible, without waiting
322 : * between iteration.
323 : */
324 : class OfflineClockDriver : public ThreadedDriver
325 : {
326 : public:
327 : OfflineClockDriver(MediaStreamGraphImpl* aGraphImpl, GraphTime aSlice);
328 : virtual ~OfflineClockDriver();
329 : MediaTime GetIntervalForIteration() override;
330 : void WaitForNextIteration() override;
331 : void WakeUp() override;
332 : TimeStamp GetCurrentTimeStamp() override;
333 0 : OfflineClockDriver* AsOfflineClockDriver() override {
334 0 : return this;
335 : }
336 :
337 : private:
338 : // Time, in GraphTime, for each iteration
339 : GraphTime mSlice;
340 : };
341 :
342 0 : struct StreamAndPromiseForOperation
343 : {
344 : StreamAndPromiseForOperation(MediaStream* aStream,
345 : void* aPromise,
346 : dom::AudioContextOperation aOperation);
347 : RefPtr<MediaStream> mStream;
348 : void* mPromise;
349 : dom::AudioContextOperation mOperation;
350 : };
351 :
352 : enum AsyncCubebOperation {
353 : INIT,
354 : SHUTDOWN
355 : };
356 :
357 : /**
358 : * This is a graph driver that is based on callback functions called by the
359 : * audio api. This ensures minimal audio latency, because it means there is no
360 : * buffering happening: the audio is generated inside the callback.
361 : *
362 : * This design is less flexible than running our own thread:
363 : * - We have no control over the thread:
364 : * - It cannot block, and it has to run for a shorter amount of time than the
365 : * buffer it is going to fill, or an under-run is going to occur (short burst
366 : * of silence in the final audio output).
367 : * - We can't know for sure when the callback function is going to be called
368 : * (although we compute an estimation so we can schedule video frames)
369 : * - Creating and shutting the thread down is a blocking operation, that can
370 : * take _seconds_ in some cases (because IPC has to be set up, and
371 : * sometimes hardware components are involved and need to be warmed up)
372 : * - We have no control on how much audio we generate, we have to return exactly
373 : * the number of frames asked for by the callback. Since for the Web Audio
374 : * API, we have to do block processing at 128 frames per block, we need to
375 : * keep a little spill buffer to store the extra frames.
376 : */
377 : class AudioCallbackDriver : public GraphDriver,
378 : public MixerCallbackReceiver
379 : {
380 : public:
381 : explicit AudioCallbackDriver(MediaStreamGraphImpl* aGraphImpl);
382 : virtual ~AudioCallbackDriver();
383 :
384 : void Destroy() override;
385 : void Start() override;
386 : void Stop() override;
387 : void Resume() override;
388 : void Revive() override;
389 : void RemoveCallback() override;
390 : void WaitForNextIteration() override;
391 : void WakeUp() override;
392 :
393 : /* Static wrapper function cubeb calls back. */
394 : static long DataCallback_s(cubeb_stream * aStream,
395 : void * aUser,
396 : const void * aInputBuffer,
397 : void * aOutputBuffer,
398 : long aFrames);
399 : static void StateCallback_s(cubeb_stream* aStream, void * aUser,
400 : cubeb_state aState);
401 : static void DeviceChangedCallback_s(void * aUser);
402 : /* This function is called by the underlying audio backend when a refill is
403 : * needed. This is what drives the whole graph when it is used to output
404 : * audio. If the return value is exactly aFrames, this function will get
405 : * called again. If it is less than aFrames, the stream will go in draining
406 : * mode, and this function will not be called again. */
407 : long DataCallback(const AudioDataValue* aInputBuffer, AudioDataValue* aOutputBuffer, long aFrames);
408 : /* This function is called by the underlying audio backend, but is only used
409 : * for informational purposes at the moment. */
410 : void StateCallback(cubeb_state aState);
411 : /* This is an approximation of the number of millisecond there are between two
412 : * iterations of the graph. */
413 : uint32_t IterationDuration() override;
414 :
415 : /* This function gets called when the graph has produced the audio frames for
416 : * this iteration. */
417 : void MixerCallback(AudioDataValue* aMixedBuffer,
418 : AudioSampleFormat aFormat,
419 : uint32_t aChannels,
420 : uint32_t aFrames,
421 : uint32_t aSampleRate) override;
422 :
423 : // These are invoked on the MSG thread (we don't call this if not LIFECYCLE_RUNNING)
424 0 : virtual void SetInputListener(AudioDataListener *aListener) {
425 0 : MOZ_ASSERT(OnThread());
426 0 : mAudioInput = aListener;
427 0 : }
428 : // XXX do we need the param? probably no
429 0 : virtual void RemoveInputListener(AudioDataListener *aListener) {
430 0 : MOZ_ASSERT(OnThread());
431 0 : mAudioInput = nullptr;
432 0 : }
433 :
434 0 : AudioCallbackDriver* AsAudioCallbackDriver() override {
435 0 : return this;
436 : }
437 :
438 : /* Enqueue a promise that is going to be resolved when a specific operation
439 : * occurs on the cubeb stream. */
440 : void EnqueueStreamAndPromiseForOperation(MediaStream* aStream,
441 : void* aPromise,
442 : dom::AudioContextOperation aOperation);
443 :
444 : /**
445 : * Whether the audio callback is processing. This is for asserting only.
446 : */
447 : bool InCallback();
448 :
449 0 : bool OnThread() override { return !mStarted || InCallback(); }
450 :
451 : /* Whether the underlying cubeb stream has been started. See comment for
452 : * mStarted for details. */
453 : bool IsStarted();
454 :
455 : /* Tell the driver whether this process is using a microphone or not. This is
456 : * thread safe. */
457 : void SetMicrophoneActive(bool aActive);
458 :
459 : void CompleteAudioContextOperations(AsyncCubebOperation aOperation);
460 : private:
461 : /**
462 : * On certain MacBookPro, the microphone is located near the left speaker.
463 : * We need to pan the sound output to the right speaker if we are using the
464 : * mic and the built-in speaker, or we will have terrible echo. */
465 : void PanOutputIfNeeded(bool aMicrophoneActive);
466 : /**
467 : * This is called when the output device used by the cubeb stream changes. */
468 : void DeviceChangedCallback();
469 : /* Start the cubeb stream */
470 : bool StartStream();
471 : friend class AsyncCubebTask;
472 : bool Init();
473 : /* MediaStreamGraphs are always down/up mixed to stereo for now. */
474 : static const uint32_t ChannelCount = 2;
475 : /* The size of this buffer comes from the fact that some audio backends can
476 : * call back with a number of frames lower than one block (128 frames), so we
477 : * need to keep at most two block in the SpillBuffer, because we always round
478 : * up to block boundaries during an iteration.
479 : * This is only ever accessed on the audio callback thread. */
480 : SpillBuffer<AudioDataValue, WEBAUDIO_BLOCK_SIZE * 2, ChannelCount> mScratchBuffer;
481 : /* Wrapper to ensure we write exactly the number of frames we need in the
482 : * audio buffer cubeb passes us. This is only ever accessed on the audio
483 : * callback thread. */
484 : AudioCallbackBufferWrapper<AudioDataValue, ChannelCount> mBuffer;
485 : /* cubeb stream for this graph. This is guaranteed to be non-null after Init()
486 : * has been called, and is synchronized internaly. */
487 : nsAutoRef<cubeb_stream> mAudioStream;
488 : /* The sample rate for the aforementionned cubeb stream. This is set on
489 : * initialization and can be read safely afterwards. */
490 : uint32_t mSampleRate;
491 : /* The number of input channels from cubeb. Should be set before opening cubeb
492 : * and then be static. */
493 : uint32_t mInputChannels;
494 : /* Approximation of the time between two callbacks. This is used to schedule
495 : * video frames. This is in milliseconds. Only even used (after
496 : * inizatialization) on the audio callback thread. */
497 : uint32_t mIterationDurationMS;
498 : /* cubeb_stream_init calls the audio callback to prefill the buffers. The
499 : * previous driver has to be kept alive until the audio stream has been
500 : * started, because it is responsible to call cubeb_stream_start, so we delay
501 : * the cleanup of the previous driver until it has started the audio stream.
502 : * Otherwise, there is a race where we kill the previous driver thread
503 : * between cubeb_stream_init and cubeb_stream_start,
504 : * and callbacks after the prefill never get called.
505 : * This is written on the previous driver's thread (if switching) or main
506 : * thread (if this driver is the first one).
507 : * This is read on previous driver's thread (during callbacks from
508 : * cubeb_stream_init) and the audio thread (when switching away from this
509 : * driver back to a SystemClockDriver).
510 : * This is synchronized by the Graph's monitor.
511 : * */
512 : bool mStarted;
513 : /* Listener for mic input, if any. */
514 : RefPtr<AudioDataListener> mAudioInput;
515 :
516 : struct AutoInCallback
517 : {
518 : explicit AutoInCallback(AudioCallbackDriver* aDriver);
519 : ~AutoInCallback();
520 : AudioCallbackDriver* mDriver;
521 : };
522 :
523 : /* Thread for off-main-thread initialization and
524 : * shutdown of the audio stream. */
525 : nsCOMPtr<nsIThread> mInitShutdownThread;
526 : /* This must be accessed with the graph monitor held. */
527 : AutoTArray<StreamAndPromiseForOperation, 1> mPromisesForOperation;
528 : /* This is set during initialization, and can be read safely afterwards. */
529 : dom::AudioChannel mAudioChannel;
530 : /* Used to queue us to add the mixer callback on first run. */
531 : bool mAddedMixer;
532 :
533 : /* This is atomic and is set by the audio callback thread. It can be read by
534 : * any thread safely. */
535 : Atomic<bool> mInCallback;
536 : /**
537 : * True if microphone is being used by this process. This is synchronized by
538 : * the graph's monitor. */
539 : Atomic<bool> mMicrophoneActive;
540 : /* True if this driver was created from a driver created because of a previous
541 : * AudioCallbackDriver failure. */
542 : bool mFromFallback;
543 : };
544 :
545 : class AsyncCubebTask : public Runnable
546 : {
547 : public:
548 :
549 : AsyncCubebTask(AudioCallbackDriver* aDriver, AsyncCubebOperation aOperation);
550 :
551 0 : nsresult Dispatch(uint32_t aFlags = NS_DISPATCH_NORMAL)
552 : {
553 0 : nsresult rv = EnsureThread();
554 0 : if (!NS_FAILED(rv)) {
555 0 : rv = sThreadPool->Dispatch(this, aFlags);
556 : }
557 0 : return rv;
558 : }
559 :
560 : protected:
561 : virtual ~AsyncCubebTask();
562 :
563 : private:
564 : static nsresult EnsureThread();
565 :
566 : NS_IMETHOD Run() override final;
567 : static StaticRefPtr<nsIThreadPool> sThreadPool;
568 : RefPtr<AudioCallbackDriver> mDriver;
569 : AsyncCubebOperation mOperation;
570 : RefPtr<MediaStreamGraphImpl> mShutdownGrip;
571 : };
572 :
573 : } // namespace mozilla
574 :
575 : #endif // GRAPHDRIVER_H_
|
